Mercurial > piecrust2

comparison tests/mockutil.py @ 411:e7b865f8f335

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Enable multiprocess baking. Baking is now done by running a worker per CPU, and sending jobs to them. This changes several things across the codebase: * Ability to not cache things related to pages other than the 'main' page (i.e. the page at the bottom of the execution stack). * Decouple the baking process from the bake records, so only the main process keeps track (and modifies) the bake record. * Remove the need for 'batch page getters' and loading a page directly from the page factories. There are various smaller changes too included here, including support for scope performance timers that are saved with the bake record and can be printed out to the console. Yes I got carried away. For testing, the in-memory 'mock' file-system doesn't work anymore, since we're spawning processes, so this is replaced by a 'tmpfs' file-system which is saved in temporary files on disk and deleted after tests have run.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 Jun 2015 17:09:19 -0700
parents 65db6df28120
children fd694f1297c7
comparison
equal deleted inserted replaced
410:d1a472464e57 411:e7b865f8f335
1 import io
2 import time
3 import errno
4 import random
5 import codecs
6 import shutil
7 import os.path 1 import os.path
8 import mock 2 import mock
9 import yaml
10 from piecrust.app import PieCrust, PieCrustConfiguration 3 from piecrust.app import PieCrust, PieCrustConfiguration
11 from piecrust.page import Page 4 from piecrust.page import Page
12 from piecrust.rendering import QualifiedPage, PageRenderingContext, render_page 5 from piecrust.rendering import QualifiedPage, PageRenderingContext, render_page
13
14
15 resources_path = os.path.abspath(
16 os.path.join(
17 os.path.dirname(__file__),
18 '..', 'piecrust', 'resources'))
19 6
20 7
21 def get_mock_app(config=None): 8 def get_mock_app(config=None):
22 app = mock.MagicMock(spec=PieCrust) 9 app = mock.MagicMock(spec=PieCrust)
23 app.config = PieCrustConfiguration() 10 app.config = PieCrustConfiguration()
35 ctx = PageRenderingContext(qp) 22 ctx = PageRenderingContext(qp)
36 rp = render_page(ctx) 23 rp = render_page(ctx)
37 return rp.content 24 return rp.content
38 25
39 26
40 class _MockFsEntry(object): 27 from .tmpfs import (
41 def __init__(self, contents): 28 TempDirFileSystem as mock_fs,
42 self.contents = contents 29 TempDirScope as mock_fs_scope)
43 self.metadata = {'mtime': time.time()}
44 30
45
46 class _MockFsEntryWriter(object):
47 def __init__(self, entry, mode='rt'):
48 self._entry = entry
49 self._mode = mode
50
51 if 'b' in mode:
52 data = entry.contents
53 if isinstance(data, str):
54 data = data.encode('utf8')
55 self._stream = io.BytesIO(data)
56 else:
57 self._stream = io.StringIO(entry.contents)
58
59 def __getattr__(self, name):
60 return getattr(self._stream, name)
61
62 def __enter__(self):
63 return self
64
65 def __exit__(self, exc_type, exc_value, exc_tb):
66 if 'w' in self._mode:
67 if 'a' in self._mode:
68 self._entry.contents += self._stream.getvalue()
69 else:
70 self._entry.contents = self._stream.getvalue()
71 self._entry.metadata['mtime'] = time.time()
72 self._stream.close()
73
74
75 class mock_fs(object):
76 def __init__(self, default_spec=True):
77 self._root = 'root_%d' % random.randrange(1000)
78 self._fs = {self._root: {}}
79 if default_spec:
80 self.withDir('counter')
81 self.withFile('kitchen/config.yml',
82 "site:\n title: Mock Website\n")
83
84 def path(self, p):
85 p = p.replace('\\', '/')
86 if p in ['/', '', None]:
87 return '/%s' % self._root
88 return '/%s/%s' % (self._root, p.lstrip('/'))
89
90 def getApp(self, cache=True):
91 root_dir = self.path('/kitchen')
92 return PieCrust(root_dir, cache=cache, debug=True)
93
94 def withDir(self, path):
95 path = path.replace('\\', '/')
96 path = path.lstrip('/')
97 path = '/%s/%s' % (self._root, path)
98 self._createDir(path)
99 return self
100
101 def withFile(self, path, contents):
102 path = path.replace('\\', '/')
103 path = path.lstrip('/')
104 path = '/%s/%s' % (self._root, path)
105 self._createFile(path, contents)
106 return self
107
108 def withAsset(self, path, contents):
109 return self.withFile('kitchen/' + path, contents)
110
111 def withAssetDir(self, path):
112 return self.withDir('kitchen/' + path)
113
114 def withConfig(self, config):
115 return self.withFile(
116 'kitchen/config.yml',
117 yaml.dump(config))
118
119 def withThemeConfig(self, config):
120 return self.withFile(
121 'kitchen/theme/theme_config.yml',
122 yaml.dump(config))
123
124 def withPage(self, url, config=None, contents=None):
125 config = config or {}
126 contents = contents or "A test page."
127 text = "---\n"
128 text += yaml.dump(config)
129 text += "---\n"
130 text += contents
131
132 name, ext = os.path.splitext(url)
133 if not ext:
134 url += '.md'
135 url = url.lstrip('/')
136 return self.withAsset(url, text)
137
138 def withPageAsset(self, page_url, name, contents=None):
139 contents = contents or "A test asset."
140 url_base, ext = os.path.splitext(page_url)
141 dirname = url_base + '-assets'
142 return self.withAsset('%s/%s' % (dirname, name),
143 contents)
144
145 def withPages(self, num, url_factory, config_factory=None,
146 contents_factory=None):
147 for i in range(num):
148 if isinstance(url_factory, str):
149 url = url_factory.format(idx=i, idx1=(i + 1))
150 else:
151 url = url_factory(i)
152
153 config = None
154 if config_factory:
155 config = config_factory(i)
156
157 contents = None
158 if contents_factory:
159 contents = contents_factory(i)
160
161 self.withPage(url, config, contents)
162 return self
163
164 def getStructure(self, path=None):
165 root = self._fs[self._root]
166 if path:
167 root = self._getEntry(self.path(path))
168 if root is None:
169 raise Exception("No such path: %s" % path)
170 if not isinstance(root, dict):
171 raise Exception("Path is not a directory: %s" % path)
172
173 res = {}
174 for k, v in root.items():
175 self._getStructureRecursive(v, res, k)
176 return res
177
178 def getFileEntry(self, path):
179 entry = self._getEntry(self.path(path))
180 if entry is None:
181 raise Exception("No such file: %s" % path)
182 if not isinstance(entry, _MockFsEntry):
183 raise Exception("Path is not a file: %s" % path)
184 return entry.contents
185
186 def _getStructureRecursive(self, src, target, name):
187 if isinstance(src, _MockFsEntry):
188 target[name] = src.contents
189 return
190
191 e = {}
192 for k, v in src.items():
193 self._getStructureRecursive(v, e, k)
194 target[name] = e
195
196 def _getEntry(self, path):
197 cur = self._fs
198 path = path.replace('\\', '/').lstrip('/')
199 bits = path.split('/')
200 for p in bits:
201 try:
202 cur = cur[p]
203 except KeyError:
204 return None
205 return cur
206
207 def _createDir(self, path):
208 cur = self._fs
209 path = path.replace('\\', '/').strip('/')
210 bits = path.split('/')
211 for b in bits:
212 if b not in cur:
213 cur[b] = {}
214 cur = cur[b]
215 return self
216
217 def _createFile(self, path, contents):
218 cur = self._fs
219 path = path.replace('\\', '/').lstrip('/')
220 bits = path.split('/')
221 for b in bits[:-1]:
222 if b not in cur:
223 cur[b] = {}
224 cur = cur[b]
225 cur[bits[-1]] = _MockFsEntry(contents)
226 return self
227
228 def _deleteEntry(self, path):
229 parent = self._getEntry(os.path.dirname(path))
230 assert parent is not None
231 name = os.path.basename(path)
232 assert name in parent
233 del parent[name]
234
235
236 class mock_fs_scope(object):
237 def __init__(self, fs, open_patches=None):
238 self.open_patches = open_patches or []
239 self._fs = fs
240 self._patchers = []
241 self._originals = {}
242
243 @property
244 def root(self):
245 return self._fs._root
246
247 def __enter__(self):
248 self._startMock()
249 return self
250
251 def __exit__(self, type, value, traceback):
252 self._endMock()
253
254 def _startMock(self):
255 # TODO: sadly, there seems to be no way to replace `open` everywhere?
256 modules = self.open_patches + [
257 '__main__',
258 'piecrust.records',
259 'jinja2.utils']
260 for m in modules:
261 self._createMock('%s.open' % m, open, self._open, create=True)
262
263 self._createMock('codecs.open', codecs.open, self._codecsOpen)
264 self._createMock('os.listdir', os.listdir, self._listdir)
265 self._createMock('os.makedirs', os.makedirs, self._makedirs)
266 self._createMock('os.remove', os.remove, self._remove)
267 self._createMock('os.rename', os.rename, self._rename)
268 self._createMock('os.path.exists', os.path.exists, self._exists)
269 self._createMock('os.path.isdir', os.path.isdir, self._isdir)
270 self._createMock('os.path.isfile', os.path.isfile, self._isfile)
271 self._createMock('os.path.islink', os.path.islink, self._islink)
272 self._createMock('os.path.getmtime', os.path.getmtime, self._getmtime)
273 self._createMock('shutil.copyfile', shutil.copyfile, self._copyfile)
274 self._createMock('shutil.rmtree', shutil.rmtree, self._rmtree)
275 for p in self._patchers:
276 p.start()
277
278 def _endMock(self):
279 for p in self._patchers:
280 p.stop()
281
282 def _createMock(self, name, orig, func, **kwargs):
283 self._originals[name] = orig
284 self._patchers.append(mock.patch(name, func, **kwargs))
285
286 def _doOpen(self, orig_name, path, mode, *args, **kwargs):
287 path = os.path.normpath(path)
288 if path.startswith(resources_path):
289 return self._originals[orig_name](path, mode, *args, **kwargs)
290
291 if 'r' in mode:
292 e = self._getFsEntry(path)
293 elif 'w' in mode or 'x' in mode or 'a' in mode:
294 e = self._getFsEntry(path)
295 if e is None:
296 contents = ''
297 if 'b' in mode:
298 contents = bytes()
299 self._fs._createFile(path, contents)
300 e = self._getFsEntry(path)
301 assert e is not None
302 elif 'x' in mode:
303 err = IOError("File '%s' already exists" % path)
304 err.errno = errno.EEXIST
305 raise err
306 else:
307 err = IOError("Unsupported open mode: %s" % mode)
308 err.errno = errno.EINVAL
309 raise err
310
311 if e is None:
312 err = IOError("No such file: %s" % path)
313 err.errno = errno.ENOENT
314 raise err
315 if not isinstance(e, _MockFsEntry):
316 err = IOError("'%s' is not a file %s" % (path, e))
317 err.errno = errno.EISDIR
318 raise err
319
320 return _MockFsEntryWriter(e, mode)
321
322 def _open(self, path, mode, *args, **kwargs):
323 return self._doOpen('__main__.open', path, mode, *args, **kwargs)
324
325 def _codecsOpen(self, path, mode, *args, **kwargs):
326 return self._doOpen('codecs.open', path, mode, *args, **kwargs)
327
328 def _listdir(self, path):
329 path = os.path.normpath(path)
330 if path.startswith(resources_path):
331 return self._originals['os.listdir'](path)
332
333 e = self._getFsEntry(path)
334 if e is None:
335 raise OSError("No such directory: %s" % path)
336 if not isinstance(e, dict):
337 raise OSError("'%s' is not a directory." % path)
338 return list(e.keys())
339
340 def _makedirs(self, path, mode=0o777):
341 if not path.replace('\\', '/').startswith('/' + self.root):
342 raise Exception("Shouldn't create directory: %s" % path)
343 self._fs._createDir(path)
344
345 def _remove(self, path):
346 path = os.path.normpath(path)
347 self._fs._deleteEntry(path)
348
349 def _exists(self, path):
350 path = os.path.normpath(path)
351 if path.startswith(resources_path):
352 return self._originals['os.path.isdir'](path)
353 e = self._getFsEntry(path)
354 return e is not None
355
356 def _isdir(self, path):
357 path = os.path.normpath(path)
358 if path.startswith(resources_path):
359 return self._originals['os.path.isdir'](path)
360 e = self._getFsEntry(path)
361 return e is not None and isinstance(e, dict)
362
363 def _isfile(self, path):
364 path = os.path.normpath(path)
365 if path.startswith(resources_path):
366 return self._originals['os.path.isfile'](path)
367 e = self._getFsEntry(path)
368 return e is not None and isinstance(e, _MockFsEntry)
369
370 def _islink(self, path):
371 path = os.path.normpath(path)
372 if path.startswith(resources_path):
373 return self._originals['os.path.islink'](path)
374 return False
375
376 def _getmtime(self, path):
377 path = os.path.normpath(path)
378 if path.startswith(resources_path):
379 return self._originals['os.path.getmtime'](path)
380 e = self._getFsEntry(path)
381 if e is None:
382 raise OSError("No such file: %s" % path)
383 return e.metadata['mtime']
384
385 def _copyfile(self, src, dst):
386 src = os.path.normpath(src)
387 if src.startswith(resources_path):
388 with self._originals['__main__.open'](src, 'r') as fp:
389 src_text = fp.read()
390 else:
391 e = self._getFsEntry(src)
392 src_text = e.contents
393 if not dst.replace('\\', '/').startswith('/' + self.root):
394 raise Exception("Shouldn't copy to: %s" % dst)
395 self._fs._createFile(dst, src_text)
396
397 def _rename(self, src, dst):
398 src = os.path.normpath(src)
399 if src.startswith(resources_path) or dst.startswith(resources_path):
400 raise Exception("Shouldn't rename files in the resources path.")
401 self._copyfile(src, dst)
402 self._remove(src)
403
404 def _rmtree(self, path):
405 if not path.replace('\\', '/').startswith('/' + self.root):
406 raise Exception("Shouldn't delete trees from: %s" % path)
407 e = self._fs._getEntry(os.path.dirname(path))
408 del e[os.path.basename(path)]
409
410 def _getFsEntry(self, path):
411 return self._fs._getEntry(path)
412