Mercurial > piecrust2

comparison tests/memfs.py @ 411:e7b865f8f335

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Enable multiprocess baking. Baking is now done by running a worker per CPU, and sending jobs to them. This changes several things across the codebase: * Ability to not cache things related to pages other than the 'main' page (i.e. the page at the bottom of the execution stack). * Decouple the baking process from the bake records, so only the main process keeps track (and modifies) the bake record. * Remove the need for 'batch page getters' and loading a page directly from the page factories. There are various smaller changes too included here, including support for scope performance timers that are saved with the bake record and can be printed out to the console. Yes I got carried away. For testing, the in-memory 'mock' file-system doesn't work anymore, since we're spawning processes, so this is replaced by a 'tmpfs' file-system which is saved in temporary files on disk and deleted after tests have run.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 Jun 2015 17:09:19 -0700
parents
children f987b29d6fab
comparison
equal deleted inserted replaced
410:d1a472464e57 411:e7b865f8f335
1 import os.path
2 import io
3 import time
4 import errno
5 import random
6 import codecs
7 import shutil
8 import mock
9 from piecrust import RESOURCES_DIR
10 from .basefs import TestFileSystemBase
11
12
13 class _MockFsEntry(object):
14 def __init__(self, contents):
15 self.contents = contents
16 self.metadata = {'mtime': time.time()}
17
18
19 class _MockFsEntryWriter(object):
20 def __init__(self, entry, mode='rt'):
21 self._entry = entry
22 self._mode = mode
23
24 if 'b' in mode:
25 data = entry.contents
26 if isinstance(data, str):
27 data = data.encode('utf8')
28 self._stream = io.BytesIO(data)
29 else:
30 self._stream = io.StringIO(entry.contents)
31
32 def __getattr__(self, name):
33 return getattr(self._stream, name)
34
35 def __enter__(self):
36 return self
37
38 def __exit__(self, exc_type, exc_value, exc_tb):
39 if 'w' in self._mode:
40 if 'a' in self._mode:
41 self._entry.contents += self._stream.getvalue()
42 else:
43 self._entry.contents = self._stream.getvalue()
44 self._entry.metadata['mtime'] = time.time()
45 self._stream.close()
46
47
48 class MemoryFileSystem(TestFileSystemBase):
49 def __init__(self, default_spec=True):
50 self._root = 'root_%d' % random.randrange(1000)
51 self._fs = {self._root: {}}
52 if default_spec:
53 self._initDefaultSpec()
54
55 def path(self, p):
56 p = p.replace('\\', '/')
57 if p in ['/', '', None]:
58 return '/%s' % self._root
59 return '/%s/%s' % (self._root, p.lstrip('/'))
60
61 def getStructure(self, path=None):
62 root = self._fs[self._root]
63 if path:
64 root = self._getEntry(self.path(path))
65 if root is None:
66 raise Exception("No such path: %s" % path)
67 if not isinstance(root, dict):
68 raise Exception("Path is not a directory: %s" % path)
69
70 res = {}
71 for k, v in root.items():
72 self._getStructureRecursive(v, res, k)
73 return res
74
75 def getFileEntry(self, path):
76 entry = self._getEntry(self.path(path))
77 if entry is None:
78 raise Exception("No such file: %s" % path)
79 if not isinstance(entry, _MockFsEntry):
80 raise Exception("Path is not a file: %s" % path)
81 return entry.contents
82
83 def _getStructureRecursive(self, src, target, name):
84 if isinstance(src, _MockFsEntry):
85 target[name] = src.contents
86 return
87
88 e = {}
89 for k, v in src.items():
90 self._getStructureRecursive(v, e, k)
91 target[name] = e
92
93 def _getEntry(self, path):
94 cur = self._fs
95 path = path.replace('\\', '/').lstrip('/')
96 bits = path.split('/')
97 for p in bits:
98 try:
99 cur = cur[p]
100 except KeyError:
101 return None
102 return cur
103
104 def _createDir(self, path):
105 cur = self._fs
106 path = path.replace('\\', '/').strip('/')
107 bits = path.split('/')
108 for b in bits:
109 if b not in cur:
110 cur[b] = {}
111 cur = cur[b]
112 return self
113
114 def _createFile(self, path, contents):
115 cur = self._fs
116 path = path.replace('\\', '/').lstrip('/')
117 bits = path.split('/')
118 for b in bits[:-1]:
119 if b not in cur:
120 cur[b] = {}
121 cur = cur[b]
122 cur[bits[-1]] = _MockFsEntry(contents)
123 return self
124
125 def _deleteEntry(self, path):
126 parent = self._getEntry(os.path.dirname(path))
127 assert parent is not None
128 name = os.path.basename(path)
129 assert name in parent
130 del parent[name]
131
132
133 class MemoryScope(object):
134 def __init__(self, fs, open_patches=None):
135 self.open_patches = open_patches or []
136 self._fs = fs
137 self._patchers = []
138 self._originals = {}
139
140 @property
141 def root(self):
142 return self._fs._root
143
144 def __enter__(self):
145 self._startMock()
146 return self
147
148 def __exit__(self, type, value, traceback):
149 self._endMock()
150
151 def _startMock(self):
152 # TODO: sadly, there seems to be no way to replace `open` everywhere?
153 modules = self.open_patches + [
154 '__main__',
155 'piecrust.records',
156 'jinja2.utils']
157 for m in modules:
158 self._createMock('%s.open' % m, open, self._open, create=True)
159
160 self._createMock('codecs.open', codecs.open, self._codecsOpen)
161 self._createMock('os.listdir', os.listdir, self._listdir)
162 self._createMock('os.makedirs', os.makedirs, self._makedirs)
163 self._createMock('os.remove', os.remove, self._remove)
164 self._createMock('os.rename', os.rename, self._rename)
165 self._createMock('os.path.exists', os.path.exists, self._exists)
166 self._createMock('os.path.isdir', os.path.isdir, self._isdir)
167 self._createMock('os.path.isfile', os.path.isfile, self._isfile)
168 self._createMock('os.path.islink', os.path.islink, self._islink)
169 self._createMock('os.path.getmtime', os.path.getmtime, self._getmtime)
170 self._createMock('shutil.copyfile', shutil.copyfile, self._copyfile)
171 self._createMock('shutil.rmtree', shutil.rmtree, self._rmtree)
172 for p in self._patchers:
173 p.start()
174
175 def _endMock(self):
176 for p in self._patchers:
177 p.stop()
178
179 def _createMock(self, name, orig, func, **kwargs):
180 self._originals[name] = orig
181 self._patchers.append(mock.patch(name, func, **kwargs))
182
183 def _doOpen(self, orig_name, path, mode, *args, **kwargs):
184 path = os.path.normpath(path)
185 if path.startswith(RESOURCES_DIR):
186 return self._originals[orig_name](path, mode, *args, **kwargs)
187
188 if 'r' in mode:
189 e = self._getFsEntry(path)
190 elif 'w' in mode or 'x' in mode or 'a' in mode:
191 e = self._getFsEntry(path)
192 if e is None:
193 contents = ''
194 if 'b' in mode:
195 contents = bytes()
196 self._fs._createFile(path, contents)
197 e = self._getFsEntry(path)
198 assert e is not None
199 elif 'x' in mode:
200 err = IOError("File '%s' already exists" % path)
201 err.errno = errno.EEXIST
202 raise err
203 else:
204 err = IOError("Unsupported open mode: %s" % mode)
205 err.errno = errno.EINVAL
206 raise err
207
208 if e is None:
209 err = IOError("No such file: %s" % path)
210 err.errno = errno.ENOENT
211 raise err
212 if not isinstance(e, _MockFsEntry):
213 err = IOError("'%s' is not a file %s" % (path, e))
214 err.errno = errno.EISDIR
215 raise err
216
217 return _MockFsEntryWriter(e, mode)
218
219 def _open(self, path, mode, *args, **kwargs):
220 return self._doOpen('__main__.open', path, mode, *args, **kwargs)
221
222 def _codecsOpen(self, path, mode, *args, **kwargs):
223 return self._doOpen('codecs.open', path, mode, *args, **kwargs)
224
225 def _listdir(self, path):
226 path = os.path.normpath(path)
227 if path.startswith(RESOURCES_DIR):
228 return self._originals['os.listdir'](path)
229
230 e = self._getFsEntry(path)
231 if e is None:
232 raise OSError("No such directory: %s" % path)
233 if not isinstance(e, dict):
234 raise OSError("'%s' is not a directory." % path)
235 return list(e.keys())
236
237 def _makedirs(self, path, mode=0o777):
238 if not path.replace('\\', '/').startswith('/' + self.root):
239 raise Exception("Shouldn't create directory: %s" % path)
240 self._fs._createDir(path)
241
242 def _remove(self, path):
243 path = os.path.normpath(path)
244 self._fs._deleteEntry(path)
245
246 def _exists(self, path):
247 path = os.path.normpath(path)
248 if path.startswith(RESOURCES_DIR):
249 return self._originals['os.path.isdir'](path)
250 e = self._getFsEntry(path)
251 return e is not None
252
253 def _isdir(self, path):
254 path = os.path.normpath(path)
255 if path.startswith(RESOURCES_DIR):
256 return self._originals['os.path.isdir'](path)
257 e = self._getFsEntry(path)
258 return e is not None and isinstance(e, dict)
259
260 def _isfile(self, path):
261 path = os.path.normpath(path)
262 if path.startswith(RESOURCES_DIR):
263 return self._originals['os.path.isfile'](path)
264 e = self._getFsEntry(path)
265 return e is not None and isinstance(e, _MockFsEntry)
266
267 def _islink(self, path):
268 path = os.path.normpath(path)
269 if path.startswith(RESOURCES_DIR):
270 return self._originals['os.path.islink'](path)
271 return False
272
273 def _getmtime(self, path):
274 path = os.path.normpath(path)
275 if path.startswith(RESOURCES_DIR):
276 return self._originals['os.path.getmtime'](path)
277 e = self._getFsEntry(path)
278 if e is None:
279 raise OSError("No such file: %s" % path)
280 return e.metadata['mtime']
281
282 def _copyfile(self, src, dst):
283 src = os.path.normpath(src)
284 if src.startswith(RESOURCES_DIR):
285 with self._originals['__main__.open'](src, 'r') as fp:
286 src_text = fp.read()
287 else:
288 e = self._getFsEntry(src)
289 src_text = e.contents
290 if not dst.replace('\\', '/').startswith('/' + self.root):
291 raise Exception("Shouldn't copy to: %s" % dst)
292 self._fs._createFile(dst, src_text)
293
294 def _rename(self, src, dst):
295 src = os.path.normpath(src)
296 if src.startswith(RESOURCES_DIR) or dst.startswith(RESOURCES_DIR):
297 raise Exception("Shouldn't rename files in the resources path.")
298 self._copyfile(src, dst)
299 self._remove(src)
300
301 def _rmtree(self, path):
302 if not path.replace('\\', '/').startswith('/' + self.root):
303 raise Exception("Shouldn't delete trees from: %s" % path)
304 e = self._fs._getEntry(os.path.dirname(path))
305 del e[os.path.basename(path)]
306
307 def _getFsEntry(self, path):
308 return self._fs._getEntry(path)
309