Mercurial > piecrust2
annotate tests/tmpfs.py @ 415:0e9a94b7fdfa
bake: Improve bake record information.
* Store things in the bake record that require less interaction between the
master process and the workers. For instance, don't store the paginator
object in the render pass info -- instead, just store whether pagination
was used, and whether it had more items.
* Simplify information passing between workers and bake passes by saving the
rendering info to the JSON cache. This means the "render first sub" job
doesn't have to return anything except errors now.
* Add more performance counter info.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 20 Jun 2015 19:23:16 -0700 |
parents | e7b865f8f335 |
children | bab91fcef741 |
rev | line source |
---|---|
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import os |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import os.path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 import shutil |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 import random |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 from .basefs import TestFileSystemBase |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 class TempDirFileSystem(TestFileSystemBase): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 def __init__(self, default_spec=True): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 self._root = os.path.join( |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 os.path.dirname(__file__), |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 '__tmpfs__', |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
13 '%d' % random.randrange(1000)) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
14 self._done = False |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
15 if default_spec: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
16 self._initDefaultSpec() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 def path(self, p): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 p = p.lstrip('/\\') |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 return os.path.join(self._root, p) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 def getStructure(self, path=None): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 path = self.path(path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 if not os.path.exists(path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 raise Exception("No such path: %s" % path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 if not os.path.isdir(path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 raise Exception("Path is not a directory: %s" % path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
29 res = {} |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 for item in os.listdir(path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 self._getStructureRecursive(res, path, item) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 return res |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 def getFileEntry(self, path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 path = self.path(path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 with open(path, 'r', encoding='utf8') as fp: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 return fp.read() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
39 def _getStructureRecursive(self, target, parent, cur): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
40 full_cur = os.path.join(parent, cur) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
41 if os.path.isdir(full_cur): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 e = {} |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
43 for item in os.listdir(full_cur): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
44 self._getStructureRecursive(e, full_cur, item) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 target[cur] = e |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 else: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 with open(full_cur, 'r', encoding='utf8') as fp: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 target[cur] = fp.read() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 def _createDir(self, path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 if not os.path.exists(path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 os.makedirs(path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 def _createFile(self, path, contents): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 dirpath = os.path.dirname(path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 if not os.path.exists(dirpath): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
57 os.makedirs(dirpath) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 with open(path, 'w', encoding='utf8') as fp: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
59 fp.write(contents) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
60 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
61 if not self._done: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
62 import traceback |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
63 with open(os.path.join(self._root, 'where.txt'), 'w') as fp: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
64 fp.write('\n'.join(traceback.format_stack(limit=10))) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
65 self._done = True |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
66 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
67 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
68 class TempDirScope(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 def __init__(self, fs, open_patches=None): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 self._fs = fs |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 self._open = open |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 @property |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 def root(self): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 return self._fs._root |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 def __enter__(self): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
78 return self |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
79 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
80 def __exit__(self, type, value, traceback): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
81 shutil.rmtree(self.root) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 |