Mercurial > piecrust2
annotate tests/tmpfs.py @ 411:e7b865f8f335
bake: Enable multiprocess baking.
Baking is now done by running a worker per CPU, and sending jobs to them.
This changes several things across the codebase:
* Ability to not cache things related to pages other than the 'main' page
(i.e. the page at the bottom of the execution stack).
* Decouple the baking process from the bake records, so only the main process
keeps track (and modifies) the bake record.
* Remove the need for 'batch page getters' and loading a page directly from
the page factories.
There are various smaller changes too included here, including support for
scope performance timers that are saved with the bake record and can be
printed out to the console. Yes I got carried away.
For testing, the in-memory 'mock' file-system doesn't work anymore, since
we're spawning processes, so this is replaced by a 'tmpfs' file-system which
is saved in temporary files on disk and deleted after tests have run.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Fri, 12 Jun 2015 17:09:19 -0700 |
parents | |
children | bab91fcef741 |
rev | line source |
---|---|
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import os |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import os.path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 import shutil |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 import random |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 from .basefs import TestFileSystemBase |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 class TempDirFileSystem(TestFileSystemBase): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 def __init__(self, default_spec=True): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 self._root = os.path.join( |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 os.path.dirname(__file__), |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 '__tmpfs__', |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
13 '%d' % random.randrange(1000)) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
14 self._done = False |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
15 if default_spec: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
16 self._initDefaultSpec() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 def path(self, p): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 p = p.lstrip('/\\') |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 return os.path.join(self._root, p) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 def getStructure(self, path=None): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 path = self.path(path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 if not os.path.exists(path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 raise Exception("No such path: %s" % path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 if not os.path.isdir(path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 raise Exception("Path is not a directory: %s" % path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
29 res = {} |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 for item in os.listdir(path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 self._getStructureRecursive(res, path, item) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 return res |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 def getFileEntry(self, path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 path = self.path(path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 with open(path, 'r', encoding='utf8') as fp: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 return fp.read() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
39 def _getStructureRecursive(self, target, parent, cur): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
40 full_cur = os.path.join(parent, cur) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
41 if os.path.isdir(full_cur): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 e = {} |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
43 for item in os.listdir(full_cur): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
44 self._getStructureRecursive(e, full_cur, item) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 target[cur] = e |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 else: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 with open(full_cur, 'r', encoding='utf8') as fp: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 target[cur] = fp.read() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 def _createDir(self, path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 if not os.path.exists(path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 os.makedirs(path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 def _createFile(self, path, contents): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 dirpath = os.path.dirname(path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 if not os.path.exists(dirpath): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
57 os.makedirs(dirpath) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 with open(path, 'w', encoding='utf8') as fp: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
59 fp.write(contents) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
60 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
61 if not self._done: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
62 import traceback |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
63 with open(os.path.join(self._root, 'where.txt'), 'w') as fp: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
64 fp.write('\n'.join(traceback.format_stack(limit=10))) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
65 self._done = True |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
66 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
67 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
68 class TempDirScope(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 def __init__(self, fs, open_patches=None): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 self._fs = fs |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 self._open = open |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 @property |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 def root(self): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 return self._fs._root |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 def __enter__(self): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
78 return self |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
79 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
80 def __exit__(self, type, value, traceback): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
81 shutil.rmtree(self.root) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 |