Mercurial > piecrust2

annotate tests/tmpfs.py @ 1188:a7c43131d871

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Fix file write flushing problem with Python 3.8+ Writing the cache files fails in Python 3.8 because it looks like flushing behaviour has changed. We need to explicitly flush. And even then, in very rare occurrences, it looks like it can still run into racing conditions, so we do a very hacky and ugly "retry" loop when fetching cached data :(
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 15 Jun 2021 22:36:23 -0700
parents 45ad976712ec
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import shutil
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import random
520
bab91fcef741 bake/serve: Improve support for unicode, add slugification options.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
5 import unicodedata
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 from .basefs import TestFileSystemBase
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 class TempDirFileSystem(TestFileSystemBase):
674
f987b29d6fab tests: Add ability to run tests with a theme site.
Ludovic Chabant <ludovic@chabant.com>
parents: 673
diff changeset
10 def __init__(self):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 self._root = os.path.join(
974
72f17534d58e tests: First pass on making unit tests work again.
Ludovic Chabant <ludovic@chabant.com>
parents: 674
diff changeset
12 os.path.dirname(__file__),
72f17534d58e tests: First pass on making unit tests work again.
Ludovic Chabant <ludovic@chabant.com>
parents: 674
diff changeset
13 '__tmpfs__',
72f17534d58e tests: First pass on making unit tests work again.
Ludovic Chabant <ludovic@chabant.com>
parents: 674
diff changeset
14 '%d' % random.randrange(1000))
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 self._done = False
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 def path(self, p):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 p = p.lstrip('/\\')
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 return os.path.join(self._root, p)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 974
diff changeset
21 def getStructure(self, path=''):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 path = self.path(path)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 if not os.path.exists(path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 raise Exception("No such path: %s" % path)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 if not os.path.isdir(path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 raise Exception("Path is not a directory: %s" % path)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 res = {}
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 for item in os.listdir(path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 self._getStructureRecursive(res, path, item)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 return res
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 def getFileEntry(self, path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 path = self.path(path)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 with open(path, 'r', encoding='utf8') as fp:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 return fp.read()
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 def _getStructureRecursive(self, target, parent, cur):
520
bab91fcef741 bake/serve: Improve support for unicode, add slugification options.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
39 cur = unicodedata.normalize('NFC', cur)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 full_cur = os.path.join(parent, cur)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 if os.path.isdir(full_cur):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 e = {}
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 for item in os.listdir(full_cur):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 self._getStructureRecursive(e, full_cur, item)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 target[cur] = e
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 else:
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 974
diff changeset
47 try:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 974
diff changeset
48 with open(full_cur, 'r', encoding='utf8') as fp:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 974
diff changeset
49 target[cur] = fp.read()
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 974
diff changeset
50 except Exception as ex:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 974
diff changeset
51 target[cur] = "ERROR: CAN'T READ '%s': %s" % (full_cur, ex)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 def _createDir(self, path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 if not os.path.exists(path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 os.makedirs(path)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 def _createFile(self, path, contents):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 dirpath = os.path.dirname(path)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 if not os.path.exists(dirpath):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 os.makedirs(dirpath)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 with open(path, 'w', encoding='utf8') as fp:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 fp.write(contents)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 if not self._done:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 import traceback
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 with open(os.path.join(self._root, 'where.txt'), 'w') as fp:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 fp.write('\n'.join(traceback.format_stack(limit=10)))
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 self._done = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 class TempDirScope(object):
673
d6403c21bdea tests: Improve failure reporting.
Ludovic Chabant <ludovic@chabant.com>
parents: 520
diff changeset
72 def __init__(self, fs, open_patches=None, keep=False):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 self._fs = fs
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 self._open = open
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 974
diff changeset
75 self._keep = keep or TestFileSystemBase._leave_mockfs
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 @property
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 def root(self):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 return self._fs._root
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 def __enter__(self):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 return self
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 def __exit__(self, type, value, traceback):
673
d6403c21bdea tests: Improve failure reporting.
Ludovic Chabant <ludovic@chabant.com>
parents: 520
diff changeset
85 if not self._keep:
d6403c21bdea tests: Improve failure reporting.
Ludovic Chabant <ludovic@chabant.com>
parents: 520
diff changeset
86 shutil.rmtree(self.root)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87