Mercurial > piecrust2

comparison piecrust/cache.py @ 1188:a7c43131d871

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Fix file write flushing problem with Python 3.8+ Writing the cache files fails in Python 3.8 because it looks like flushing behaviour has changed. We need to explicitly flush. And even then, in very rare occurrences, it looks like it can still run into racing conditions, so we do a very hacky and ugly "retry" loop when fetching cached data :(
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 15 Jun 2021 22:36:23 -0700
parents cd0345e4001e
children
comparison
equal deleted inserted replaced
1187:161cba5d031a 1188:a7c43131d871
91 return open(cache_path, mode=mode, encoding=encoding) 91 return open(cache_path, mode=mode, encoding=encoding)
92 92
93 def write(self, path, content): 93 def write(self, path, content):
94 with self.openWrite(path, mode='w', encoding='utf8') as fp: 94 with self.openWrite(path, mode='w', encoding='utf8') as fp:
95 fp.write(content) 95 fp.write(content)
96 fp.flush()
97 os.fsync(fp)
96 98
97 def openWrite(self, path, mode='w', encoding=None): 99 def openWrite(self, path, mode='w', encoding=None):
98 cache_path = self.getCachePath(path) 100 cache_path = self.getCachePath(path)
99 cache_dir = os.path.dirname(cache_path) 101 cache_dir = os.path.dirname(cache_path)
100 if not os.path.isdir(cache_dir): 102 if not os.path.isdir(cache_dir):
184 self.cache.put(key, item) 186 self.cache.put(key, item)
185 if self.fs_cache and save_to_fs: 187 if self.fs_cache and save_to_fs:
186 fs_key = _make_fs_cache_key(key) 188 fs_key = _make_fs_cache_key(key)
187 with self.fs_cache.openWrite(fs_key, mode='wb') as fp: 189 with self.fs_cache.openWrite(fs_key, mode='wb') as fp:
188 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL) 190 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL)
191 fp.flush()
192 os.fsync(fp)
189 193
190 def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True): 194 def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
191 self._last_access_hit = True 195 self._last_access_hit = True
192 item = self.cache.get(key) 196 item = self.cache.get(key)
193 if item is not None: 197 if item is not None:
219 223
220 # Save to the file-system if needed. 224 # Save to the file-system if needed.
221 if self.fs_cache is not None and save_to_fs: 225 if self.fs_cache is not None and save_to_fs:
222 with self.fs_cache.openWrite(fs_key, mode='wb') as fp: 226 with self.fs_cache.openWrite(fs_key, mode='wb') as fp:
223 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL) 227 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL)
228 fp.flush()
229 os.fsync(fp)
224 230
225 return item 231 return item
226 232