Mercurial > piecrust2

changeset 1188:a7c43131d871

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Fix file write flushing problem with Python 3.8+ Writing the cache files fails in Python 3.8 because it looks like flushing behaviour has changed. We need to explicitly flush. And even then, in very rare occurrences, it looks like it can still run into racing conditions, so we do a very hacky and ugly "retry" loop when fetching cached data :(
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 15 Jun 2021 22:36:23 -0700
parents 161cba5d031a
children bfa470063ee2
files piecrust/cache.py piecrust/page.py
diffstat 2 files changed, 17 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- a/piecrust/cache.py	Tue Jun 15 22:34:51 2021 -0700
+++ b/piecrust/cache.py	Tue Jun 15 22:36:23 2021 -0700
@@ -93,6 +93,8 @@
     def write(self, path, content):
         with self.openWrite(path, mode='w', encoding='utf8') as fp:
             fp.write(content)
+            fp.flush()
+            os.fsync(fp)
 
     def openWrite(self, path, mode='w', encoding=None):
         cache_path = self.getCachePath(path)
@@ -186,6 +188,8 @@
             fs_key = _make_fs_cache_key(key)
             with self.fs_cache.openWrite(fs_key, mode='wb') as fp:
                 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL)
+                fp.flush()
+                os.fsync(fp)
 
     def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
         self._last_access_hit = True
@@ -221,6 +225,8 @@
         if self.fs_cache is not None and save_to_fs:
             with self.fs_cache.openWrite(fs_key, mode='wb') as fp:
                 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL)
+                fp.flush()
+                os.fsync(fp)
 
         return item
 
--- a/piecrust/page.py	Tue Jun 15 22:34:51 2021 -0700
+++ b/piecrust/page.py	Tue Jun 15 22:36:23 2021 -0700
@@ -271,8 +271,18 @@
     page_time = source.getItemMtime(content_item)
     if cache.isValid(cache_path, page_time):
         try:
+            cache_raw = cache.read(cache_path)
+            if not cache_raw:
+                for i in range(5):
+                    cache_raw = cache.read(cache_path)
+                    if cache_raw:
+                        logger.warn("Had to re-pull the cache %d time(s)!" % (i + 1))
+                        break
+                if not cache_raw:
+                    raise Exception("Cache is busted!")
+
             cache_data = json.loads(
-                cache.read(cache_path),
+                cache_raw,
                 object_pairs_hook=collections.OrderedDict)
             config = PageConfiguration(
                 values=cache_data['config'],