changeset 49:fce061f8c2ed

Fix cache validation issue with rendered segments, limit disk access.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 22 Aug 2014 20:26:05 -0700
parents 002fa58f54dc
children 2fec3ee1298f
files piecrust/baking/baker.py piecrust/environment.py piecrust/page.py piecrust/rendering.py
diffstat 4 files changed, 27 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/piecrust/baking/baker.py	Fri Aug 22 20:25:41 2014 -0700
+++ b/piecrust/baking/baker.py	Fri Aug 22 20:26:05 2014 -0700
@@ -127,7 +127,7 @@
             do_bake = True
             if not self.force and prev_record_entry:
                 try:
-                    in_path_time = os.path.getmtime(page.path)
+                    in_path_time = page.path_mtime
                     out_path_time = os.path.getmtime(out_path)
                     if out_path_time > in_path_time:
                         do_bake = False
@@ -157,6 +157,8 @@
                 ctx, rp = self._bakeSingle(page, sub_uri, cur_sub, out_path,
                         pagination_filter, custom_data)
             except Exception as ex:
+                if self.app.debug:
+                    logger.exception(ex)
                 raise Exception("Error baking page '%s' for URL '%s'." %
                         (page.ref_spec, uri)) from ex
 
--- a/piecrust/environment.py	Fri Aug 22 20:25:41 2014 -0700
+++ b/piecrust/environment.py	Fri Aug 22 20:26:05 2014 -0700
@@ -25,25 +25,24 @@
         self.lock = threading.RLock()
         self.fs_cache = None
 
-    def get(self, key, item_maker):
+    def get(self, key, item_maker, fs_cache_time=None):
         item = self.cache.get(key)
         if item is None:
             logger.debug("Acquiring lock for: %s" % key)
             with self.lock:
                 item = self.cache.get(key)
                 if item is None:
-                    if self.fs_cache is not None:
+                    if (self.fs_cache is not None and
+                            fs_cache_time is not None):
                         # Try first from the file-system cache.
                         fs_key = _make_fs_cache_key(key)
-                        logger.debug("'%s' not found in cache, trying the "
-                                     "file-system: %s" % (key, fs_key))
-                        try:
+                        if self.fs_cache.isValid(fs_key, fs_cache_time):
+                            logger.debug("'%s' found in file-system cache." %
+                                         key)
                             item_raw = self.fs_cache.read(fs_key)
                             item = json.loads(item_raw)
                             self.cache.put(key, item)
                             return item
-                        except:
-                            pass
 
                     # Look into the mem-cache.
                     logger.debug("'%s' not found in cache, must build." % key)
--- a/piecrust/page.py	Fri Aug 22 20:25:41 2014 -0700
+++ b/piecrust/page.py	Fri Aug 22 20:26:05 2014 -0700
@@ -7,6 +7,7 @@
 import logging
 import datetime
 import dateutil.parser
+from werkzeug.utils import cached_property
 from piecrust.configuration import (Configuration, ConfigurationError,
         parse_config_header)
 from piecrust.environment import PHASE_PAGE_PARSING
@@ -36,7 +37,6 @@
         self.source = source
         self.source_metadata = source_metadata
         self.rel_path = rel_path
-        self.path = source.resolveRef(rel_path)
         self._config = None
         self._raw_content = None
         self._datetime = None
@@ -49,6 +49,14 @@
     def ref_spec(self):
         return '%s:%s' % (self.source.name, self.rel_path)
 
+    @cached_property
+    def path(self):
+        return self.source.resolveRef(self.rel_path)
+
+    @cached_property
+    def path_mtime(self):
+        return os.path.getmtime(self.path)
+
     @property
     def config(self):
         self._load()
@@ -79,7 +87,7 @@
                     page_time = datetime.time(0, 0, 0)
                 self._datetime = datetime.datetime.combine(page_date, page_time)
             else:
-                self._datetime = datetime.datetime.fromtimestamp(os.path.getmtime(self.path))
+                self._datetime = datetime.datetime.fromtimestamp(self.path_mtime)
         return self._datetime
 
     @datetime.setter
@@ -96,7 +104,7 @@
         eis = self.app.env.exec_info_stack
         eis.pushPage(self, PHASE_PAGE_PARSING, None)
         try:
-            config, content = load_page(self.app, self.path)
+            config, content = load_page(self.app, self.path, self.path_mtime)
             self._config = config
             self._raw_content = content
         finally:
@@ -154,9 +162,9 @@
     return data
 
 
-def load_page(app, path):
+def load_page(app, path, path_mtime=None):
     try:
-        return _do_load_page(app, path)
+        return _do_load_page(app, path, path_mtime)
     except Exception as e:
         logger.exception("Error loading page: %s" %
                 os.path.relpath(path, app.root_dir))
@@ -164,7 +172,7 @@
         raise PageLoadingError(path, e).with_traceback(traceback)
 
 
-def _do_load_page(app, path):
+def _do_load_page(app, path, path_mtime):
     exec_info = app.env.exec_info_stack.current_page_info
     if exec_info is None:
         raise Exception("Loading page '%s' but not execution context has "
@@ -173,7 +181,7 @@
     # Check the cache first.
     cache = app.cache.getCache('pages')
     cache_path = "%s.json" % hashlib.md5(path.encode('utf8')).hexdigest()
-    page_time = os.path.getmtime(path)
+    page_time = path_mtime or os.path.getmtime(path)
     if cache.isValid(cache_path, page_time):
         exec_info.was_cache_valid = True
         cache_data = json.loads(cache.read(cache_path))
--- a/piecrust/rendering.py	Fri Aug 22 20:25:41 2014 -0700
+++ b/piecrust/rendering.py	Fri Aug 22 20:26:05 2014 -0700
@@ -90,8 +90,10 @@
         repo = ctx.app.env.rendered_segments_repository
         if repo:
             cache_key = '%s:%s' % (ctx.uri, ctx.page_num)
+            page_time = page.path_mtime
             contents = repo.get(cache_key,
-                    lambda: _do_render_page_segments(page, page_data))
+                    lambda: _do_render_page_segments(page, page_data),
+                    fs_cache_time=page_time)
         else:
             contents = _do_render_page_segments(page, page_data)