Mercurial > piecrust2
changeset 49:fce061f8c2ed
Fix cache validation issue with rendered segments, limit disk access.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Fri, 22 Aug 2014 20:26:05 -0700 |
parents | 002fa58f54dc |
children | 2fec3ee1298f |
files | piecrust/baking/baker.py piecrust/environment.py piecrust/page.py piecrust/rendering.py |
diffstat | 4 files changed, 27 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/baking/baker.py Fri Aug 22 20:25:41 2014 -0700 +++ b/piecrust/baking/baker.py Fri Aug 22 20:26:05 2014 -0700 @@ -127,7 +127,7 @@ do_bake = True if not self.force and prev_record_entry: try: - in_path_time = os.path.getmtime(page.path) + in_path_time = page.path_mtime out_path_time = os.path.getmtime(out_path) if out_path_time > in_path_time: do_bake = False @@ -157,6 +157,8 @@ ctx, rp = self._bakeSingle(page, sub_uri, cur_sub, out_path, pagination_filter, custom_data) except Exception as ex: + if self.app.debug: + logger.exception(ex) raise Exception("Error baking page '%s' for URL '%s'." % (page.ref_spec, uri)) from ex
--- a/piecrust/environment.py Fri Aug 22 20:25:41 2014 -0700 +++ b/piecrust/environment.py Fri Aug 22 20:26:05 2014 -0700 @@ -25,25 +25,24 @@ self.lock = threading.RLock() self.fs_cache = None - def get(self, key, item_maker): + def get(self, key, item_maker, fs_cache_time=None): item = self.cache.get(key) if item is None: logger.debug("Acquiring lock for: %s" % key) with self.lock: item = self.cache.get(key) if item is None: - if self.fs_cache is not None: + if (self.fs_cache is not None and + fs_cache_time is not None): # Try first from the file-system cache. fs_key = _make_fs_cache_key(key) - logger.debug("'%s' not found in cache, trying the " - "file-system: %s" % (key, fs_key)) - try: + if self.fs_cache.isValid(fs_key, fs_cache_time): + logger.debug("'%s' found in file-system cache." % + key) item_raw = self.fs_cache.read(fs_key) item = json.loads(item_raw) self.cache.put(key, item) return item - except: - pass # Look into the mem-cache. logger.debug("'%s' not found in cache, must build." % key)
--- a/piecrust/page.py Fri Aug 22 20:25:41 2014 -0700 +++ b/piecrust/page.py Fri Aug 22 20:26:05 2014 -0700 @@ -7,6 +7,7 @@ import logging import datetime import dateutil.parser +from werkzeug.utils import cached_property from piecrust.configuration import (Configuration, ConfigurationError, parse_config_header) from piecrust.environment import PHASE_PAGE_PARSING @@ -36,7 +37,6 @@ self.source = source self.source_metadata = source_metadata self.rel_path = rel_path - self.path = source.resolveRef(rel_path) self._config = None self._raw_content = None self._datetime = None @@ -49,6 +49,14 @@ def ref_spec(self): return '%s:%s' % (self.source.name, self.rel_path) + @cached_property + def path(self): + return self.source.resolveRef(self.rel_path) + + @cached_property + def path_mtime(self): + return os.path.getmtime(self.path) + @property def config(self): self._load() @@ -79,7 +87,7 @@ page_time = datetime.time(0, 0, 0) self._datetime = datetime.datetime.combine(page_date, page_time) else: - self._datetime = datetime.datetime.fromtimestamp(os.path.getmtime(self.path)) + self._datetime = datetime.datetime.fromtimestamp(self.path_mtime) return self._datetime @datetime.setter @@ -96,7 +104,7 @@ eis = self.app.env.exec_info_stack eis.pushPage(self, PHASE_PAGE_PARSING, None) try: - config, content = load_page(self.app, self.path) + config, content = load_page(self.app, self.path, self.path_mtime) self._config = config self._raw_content = content finally: @@ -154,9 +162,9 @@ return data -def load_page(app, path): +def load_page(app, path, path_mtime=None): try: - return _do_load_page(app, path) + return _do_load_page(app, path, path_mtime) except Exception as e: logger.exception("Error loading page: %s" % os.path.relpath(path, app.root_dir)) @@ -164,7 +172,7 @@ raise PageLoadingError(path, e).with_traceback(traceback) -def _do_load_page(app, path): +def _do_load_page(app, path, path_mtime): exec_info = app.env.exec_info_stack.current_page_info if exec_info is None: raise Exception("Loading page '%s' but not execution context has " @@ -173,7 +181,7 @@ # Check the cache first. cache = app.cache.getCache('pages') cache_path = "%s.json" % hashlib.md5(path.encode('utf8')).hexdigest() - page_time = os.path.getmtime(path) + page_time = path_mtime or os.path.getmtime(path) if cache.isValid(cache_path, page_time): exec_info.was_cache_valid = True cache_data = json.loads(cache.read(cache_path))
--- a/piecrust/rendering.py Fri Aug 22 20:25:41 2014 -0700 +++ b/piecrust/rendering.py Fri Aug 22 20:26:05 2014 -0700 @@ -90,8 +90,10 @@ repo = ctx.app.env.rendered_segments_repository if repo: cache_key = '%s:%s' % (ctx.uri, ctx.page_num) + page_time = page.path_mtime contents = repo.get(cache_key, - lambda: _do_render_page_segments(page, page_data)) + lambda: _do_render_page_segments(page, page_data), + fs_cache_time=page_time) else: contents = _do_render_page_segments(page, page_data)