Mercurial > piecrust2
changeset 371:c2ca72fb7f0b 2.0.0a8
caching: Use separate caches for config variants and other contexts.
* The `_cache` directory is now organized in multiple "sub-caches" for
different contexts.
* A new context is created when config variants or overrides are applied.
* `serve` context uses a different context that the other commends, to prevent
the `bake` command's output from messing up the preview server (e.g. with
how asset URLs are generated differently between the two).
* Fix a few places where the cache directory was referenced directly.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Sun, 03 May 2015 23:59:46 -0700 |
| parents | a1bbe66cba03 |
| children | 115668bb447d |
| files | piecrust/app.py piecrust/baking/baker.py piecrust/cache.py piecrust/commands/base.py piecrust/commands/builtin/serving.py piecrust/commands/builtin/util.py piecrust/environment.py piecrust/main.py piecrust/processing/base.py piecrust/processing/sass.py piecrust/serving.py |
| diffstat | 11 files changed, 156 insertions(+), 66 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/app.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/app.py Sun May 03 23:59:46 2015 -0700 @@ -403,7 +403,8 @@ self.plugin_loader = PluginLoader(self) if cache: - self.cache = ExtensibleCache(self.cache_dir) + cache_dir = os.path.join(self.cache_dir, 'default') + self.cache = ExtensibleCache(cache_dir) else: self.cache = NullExtensibleCache() @@ -494,6 +495,12 @@ def cache_dir(self): return os.path.join(self.root_dir, CACHE_DIR) + @property # Not a cached property because its result can change. + def sub_cache_dir(self): + if self.cache.enabled: + return self.cache.base_dir + return None + @cached_property def sources(self): defs = {} @@ -557,6 +564,18 @@ return tax return None + def useSubCache(self, cache_name, cache_key): + cache_hash = hashlib.md5(cache_key.encode('utf8')).hexdigest() + cache_dir = os.path.join(self.cache_dir, + '%s_%s' % (cache_name, cache_hash)) + self._useSubCacheDir(cache_dir) + + def _useSubCacheDir(self, cache_dir): + assert cache_dir + logger.debug("Moving cache to: %s" % cache_dir) + self.cache = ExtensibleCache(cache_dir) + self.env._onSubCacheDirChanged(self) + def _get_dir(self, default_rel_dir): abs_dir = os.path.join(self.root_dir, default_rel_dir) if os.path.isdir(abs_dir):
--- a/piecrust/baking/baker.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/baking/baker.py Sun May 03 23:59:46 2015 -0700 @@ -1,6 +1,5 @@ import time import os.path -import shutil import hashlib import logging import threading @@ -10,7 +9,6 @@ from piecrust.baking.single import (BakingError, PageBaker) from piecrust.chefutil import format_timed, log_friendly_exception from piecrust.sources.base import ( - PageFactory, REALM_NAMES, REALM_USER, REALM_THEME) @@ -141,12 +139,7 @@ if reason is not None: # We have to bake everything from scratch. - for cache_name in self.app.cache.getCacheNames( - except_names=['app']): - cache_dir = self.app.cache.getCacheDir(cache_name) - if os.path.isdir(cache_dir): - logger.debug("Cleaning baker cache: %s" % cache_dir) - shutil.rmtree(cache_dir) + self.app.cache.clearCaches(except_names=['app']) self.force = True record.incremental_count = 0 record.clearPrevious()
--- a/piecrust/cache.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/cache.py Sun May 03 23:59:46 2015 -0700 @@ -1,5 +1,6 @@ import os import os.path +import shutil import codecs import logging import threading @@ -41,6 +42,16 @@ return dirnames return [dn for dn in dirnames if dn not in except_names] + def clearCache(self, name): + cache_dir = self.getCacheDir(name) + if os.path.isdir(cache_dir): + logger.debug("Cleaning cache: %s" % cache_dir) + shutil.rmtree(cache_dir) + + def clearCaches(self, except_names=None): + for name in self.getCacheNames(except_names=except_names): + self.clearCache(name) + class SimpleCache(object): def __init__(self, base_dir): @@ -122,3 +133,15 @@ def getCache(self, name): return self.null_cache + def getCacheDir(self, name): + raise NotImplementedError() + + def getCacheNames(self, except_names=None): + return [] + + def clearCache(self, name): + pass + + def clearCaches(self, except_names=None): + pass +
--- a/piecrust/commands/base.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/commands/base.py Sun May 03 23:59:46 2015 -0700 @@ -19,6 +19,7 @@ self.name = '__unknown__' self.description = '__unknown__' self.requires_website = True + self.cache_name = 'default' def setupParser(self, parser, app): raise NotImplementedError()
--- a/piecrust/commands/builtin/serving.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/commands/builtin/serving.py Sun May 03 23:59:46 2015 -0700 @@ -11,6 +11,7 @@ super(ServeCommand, self).__init__() self.name = 'serve' self.description = "Runs a local web server to serve your website." + self.cache_name = 'server' def setupParser(self, parser, app): parser.add_argument( @@ -43,6 +44,7 @@ server = Server( ctx.app.root_dir, debug=debug, + sub_cache_dir=ctx.app.sub_cache_dir, use_reloader=ctx.args.use_reloader) app = server.getWsgiApp()
--- a/piecrust/commands/builtin/util.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/commands/builtin/util.py Sun May 03 23:59:46 2015 -0700 @@ -59,8 +59,8 @@ pass def run(self, ctx): - cache_dir = ctx.app.cache_dir - if os.path.isdir(cache_dir): + cache_dir = ctx.app.sub_cache_dir + if cache_dir and os.path.isdir(cache_dir): logger.info("Purging cache: %s" % cache_dir) shutil.rmtree(cache_dir)
--- a/piecrust/environment.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/environment.py Sun May 03 23:59:46 2015 -0700 @@ -137,7 +137,9 @@ self.exec_info_stack.clear() self.was_cache_cleaned = False self.base_asset_url_format = '%uri%' + self._onSubCacheDirChanged(app) + def _onSubCacheDirChanged(self, app): for name, repo in self.fs_caches.items(): cache = app.cache.getCache(name) repo.fs_cache = cache
--- a/piecrust/main.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/main.py Sun May 03 23:59:46 2015 -0700 @@ -181,14 +181,19 @@ else: app = PieCrust(root, cache=pre_args.cache, debug=pre_args.debug) + # Build a hash for a custom cache directory. + cache_key = 'default' + # Handle a configuration variant. if pre_args.config_variant is not None: if not root: raise SiteNotFoundError("Can't apply any variant.") app.config.applyVariant('variants/' + pre_args.config_variant) + cache_key += ',variant=%s' % pre_args.config_variant for name, value in pre_args.config_values: logger.debug("Setting configuration '%s' to: %s" % (name, value)) app.config.set(name, value) + cache_key += ',%s=%s' % (name, value) # Setup the arg parser. parser = argparse.ArgumentParser( @@ -234,6 +239,7 @@ p = subparsers.add_parser(c.name, help=c.description) c.setupParser(p, app) p.set_defaults(func=c.checkedRun) + p.set_defaults(cache_name=c.cache_name) help_cmd = next(filter(lambda c: c.name == 'help', commands), None) if help_cmd and help_cmd.has_topics: @@ -253,6 +259,10 @@ parser.print_help() return 0 + # Use a customized cache for the command and current config. + if result.cache_name != 'default' or cache_key != 'default': + app.useSubCache(result.cache_name, cache_key) + # Run the command! ctx = CommandContext(app, parser, result) exit_code = result.func(ctx)
--- a/piecrust/processing/base.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/processing/base.py Sun May 03 23:59:46 2015 -0700 @@ -132,7 +132,7 @@ self.out_dir = out_dir self.force = force - tmp_dir = app.cache_dir + tmp_dir = app.sub_cache_dir if not tmp_dir: import tempfile tmp_dir = os.path.join(tempfile.gettempdir(), 'piecrust')
--- a/piecrust/processing/sass.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/processing/sass.py Sun May 03 23:59:46 2015 -0700 @@ -128,7 +128,7 @@ cache_dir = None if self.app.cache.enabled: - cache_dir = os.path.join(self.app.cache_dir, 'sass') + cache_dir = os.path.join(self.app.sub_cache_dir, 'sass') self._conf.setdefault('cache_dir', cache_dir) def _getMapPath(self, path):
--- a/piecrust/serving.py Sun May 03 23:45:32 2015 -0700 +++ b/piecrust/serving.py Sun May 03 23:59:46 2015 -0700 @@ -66,9 +66,11 @@ class Server(object): def __init__(self, root_dir, - debug=False, use_reloader=False, static_preview=True): + debug=False, sub_cache_dir=None, + use_reloader=False, static_preview=True): self.root_dir = root_dir self.debug = debug + self.sub_cache_dir = sub_cache_dir self.use_reloader = use_reloader self.static_preview = static_preview self._out_dir = None @@ -80,7 +82,8 @@ # Bake all the assets so we know what we have, and so we can serve # them to the client. We need a temp app for this. app = PieCrust(root_dir=self.root_dir, debug=self.debug) - self._out_dir = os.path.join(app.cache_dir, 'server') + app._useSubCacheDir(self.sub_cache_dir) + self._out_dir = os.path.join(app.sub_cache_dir, 'server') self._page_record = ServeRecord() if (not self.use_reloader or @@ -129,6 +132,7 @@ # Create the app for this request. app = PieCrust(root_dir=self.root_dir, debug=self.debug) + app._useSubCacheDir(self.sub_cache_dir) app.config.set('site/root', '/') app.config.set('server/is_serving', True) if (app.config.get('site/enable_debug_info') and @@ -226,67 +230,35 @@ if len(routes) == 0: raise RouteNotFoundError("Can't find route for: %s" % req_path) - taxonomy = None - tax_terms = None + rendered_page = None + first_not_found = None for route, route_metadata in routes: - source = app.getSource(route.source_name) - if route.taxonomy_name is None: - factory = source.findPageFactory(route_metadata, MODE_PARSING) - if factory is not None: + try: + logger.debug("Trying to render match from source '%s'." % + route.source_name) + rendered_page = self._try_render_page( + app, route, route_metadata, page_num, req_path) + if rendered_page is not None: break - else: - taxonomy = app.getTaxonomy(route.taxonomy_name) - route_terms = route_metadata.get(taxonomy.term_name) - if route_terms is not None: - tax_page_ref = taxonomy.getPageRef(source.name) - factory = tax_page_ref.getFactory() - tax_terms = route.unslugifyTaxonomyTerm(route_terms) - factory.metadata[taxonomy.term_name] = tax_terms - break + except NotFound as nfe: + if first_not_found is None: + first_not_found = nfe else: raise SourceNotFoundError( "Can't find path for: %s (looked in: %s)" % (req_path, [r.source_name for r, _ in routes])) - # Build the page. - page = factory.buildPage() - # We force the rendering of the page because it could not have - # changed, but include pages that did change. - qp = QualifiedPage(page, route, route_metadata) - render_ctx = PageRenderingContext(qp, - page_num=page_num, - force_render=True) - if taxonomy is not None: - render_ctx.setTaxonomyFilter(taxonomy, tax_terms) + # If we haven't found any good match, raise whatever exception we + # first got. Otherwise, raise a generic exception. + if rendered_page is None: + first_not_found = first_not_found or NotFound( + "This page couldn't be found.") + raise first_not_found - # See if this page is known to use sources. If that's the case, - # just don't use cached rendered segments for that page (but still - # use them for pages that are included in it). - entry = self._page_record.getEntry(req_path, page_num) - if (taxonomy is not None or entry is None or - entry.used_source_names): - cache_key = '%s:%s' % (req_path, page_num) - app.env.rendered_segments_repository.invalidate(cache_key) - - # Render the page. - rendered_page = render_page(render_ctx) + # Start doing stuff. + page = rendered_page.page rp_content = rendered_page.content - if taxonomy is not None: - paginator = rendered_page.data.get('pagination') - if (paginator and paginator.is_loaded and - len(paginator.items) == 0): - message = ("This URL matched a route for taxonomy '%s' but " - "no pages have been found to have it. This page " - "won't be generated by a bake." % taxonomy.name) - raise NotFound(message) - - if entry is None: - entry = ServeRecordPageEntry(req_path, page_num) - self._page_record.addEntry(entry) - for p, pinfo in render_ctx.render_passes.items(): - entry.used_source_names |= pinfo.used_source_names - # Profiling. if app.config.get('site/show_debug_info'): now_time = time.clock() @@ -341,6 +313,74 @@ return response + def _try_render_page(self, app, route, route_metadata, page_num, req_path): + # Match the route to an actual factory. + taxonomy_info = None + source = app.getSource(route.source_name) + if route.taxonomy_name is None: + factory = source.findPageFactory(route_metadata, MODE_PARSING) + if factory is None: + return None + else: + taxonomy = app.getTaxonomy(route.taxonomy_name) + route_terms = route_metadata.get(taxonomy.term_name) + if route_terms is None: + return None + + tax_page_ref = taxonomy.getPageRef(source.name) + factory = tax_page_ref.getFactory() + tax_terms = route.unslugifyTaxonomyTerm(route_terms) + route_metadata[taxonomy.term_name] = tax_terms + taxonomy_info = (taxonomy, tax_terms) + + # Build the page. + page = factory.buildPage() + # We force the rendering of the page because it could not have + # changed, but include pages that did change. + qp = QualifiedPage(page, route, route_metadata) + render_ctx = PageRenderingContext(qp, + page_num=page_num, + force_render=True) + if taxonomy_info is not None: + taxonomy, tax_terms = taxonomy_info + render_ctx.setTaxonomyFilter(taxonomy, tax_terms) + + # See if this page is known to use sources. If that's the case, + # just don't use cached rendered segments for that page (but still + # use them for pages that are included in it). + uri = qp.getUri() + assert uri == req_path + entry = self._page_record.getEntry(uri, page_num) + if (taxonomy_info is not None or entry is None or + entry.used_source_names): + cache_key = '%s:%s' % (uri, page_num) + app.env.rendered_segments_repository.invalidate(cache_key) + + # Render the page. + rendered_page = render_page(render_ctx) + + # Check if this page is a taxonomy page that actually doesn't match + # anything. + if taxonomy_info is not None: + paginator = rendered_page.data.get('pagination') + if (paginator and paginator.is_loaded and + len(paginator.items) == 0): + taxonomy = taxonomy_info[0] + message = ("This URL matched a route for taxonomy '%s' but " + "no pages have been found to have it. This page " + "won't be generated by a bake." % taxonomy.name) + raise NotFound(message) + + # Remember stuff for next time. + if entry is None: + entry = ServeRecordPageEntry(req_path, page_num) + self._page_record.addEntry(entry) + for p, pinfo in render_ctx.render_passes.items(): + entry.used_source_names |= pinfo.used_source_names + + # Ok all good. + return rendered_page + def _make_wrapped_file_response(self, environ, request, path): logger.debug("Serving %s" % path)