Mercurial > piecrust2
changeset 666:81d9c3a3a0b5
internal: Get rid of the whole "sub cache" business.
* Compute cache keys up front, so the cache directory is only chosen once.
* Buffer up config variants to apply before loading the config. Makes it
possible to cache variant-resulting configs, too.
* Make a factory class to reuse the logic that creates the `PieCrust` object
correctly for multi-process workers and such.
* Add a test.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Thu, 03 Mar 2016 08:22:41 -0800 |
| parents | 5dc13c816045 |
| children | fc761964e1a7 |
| files | piecrust/app.py piecrust/appconfig.py piecrust/baking/baker.py piecrust/baking/worker.py piecrust/commands/builtin/baking.py piecrust/commands/builtin/serving.py piecrust/commands/builtin/util.py piecrust/configuration.py piecrust/environment.py piecrust/main.py piecrust/processing/pipeline.py piecrust/processing/sass.py piecrust/processing/worker.py piecrust/serving/middlewares.py piecrust/serving/procloop.py piecrust/serving/server.py piecrust/serving/util.py piecrust/serving/wrappers.py piecrust/wsgiutil/__init__.py tests/bakes/test_variant.yaml tests/conftest.py |
| diffstat | 21 files changed, 297 insertions(+), 237 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/app.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/app.py Thu Mar 03 08:22:41 2016 -0800 @@ -23,15 +23,15 @@ class PieCrust(object): def __init__(self, root_dir, cache=True, debug=False, theme_site=False, - env=None): + env=None, cache_key=None): self.root_dir = root_dir self.debug = debug self.theme_site = theme_site self.plugin_loader = PluginLoader(self) + self.cache_key = cache_key or 'default' if cache: - cache_dir = os.path.join(self.cache_dir, 'default') - self.cache = ExtensibleCache(cache_dir) + self.cache = ExtensibleCache(self.cache_dir) else: self.cache = NullExtensibleCache() @@ -82,7 +82,7 @@ if srcc is not None: for sn, sc in srcc.items(): sc['realm'] = REALM_THEME - config.fixups.append(_fixupThemeSources) + config.addFixup(_fixupThemeSources) self.env.stepTimer('SiteConfigLoad', time.perf_counter() - start_time) return config @@ -124,13 +124,7 @@ @cached_property def cache_dir(self): - return os.path.join(self.root_dir, CACHE_DIR) - - @property # Not a cached property because its result can change. - def sub_cache_dir(self): - if self.cache.enabled: - return self.cache.base_dir - return None + return os.path.join(self.root_dir, CACHE_DIR, self.cache_key) @cached_property def sources(self): @@ -197,18 +191,6 @@ return tax return None - def useSubCache(self, cache_name, cache_key): - cache_hash = hashlib.md5(cache_key.encode('utf8')).hexdigest() - cache_dir = os.path.join(self.cache_dir, - '%s_%s' % (cache_name, cache_hash)) - self._useSubCacheDir(cache_dir) - - def _useSubCacheDir(self, cache_dir): - assert cache_dir - logger.debug("Moving cache to: %s" % cache_dir) - self.cache = ExtensibleCache(cache_dir) - self.env._onSubCacheDirChanged(self) - def _get_dir(self, default_rel_dir): abs_dir = os.path.join(self.root_dir, default_rel_dir) if os.path.isdir(abs_dir): @@ -234,13 +216,42 @@ return dirs + def apply_variant_and_values(app, config_variant=None, config_values=None): if config_variant is not None: - logger.debug("Applying configuration variant '%s'." % config_variant) - app.config.applyVariant('variants/' + config_variant) + logger.debug("Adding configuration variant '%s'." % config_variant) + variant_path = os.path.join( + app.root_dir, 'configs', '%s.yml' % config_variant) + app.config.addVariant(variant_path) if config_values is not None: for name, value in config_values: - logger.debug("Setting configuration '%s' to: %s" % (name, value)) - app.config.set(name, value) + logger.debug("Adding configuration override '%s': %s" % (name, value)) + app.config.addVariantValue(name, value) + +class PieCrustFactory(object): + def __init__( + self, root_dir, *, + cache=True, cache_key=None, + config_variant=None, config_values=None, + debug=False, theme_site=False): + self.root_dir = root_dir + self.cache = cache + self.cache_key = cache_key + self.config_variant = config_variant + self.config_values = config_values + self.debug = debug + self.theme_site = theme_site + + def create(self): + app = PieCrust( + self.root_dir, + cache=self.cache, + cache_key=self.cache_key, + debug=self.debug, + theme_site=self.theme_site) + apply_variant_and_values( + app, self.config_variant, self.config_values) + return app +
--- a/piecrust/appconfig.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/appconfig.py Thu Mar 03 08:22:41 2016 -0800 @@ -14,7 +14,7 @@ from piecrust.cache import NullCache from piecrust.configuration import ( Configuration, ConfigurationError, ConfigurationLoader, - merge_dicts, visit_dict) + get_dict_value, set_dict_value, merge_dicts, visit_dict) from piecrust.sources.base import REALM_USER, REALM_THEME @@ -22,55 +22,100 @@ class VariantNotFoundError(Exception): - def __init__(self, variant_path, message=None): + def __init__(self, variant_name, message=None): super(VariantNotFoundError, self).__init__( message or ("No such configuration variant: %s" % - variant_path)) + variant_name)) + + +def _make_variant_fixup(variant_name, raise_if_not_found): + def _variant_fixup(index, config): + if index != -1: + return + try: + try: + v = get_dict_value(config, 'variants/%s' % variant_name) + except KeyError: + raise VariantNotFoundError(variant_name) + if not isinstance(v, dict): + raise VariantNotFoundError( + variant_name, + "Configuration variant '%s' is not an array. " + "Check your configuration file." % variant_name) + merge_dicts(config, v) + except VariantNotFoundError: + if raise_if_not_found: + raise + + return _variant_fixup class PieCrustConfiguration(Configuration): def __init__(self, paths=None, cache=None, values=None, validate=True, theme_config=False): super(PieCrustConfiguration, self).__init__() - self.paths = paths - self.cache = cache or NullCache() - self.fixups = [] + self._paths = paths + self._cache = cache or NullCache() + self._fixups = [] self.theme_config = theme_config # Set the values after we set the rest, since our validation needs # our attributes. if values: self.setAll(values, validate=validate) - def applyVariant(self, variant_path, raise_if_not_found=True): - variant = self.get(variant_path) - if variant is None: - if raise_if_not_found: - raise VariantNotFoundError(variant_path) - return - if not isinstance(variant, dict): - raise VariantNotFoundError( - variant_path, - "Configuration variant '%s' is not an array. " - "Check your configuration file." % variant_path) - self.merge(variant) + def addFixup(self, f): + self._ensureNotLoaded() + self._fixups.append(f) + + def addPath(self, p, first=False): + self._ensureNotLoaded() + if not first: + self._paths.append(p) + else: + self._paths.insert(0, p) + + def addVariant(self, variant_path, raise_if_not_found=True): + self._ensureNotLoaded() + if os.path.isfile(variant_path): + self.addPath(variant_path) + else: + name, _ = os.path.splitext(os.path.basename(variant_path)) + fixup = _make_variant_fixup(name, raise_if_not_found) + self.addFixup(fixup) + + logger.warning( + "Configuration variants should now be `.yml` files located " + "in the `configs/` directory of your website.") + logger.warning( + "Variants defined in the site configuration will be " + "deprecated in a future version of PieCrust.") + + def addVariantValue(self, path, value): + def _fixup(index, config): + set_dict_value(config, path, value) + self.addFixup(_fixup) + + def _ensureNotLoaded(self): + if self._values is not None: + raise Exception("The configurations has been loaded.") def _load(self): - if self.paths is None: + if self._paths is None: self._values = self._validateAll({}) return - path_times = [os.path.getmtime(p) for p in self.paths] + path_times = [os.path.getmtime(p) for p in self._paths] cache_key_hash = hashlib.md5( ("version=%s&cache=%d" % ( APP_VERSION, CACHE_VERSION)).encode('utf8')) - for p in self.paths: + for p in self._paths: cache_key_hash.update(("&path=%s" % p).encode('utf8')) cache_key = cache_key_hash.hexdigest() - if self.cache.isValid('config.json', path_times): + if self._cache.isValid('config.json', path_times): logger.debug("Loading configuration from cache...") - config_text = self.cache.read('config.json') + config_text = self._cache.read('config.json') self._values = json.loads( config_text, object_pairs_hook=collections.OrderedDict) @@ -82,32 +127,32 @@ logger.debug("Outdated cache key '%s' (expected '%s')." % ( actual_cache_key, cache_key)) - logger.debug("Loading configuration from: %s" % self.paths) + logger.debug("Loading configuration from: %s" % self._paths) values = {} try: - for i, p in enumerate(self.paths): + for i, p in enumerate(self._paths): with open(p, 'r', encoding='utf-8') as fp: loaded_values = yaml.load( fp.read(), Loader=ConfigurationLoader) if loaded_values is None: loaded_values = {} - for fixup in self.fixups: + for fixup in self._fixups: fixup(i, loaded_values) merge_dicts(values, loaded_values) - for fixup in self.fixups: - fixup(len(self.paths), values) + for fixup in self._fixups: + fixup(-1, values) self._values = self._validateAll(values) except Exception as ex: raise Exception("Error loading configuration from: %s" % - ', '.join(self.paths)) from ex + ', '.join(self._paths)) from ex logger.debug("Caching configuration...") self._values['__cache_key'] = cache_key config_text = json.dumps(self._values) - self.cache.write('config.json', config_text) + self._cache.write('config.json', config_text) self._values['__cache_valid'] = False
--- a/piecrust/baking/baker.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/baking/baker.py Thu Mar 03 08:22:41 2016 -0800 @@ -540,19 +540,27 @@ logger.error(" " + e) def _createWorkerPool(self, previous_record_path): + from piecrust.app import PieCrustFactory from piecrust.workerpool import WorkerPool from piecrust.baking.worker import BakeWorkerContext, BakeWorker + appfactory = PieCrustFactory( + self.app.root_dir, + cache=self.app.cache.enabled, + cache_key=self.app.cache_key, + config_variant=self.applied_config_variant, + config_values=self.applied_config_values, + debug=self.app.debug, + theme_site=self.app.theme_site) + worker_count = self.app.config.get('baker/workers') batch_size = self.app.config.get('baker/batch_size') ctx = BakeWorkerContext( - self.app.root_dir, self.app.cache.base_dir, self.out_dir, - previous_record_path=previous_record_path, - config_variant=self.applied_config_variant, - config_values=self.applied_config_values, - force=self.force, debug=self.app.debug, - theme_site=self.app.theme_site) + appfactory, + self.out_dir, + force=self.force, + previous_record_path=previous_record_path) pool = WorkerPool( worker_count=worker_count, batch_size=batch_size,
--- a/piecrust/baking/worker.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/baking/worker.py Thu Mar 03 08:22:41 2016 -0800 @@ -15,19 +15,12 @@ class BakeWorkerContext(object): - def __init__(self, root_dir, sub_cache_dir, out_dir, *, - previous_record_path=None, - config_variant=None, config_values=None, - force=False, debug=False, theme_site=False): - self.root_dir = root_dir - self.sub_cache_dir = sub_cache_dir + def __init__(self, appfactory, out_dir, *, + force=False, previous_record_path=None): + self.appfactory = appfactory self.out_dir = out_dir + self.force = force self.previous_record_path = previous_record_path - self.config_variant = config_variant - self.config_values = config_values - self.force = force - self.debug = debug - self.theme_site = theme_site self.app = None self.previous_record = None self.previous_record_index = None @@ -40,9 +33,7 @@ def initialize(self): # Create the app local to this worker. - app = PieCrust(self.ctx.root_dir, debug=self.ctx.debug, - theme_site=self.ctx.theme_site) - app._useSubCacheDir(self.ctx.sub_cache_dir) + app = self.ctx.appfactory.create() app.config.set('baker/is_baking', True) app.config.set('baker/worker_id', self.wid) app.env.base_asset_url_format = '%uri%' @@ -50,8 +41,6 @@ app.env.registerTimer("BakeWorker_%d_Total" % self.wid) app.env.registerTimer("BakeWorkerInit") app.env.registerTimer("JobReceive") - apply_variant_and_values(app, self.ctx.config_variant, - self.ctx.config_values) self.ctx.app = app # Load previous record @@ -139,7 +128,7 @@ except Exception as ex: logger.debug("Got loading error. Sending it to master.") result['errors'] = _get_errors(ex) - if self.ctx.debug: + if self.ctx.app.debug: logger.exception(ex) return result @@ -223,7 +212,7 @@ except BakingError as ex: logger.debug("Got baking error. Sending it to master.") result['errors'] = _get_errors(ex) - if self.ctx.debug: + if self.ctx.app.debug: logger.exception(ex) return result
--- a/piecrust/commands/builtin/baking.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/commands/builtin/baking.py Thu Mar 03 08:22:41 2016 -0800 @@ -109,7 +109,9 @@ def _bakeAssets(self, ctx, out_dir): proc = ProcessorPipeline( ctx.app, out_dir, - force=ctx.args.force) + force=ctx.args.force, + applied_config_variant=ctx.config_variant, + applied_config_values=ctx.config_values) record = proc.run() _merge_timers(record.timers, ctx.timers) return record.success
--- a/piecrust/commands/builtin/serving.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/commands/builtin/serving.py Thu Mar 03 08:22:41 2016 -0800 @@ -42,12 +42,19 @@ port = int(ctx.args.port) debug = ctx.args.debug or ctx.args.use_debugger + from piecrust.app import PieCrustFactory + appfactory = PieCrustFactory( + ctx.app.root_dir, + cache=ctx.app.cache.enabled, + cache_key=ctx.app.cache_key, + config_variant=ctx.config_variant, + config_values=ctx.config_values, + debug=ctx.app.debug, + theme_site=ctx.app.theme_site) + if ctx.args.wsgi == 'werkzeug': run_werkzeug_server( - root_dir, host, port, - debug_piecrust=debug, - theme_site=ctx.args.theme, - sub_cache_dir=ctx.app.sub_cache_dir, + appfactory, host, port, use_debugger=debug, use_reloader=ctx.args.use_reloader) @@ -60,10 +67,5 @@ options['loglevel'] = 'debug' if ctx.args.use_reloader: options['reload'] = True - run_gunicorn_server( - root_dir, - debug_piecrust=debug, - theme_site=ctx.args.theme, - sub_cache_dir=ctx.app.sub_cache_dir, - gunicorn_options=options) + run_gunicorn_server(appfactory, gunicorn_options=options)
--- a/piecrust/commands/builtin/util.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/commands/builtin/util.py Thu Mar 03 08:22:41 2016 -0800 @@ -63,7 +63,7 @@ pass def run(self, ctx): - cache_dir = ctx.app.sub_cache_dir + cache_dir = ctx.app.cache_dir if cache_dir and os.path.isdir(cache_dir): logger.info("Purging cache: %s" % cache_dir) shutil.rmtree(cache_dir)
--- a/piecrust/configuration.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/configuration.py Thu Mar 03 08:22:41 2016 -0800 @@ -28,28 +28,15 @@ def __getitem__(self, key): self._ensureLoaded() - bits = key.split('/') - cur = self._values - for b in bits: - try: - cur = cur[b] - except KeyError: - raise KeyError("No such item: %s" % key) - return cur + try: + return get_dict_value(self._values, key) + except KeyError: + raise KeyError("No such item: %s" % key) def __setitem__(self, key, value): self._ensureLoaded() value = self._validateValue(key, value) - bits = key.split('/') - bitslen = len(bits) - cur = self._values - for i, b in enumerate(bits): - if i == bitslen - 1: - cur[b] = value - else: - if b not in cur: - cur[b] = {} - cur = cur[b] + set_dict_value(self._values, key, value) def __delitem__(self, key): raise NotImplementedError() @@ -129,6 +116,27 @@ return value +def get_dict_value(d, key): + bits = key.split('/') + cur = d + for b in bits: + cur = cur[b] + return cur + + +def set_dict_value(d, key, value): + bits = key.split('/') + bitslen = len(bits) + cur = d + for i, b in enumerate(bits): + if i == bitslen - 1: + cur[b] = value + else: + if b not in cur: + cur[b] = {} + cur = cur[b] + + def merge_dicts(source, merging, validator=None, *args): _recurse_merge_dicts(source, merging, None, validator) for other in args:
--- a/piecrust/environment.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/environment.py Thu Mar 03 08:22:41 2016 -0800 @@ -89,7 +89,9 @@ self.was_cache_cleaned = False self.base_asset_url_format = '%uri%' - self._onSubCacheDirChanged(app) + for name, repo in self.fs_caches.items(): + cache = app.cache.getCache(name) + repo.fs_cache = cache def registerTimer(self, category, *, raise_if_registered=True): if raise_if_registered and category in self._timers: @@ -109,11 +111,6 @@ def stepTimerSince(self, category, since): self.stepTimer(category, time.perf_counter() - since) - def _onSubCacheDirChanged(self, app): - for name, repo in self.fs_caches.items(): - cache = app.cache.getCache(name) - repo.fs_cache = cache - class StandardEnvironment(Environment): def __init__(self):
--- a/piecrust/main.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/main.py Thu Mar 03 08:22:41 2016 -0800 @@ -3,6 +3,7 @@ import io import sys import time +import hashlib import logging import argparse import colorama @@ -51,9 +52,6 @@ self.plugin_loader = PluginLoader(self) self.env = None - def useSubCache(self, cache_name, cache_key): - pass - def main(): if sys.platform == 'darwin': @@ -135,6 +133,15 @@ help="Write a PID file for the current process.") +""" Kinda hacky, but we want the `serve` command to use a different cache + so that PieCrust doesn't need to re-render all the pages when going + between `serve` and `bake` (or, worse, *not* re-render them all correctly + and end up serving or baking the wrong version). +""" +_command_caches = { + 'serve': 'server'} + + def _pre_parse_chef_args(argv): # We need to parse some arguments before we can build the actual argument # parser, because it can affect which plugins will be loaded. Also, log- @@ -142,7 +149,7 @@ # from the beginning. parser = argparse.ArgumentParser() _setup_main_parser_arguments(parser) - parser.add_argument('args', nargs=argparse.REMAINDER) + parser.add_argument('extra_args', nargs=argparse.REMAINDER) res, _ = parser.parse_known_args(argv) # Setup the logger. @@ -196,6 +203,23 @@ return res +def _build_cache_key(pre_args): + cache_key_str = 'default' + if pre_args.extra_args: + cmd_name = pre_args.extra_args[0] + if cmd_name in _command_caches: + cache_key_str = _command_caches[cmd_name] + if pre_args.config_variant is not None: + cache_key_str += ',variant=%s' % pre_args.config_variant + if pre_args.config_values: + for name, value in pre_args.config_values: + cache_key_str += ',%s=%s' % (name, value) + + logger.debug("Using cache key: %s" % cache_key_str) + cache_key = hashlib.md5(cache_key_str.encode('utf8')).hexdigest() + return cache_key + + def _run_chef(pre_args, argv): # Setup the app. start_time = time.perf_counter() @@ -208,33 +232,27 @@ except SiteNotFoundError: root = None - if not root: - app = NullPieCrust( - theme_site=pre_args.theme) - else: + # Can't apply custom configuration stuff if there's no website. + if (pre_args.config_variant or pre_args.config_values) and not root: + raise SiteNotFoundError( + "Can't apply any configuration variant or value overrides, " + "there is no website here.") + + if root: + cache_key = None + if not pre_args.no_cache: + cache_key = _build_cache_key(pre_args) app = PieCrust( root, theme_site=pre_args.theme, cache=(not pre_args.no_cache), + cache_key=cache_key, debug=pre_args.debug) - - # Build a hash for a custom cache directory. - cache_key = 'default' - - # Handle custom configurations. - if (pre_args.config_variant or pre_args.config_values) and not root: - raise SiteNotFoundError( - "Can't apply any configuration variant or value overrides, " - "there is no website here.") - apply_variant_and_values(app, pre_args.config_variant, - pre_args.config_values) - - # Adjust the cache key. - if pre_args.config_variant is not None: - cache_key += ',variant=%s' % pre_args.config_variant - if pre_args.config_values: - for name, value in pre_args.config_values: - cache_key += ',%s=%s' % (name, value) + apply_variant_and_values( + app, pre_args.config_variant, pre_args.config_values) + else: + app = NullPieCrust( + theme_site=pre_args.theme) # Setup the arg parser. parser = argparse.ArgumentParser( @@ -270,10 +288,6 @@ parser.print_help() return 0 - # Use a customized cache for the command and current config. - if result.cache_name != 'default' or cache_key != 'default': - app.useSubCache(result.cache_name, cache_key) - # Run the command! ctx = CommandContext(app, parser, result) ctx.config_variant = pre_args.config_variant
--- a/piecrust/processing/pipeline.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/processing/pipeline.py Thu Mar 03 08:22:41 2016 -0800 @@ -27,13 +27,17 @@ class ProcessorPipeline(object): - def __init__(self, app, out_dir, force=False): + def __init__(self, app, out_dir, force=False, + applied_config_variant=None, + applied_config_values=None): assert app and out_dir self.app = app self.out_dir = out_dir self.force = force + self.applied_config_variant = applied_config_variant + self.applied_config_values = applied_config_values - tmp_dir = app.sub_cache_dir + tmp_dir = app.cache_dir if not tmp_dir: import tempfile tmp_dir = os.path.join(tempfile.gettempdir(), 'piecrust') @@ -246,14 +250,24 @@ ctx.jobs.append(job) def _createWorkerPool(self): + from piecrust.app import PieCrustFactory from piecrust.workerpool import WorkerPool from piecrust.processing.worker import ( ProcessingWorkerContext, ProcessingWorker) + appfactory = PieCrustFactory( + self.app.root_dir, + cache=self.app.cache.enabled, + cache_key=self.app.cache_key, + config_variant=self.applied_config_variant, + config_values=self.applied_config_values, + debug=self.app.debug, + theme_site=self.app.theme_site) + ctx = ProcessingWorkerContext( - self.app.root_dir, self.out_dir, self.tmp_dir, - force=self.force, debug=self.app.debug, - theme_site=self.app.theme_site) + appfactory, + self.out_dir, self.tmp_dir, + force=self.force) ctx.enabled_processors = self.enabled_processors if self.additional_processors_factories is not None: ctx.additional_processors = [
--- a/piecrust/processing/sass.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/processing/sass.py Thu Mar 03 08:22:41 2016 -0800 @@ -129,7 +129,7 @@ cache_dir = None if self.app.cache.enabled: - cache_dir = os.path.join(self.app.sub_cache_dir, 'sass') + cache_dir = os.path.join(self.app.cache_dir, 'sass') self._conf.setdefault('cache_dir', cache_dir) def _getMapPath(self, path):
--- a/piecrust/processing/worker.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/processing/worker.py Thu Mar 03 08:22:41 2016 -0800 @@ -2,7 +2,7 @@ import os.path import time import logging -from piecrust.app import PieCrust +from piecrust.app import PieCrust, apply_variant_and_values from piecrust.processing.base import PipelineContext from piecrust.processing.records import ( FLAG_NONE, FLAG_PREPARED, FLAG_PROCESSED, @@ -23,14 +23,12 @@ class ProcessingWorkerContext(object): - def __init__(self, root_dir, out_dir, tmp_dir, *, - force=False, debug=False, theme_site=False): - self.root_dir = root_dir + def __init__(self, appfactory, out_dir, tmp_dir, *, + force=False): + self.appfactory = appfactory self.out_dir = out_dir self.tmp_dir = tmp_dir self.force = force - self.debug = debug - self.theme_site = theme_site self.is_profiling = False self.enabled_processors = None self.additional_processors = None @@ -60,8 +58,7 @@ def initialize(self): # Create the app local to this worker. - app = PieCrust(self.ctx.root_dir, debug=self.ctx.debug, - theme_site=self.ctx.theme_site) + app = self.ctx.appfactory.create() app.env.registerTimer("PipelineWorker_%d_Total" % self.wid) app.env.registerTimer("PipelineWorkerInit") app.env.registerTimer("JobReceive")
--- a/piecrust/serving/middlewares.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/serving/middlewares.py Thu Mar 03 08:22:41 2016 -0800 @@ -40,18 +40,14 @@ class PieCrustDebugMiddleware(object): """ WSGI middleware that handles debugging of PieCrust stuff. """ - def __init__(self, app, root_dir, debug=False, theme_site=False, - sub_cache_dir=None, run_sse_check=None): + def __init__(self, app, appfactory, + run_sse_check=None): self.app = app - self.root_dir = root_dir - self.debug = debug - self.theme_site = theme_site - self.sub_cache_dir = sub_cache_dir + self.appfactory = appfactory self.run_sse_check = run_sse_check self._proc_loop = None - self._out_dir = os.path.join(root_dir, CACHE_DIR, 'server') - if sub_cache_dir: - self._out_dir = os.path.join(sub_cache_dir, 'server') + self._out_dir = os.path.join( + root_dir, CACHE_DIR, appfactory.cache_key, 'server') self._handlers = { 'debug_info': self._getDebugInfo, 'werkzeug_shutdown': self._shutdownWerkzeug, @@ -63,10 +59,7 @@ # to start the pipeline loop in the inner process most of the # time so we let the implementation tell us if this is OK. from piecrust.serving.procloop import ProcessingLoop - self._proc_loop = ProcessingLoop(root_dir, self._out_dir, - theme_site=theme_site, - sub_cache_dir=sub_cache_dir, - debug=debug) + self._proc_loop = ProcessingLoop(self.appfactory, self._out_dir) self._proc_loop.start() def __call__(self, environ, start_response): @@ -82,8 +75,7 @@ return self.app(environ, start_response) def _getDebugInfo(self, request, start_response): - app = get_app_for_server(self.root_dir, debug=self.debug, - sub_cache_dir=self.sub_cache_dir) + app = get_app_for_server(self.appfactory) if not app.config.get('site/enable_debug_info'): return Forbidden()
--- a/piecrust/serving/procloop.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/serving/procloop.py Thu Mar 03 08:22:41 2016 -0800 @@ -75,15 +75,10 @@ class ProcessingLoop(threading.Thread): - def __init__(self, root_dir, out_dir, sub_cache_dir=None, - theme_site=False, debug=False): + def __init__(self, appfactory): super(ProcessingLoop, self).__init__( name='pipeline-reloader', daemon=True) - self.root_dir = root_dir - self.out_dir = out_dir - self.sub_cache_dir = sub_cache_dir - self.debug = debug - self.theme_site = theme_site + self.appfactory = appfactory self.last_status_id = 0 self.interval = 1 self.app = None @@ -95,7 +90,7 @@ self._last_config_mtime = 0 self._obs = [] self._obs_lock = threading.Lock() - if theme_site: + if appfactory.theme_site: self._config_path = os.path.join(root_dir, THEME_CONFIG_PATH) else: self._config_path = os.path.join(root_dir, CONFIG_PATH) @@ -162,10 +157,7 @@ def _initPipeline(self): # Create the app and pipeline. - self.app = PieCrust(root_dir=self.root_dir, debug=self.debug, - theme_site=self.theme_site) - if self.sub_cache_dir: - self.app._useSubCacheDir(self.sub_cache_dir) + self.app = self.appfactory.create() self.pipeline = ProcessorPipeline(self.app, self.out_dir) # Get the list of assets directories.
--- a/piecrust/serving/server.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/serving/server.py Thu Mar 03 08:22:41 2016 -0800 @@ -22,8 +22,8 @@ class WsgiServer(object): - def __init__(self, root_dir, **kwargs): - self.server = Server(root_dir, **kwargs) + def __init__(self, appfactory, **kwargs): + self.server = Server(appfactory, **kwargs) def __call__(self, environ, start_response): return self.server._run_request(environ, start_response) @@ -70,21 +70,20 @@ class Server(object): - def __init__(self, root_dir, - debug=False, theme_site=False, - sub_cache_dir=None, enable_debug_info=True, - root_url='/', static_preview=True): - self.root_dir = root_dir - self.debug = debug - self.theme_site = theme_site - self.sub_cache_dir = sub_cache_dir + def __init__(self, appfactory, + enable_debug_info=True, + root_url='/', + static_preview=True): + self.appfactory = appfactory self.enable_debug_info = enable_debug_info self.root_url = root_url self.static_preview = static_preview self._page_record = ServeRecord() - self._out_dir = os.path.join(root_dir, CACHE_DIR, 'server') - if sub_cache_dir: - self._out_dir = os.path.join(sub_cache_dir, 'server') + self._out_dir = os.path.join( + appfactory.root_dir, + CACHE_DIR, + (appfactory.cache_key or 'default'), + 'server') def _run_request(self, environ, start_response): try: @@ -111,9 +110,7 @@ return response # Create the app for this request. - app = get_app_for_server(self.root_dir, debug=self.debug, - theme_site=self.theme_site, - sub_cache_dir=self.sub_cache_dir, + app = get_app_for_server(self.appfactory, root_url=self.root_url) if (app.config.get('site/enable_debug_info') and self.enable_debug_info and
--- a/piecrust/serving/util.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/serving/util.py Thu Mar 03 08:22:41 2016 -0800 @@ -5,7 +5,7 @@ import datetime from werkzeug.wrappers import Response from werkzeug.wsgi import wrap_file -from piecrust.app import PieCrust +from piecrust.app import PieCrust, apply_variant_and_values from piecrust.rendering import QualifiedPage from piecrust.routing import RouteNotFoundError from piecrust.sources.base import MODE_PARSING @@ -16,11 +16,8 @@ logger = logging.getLogger(__name__) -def get_app_for_server(root_dir, debug=False, theme_site=False, - sub_cache_dir=None, root_url='/'): - app = PieCrust(root_dir=root_dir, debug=debug, theme_site=theme_site) - if sub_cache_dir: - app._useSubCacheDir(sub_cache_dir) +def get_app_for_server(appfactory, root_url='/'): + app = appfactory.create() app.config.set('site/root', root_url) app.config.set('server/is_serving', True) return app
--- a/piecrust/serving/wrappers.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/serving/wrappers.py Thu Mar 03 08:22:41 2016 -0800 @@ -8,9 +8,7 @@ logger = logging.getLogger(__name__) -def run_werkzeug_server(root_dir, host, port, - debug_piecrust=False, theme_site=False, - sub_cache_dir=None, +def run_werkzeug_server(appfactory, host, port, use_debugger=False, use_reloader=False): from werkzeug.serving import run_simple @@ -24,10 +22,7 @@ return (not use_reloader or os.environ.get('WERKZEUG_RUN_MAIN') == 'true') - app = _get_piecrust_server(root_dir, - debug=debug_piecrust, - theme_site=theme_site, - sub_cache_dir=sub_cache_dir, + app = _get_piecrust_server(appfactory, run_sse_check=_run_sse_check) # We need to do a few things to get Werkzeug to properly shutdown or @@ -79,10 +74,7 @@ raise -def run_gunicorn_server(root_dir, - debug_piecrust=False, theme_site=False, - sub_cache_dir=None, - gunicorn_options=None): +def run_gunicorn_server(appfactory, gunicorn_options=None): from gunicorn.app.base import BaseApplication class PieCrustGunicornApplication(BaseApplication): @@ -99,28 +91,20 @@ def load(self): return self.app - app = _get_piecrust_server(root_dir, - debug=debug_piecrust, - theme_site=theme_site, - sub_cache_dir=sub_cache_dir) + app = _get_piecrust_server(appfactory) gunicorn_options = gunicorn_options or {} app_wrapper = PieCrustGunicornApplication(app, gunicorn_options) app_wrapper.run() -def _get_piecrust_server( - root_dir, debug=False, theme_site=False, - sub_cache_dir=None, run_sse_check=None): +def _get_piecrust_server(appfactory, run_sse_check=None): from piecrust.serving.middlewares import ( StaticResourcesMiddleware, PieCrustDebugMiddleware) from piecrust.serving.server import WsgiServer - app = WsgiServer(root_dir, debug=debug, theme_site=theme_site, - sub_cache_dir=sub_cache_dir) + app = WsgiServer(appfactory) app = StaticResourcesMiddleware(app) - app = PieCrustDebugMiddleware(app, root_dir, - theme_site=theme_site, - sub_cache_dir=sub_cache_dir, - run_sse_check=run_sse_check) + app = PieCrustDebugMiddleware( + app, appfactory, run_sse_check=run_sse_check) return app
--- a/piecrust/wsgiutil/__init__.py Thu Mar 03 08:19:28 2016 -0800 +++ b/piecrust/wsgiutil/__init__.py Thu Mar 03 08:22:41 2016 -0800 @@ -1,9 +1,9 @@ from piecrust.serving.server import WsgiServer -def get_app(root_dir, sub_cache_dir='prod', enable_debug_info=False): +def get_app(root_dir, cache_key='prod', enable_debug_info=False): app = WsgiServer(root_dir, - sub_cache_dir=sub_cache_dir, + cache_key=cache_key, enable_debug_info=enable_debug_info) return app
--- a/tests/bakes/test_variant.yaml Thu Mar 03 08:19:28 2016 -0800 +++ b/tests/bakes/test_variant.yaml Thu Mar 03 08:22:41 2016 -0800 @@ -12,6 +12,15 @@ --- config: what: not good +config_variant: test +in: + pages/_index.md: 'This is {{what}}.' + configs/test.yml: 'what: awesome' +out: + index.html: 'This is awesome.' +--- +config: + what: not good config_values: what: awesome in:
--- a/tests/conftest.py Thu Mar 03 08:19:28 2016 -0800 +++ b/tests/conftest.py Thu Mar 03 08:22:41 2016 -0800 @@ -322,9 +322,11 @@ from werkzeug.test import Client from werkzeug.wrappers import BaseResponse + from piecrust.app import PieCrustFactory from piecrust.serving.server import Server with mock_fs_scope(fs): - server = Server(fs.path('/kitchen')) + appfactory = PieCrustFactory(fs.path('/kitchen')) + server = Server(appfactory) test_app = self._TestApp(server) client = Client(test_app, BaseResponse) resp = client.get(url)