Mercurial > piecrust2
changeset 979:45ad976712ec
tests: Big push to get the tests to pass again.
- Lots of fixes everywhere in the code.
- Try to handle debug logging in the multiprocessing worker pool when running in pytest. Not perfect, but usable for now.
- Replace all `.md` test files with `.html` since now a auto-format extension always sets the format.
- Replace `out` with `outfiles` in most places since now blog archives are added to the bake output and I don't want to add expected outputs for blog archives everywhere.
line wrap: on
line diff
--- a/piecrust/admin/web.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/admin/web.py Sun Oct 29 22:51:57 2017 -0700 @@ -1,7 +1,6 @@ import os.path import logging from flask import Flask -from werkzeug import SharedDataMiddleware logger = logging.getLogger(__name__) @@ -38,14 +37,14 @@ if app.config.get('FOODTRUCK_DEBUG_404'): @app.errorhandler(404) def page_not_found(e): - return _debug_page_not_found(e) + return _debug_page_not_found(app, e) logger.debug("Created FoodTruck app with admin root: %s" % root_dir) return app -def _debug_page_not_found(e): +def _debug_page_not_found(app, e): from flask import request, url_for output = [] for rule in app.url_map.iter_rules(): @@ -60,7 +59,8 @@ line = ("{:50s} {:20s} {}".format(rule.endpoint, methods, url)) output.append(line) - resp = 'FOODTRUCK_ROOT_URL=%s<br/>\n' % str(app.config['FOODTRUCK_ROOT_URL']) + resp = 'FOODTRUCK_ROOT_URL=%s<br/>\n' % str( + app.config['FOODTRUCK_ROOT_URL']) resp += 'PATH=%s<br/>\n' % request.path resp += 'ENVIRON=%s<br/>\n' % str(request.environ) resp += 'URL RULES:<br/>\n'
--- a/piecrust/appconfig.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/appconfig.py Sun Oct 29 22:51:57 2017 -0700 @@ -391,6 +391,7 @@ sc.setdefault('ignore_missing_dir', False) sc.setdefault('data_endpoint', None) sc.setdefault('data_type', None) + sc.setdefault('default_layout', 'default') sc.setdefault('item_name', sn) sc.setdefault('items_per_page', 5) sc.setdefault('date_format', DEFAULT_DATE_FORMAT)
--- a/piecrust/appconfigdefaults.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/appconfigdefaults.py Sun Oct 29 22:51:57 2017 -0700 @@ -189,6 +189,7 @@ data_endpoint = 'blog' item_name = 'post' tpl_func_prefix = 'pc' + year_archive_tpl = '_year.html' if theme_site: # If this is a theme site, show posts from a `sample` directory @@ -208,6 +209,7 @@ (site_values, '%s/func_prefix' % blog_name), (values, '%s/func_prefix' % blog_name), default=('pc%s' % blog_name)) + year_archive_tpl = '%s_year.html,_year.html' % page_prefix # Figure out the settings values for this blog, specifically. # The value could be set on the blog config itself, globally, or left at @@ -224,7 +226,6 @@ default_layout = blog_values['default_post_layout'] post_url = '/' + url_prefix + blog_values['post_url'].lstrip('/') year_url = '/' + url_prefix + blog_values['year_url'].lstrip('/') - year_archive_tpl = '%s_year.html' % page_prefix cfg = collections.OrderedDict({ 'site': collections.OrderedDict({
--- a/piecrust/baking/baker.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/baking/baker.py Sun Oct 29 22:51:57 2017 -0700 @@ -225,17 +225,20 @@ src = ppinfo.source pp = ppinfo.pipeline - logger.debug( - "Queuing jobs for source '%s' using pipeline '%s' " - "(%s, step 0)." % - (src.name, pp.PIPELINE_NAME, realm_name)) - next_step_jobs[src.name] = [] jcctx = PipelineJobCreateContext(pp_pass_num, record_histories) jobs = pp.createJobs(jcctx) if jobs is not None: - job_count += len(jobs) + new_job_count = len(jobs) + job_count += new_job_count pool.queueJobs(jobs) + else: + new_job_count = 0 + + logger.debug( + "Queued %d jobs for source '%s' using pipeline '%s' " + "(%s, step 0)." % + (new_job_count, src.name, pp.PIPELINE_NAME, realm_name)) stats.stepTimer('WorkerTaskPut', time.perf_counter() - start_time)
--- a/piecrust/commands/builtin/baking.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/commands/builtin/baking.py Sun Oct 29 22:51:57 2017 -0700 @@ -95,11 +95,17 @@ raise Exception( "Can't specify `--html-only` or `--assets-only` with " "`--pipelines`.") + allowed_pipelines = [] + forbidden_pipelines = [] for p in ctx.args.pipelines: if p[0] == '-': forbidden_pipelines.append(p) else: allowed_pipelines.append(p) + if not allowed_pipelines: + allowed_pipelines = None + if not forbidden_pipelines: + forbidden_pipelines = None baker = Baker( ctx.appfactory, ctx.app, out_dir, @@ -114,14 +120,14 @@ class ShowRecordCommand(ChefCommand): def __init__(self): super(ShowRecordCommand, self).__init__() - self.name = 'showrecord' - self.description = ("Shows the bake record for a given output " + self.name = 'showrecords' + self.description = ("Shows the bake records for a given output " "directory.") def setupParser(self, parser, app): parser.add_argument( '-o', '--output', - help="The output directory for which to show the bake record " + help="The output directory for which to show the bake records " "(defaults to `_counter`)", nargs='?') parser.add_argument( @@ -140,7 +146,10 @@ '--last', type=int, default=0, - help="Show the last Nth bake record.") + help="Show the last Nth bake records.") + parser.add_argument( + '--records', + help="Load the specified records file.") parser.add_argument( '--html-only', action='store_true', @@ -157,23 +166,31 @@ parser.add_argument( '--show-stats', action='store_true', - help="Show stats from the record.") + help="Show stats from the records.") parser.add_argument( '--show-manifest', - help="Show manifest entries from the record.") + help="Show manifest entries from the records.") def run(self, ctx): import fnmatch from piecrust.baking.baker import get_bake_records_path from piecrust.pipelines.records import load_records - out_dir = ctx.args.output or os.path.join(ctx.app.root_dir, '_counter') - suffix = '' if ctx.args.last == 0 else '.%d' % ctx.args.last - records_path = get_bake_records_path(ctx.app, out_dir, suffix=suffix) - records = load_records(records_path) + records_path = ctx.args.records + if records_path is None: + out_dir = ctx.args.output or os.path.join(ctx.app.root_dir, + '_counter') + suffix = '' if ctx.args.last == 0 else '.%d' % ctx.args.last + records_path = get_bake_records_path(ctx.app, out_dir, + suffix=suffix) + logger.info("Bake records for output: %s" % out_dir) + else: + logger.info("Bake records from: %s" % records_path) + + records = load_records(records_path, True) if records.invalidated: raise Exception( - "The bake record was saved by a previous version of " + "The bake records were saved by a previous version of " "PieCrust and can't be shown.") in_pattern = None @@ -185,15 +202,12 @@ out_pattern = '*%s*' % ctx.args.out_path.strip('*') pipelines = ctx.args.pipelines - if not pipelines: - pipelines = [p.PIPELINE_NAME - for p in ctx.app.plugin_loader.getPipelines()] - if ctx.args.assets_only: - pipelines = ['asset'] - if ctx.args.html_only: - pipelines = ['page'] + if pipelines is None: + if ctx.args.assets_only: + pipelines = ['asset'] + if ctx.args.html_only: + pipelines = ['page'] - logger.info("Bake record for: %s" % out_dir) logger.info("Status: %s" % ('SUCCESS' if records.success else 'FAILURE')) logger.info("Date/time: %s" % @@ -206,10 +220,17 @@ if not ctx.args.show_stats and not ctx.args.show_manifest: for rec in records.records: if ctx.args.fails and rec.success: + logger.debug( + "Ignoring record '%s' because it was successful, " + "and `--fail` was passed." % rec.name) continue ppname = rec.name[rec.name.index('@') + 1:] - if ppname not in pipelines: + if pipelines is not None and ppname not in pipelines: + logging.debug( + "Ignoring record '%s' because it was created by " + "pipeline '%s', which isn't listed in " + "`--pipelines`." % (rec.name, ppname)) continue entries_to_show = []
--- a/piecrust/data/linker.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/data/linker.py Sun Oct 29 22:51:57 2017 -0700 @@ -65,9 +65,12 @@ src = self._source app = src.app for i in self._getAllSiblings(): - if not i.is_group and i.spec != self._content_item.spec: + if not i.is_group: ipage = app.getPage(src, i) - yield PaginationData(ipage) + ipage_data = PaginationData(ipage) + ipage_data._setValue('is_self', + i.spec == self._content_item.spec) + yield ipage_data @property def children(self):
--- a/piecrust/data/pagedata.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/data/pagedata.py Sun Oct 29 22:51:57 2017 -0700 @@ -1,3 +1,4 @@ +import copy import time import logging import collections.abc @@ -165,13 +166,21 @@ self._ctx = ctx def _load(self): + from piecrust.uriutil import split_uri + page = self._page set_val = self._setValue + page_url = page.getUri(self._ctx.sub_num) + _, rel_url = split_uri(page.app, page_url) + dt = page.datetime for k, v in page.source_metadata.items(): set_val(k, v) - set_val('url', page.getUri(self._ctx.sub_num)) + set_val('url', page_url) + set_val('rel_url', rel_url) + set_val('route', copy.deepcopy(page.source_metadata['route_params'])) + set_val('timestamp', time.mktime(dt.timetuple())) set_val('datetime', { 'year': dt.year, 'month': dt.month, 'day': dt.day,
--- a/piecrust/data/paginationdata.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/data/paginationdata.py Sun Oct 29 22:51:57 2017 -0700 @@ -51,7 +51,7 @@ def _load_datetime(data, name): - dt = data_page.datetime + dt = data._page.datetime return { 'year': dt.year, 'month': dt.month, 'day': dt.day, 'hour': dt.hour, 'minute': dt.minute, 'second': dt.second}
--- a/piecrust/data/paginator.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/data/paginator.py Sun Oct 29 22:51:57 2017 -0700 @@ -208,6 +208,7 @@ self._iterator.slice(offset, limit) self._iterator._lockIterator() + self._iterator._load() if isinstance(self._source, ContentSource): self._onIteration(self._iterator)
--- a/piecrust/data/providersdata.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/data/providersdata.py Sun Oct 29 22:51:57 2017 -0700 @@ -31,7 +31,7 @@ self._dict = {} for source in self._page.app.sources: - pname = source.config.get('data_type') + pname = source.config.get('data_type') or 'page_iterator' pendpoint = source.config.get('data_endpoint') if not pname or not pendpoint: continue @@ -48,11 +48,16 @@ provider = build_data_provider(pname, source, self._page) endpoint[endpoint_bits[-1]] = provider elif isinstance(existing, DataProvider): - if existing.PROVIDER_NAME != pname: + existing_source = existing._sources[0] + if (existing.PROVIDER_NAME != pname or + existing_source.SOURCE_NAME != source.SOURCE_NAME): raise ConfigurationError( - "Can't combine data providers '%s' and '%' on " - "endpoint '%s'." % - (existing.PROVIDER_NAME, pname, pendpoint)) + "Can't combine data providers '%s' and '%' " + "(using sources '%s' and '%s') " + "on endpoint '%s'." % + (existing.PROVIDER_NAME, pname, + existing_source.SOURCE_NAME, source.SOURCE_NAME, + pendpoint)) existing._addSource(source) else: raise ConfigurationError(
--- a/piecrust/dataproviders/blog.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/dataproviders/blog.py Sun Oct 29 22:51:57 2017 -0700 @@ -56,7 +56,8 @@ def __iter__(self): self._buildPosts() self._buildArchives() - return ['posts', 'years', 'months'] + list(self._taxonomies.keys()) + return ['posts', 'years', 'months'] + list( + sorted(self._taxonomies.keys())) def __len__(self): self._buildPosts() @@ -111,7 +112,7 @@ (post_dt.year, post_dt.month, 1, 0, 0, 0, 0, 0, -1)) posts_this_month = BlogArchiveEntry( - source, page, month, timestamp) + source, page, month[0], timestamp) monthly_index[month] = posts_this_month posts_this_month._items.append(post.content_item) @@ -144,7 +145,8 @@ self._taxonomies = {} for tax_name, entries in tax_index.items(): - self._taxonomies[tax_name] = list(entries.values()) + self._taxonomies[tax_name] = list( + sorted(entries.values(), key=lambda i: i.term)) self._onIteration(None) @@ -171,7 +173,7 @@ self._iterator = None def __str__(self): - return self.name + return str(self.name) def __int__(self): return int(self.name)
--- a/piecrust/dataproviders/pageiterator.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/dataproviders/pageiterator.py Sun Oct 29 22:51:57 2017 -0700 @@ -9,11 +9,39 @@ logger = logging.getLogger(__name__) -class _ItInfo: - def __init__(self): +class _CombinedSource: + def __init__(self, sources): + self.sources = sources + self.app = sources[0].app + self.name = None + + # This is for recursive traversal of the iterator chain. + # See later in `PageIterator`. self.it = None - self.iterated = False - self.source_name = None + + def __iter__(self): + sources = self.sources + + if len(sources) == 1: + source = sources[0] + self.name = source.name + yield from source.getAllPages() + self.name = None + return + + # Return the pages from all the combined sources, but skip + # those that are "overridden" -- e.g. a theme page that gets + # replaced by a user page of the same name. + used_uris = set() + for source in sources: + self.name = source.name + for page in source.getAllPages(): + page_uri = page.getUri() + if page_uri not in used_uris: + used_uris.add(page_uri) + yield page + + self.name = None class PageIteratorDataProvider(DataProvider): @@ -31,36 +59,37 @@ def __init__(self, source, page): super().__init__(source, page) - self._its = None self._app = source.app + self._it = None + self._iterated = False def __len__(self): self._load() - return sum([len(i.it) for i in self._its]) + return len(self._it) def __iter__(self): self._load() - for i in self._its: - yield from i.it + yield from self._it def _load(self): - if self._its is not None: + if self._it is not None: return - self._its = [] - for source in self._sources: - i = _ItInfo() - i.it = PageIterator(source, current_page=self._page) - i.it._iter_event += self._onIteration - i.source_name = source.name - self._its.append(i) + combined_source = _CombinedSource(list(reversed(self._sources))) + self._it = PageIterator(combined_source, current_page=self._page) + self._it._iter_event += self._onIteration def _onIteration(self, it): - ii = next(filter(lambda i: i.it == it, self._its)) - if not ii.iterated: + if not self._iterated: rcs = self._app.env.render_ctx_stack - rcs.current_ctx.addUsedSource(ii.source_name) - ii.iterated = True + rcs.current_ctx.addUsedSource(it._source) + self._iterated = True + + def _addSource(self, source): + if self._it is not None: + raise Exception("Can't add sources after the data provider " + "has been loaded.") + super()._addSource(source) def _debugRenderDoc(self): return 'Provides a list of %d items' % len(self) @@ -69,7 +98,8 @@ class PageIterator: def __init__(self, source, *, current_page=None): self._source = source - self._is_content_source = isinstance(source, ContentSource) + self._is_content_source = isinstance( + source, (ContentSource, _CombinedSource)) self._cache = None self._pagination_slicer = None self._has_sorter = False @@ -150,14 +180,11 @@ (filter_name, self._current_page.path)) return self._simpleNonSortedWrap(SettingFilterIterator, filter_conf) - def sort(self, setting_name, reverse=False): - if not setting_name: - raise Exception("You need to specify a configuration setting " - "to sort by.") - self._ensureUnlocked() - self._ensureUnloaded() - self._pages = SettingSortIterator(self._pages, setting_name, reverse) - self._has_sorter = True + def sort(self, setting_name=None, reverse=False): + if setting_name: + self._wrapAsSort(SettingSortIterator, setting_name, reverse) + else: + self._wrapAsSort(NaturalSortIterator, reverse) return self def reset(self): @@ -171,12 +198,15 @@ @property def _has_more(self): - if self._cache is None: - return False + self._load() if self._pagination_slicer: return self._pagination_slicer.has_more return False + @property + def _is_loaded_and_has_more(self): + return self._is_loaded and self._has_more + def _simpleWrap(self, it_class, *args, **kwargs): self._ensureUnlocked() self._ensureUnloaded() @@ -226,7 +256,11 @@ def _initIterator(self): if self._is_content_source: - self._it = PageContentSourceIterator(self._source) + if isinstance(self._source, _CombinedSource): + self._it = self._source + else: + self._it = PageContentSourceIterator(self._source) + app = self._source.app if app.config.get('baker/is_baking'): # While baking, automatically exclude any page with @@ -333,6 +367,15 @@ return iter(self._cache) +class NaturalSortIterator: + def __init__(self, it, reverse=False): + self.it = it + self.reverse = reverse + + def __iter__(self): + return iter(sorted(self.it, reverse=self.reverse)) + + class SettingSortIterator: def __init__(self, it, name, reverse=False): self.it = it @@ -344,7 +387,7 @@ reverse=self.reverse)) def _key_getter(self, item): - key = item.config.get(item) + key = item.config.get(self.name) if key is None: return 0 return key
--- a/piecrust/main.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/main.py Sun Oct 29 22:51:57 2017 -0700 @@ -143,7 +143,16 @@ 'serve': 'server'} -def _pre_parse_chef_args(argv): +def _make_chef_state(): + return [] + + +def _recover_pre_chef_state(state): + for s in state: + s() + + +def _pre_parse_chef_args(argv, *, bypass_setup=False, state=None): # We need to parse some arguments before we can build the actual argument # parser, because it can affect which plugins will be loaded. Also, log- # related arguments must be parsed first because we want to log everything @@ -152,6 +161,8 @@ _setup_main_parser_arguments(parser) parser.add_argument('extra_args', nargs=argparse.REMAINDER) res, _ = parser.parse_known_args(argv) + if bypass_setup: + return res # Setup the logger. if res.debug and res.quiet: @@ -163,13 +174,20 @@ colorama.init(strip=strip_colors) root_logger = logging.getLogger() + previous_level = root_logger.level root_logger.setLevel(logging.INFO) if res.debug or res.log_debug: root_logger.setLevel(logging.DEBUG) + if state is not None: + state.append(lambda: root_logger.setLevel(previous_level)) if res.debug_only: for n in res.debug_only: - logging.getLogger(n).setLevel(logging.DEBUG) + sub_logger = logging.getLogger(n) + previous_level = sub_logger.level + sub_logger.setLevel(logging.DEBUG) + if state is not None: + state.append(lambda: sub_logger.setLevel(previous_level)) log_handler = logging.StreamHandler(sys.stdout) if res.debug or res.debug_only: @@ -182,12 +200,16 @@ log_handler.setLevel(logging.INFO) log_handler.setFormatter(ColoredFormatter("%(message)s")) root_logger.addHandler(log_handler) + if state is not None: + state.append(lambda: root_logger.removeHandler(log_handler)) if res.log_file: file_handler = logging.FileHandler(res.log_file, mode='w') root_logger.addHandler(file_handler) if res.log_debug: file_handler.setLevel(logging.DEBUG) + if state is not None: + state.append(lambda: root_logger.removeHandler(file_handler)) # PID file. if res.pid_file:
--- a/piecrust/page.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/page.py Sun Oct 29 22:51:57 2017 -0700 @@ -3,7 +3,6 @@ import hashlib import logging import datetime -import dateutil.parser import collections from werkzeug.utils import cached_property from piecrust.configuration import ( @@ -176,6 +175,7 @@ return None if isinstance(page_date, str): + import dateutil.parser try: parsed_d = dateutil.parser.parse(page_date) except Exception as ex: @@ -197,6 +197,7 @@ return page_time if isinstance(page_time, str): + import dateutil.parser try: parsed_t = dateutil.parser.parse(page_time) except Exception as ex: @@ -306,7 +307,7 @@ line_count = 1 while True: nex = txt.find('\n', cur) - if nex < 0: + if nex < 0 or (end >= 0 and nex >= end): break cur = nex + 1 @@ -374,7 +375,7 @@ # Handle text past the last match. lastm = matches[-1] - last_seg_start = lastm.end() + last_seg_start = lastm.end() + 1 seg = ContentSegment( raw[last_seg_start:],
--- a/piecrust/pipelines/_pagebaker.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/pipelines/_pagebaker.py Sun Oct 29 22:51:57 2017 -0700 @@ -13,6 +13,22 @@ logger = logging.getLogger(__name__) +def get_output_path(app, out_dir, uri, pretty_urls): + uri_root, uri_path = split_uri(app, uri) + + bake_path = [out_dir] + decoded_uri = urllib.parse.unquote(uri_path) + if pretty_urls: + bake_path.append(decoded_uri) + bake_path.append('index.html') + elif decoded_uri == '': + bake_path.append('index.html') + else: + bake_path.append(decoded_uri) + + return os.path.normpath(os.path.join(*bake_path)) + + class BakingError(Exception): pass @@ -51,24 +67,11 @@ with open(out_path, 'w', encoding='utf8') as fp: fp.write(content) - def getOutputPath(self, uri, pretty_urls): - uri_root, uri_path = split_uri(self.app, uri) - - bake_path = [self.out_dir] - decoded_uri = urllib.parse.unquote(uri_path) - if pretty_urls: - bake_path.append(decoded_uri) - bake_path.append('index.html') - elif decoded_uri == '': - bake_path.append('index.html') - else: - bake_path.append(decoded_uri) - - return os.path.normpath(os.path.join(*bake_path)) - def bake(self, page, prev_entry, cur_entry): cur_sub = 1 has_more_subs = True + app = self.app + out_dir = self.out_dir pretty_urls = page.config.get('pretty_urls', self.pretty_urls) # Start baking the sub-pages. @@ -76,7 +79,7 @@ sub_uri = page.getUri(sub_num=cur_sub) logger.debug("Baking '%s' [%d]..." % (sub_uri, cur_sub)) - out_path = self.getOutputPath(sub_uri, pretty_urls) + out_path = get_output_path(app, out_dir, sub_uri, pretty_urls) # Create the sub-entry for the bake record. cur_sub_entry = SubPagePipelineRecordEntry(sub_uri, out_path) @@ -204,6 +207,8 @@ # Easy test. if force: + cur_sub_entry.flags |= \ + SubPagePipelineRecordEntry.FLAG_FORCED_BY_GENERAL_FORCE return STATUS_BAKE # Check for up-to-date outputs. @@ -212,6 +217,8 @@ out_path_time = os.path.getmtime(out_path) except OSError: # File doesn't exist, we'll need to bake. + cur_sub_entry.flags |= \ + SubPagePipelineRecordEntry.FLAG_FORCED_BY_NO_PREVIOUS return STATUS_BAKE if out_path_time <= in_path_time:
--- a/piecrust/pipelines/_pagerecords.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/pipelines/_pagerecords.py Sun Oct 29 22:51:57 2017 -0700 @@ -8,7 +8,8 @@ FLAG_FORCED_BY_SOURCE = 2**1 FLAG_FORCED_BY_NO_PREVIOUS = 2**2 FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3 - FLAG_FORMATTING_INVALIDATED = 2**4 + FLAG_FORCED_BY_GENERAL_FORCE = 2**4 + FLAG_FORMATTING_INVALIDATED = 2**5 def __init__(self, out_uri, out_path): self.out_uri = out_uri @@ -133,6 +134,8 @@ SubPagePipelineRecordEntry.FLAG_FORCED_BY_NO_PREVIOUS: 'forced b/c new', SubPagePipelineRecordEntry.FLAG_FORCED_BY_PREVIOUS_ERRORS: 'forced by errors', + SubPagePipelineRecordEntry.FLAG_FORCED_BY_GENERAL_FORCE: + 'manually forced', SubPagePipelineRecordEntry.FLAG_FORMATTING_INVALIDATED: 'formatting invalidated' }
--- a/piecrust/pipelines/asset.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/pipelines/asset.py Sun Oct 29 22:51:57 2017 -0700 @@ -25,19 +25,18 @@ "The asset pipeline only support file-system sources.") super().__init__(source, ppctx) - self.enabled_processors = None - self.ignore_patterns = [] + self._ignore_patterns = [] self._processors = None self._base_dir = source.fs_endpoint_path def initialize(self): # Get the list of processors for this run. processors = self.app.plugin_loader.getProcessors() - if self.enabled_processors is not None: - logger.debug("Filtering processors to: %s" % - self.enabled_processors) + enabled_processors = self.app.config.get('pipelines/asset/processors') + if enabled_processors is not None: + logger.debug("Filtering processors to: %s" % enabled_processors) processors = get_filtered_processors(processors, - self.enabled_processors) + enabled_processors) # Invoke pre-processors. proc_ctx = ProcessorContext(self) @@ -55,7 +54,9 @@ # Pre-processors can define additional ignore patterns so let's # add them to what we had already. - self.ignore_patterns += make_re(proc_ctx.ignore_patterns) + ignores = self.app.config.get('pipelines/asset/ignore', []) + ignores += proc_ctx.ignore_patterns + self._ignore_patterns += make_re(ignores) # Register timers. stats = self.app.env.stats @@ -65,7 +66,7 @@ def run(self, job, ctx, result): # See if we need to ignore this item. rel_path = os.path.relpath(job.content_item.spec, self._base_dir) - if re_matchany(rel_path, self.ignore_patterns): + if re_matchany(rel_path, self._ignore_patterns): return record_entry = result.record_entry
--- a/piecrust/pipelines/page.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/pipelines/page.py Sun Oct 29 22:51:57 2017 -0700 @@ -1,6 +1,6 @@ import logging from piecrust.pipelines.base import ContentPipeline -from piecrust.pipelines._pagebaker import PageBaker +from piecrust.pipelines._pagebaker import PageBaker, get_output_path from piecrust.pipelines._pagerecords import PagePipelineRecordEntry from piecrust.sources.base import AbortedSourceUseError @@ -39,18 +39,21 @@ used_paths[p] = (src_name, e) jobs = [] + app = self.app route = self.source.route - pretty_urls = self.app.config.get('site/pretty_urls') + out_dir = self.ctx.out_dir + pretty_urls = app.config.get('site/pretty_urls') record = ctx.record_histories.current.getRecord(self.record_name) for item in self.source.getAllContents(): route_params = item.metadata['route_params'] uri = route.getUri(route_params) - path = self._pagebaker.getOutputPath(uri, pretty_urls) + path = get_output_path(app, out_dir, uri, pretty_urls) override = used_paths.get(path) + if override is not None: override_source_name, override_entry = override - override_source = self.app.getSource(override_source_name) + override_source = app.getSource(override_source_name) if override_source.config['realm'] == \ self.source.config['realm']: logger.error( @@ -78,9 +81,9 @@ def mergeRecordEntry(self, record_entry, ctx): existing = ctx.record.getEntry(record_entry.item_spec) + existing.flags |= record_entry.flags existing.errors += record_entry.errors - existing.flags |= record_entry.flags - existing.subs = record_entry.subs + existing.subs += record_entry.subs def run(self, job, ctx, result): step_num = job.step_num
--- a/piecrust/pipelines/records.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/pipelines/records.py Sun Oct 29 22:51:57 2017 -0700 @@ -113,10 +113,17 @@ multi_record._record_version == MultiRecord.RECORD_VERSION) -def load_records(path): +def load_records(path, raise_errors=False): try: multi_record = MultiRecord.load(path) + except FileNotFoundError: + if raise_errors: + raise + logger.debug("No existing records found at: %s" % path) + multi_record = None except Exception as ex: + if raise_errors: + raise logger.debug("Error loading records from: %s" % path) logger.debug(ex) logger.debug("Will use empty records.")
--- a/piecrust/plugins/base.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/plugins/base.py Sun Oct 29 22:51:57 2017 -0700 @@ -171,6 +171,8 @@ all_components = [] for plugin in self.plugins: plugin_components = getattr(plugin, name)(*args) + # Make sure it's a list in case it was an iterator. + plugin_components = list(plugin_components) all_components += plugin_components if initialize:
--- a/piecrust/processing/sitemap.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/processing/sitemap.py Sun Oct 29 22:51:57 2017 -0700 @@ -11,17 +11,17 @@ SITEMAP_HEADER = \ - """<?xml version="1.0" encoding="utf-8"?> - <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> - """ +"""<?xml version="1.0" encoding="utf-8"?> +<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> +""" SITEMAP_FOOTER = "</urlset>\n" -SITEURL_HEADER = " <url>\n" -SITEURL_LOC = " <loc>%s</loc>\n" -SITEURL_LASTMOD = " <lastmod>%s</lastmod>\n" -SITEURL_CHANGEFREQ = " <changefreq>%s</changefreq>\n" -SITEURL_PRIORITY = " <priority>%0.1f</priority>\n" -SITEURL_FOOTER = " </url>\n" +SITEURL_HEADER = " <url>\n" # NOQA: E222 +SITEURL_LOC = " <loc>%s</loc>\n" # NOQA: E222 +SITEURL_LASTMOD = " <lastmod>%s</lastmod>\n" # NOQA: E222 +SITEURL_CHANGEFREQ = " <changefreq>%s</changefreq>\n" # NOQA: E222 +SITEURL_PRIORITY = " <priority>%0.1f</priority>\n" # NOQA: E222 +SITEURL_FOOTER = " </url>\n" # NOQA: E222 class SitemapProcessor(SimpleFileProcessor):
--- a/piecrust/rendering.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/rendering.py Sun Oct 29 22:51:57 2017 -0700 @@ -70,9 +70,7 @@ def setCustomInfo(self, key, info): self._custom_info[key] = info - def getCustomInfo(self, key, default=None, create_if_missing=False): - if create_if_missing: - return self._custom_info.setdefault(key, default) + def getCustomInfo(self, key, default=None): return self._custom_info.get(key, default)
--- a/piecrust/routing.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/routing.py Sun Oct 29 22:51:57 2017 -0700 @@ -46,7 +46,7 @@ self.source_name = cfg['source'] self.uri_pattern = cfg['url'].lstrip('/') - self.pass_num = cfg['pass'] + self.pass_num = cfg.get('pass', 1) self.supported_params = self.source.getSupportedRouteParameters()
--- a/piecrust/serving/server.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/serving/server.py Sun Oct 29 22:51:57 2017 -0700 @@ -102,7 +102,7 @@ # Create the app for this request. app = get_app_for_server(self.appfactory, root_url=self.root_url) - if (app.config.get('site/enable_debug_info') and + if (app.config.get('server/enable_debug_info') and self.enable_debug_info and '!debug' in request.args): app.config.set('site/show_debug_info', True)
--- a/piecrust/serving/util.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/serving/util.py Sun Oct 29 22:51:57 2017 -0700 @@ -30,9 +30,14 @@ self.not_found_errors = [] -def find_routes(routes, uri, uri_no_sub, sub_num=1): +def find_routes(routes, uri, decomposed_uri=None): """ Returns routes matching the given URL. """ + sub_num = 0 + uri_no_sub = None + if decomposed_uri is not None: + uri_no_sub, sub_num = decomposed_uri + res = [] for route in routes: route_params = route.matchUri(uri) @@ -56,7 +61,7 @@ # It could also be a sub-page (i.e. the URL ends with a page number), so # we try to also match the base URL (without the number). req_path_no_sub, sub_num = split_sub_uri(app, req_path) - routes = find_routes(app.routes, req_path, req_path_no_sub, sub_num) + routes = find_routes(app.routes, req_path, (req_path_no_sub, sub_num)) if len(routes) == 0: raise RouteNotFoundError("Can't find route for: %s" % req_path)
--- a/piecrust/sources/autoconfig.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/sources/autoconfig.py Sun Oct 29 22:51:57 2017 -0700 @@ -26,17 +26,19 @@ name) def _finalizeContent(self, parent_group, items, groups): - DefaultContentSource._finalizeContent(parent_group, items, groups) + super()._finalizeContent(parent_group, items, groups) # If `capture_mode` is `dirname`, we don't need to recompute it # for each filename, so we do it here. if self.capture_mode == 'dirname': - rel_dirpath = os.path.relpath(parent_group.spec, - self.fs_endpoint_path) + rel_dirpath = '.' + if parent_group is not None: + rel_dirpath = os.path.relpath(parent_group.spec, + self.fs_endpoint_path) config = self._extractConfigFragment(rel_dirpath) for i in items: - # Compute the confif for the other capture modes. + # Compute the config for the other capture modes. if self.capture_mode == 'path': rel_path = os.path.relpath(i.spec, self.fs_endpoint_path) config = self._extractConfigFragment(rel_path) @@ -60,7 +62,7 @@ def __init__(self, app, name, config): config['capture_mode'] = 'dirname' - AutoConfigContentSourceBase.__init__(app, name, config) + super().__init__(app, name, config) self.setting_name = config.get('setting_name', name) self.only_single_values = config.get('only_single_values', False) @@ -108,6 +110,10 @@ return ContentItem(path, metadata) return None + def _makeSlug(self, path): + slug = super()._makeSlug(path) + return os.path.basename(slug) + class OrderedContentSource(AutoConfigContentSourceBase): """ A content source that assigns an "order" to its pages based on a @@ -120,7 +126,7 @@ def __init__(self, app, name, config): config['capture_mode'] = 'path' - AutoConfigContentSourceBase.__init__(app, name, config) + super().__init__(app, name, config) self.setting_name = config.get('setting_name', 'order') self.default_value = config.get('default_value', 0)
--- a/piecrust/sources/base.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/sources/base.py Sun Oct 29 22:51:57 2017 -0700 @@ -78,6 +78,7 @@ class ContentSource: """ A source for content. """ + SOURCE_NAME = None DEFAULT_PIPELINE_NAME = None def __init__(self, app, name, config):
--- a/piecrust/sources/fs.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/sources/fs.py Sun Oct 29 22:51:57 2017 -0700 @@ -152,7 +152,7 @@ # page file with the same name as the folder. if not item.is_group: raise ValueError() - parent_glob = os.path.join(item.spec, '*') + parent_glob = item.spec.rstrip('/\\') + '.*' for n in glob.iglob(parent_glob): if os.path.isfile(n): metadata = self._createItemMetadata(n)
--- a/piecrust/sources/taxonomy.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/sources/taxonomy.py Sun Oct 29 22:51:57 2017 -0700 @@ -161,9 +161,11 @@ def onRouteFunctionUsed(self, route_params): # Get the values, and slugify them appropriately. + # If this is a "multiple" taxonomy, `values` will be a tuple of + # terms. If not, `values` will just be a term. values = route_params[self.taxonomy.term_name] - if self.taxonomy.is_multiple: - # TODO: here we assume the route has been properly configured. + tax_is_multiple = self.taxonomy.is_multiple + if tax_is_multiple: slugified_values = self.slugifyMultiple((str(v) for v in values)) route_val = self.taxonomy.separator.join(slugified_values) else: @@ -174,8 +176,13 @@ rcs = self.app.env.render_ctx_stack cpi = rcs.current_ctx.current_pass_info if cpi: - utt = cpi.getCustomInfo('used_taxonomy_terms', [], True) - utt.append(slugified_values) + utt = cpi.getCustomInfo('used_taxonomy_terms') + if utt is None: + utt = set() + utt.add(slugified_values) + cpi.setCustomInfo('used_taxonomy_terms', utt) + else: + utt.add(slugified_values) # Put the slugified values in the route metadata so they're used to # generate the URL. @@ -407,14 +414,25 @@ # # Add the combinations to that list. We get those combinations from # wherever combinations were used, so they're coming from the - # `onRouteFunctionUsed` method. + # `onRouteFunctionUsed` method. And because combinations can be used + # by any page in the website (anywhere someone can ask for an URL + # to the combination page), it means we check all the records, not + # just the record for our source. if taxonomy.is_multiple: known_combinations = set() - for cur_entry in cur_rec.getEntries(): - used_terms = _get_all_entry_taxonomy_terms(cur_entry) - for terms in used_terms: - if len(terms) > 1: - known_combinations.add(terms) + for rec in current_records.records: + # Cheap way to test if a record contains entries that + # are sub-types of a page entry: test the first one. + first_entry = next(iter(rec.getEntries()), None) + if (first_entry is None or + not isinstance(first_entry, PagePipelineRecordEntry)): + continue + + for cur_entry in rec.getEntries(): + used_terms = _get_all_entry_taxonomy_terms(cur_entry) + for terms in used_terms: + if len(terms) > 1: + known_combinations.add(terms) dcc = 0 for terms in known_combinations:
--- a/piecrust/workerpool.py Sun Oct 29 22:46:41 2017 -0700 +++ b/piecrust/workerpool.py Sun Oct 29 22:51:57 2017 -0700 @@ -88,6 +88,30 @@ _CRITICAL_WORKER_ERROR, None, False, params.wid, msg)) +def _pre_parse_pytest_args(): + # If we are unit-testing, we need to translate our test logging + # arguments into something Chef can understand. + import argparse + parser = argparse.ArgumentParser() + # This is adapted from our `conftest.py`. + parser.add_argument('--log-debug', action='store_true') + parser.add_argument('--log-file') + res, _ = parser.parse_known_args(sys.argv[1:]) + + chef_args = [] + if res.log_debug: + chef_args.append('--debug') + if res.log_file: + chef_args += ['--log', res.log_file] + + root_logger = logging.getLogger() + while len(root_logger.handlers) > 0: + root_logger.removeHandler(root_logger.handlers[0]) + + from piecrust.main import _pre_parse_chef_args + _pre_parse_chef_args(chef_args) + + def _real_worker_func_unsafe(params): wid = params.wid @@ -95,33 +119,18 @@ stats.registerTimer('WorkerInit') init_start_time = time.perf_counter() - # If we are unit-testing, we didn't setup all the logging environment - # yet, since the executable is `py.test`. We need to translate our - # test logging arguments into something Chef can understand. - if params.is_unit_testing: - import argparse - parser = argparse.ArgumentParser() - # This is adapted from our `conftest.py`. - parser.add_argument('--log-debug', action='store_true') - parser.add_argument('--log-file') - res, _ = parser.parse_known_args(sys.argv[1:]) - - chef_args = [] - if res.log_debug: - chef_args.append('--debug') - if res.log_file: - chef_args += ['--log', res.log_file] - - from piecrust.main import _pre_parse_chef_args - _pre_parse_chef_args(chef_args) - # In a context where `multiprocessing` is using the `spawn` forking model, # the new process doesn't inherit anything, so we lost all our logging # configuration here. Let's set it up again. - elif (hasattr(multiprocessing, 'get_start_method') and + if (hasattr(multiprocessing, 'get_start_method') and multiprocessing.get_start_method() == 'spawn'): - from piecrust.main import _pre_parse_chef_args - _pre_parse_chef_args(sys.argv[1:]) + if not params.is_unit_testing: + from piecrust.main import _pre_parse_chef_args + _pre_parse_chef_args(sys.argv[1:]) + else: + _pre_parse_pytest_args() + elif params.is_unit_testing: + _pre_parse_pytest_args() from piecrust.main import ColoredFormatter root_logger = logging.getLogger() @@ -295,6 +304,11 @@ self._event.clear() for job in jobs: self._quick_put((TASK_JOB, job)) + else: + with self._lock_jobs_left: + done = (self._jobs_left == 0) + if done: + self._event.set() def wait(self, timeout=None): if self._closed: @@ -308,8 +322,10 @@ def close(self): if self._closed: raise Exception("This worker pool has been closed.") - if self._jobs_left > 0 or not self._event.is_set(): + if self._jobs_left > 0: raise Exception("A previous job queue has not finished yet.") + if not self._event.is_set(): + raise Exception("A previous job queue hasn't been cleared.") logger.debug("Closing worker pool...") live_workers = list(filter(lambda w: w is not None, self._pool))
--- a/tests/bakes/test_archives.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_archives.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,6 +1,6 @@ --- in: - pages/_year.html: | + templates/_year.html: | Posts in {{year}} {% for post in pagination.posts -%} {{post.url}} @@ -27,7 +27,7 @@ /2016/01/01/post1.html --- in: - pages/_year.html: | + templates/_year.html: | Posts in {{year}} {% for post in archives -%} {{post.url}}
--- a/tests/bakes/test_assets.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_assets.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,15 +1,11 @@ --- in: posts/2010-01-01_post1-assets/blah.png: 'fake image' - posts/2010-01-01_post1.md: 'my image: {{assets.blah}}' - pages/_index.md: 'something' -out: - '2010': - '01': - '01': - post1.html: 'my image: /2010/01/01/post1/blah.png' - post1: - blah.png: 'fake image' + posts/2010-01-01_post1.html: 'my image: {{assets.blah}}' + pages/_index.html: 'something' +outfiles: + 2010/01/01/post1.html: 'my image: /2010/01/01/post1/blah.png' + 2010/01/01/post1/blah.png: 'fake image' index.html: 'something' --- config: @@ -17,14 +13,10 @@ pretty_urls: true in: posts/2010-01-01_post1-assets/blah.png: 'fake image' - posts/2010-01-01_post1.md: 'my image: {{assets.blah}}' - pages/_index.md: 'something' -out: - '2010': - '01': - '01': - 'post1': - index.html: 'my image: /2010/01/01/post1/blah.png' - blah.png: 'fake image' + posts/2010-01-01_post1.html: 'my image: {{assets.blah}}' + pages/_index.html: 'something' +outfiles: + 2010/01/01/post1/index.html: 'my image: /2010/01/01/post1/blah.png' + 2010/01/01/post1/blah.png: 'fake image' index.html: 'something'
--- a/tests/bakes/test_data_provider.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_data_provider.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,10 +1,24 @@ --- in: + pages/_index.html: | + --- + date: '2010/01/05' + --- + The index pages/foo.md: | + --- + date: '2010/01/08' + --- Foo! pages/bar.md: | + --- + date: '2010/01/09' + --- Bar! - pages/allpages.md: | + pages/allpages.html: | + --- + date: '2010/01/10' + --- {% for p in site.pages -%} {{p.url}} {% endfor %} @@ -19,7 +33,7 @@ posts/2016-06-01_one.md: "One!" posts/2016-06-02_two.md: "Two!" posts/2016-06-03_three.md: "Three!" - pages/_index.md: | + pages/_index.html: | {% for p in blog.posts -%} {{p.url}} {% endfor %} @@ -36,7 +50,7 @@ posts/2016-06-01_one.md: "One!" posts/2016-06-02_two.md: "Two!" posts/2016-06-03_three.md: "Three!" - pages/_index.md: | + pages/_index.html: | {{blog.subtitle}} {% for p in blog.posts -%} {{p.url}} @@ -56,7 +70,7 @@ posts/aaa/2016-06-02_two.md: "Two!" posts/xyz/2016-06-01_one-other.md: "One Other!" posts/xyz/2016-06-02_two-other.md: "Two Other!" - pages/_index.md: | + pages/_index.html: | {% for p in aaa.posts -%} {{p.url}} {% endfor %}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/bakes/test_dotfiles.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -0,0 +1,8 @@ +--- +in: + assets/something.txt: Foo bar + assets/.htaccess: "# Apache config" +outfiles: + something.txt: Foo bar + .htaccess: "# Apache config" +
--- a/tests/bakes/test_linker.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_linker.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,6 +1,6 @@ --- in: - pages/foo.md: | + pages/foo.html: | {%for c in family.children%} {{c.title}} {%endfor%} @@ -8,15 +8,15 @@ foo.html: '' --- in: - pages/foo.md: | + pages/foo.html: | {%for c in family.children-%} {{c.title}} {%endfor%} - pages/foo/one.md: | + pages/foo/one.html: | --- title: One --- - pages/foo/two.md: | + pages/foo/two.html: | --- title: Two --- @@ -26,18 +26,18 @@ Two --- in: - pages/foo.md: | + pages/foo.html: | --- title: Foo --- {%for c in family.siblings-%} {{c.title}}{%if c.is_self%} SELFIE!{%endif%} {%endfor%} - pages/bar.md: | + pages/bar.html: | --- title: Bar --- - pages/other.md: | + pages/other.html: | --- title: Other --- @@ -48,27 +48,27 @@ Other --- in: - pages/foo.md: "---\ntitle: Foo\n---\n" - pages/foo/one.md: | + pages/foo.html: "---\ntitle: Foo\n---\n" + pages/foo/one.html: | {{family.parent.url}} {{family.parent.title}} outfiles: foo/one.html: /foo.html Foo --- in: - pages/foo.md: "---\ntitle: Foo\n---\n" - pages/foo/bar.md: "---\ntitle: Bar\n---\n" - pages/foo/bar/one.md: | + pages/foo.html: "---\ntitle: Foo\n---\n" + pages/foo/bar.html: "---\ntitle: Bar\n---\n" + pages/foo/bar/one.html: | {{family.parent.url}} {{family.parent.title}} - {{family.parent.parent.url}} {{family.parent.parent.title}} + {{family.ancestors[1].url}} {{family.ancestors[1].title}} outfiles: foo/bar/one.html: | /foo/bar.html Bar /foo.html Foo --- in: - pages/foo.md: "---\ntitle: Foo\n---\n" - pages/foo/bar.md: "---\ntitle: Bar\n---\n" - pages/foo/bar/one.md: | + pages/foo.html: "---\ntitle: Foo\n---\n" + pages/foo/bar.html: "---\ntitle: Bar\n---\n" + pages/foo/bar/one.html: | {% for p in family.ancestors -%} {{p.url}} {{p.title}} {% endfor %}
--- a/tests/bakes/test_multiblog.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_multiblog.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -15,14 +15,14 @@ site: blogs: [one, two] one: - func_prefix: pc + func_prefix: pc1 two: - func_prefix: pc + func_prefix: pc2 in: posts/one/2016-01-01_post1.html: '' posts/two/2016-01-02_post2.html: '' - pages/foo-one.html: "---\nblog: one\n---\nLink: {{pcposturl(2016, 01, 01, 'post1', 'one')}}" - pages/foo-two.html: "---\nblog: two\n---\nLink: {{pcposturl(2016, 01, 02, 'post2', 'two')}}" + pages/foo-one.html: "---\nblog: one\n---\nLink: {{pc1posturl(2016, 01, 01, 'post1', 'one')}}" + pages/foo-two.html: "---\nblog: two\n---\nLink: {{pc2posturl(2016, 01, 02, 'post2', 'two')}}" outfiles: foo-one.html: "Link: /one/2016/01/01/post1.html" foo-two.html: "Link: /two/2016/01/02/post2.html"
--- a/tests/bakes/test_pagination.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_pagination.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -4,15 +4,15 @@ posts_per_page: 3 pagination_suffix: /page%num% in: - posts/2015-03-01_post01.md: "---\ntitle: Post 01\n---\n" - posts/2015-03-02_post02.md: "---\ntitle: Post 02\n---\n" - posts/2015-03-03_post03.md: "---\ntitle: Post 03\n---\n" - posts/2015-03-04_post04.md: "---\ntitle: Post 04\n---\n" - posts/2015-03-05_post05.md: "---\ntitle: Post 05\n---\n" - posts/2015-03-06_post06.md: "---\ntitle: Post 06\n---\n" - posts/2015-03-07_post07.md: "---\ntitle: Post 07\n---\n" - pages/_index.md: '' - pages/foo.md: | + posts/2015-03-01_post01.html: "---\ntitle: Post 01\n---\n" + posts/2015-03-02_post02.html: "---\ntitle: Post 02\n---\n" + posts/2015-03-03_post03.html: "---\ntitle: Post 03\n---\n" + posts/2015-03-04_post04.html: "---\ntitle: Post 04\n---\n" + posts/2015-03-05_post05.html: "---\ntitle: Post 05\n---\n" + posts/2015-03-06_post06.html: "---\ntitle: Post 06\n---\n" + posts/2015-03-07_post07.html: "---\ntitle: Post 07\n---\n" + pages/_index.html: '' + pages/foo.html: | {%- for p in pagination.items -%} {{p.url}} {{p.title}} {% endfor -%} @@ -44,33 +44,33 @@ site: posts_per_page: 3 in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [foo] --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 tags: [foo] --- - posts/2015-03-03_post03.md: | + posts/2015-03-03_post03.html: | --- title: Post 03 tags: [foo] --- - posts/2015-03-04_post04.md: | + posts/2015-03-04_post04.html: | --- title: Post 04 tags: [foo] --- - posts/2015-03-05_post05.md: | + posts/2015-03-05_post05.html: | --- title: Post 05 tags: [foo] --- - pages/_index.md: '' - pages/_tag.md: | + pages/_index.html: '' + templates/_tag.html: | Posts with {{tag}} {% for p in pagination.items -%} {{p.url}} {{p.title}}
--- a/tests/bakes/test_relative_pagination.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_relative_pagination.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -3,9 +3,9 @@ site: default_post_layout: post in: - posts/2015-03-01_post01.md: "---\ntitle: Post 01\n---\nContent 01" - posts/2015-03-02_post02.md: "---\ntitle: Post 02\n---\nContent 02" - posts/2015-03-03_post03.md: "---\ntitle: Post 03\n---\nContent 03" + posts/2015-03-01_post01.html: "---\ntitle: Post 01\n---\nContent 01" + posts/2015-03-02_post02.html: "---\ntitle: Post 02\n---\nContent 02" + posts/2015-03-03_post03.html: "---\ntitle: Post 03\n---\nContent 03" templates/post.html: | BLAH {{content|safe}} {{pagination.prev_item.url}} {{pagination.prev_item.title}}
--- a/tests/bakes/test_simple.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_simple.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,13 +1,10 @@ --- in: - posts/2010-01-01_post1.md: 'post one' - pages/about.md: 'URL: {{page.url}}' - pages/_index.md: 'something' -out: - '2010': - '01': - '01': - post1.html: 'post one' + posts/2010-01-01_post1.html: 'post one' + pages/about.html: 'URL: {{page.url}}' + pages/_index.html: 'something' +outfiles: + 2010/01/01/post1.html: 'post one' about.html: 'URL: /about.html' index.html: 'something' --- @@ -15,19 +12,16 @@ site: root: /whatever in: - posts/2010-01-01_post1.md: 'post one' - pages/about.md: 'URL: {{page.url}}' - pages/_index.md: 'something' -out: - '2010': - '01': - '01': - post1.html: 'post one' + posts/2010-01-01_post1.html: 'post one' + pages/about.html: 'URL: {{page.url}}' + pages/_index.html: 'something' +outfiles: + 2010/01/01/post1.html: 'post one' about.html: 'URL: /whatever/about.html' index.html: 'something' --- in: - pages/foo.md: | + pages/foo.html: | This page is {{page.url}} outfiles: foo.html: | @@ -37,7 +31,7 @@ site: author: Amélie Poulain in: - pages/foo.md: 'Site by {{site.author}}' + pages/foo.html: 'Site by {{site.author}}' outfiles: foo.html: 'Site by Amélie Poulain'
--- a/tests/bakes/test_simple_categories.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_simple_categories.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -3,45 +3,39 @@ site: category_url: cat/%category% in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 category: foo --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 category: bar --- - posts/2015-03-03_post03.md: | + posts/2015-03-03_post03.html: | --- title: Post 03 category: foo --- - pages/_category.md: | + templates/_category.html: | Pages in {{category}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/link.md: 'Link: {{pccaturl("bar")}}' - pages/_index.md: '' -out: + pages/link.html: 'Link: {{pccaturl("bar")}}' + pages/_index.html: '' +outfiles: index.html: '' - '2015': - '03': - '01': - post01.html: '' - '02': - post02.html: '' - '03': - post03.html: '' + '2015/03/01/post01.html': '' + '2015/03/02/post02.html': '' + '2015/03/03/post03.html': '' link.html: 'Link: /cat/bar.html' - cat: - foo.html: | - Pages in foo - Post 03 - Post 01 - bar.html: | - Pages in bar - Post 02 + 'cat/foo.html': | + Pages in foo + Post 03 + Post 01 + 'cat/bar.html': | + Pages in bar + Post 02
--- a/tests/bakes/test_simple_tags.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_simple_tags.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,7 +1,7 @@ --- in: - posts/2015-03-01_post01.md: "---\ntitle: Post 01\n---\nContent 01" - pages/_index.md: | + posts/2015-03-01_post01.html: "---\ntitle: Post 01\n---\nContent 01" + pages/_index.html: | {%for p in pagination.items -%} {{p.content|safe}} {%if p.tags%}{{p.tags}}{%else%}No tags{%endif%} @@ -12,67 +12,61 @@ No tags --- in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [foo] --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 tags: [bar, whatever] --- - posts/2015-03-03_post03.md: | + posts/2015-03-03_post03.html: | --- title: Post 03 tags: [foo, bar] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{tag}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' -out: + pages/_index.html: '' +outfiles: index.html: '' - '2015': - '03': - '01': - post01.html: '' - '02': - post02.html: '' - '03': - post03.html: '' - tag: - foo.html: | - Pages in foo - Post 03 - Post 01 - bar.html: | - Pages in bar - Post 03 - Post 02 - whatever.html: | - Pages in whatever - Post 02 + 2015/03/01/post01.html: '' + 2015/03/02/post02.html: '' + 2015/03/03/post03.html: '' + tag/foo.html: | + Pages in foo + Post 03 + Post 01 + tag/bar.html: | + Pages in bar + Post 03 + Post 02 + tag/whatever.html: | + Pages in whatever + Post 02 --- in: - posts/2016-06-01_post01.md: | + posts/2016-06-01_post01.html: | --- title: Post 01 tags: [foo, bar] --- - posts/2016-06-02_post02.md: | + posts/2016-06-02_post02.html: | --- title: Post 02 tags: [bar, foo] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{tags|join(', ')}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/blah.md: | + pages/blah.html: | Link to: {{pctagurl('foo', 'bar')}} outfiles: blah.html: | @@ -94,17 +88,17 @@ site: slugify_mode: space_to_dash in: - posts/2016-09-01_post01.md: | + posts/2016-09-01_post01.html: | --- title: Post 01 tags: [foo bar] --- - posts/2016-09-02_post2.md: | + posts/2016-09-02_post2.html: | --- title: Post 02 tags: ['foo-bar'] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/bakes/test_sitemap.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -0,0 +1,40 @@ +--- +in: + assets/sitemap.sitemap: | + autogen: [pages, theme_pages] + pages/foo.md: This is a foo +outfiles: + sitemap.xml: | + <?xml version="1.0" encoding="utf-8"?> + <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + <url> + <loc>/foo.html</loc> + <lastmod>%test_time_iso8601%</lastmod> + </url> + <url> + <loc>/</loc> + <lastmod>%test_time_iso8601%</lastmod> + </url> + </urlset> +--- +in: + assets/sitemap.sitemap: | + autogen: [pages] + pages/foo.md: | + --- + sitemap: + changefreq: monthly + priority: 0.8 + --- + This is a foo +outfiles: + sitemap.xml: | + <?xml version="1.0" encoding="utf-8"?> + <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + <url> + <loc>/foo.html</loc> + <lastmod>%test_time_iso8601%</lastmod> + <changefreq>monthly</changefreq> + <priority>0.8</priority> + </url> + </urlset>
--- a/tests/bakes/test_special_root.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_special_root.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -3,8 +3,8 @@ site: root: /~john/public/ in: - pages/about.md: 'URL: {{page.url}}, LINK: {{pcurl("missing")}}' - pages/_index.md: 'URL: {{page.url}}' -out: + pages/about.html: 'URL: {{page.url}}, LINK: {{pcurl("missing")}}' + pages/_index.html: 'URL: {{page.url}}' +outfiles: about.html: 'URL: /%7Ejohn/public/about.html, LINK: /%7Ejohn/public/missing.html' index.html: 'URL: /%7Ejohn/public/'
--- a/tests/bakes/test_theme.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_theme.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -5,11 +5,11 @@ default_page_layout: 'none' foo: bar in: - pages/foo.md: "This is: {{foo}}, with no template" + pages/foo.html: "This is: {{foo}}, with no template" theme/theme_config.yml: "name: testtheme" - theme/pages/_index.md: "This is {{site.title}} by {{name}}, with theme template" + theme/pages/_index.html: "This is {{site.title}} by {{name}}, with theme template" theme/templates/default.html: "THEME: {{content}}" -out: +outfiles: index.html: "THEME: This is Some Test by testtheme, with theme template" foo.html: "This is: bar, with no template" --- @@ -17,14 +17,14 @@ site: default_page_layout: 'custom' in: - pages/foo.md: "FOO" - pages/bar.md: "---\nlayout: blah\n---\nBAR" + pages/foo.html: "FOO" + pages/bar.html: "---\nlayout: blah\n---\nBAR" templates/custom.html: "CUSTOM: {{content}}" theme/theme_config.yml: "site: {sources: {theme_pages: {default_layout: blah}}}" - theme/pages/_index.md: "theme index" - theme/pages/about.md: "about" + theme/pages/_index.html: "theme index" + theme/pages/about.html: "about" theme/templates/blah.html: "THEME: {{content}}" -out: +outfiles: index.html: "THEME: theme index" about.html: "THEME: about" foo.html: "CUSTOM: FOO"
--- a/tests/bakes/test_theme_site.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_theme_site.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -3,7 +3,7 @@ site: title: "Some Test Theme" in: - pages/foo.md: "This is: {{site.title}}" + pages/foo.html: "This is: {{site.title}}" outfiles: foo.html: "This is: Some Test Theme" --- @@ -11,7 +11,7 @@ site: title: "Some Test Theme" in: - pages/foo.md: "This is: {{foo}}" + pages/foo.html: "This is: {{foo}}" configs/theme_preview.yml: "foo: bar" outfiles: foo.html: "This is: bar"
--- a/tests/bakes/test_unicode.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_unicode.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,21 +1,17 @@ --- in: - posts/2010-01-01_déjà-des-accents.md: 'POST URL: {{page.url}}' - pages/présentation.md: 'PAGE URL: {{page.url}}' - pages/_index.md: '' -out: - '2010': - '01': - '01': - déjà-des-accents.html: 'POST URL: /2010/01/01/d%C3%A9j%C3%A0-des-accents.html' + posts/2010-01-01_déjà-des-accents.html: 'POST URL: {{page.url}}' + pages/présentation.html: 'PAGE URL: {{page.url}}' + pages/_index.html: '' +outfiles: + 2010/01/01/déjà-des-accents.html: 'POST URL: /2010/01/01/d%C3%A9j%C3%A0-des-accents.html' présentation.html: 'PAGE URL: /pr%C3%A9sentation.html' index.html: '' --- in: - pages/special/Это тэг.md: 'PAGE URL: {{page.url}}' - pages/_index.md: '' -out: - special: - Это тэг.html: 'PAGE URL: /special/%D0%AD%D1%82%D0%BE%20%D1%82%D1%8D%D0%B3.html' + pages/special/Это тэг.html: 'PAGE URL: {{page.url}}' + pages/_index.html: '' +outfiles: + special/Это тэг.html: 'PAGE URL: /special/%D0%AD%D1%82%D0%BE%20%D1%82%D1%8D%D0%B3.html' index.html: ''
--- a/tests/bakes/test_unicode_tags.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_unicode_tags.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,21 +1,21 @@ --- in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [étrange] --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 tags: [étrange, sévère] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} with {{tag}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' outfiles: tag/étrange.html: | Pages in /tag/%C3%A9trange.html with étrange @@ -26,17 +26,17 @@ Post 02 --- in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [Это тэг] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' outfiles: tag/Это тэг.html: | Pages in /tag/%D0%AD%D1%82%D0%BE%20%D1%82%D1%8D%D0%B3.html @@ -46,17 +46,17 @@ site: slugify_mode: lowercase,encode in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [Это тэг] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' outfiles: tag/это тэг.html: | Pages in /tag/%D1%8D%D1%82%D0%BE%20%D1%82%D1%8D%D0%B3.html @@ -66,22 +66,22 @@ site: slugify_mode: lowercase,transliterate in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [étrange] --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 tags: [étrange, sévère] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' outfiles: tag/etrange.html: | Pages in /tag/etrange.html @@ -95,17 +95,17 @@ site: slugify_mode: lowercase,transliterate,space_to_dash in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [Это тэг] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' outfiles: tag/eto-teg.html: | Pages in /tag/eto-teg.html
--- a/tests/bakes/test_variant.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/bakes/test_variant.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -3,7 +3,7 @@ what: not good config_variants: [test] in: - pages/_index.md: 'This is {{what}}.' + pages/_index.html: 'This is {{what}}.' configs/test.yml: 'what: awesome' out: index.html: 'This is awesome.' @@ -13,7 +13,7 @@ config_values: what: awesome in: - pages/_index.md: 'This is {{what}}.' + pages/_index.html: 'This is {{what}}.' out: index.html: 'This is awesome.'
--- a/tests/basefs.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/basefs.py Sun Oct 29 22:51:57 2017 -0700 @@ -1,11 +1,14 @@ import os.path import yaml from piecrust.app import PieCrust -from piecrust.main import _pre_parse_chef_args, _run_chef from piecrust.sources.base import ContentItem class TestFileSystemBase(object): + _use_chef_debug = False + _pytest_log_handler = None + _leave_mockfs = False + def __init__(self): pass @@ -99,10 +102,31 @@ def runChef(self, *args): root_dir = self.path('/kitchen') - chef_args = ['--root', root_dir] + args + chef_args = ['--root', root_dir] + if self._use_chef_debug: + chef_args += ['--debug'] + chef_args += list(args) + + import logging + from piecrust.main import ( + _make_chef_state, _recover_pre_chef_state, + _pre_parse_chef_args, _run_chef) - pre_args = _pre_parse_chef_args(chef_args) + # If py.test added a log handler, remove it because Chef will + # add its own logger. + if self._pytest_log_handler: + logging.getLogger().removeHandler( + self._pytest_log_handler) + + state = _make_chef_state() + pre_args = _pre_parse_chef_args(chef_args, state=state) exit_code = _run_chef(pre_args, chef_args) + _recover_pre_chef_state(state) + + if self._pytest_log_handler: + logging.getLogger().addHandler( + self._pytest_log_handler) + assert exit_code == 0 def getSimplePage(self, rel_path):
--- a/tests/conftest.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/conftest.py Sun Oct 29 22:51:57 2017 -0700 @@ -29,13 +29,22 @@ '--mock-debug', action='store_true', help="Prints contents of the mock file-system.") + parser.addoption( + '--leave-mockfs', + action='store_true', + help="Leave the contents of the mock file-system on disk.") def pytest_configure(config): if config.getoption('--log-debug'): + root_logger = logging.getLogger() hdl = logging.StreamHandler(stream=sys.stdout) - logging.getLogger('piecrust').addHandler(hdl) - logging.getLogger('piecrust').setLevel(logging.DEBUG) + root_logger.addHandler(hdl) + root_logger.setLevel(logging.DEBUG) + + from .basefs import TestFileSystemBase + TestFileSystemBase._use_chef_debug = True + TestFileSystemBase._pytest_log_handler = hdl log_file = config.getoption('--log-file') if log_file: @@ -43,14 +52,16 @@ stream=open(log_file, 'w', encoding='utf8')) logging.getLogger().addHandler(hdl) + if config.getoption('--leave-mockfs'): + from .basefs import TestFileSystemBase + TestFileSystemBase._leave_mockfs = True + def pytest_collect_file(parent, path): if path.ext == '.yaml' and path.basename.startswith("test"): category = os.path.basename(path.dirname) if category == 'bakes': return BakeTestFile(path, parent) - elif category == 'procs': - return PipelineTestFile(path, parent) elif category == 'cli': return ChefTestFile(path, parent) elif category == 'servings': @@ -264,11 +275,10 @@ if values is not None: values = list(values.items()) variants = self.spec.get('config_variants') - if variants is not None: - variants = list(variants.items()) apply_variants_and_values(app, variants, values) appfactory = PieCrustFactory(app.root_dir, + theme_site=self.is_theme_site, config_variants=variants, config_values=values) baker = Baker(appfactory, app, out_dir) @@ -313,69 +323,7 @@ __item_class__ = BakeTestItem -class PipelineTestItem(YamlTestItemBase): - def runtest(self): - fs = self._prepareMockFs() - - from piecrust.processing.pipeline import ProcessorPipeline - with mock_fs_scope(fs, keep=self.mock_debug): - out_dir = fs.path('kitchen/_counter') - app = fs.getApp(theme_site=self.is_theme_site) - pipeline = ProcessorPipeline(app, out_dir) - - proc_names = self.spec.get('processors') - if proc_names: - pipeline.enabled_processors = proc_names - - record = pipeline.run() - - if not record.success: - errors = [] - for e in record.entries: - errors += e.errors - raise PipelineError(errors) - - check_expected_outputs(self.spec, fs, ExpectedPipelineOutputError) - - def reportinfo(self): - return self.fspath, 0, "pipeline: %s" % self.name - - def repr_failure(self, excinfo): - if isinstance(excinfo.value, ExpectedPipelineOutputError): - return ('\n'.join( - ['Unexpected pipeline output. Left is expected output, ' - 'right is actual output'] + - excinfo.value.args[0])) - elif isinstance(excinfo.value, PipelineError): - res = ('\n'.join( - ['Errors occured during processing:'] + - excinfo.value.args[0])) - res += repr_nested_failure(excinfo) - return res - return super(PipelineTestItem, self).repr_failure(excinfo) - - -class PipelineError(Exception): - pass - - -class ExpectedPipelineOutputError(Exception): - pass - - -class PipelineTestFile(YamlTestFileBase): - __item_class__ = PipelineTestItem - - class ServeTestItem(YamlTestItemBase): - class _TestApp(object): - def __init__(self, server): - self.server = server - - def __call__(self, environ, start_response): - response = self.server._try_run_request(environ) - return response(environ, start_response) - def runtest(self): fs = self._prepareMockFs() @@ -387,28 +335,19 @@ expected_headers = self.spec.get('headers') expected_output = self.spec.get('out') expected_contains = self.spec.get('out_contains') - is_admin_test = self.spec.get('admin') is True from werkzeug.test import Client from werkzeug.wrappers import BaseResponse + from piecrust.app import PieCrustFactory + from piecrust.serving.server import PieCrustServer + with mock_fs_scope(fs, keep=self.mock_debug): - if is_admin_test: - from piecrust.admin.web import create_foodtruck_app - s = { - 'FOODTRUCK_CMDLINE_MODE': True, - 'FOODTRUCK_ROOT': fs.path('/kitchen') - } - test_app = create_foodtruck_app(s) - else: - from piecrust.app import PieCrustFactory - from piecrust.serving.server import Server - appfactory = PieCrustFactory( - fs.path('/kitchen'), - theme_site=self.is_theme_site) - server = Server(appfactory) - test_app = self._TestApp(server) + appfactory = PieCrustFactory( + fs.path('/kitchen'), + theme_site=self.is_theme_site) + server = PieCrustServer(appfactory) - client = Client(test_app, BaseResponse) + client = Client(server, BaseResponse) resp = client.get(url) assert expected_status == resp.status_code @@ -560,9 +499,15 @@ right_time_str = right[i:i + len(test_time_iso8601)] right_time = time.strptime(right_time_str, '%Y-%m-%dT%H:%M:%SZ') left_time = time.gmtime(ctx.time) + # Need to patch the daylist-savings-time flag because it can + # mess up the computation of the time difference. + right_time = (right_time[0], right_time[1], right_time[2], + right_time[3], right_time[4], right_time[5], + right_time[6], right_time[7], + left_time.tm_isdst) difference = time.mktime(left_time) - time.mktime(right_time) print("Got time difference: %d" % difference) - if abs(difference) <= 2: + if abs(difference) <= 1: print("(good enough, moving to end of timestamp)") skip_for = len(test_time_iso8601) - 1
--- a/tests/procs/test_dotfiles.yaml Sun Oct 29 22:46:41 2017 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ ---- -in: - assets/something.txt: Foo bar - assets/.htaccess: "# Apache config" -outfiles: - something.txt: Foo bar - .htaccess: "# Apache config" -
--- a/tests/procs/test_sitemap.yaml Sun Oct 29 22:46:41 2017 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,40 +0,0 @@ ---- -in: - assets/sitemap.sitemap: | - autogen: [pages, theme_pages] - pages/foo.md: This is a foo -outfiles: - sitemap.xml: | - <?xml version="1.0" encoding="utf-8"?> - <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> - <url> - <loc>/foo.html</loc> - <lastmod>%test_time_iso8601%</lastmod> - </url> - <url> - <loc>/</loc> - <lastmod>%test_time_iso8601%</lastmod> - </url> - </urlset> ---- -in: - assets/sitemap.sitemap: | - autogen: [pages] - pages/foo.md: | - --- - sitemap: - changefreq: monthly - priority: 0.8 - --- - This is a foo -outfiles: - sitemap.xml: | - <?xml version="1.0" encoding="utf-8"?> - <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> - <url> - <loc>/foo.html</loc> - <lastmod>%test_time_iso8601%</lastmod> - <changefreq>monthly</changefreq> - <priority>0.8</priority> - </url> - </urlset>
--- a/tests/servings/test_admin.yaml Sun Oct 29 22:46:41 2017 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,9 +0,0 @@ ---- -admin: true -url: / -in: - pages/one.html: '' - posts/2016-01-01_post1.html: '' -out_contains: | - 1 pages -
--- a/tests/servings/test_archives.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/servings/test_archives.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,7 +1,7 @@ --- url: /archives/2016.html in: - pages/_year.html: | + templates/_year.html: | Posts in {{year}} {% for post in pagination.posts -%} {{post.url}}
--- a/tests/servings/test_debug_info.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/servings/test_debug_info.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,14 +1,14 @@ --- url: /foo in: - pages/foo.md: | + pages/foo.html: | BLAH {{piecrust.debug_info}} out: BLAH --- url: /foo?!debug in: - pages/foo.md: | + pages/foo.html: | BLAH {{piecrust.debug_info}} out_contains: | @@ -17,12 +17,12 @@ --- url: /foo in: - pages/foo.md: BLAH {{pcurl('bar')}} + pages/foo.html: BLAH {{pcurl('bar')}} out: BLAH /bar.html --- url: /foo?!debug in: - pages/foo.md: BLAH {{pcurl('bar')}} + pages/foo.html: BLAH {{pcurl('bar')}} out: BLAH /bar.html?!debug --- url: /foo @@ -30,7 +30,7 @@ site: pretty_urls: true in: - pages/foo.md: BLAH {{pcurl('bar')}} + pages/foo.html: BLAH {{pcurl('bar')}} out: BLAH /bar --- url: /foo?!debug @@ -38,6 +38,6 @@ site: pretty_urls: true in: - pages/foo.md: BLAH {{pcurl('bar')}} + pages/foo.html: BLAH {{pcurl('bar')}} out: BLAH /bar?!debug
--- a/tests/servings/test_theme.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/servings/test_theme.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -5,7 +5,7 @@ title: "Some Test" in: theme/theme_config.yml: "name: testtheme" - theme/pages/_index.md: "This is {{site.title}} by {{name}}" + theme/pages/_index.html: "This is {{site.title}} by {{name}}" theme/templates/default.html: "THEME: {{content}}" out: "THEME: This is Some Test by testtheme" --- @@ -15,7 +15,7 @@ title: "Some Test" foo: bar in: - pages/foo.md: "This is: {{foo}} by {{name}}" + pages/foo.html: "This is: {{foo}} by {{name}}" theme/theme_config.yml: "name: testtheme" out: "This is: bar by testtheme"
--- a/tests/servings/test_theme_site.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/servings/test_theme_site.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -4,7 +4,7 @@ site: title: "Some Test Theme" in: - pages/foo.md: "This is: {{site.title}}" + pages/foo.html: "This is: {{site.title}}" out: "This is: Some Test Theme" --- url: /foo.html @@ -12,7 +12,7 @@ site: title: "Some Test Theme" in: - pages/foo.md: "This is: {{foo}}" + pages/foo.html: "This is: {{foo}}" configs/theme_preview.yml: "foo: bar" out: "This is: bar"
--- a/tests/servings/test_unicode.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/servings/test_unicode.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,16 +1,16 @@ --- url: /pr%C3%A9sentation.html in: - pages/présentation.md: 'PAGE URL: {{page.url}}' + pages/présentation.html: 'PAGE URL: {{page.url}}' out: 'PAGE URL: /pr%C3%A9sentation.html' --- url: /2010/01/01/d%C3%A9j%C3%A0-des-accents.html in: - posts/2010-01-01_déjà-des-accents.md: 'POST URL: {{page.url}}' + posts/2010-01-01_déjà-des-accents.html: 'POST URL: {{page.url}}' out: 'POST URL: /2010/01/01/d%C3%A9j%C3%A0-des-accents.html' --- url: /special/%D0%AD%D1%82%D0%BE%20%D1%82%D1%8D%D0%B3.html in: - pages/special/Это тэг.md: 'PAGE URL: {{page.url}}' + pages/special/Это тэг.html: 'PAGE URL: {{page.url}}' out: 'PAGE URL: /special/%D0%AD%D1%82%D0%BE%20%D1%82%D1%8D%D0%B3.html'
--- a/tests/servings/test_unicode_tags.yaml Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/servings/test_unicode_tags.yaml Sun Oct 29 22:51:57 2017 -0700 @@ -1,22 +1,22 @@ --- url: /tag/%C3%A9trange.html in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [étrange] --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 tags: [étrange, sévère] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' out: | Pages in /tag/%C3%A9trange.html Post 02 @@ -24,39 +24,39 @@ --- url: /tag/s%C3%A9v%C3%A8re.html in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [étrange] --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 tags: [étrange, sévère] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' out: | Pages in /tag/s%C3%A9v%C3%A8re.html Post 02 --- url: /tag/%D0%AD%D1%82%D0%BE%20%D1%82%D1%8D%D0%B3.html in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [Это тэг] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' out: | Pages in /tag/%D0%AD%D1%82%D0%BE%20%D1%82%D1%8D%D0%B3.html Post 01 @@ -66,22 +66,22 @@ slugify_mode: lowercase,transliterate url: /tag/etrange.html in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [étrange] --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 tags: [étrange, sévère] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' out: | Pages in /tag/etrange.html Post 02 @@ -92,22 +92,22 @@ slugify_mode: lowercase,transliterate url: /tag/severe.html in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [étrange] --- - posts/2015-03-02_post02.md: | + posts/2015-03-02_post02.html: | --- title: Post 02 tags: [étrange, sévère] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' out: | Pages in /tag/severe.html Post 02 @@ -117,17 +117,17 @@ slugify_mode: lowercase,transliterate,space_to_dash url: /tag/eto-teg.html in: - posts/2015-03-01_post01.md: | + posts/2015-03-01_post01.html: | --- title: Post 01 tags: [Это тэг] --- - pages/_tag.md: | + templates/_tag.html: | Pages in {{pctagurl(tag)}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} - pages/_index.md: '' + pages/_index.html: '' out: | Pages in /tag/eto-teg.html Post 01
--- a/tests/test_data_assetor.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_data_assetor.py Sun Oct 29 22:51:57 2017 -0700 @@ -40,11 +40,11 @@ assetor = Assetor(page) for en in expected.keys(): + assert en in assetor assert hasattr(assetor, en) - assert en in assetor path = site_root.rstrip('/') + '/foo/bar/%s.txt' % en + assert assetor[en] == path assert getattr(assetor, en) == path - assert assetor[en] == path def test_missing_asset():
--- a/tests/test_data_linker.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_data_linker.py Sun Oct 29 22:51:57 2017 -0700 @@ -7,12 +7,12 @@ 'fs_fac, page_path, expected', [ (lambda: mock_fs().withPage('pages/foo'), 'foo', - []), + ['/foo']), ((lambda: mock_fs() .withPage('pages/foo') .withPage('pages/bar')), 'foo', - ['/bar']), + ['/bar', '/foo']), ((lambda: mock_fs() .withPage('pages/baz') .withPage('pages/something') @@ -20,14 +20,14 @@ .withPage('pages/foo') .withPage('pages/bar')), 'foo', - ['/bar', '/baz', '/something']), + ['/bar', '/baz', '/foo', '/something']), ((lambda: mock_fs() .withPage('pages/something/else') .withPage('pages/foo') .withPage('pages/something/good') .withPage('pages/bar')), 'something/else', - ['/something/good']) + ['/something/else', '/something/good']) ]) def test_linker_siblings(fs_fac, page_path, expected): fs = fs_fac()
--- a/tests/test_data_provider.py Sun Oct 29 22:46:41 2017 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,25 +0,0 @@ -from .mockutil import mock_fs, mock_fs_scope -from .rdrutil import render_simple_page - - -def test_blog_provider(): - fs = (mock_fs() - .withConfig() - .withPage('posts/2015-03-01_one.md', - {'title': 'One', 'tags': ['Foo']}) - .withPage('posts/2015-03-02_two.md', - {'title': 'Two', 'tags': ['Foo']}) - .withPage('posts/2015-03-03_three.md', - {'title': 'Three', 'tags': ['Bar']}) - .withPage('pages/tags.md', - {'format': 'none', 'layout': 'none'}, - "{%for c in blog.tags%}\n" - "{{c.name}} ({{c.post_count}})\n" - "{%endfor%}\n")) - with mock_fs_scope(fs): - app = fs.getApp() - page = app.getSimplePage('tags.md') - actual = render_simple_page(page) - expected = "\nBar (1)\n\nFoo (2)\n" - assert actual == expected -
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_dataproviders_blog.py Sun Oct 29 22:51:57 2017 -0700 @@ -0,0 +1,123 @@ +from .mockutil import mock_fs, mock_fs_scope +from .rdrutil import render_simple_page + + +def _get_post_tokens(i, posts_per_month=2, posts_per_year=5, first_year=2001): + year = first_year + int(i / posts_per_year) + i_in_year = i % posts_per_year + month = int(i_in_year / posts_per_month) + 1 + day = i_in_year % posts_per_month + 1 + return (year, month, day, i + 1) + + +def test_blog_provider_archives(): + fs = (mock_fs() + .withConfig({ + 'site': { + 'default_layout': 'none', + 'default_format': 'none' + } + }) + .withPages( + 20, + lambda i: ('posts/%04d-%02d-%02d_post-%d.md' % + _get_post_tokens(i)), + lambda i: {'title': "Post %02d" % (i + 1), 'format': 'none'}, + lambda i: "This is post %02d" % (i + 1)) + .withPage('pages/allposts.html', + {'layout': 'none'}, + "{%for p in blog.posts-%}\n" + "{{p.title}}\n" + "{%endfor%}\n") + .withPage('pages/allyears.html', + {'layout': 'none'}, + "{%for y in blog.years-%}\n" + "YEAR={{y}}\n" + "{%for p in y.posts-%}\n" + "{{p.title}}\n" + "{%endfor%}\n" + "{%endfor%}") + .withFile('kitchen/templates/_year.html', + "YEAR={{year}}\n" + "{%for p in archives-%}\n" + "{{p.title}}\n" + "{%endfor%}\n" + "\n" + "{%for m in monthly_archives-%}\n" + "MONTH={{m.timestamp|date('%m')}}\n" + "{%for p in m.posts-%}\n" + "{{p.title}}\n" + "{%endfor%}\n" + "{%endfor%}")) + + with mock_fs_scope(fs): + fs.runChef('bake', '-o', fs.path('counter')) + + # Check `allposts`. + # Should have all the posts. Duh. + expected = '\n'.join(map(lambda i: "Post %02d" % i, + range(20, 0, -1))) + '\n' + actual = fs.getFileEntry('counter/allposts.html') + assert expected == actual + + # Check `allyears`. + # Should have all the years, each with 5 posts in reverse + # chronological order. + expected = '' + cur_index = 20 + for y in range(2004, 2000, -1): + expected += ('YEAR=%04d\n' % y) + '\n'.join( + map(lambda i: "Post %02d" % i, + range(cur_index, cur_index - 5, -1))) + '\n\n' + cur_index -= 5 + actual = fs.getFileEntry('counter/allyears.html') + assert expected == actual + + # Check each yearly page. + # Should have both the posts for that year (5 posts) in + # chronological order, followed by the months for that year + # (3 months) and the posts in each month (2, 2, and 1). + cur_index = 1 + for y in range(2001, 2005): + orig_index = cur_index + expected = ('YEAR=%04d\n' % y) + '\n'.join( + map(lambda i: "Post %02d" % i, + range(cur_index, cur_index + 5))) + '\n' + expected += "\n\n" + orig_final_index = cur_index + cur_index = orig_index + for m in range(1, 4): + expected += 'MONTH=%02d\n' % m + expected += '\n'.join( + map(lambda i: "Post %02d" % i, + range(cur_index, + min(cur_index + 2, orig_index + 5)))) + '\n' + expected += '\n' + cur_index += 2 + cur_index = orig_final_index + + actual = fs.getFileEntry('counter/archives/%04d.html' % y) + assert expected == actual + cur_index += 5 + + +def test_blog_provider_tags(): + fs = (mock_fs() + .withConfig() + .withPage('posts/2015-03-01_one.md', + {'title': 'One', 'tags': ['Foo']}) + .withPage('posts/2015-03-02_two.md', + {'title': 'Two', 'tags': ['Foo']}) + .withPage('posts/2015-03-03_three.md', + {'title': 'Three', 'tags': ['Bar']}) + .withPage('pages/tags.md', + {'format': 'none', 'layout': 'none'}, + "{%for c in blog.tags%}\n" + "{{c.name}} ({{c.post_count}})\n" + "{%endfor%}\n")) + with mock_fs_scope(fs): + page = fs.getSimplePage('tags.md') + actual = render_simple_page(page) + expected = "\nBar (1)\n\nFoo (2)\n" + assert actual == expected +
--- a/tests/test_dataproviders_pageiterator.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_dataproviders_pageiterator.py Sun Oct 29 22:51:57 2017 -0700 @@ -46,7 +46,7 @@ class TestItem(object): def __init__(self, value): self.name = str(value) - self.foo = value + self.config = {'foo': value} def __eq__(self, other): return other.name == self.name
--- a/tests/test_page.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_page.py Sun Oct 29 22:51:57 2017 -0700 @@ -6,18 +6,18 @@ test_parse_segments_data2 = ("Foo bar", {'content': 'Foo bar'}) test_parse_segments_data3 = ( """Something that spans - several lines - like this""", +several lines +like this""", {'content': """Something that spans several lines like this"""}) test_parse_segments_data4 = ( """Blah blah - ---foo--- - Something else - ---bar--- - Last thing - """, +---foo--- +Something else +---bar--- +Last thing +""", { 'content': "Blah blah\n", 'foo': "Something else\n", @@ -46,7 +46,7 @@ ('blah foo\n', 2), ('blah foo\nmore here', 2), ('blah foo\nmore here\n', 3), - ('\nblah foo\nmore here\n', 3), + ('\nblah foo\nmore here\n', 4), ]) def test_count_lines(text, expected): actual = _count_lines(text) @@ -63,5 +63,5 @@ ('\nblah foo\nmore here\n', 2, -1, 3), ]) def test_count_lines_with_offsets(text, start, end, expected): - actual = _count_lines(text) + actual = _count_lines(text, start, end) assert actual == expected
--- a/tests/test_pipelines_asset.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_pipelines_asset.py Sun Oct 29 22:51:57 2017 -0700 @@ -1,6 +1,6 @@ import time import os.path -import shutil +import random import inspect import pytest from piecrust.pipelines.asset import get_filtered_processors @@ -10,10 +10,10 @@ class FooProcessor(SimpleFileProcessor): - def __init__(self, exts=None, open_func=None): - exts = exts or {'foo', 'foo'} - super(FooProcessor, self).__init__({exts[0]: exts[1]}) - self.PROCESSOR_NAME = exts[0] + def __init__(self, name=None, exts=None, open_func=None): + self.PROCESSOR_NAME = name or 'foo' + exts = exts or {'foo': 'foo'} + super().__init__(exts) self.open_func = open_func or open def _doProcess(self, in_path, out_path): @@ -24,24 +24,20 @@ return True -class NoopProcessor(SimpleFileProcessor): - def __init__(self, exts): - super(NoopProcessor, self).__init__({exts[0]: exts[1]}) - self.PROCESSOR_NAME = exts[0] - self.processed = [] - - def _doProcess(self, in_path, out_path): - self.processed.append(in_path) - shutil.copyfile(in_path, out_path) - return True +def _get_test_plugin_name(): + return 'foo_%d' % random.randrange(1000) -def _get_test_fs(processors=None): - if processors is None: - processors = 'copy' +def _get_test_fs(*, plugins=None, processors=None): + plugins = plugins or [] + processors = processors or [] + processors.append('copy') return (mock_fs() .withDir('counter') .withConfig({ + 'site': { + 'plugins': plugins + }, 'pipelines': { 'asset': { 'processors': processors @@ -50,7 +46,7 @@ })) -def _create_test_plugin(fs, *, foo_exts=None, noop_exts=None): +def _create_test_plugin(fs, plugname, *, foo_name=None, foo_exts=None): src = [ 'from piecrust.plugins.base import PieCrustPlugin', 'from piecrust.processing.base import SimpleFileProcessor'] @@ -59,24 +55,21 @@ src += [''] src += map(lambda l: l.rstrip('\n'), foo_lines[0]) - noop_lines = inspect.getsourcelines(NoopProcessor) - src += [''] - src += map(lambda l: l.rstrip('\n'), noop_lines[0]) - src += [ '', - 'class FooNoopPlugin(PieCrustPlugin):', + 'class FooPlugin(PieCrustPlugin):', ' def getProcessors(self):', - ' yield FooProcessor(%s)' % repr(foo_exts), - ' yield NoopProcessor(%s)' % repr(noop_exts), + ' yield FooProcessor(%s, %s)' % (repr(foo_name), + repr(foo_exts)), '', - '__piecrust_plugin__ = FooNoopPlugin'] + '__piecrust_plugin__ = FooPlugin'] - fs.withFile('kitchen/plugins/foonoop.py', src) + print("Creating plugin with source:\n%s" % '\n'.join(src)) + fs.withFile('kitchen/plugins/%s.py' % plugname, '\n'.join(src)) def _bake_assets(fs): - fs.runChef('bake', '-p', 'asset') + fs.runChef('bake', '-p', 'asset', '-o', fs.path('counter')) def test_empty(): @@ -91,12 +84,12 @@ def test_one_file(): fs = (_get_test_fs() - .withFile('kitchen/assets/something.html', 'A test file.')) + .withFile('kitchen/assets/something.foo', 'A test file.')) with mock_fs_scope(fs): expected = {} assert expected == fs.getStructure('counter') _bake_assets(fs) - expected = {'something.html': 'A test file.'} + expected = {'something.foo': 'A test file.'} assert expected == fs.getStructure('counter') @@ -124,9 +117,10 @@ def test_two_levels_dirtyness(): - fs = (_get_test_fs() + plugname = _get_test_plugin_name() + fs = (_get_test_fs(plugins=[plugname], processors=['foo']) .withFile('kitchen/assets/blah.foo', 'A test file.')) - _create_test_plugin(fs, foo_exts=('foo', 'bar')) + _create_test_plugin(fs, plugname, foo_exts={'foo': 'bar'}) with mock_fs_scope(fs): _bake_assets(fs) expected = {'blah.bar': 'FOO: A test file.'} @@ -164,28 +158,32 @@ expected = { 'blah1.foo': 'A test file.'} assert expected == fs.getStructure('kitchen/assets') - _bake_assets(1) + _bake_assets(fs) assert expected == fs.getStructure('counter') def test_record_version_change(): - fs = (_get_test_fs() + plugname = _get_test_plugin_name() + fs = (_get_test_fs(plugins=[plugname], processors=['foo']) .withFile('kitchen/assets/blah.foo', 'A test file.')) - _create_test_plugin(fs, foo_exts=('foo', 'foo')) + _create_test_plugin(fs, plugname) with mock_fs_scope(fs): + time.sleep(1) _bake_assets(fs) - assert os.path.exists(fs.path('/counter/blah.foo')) is True - mtime = os.path.getmtime(fs.path('/counter/blah.foo')) + time.sleep(0.1) + mtime = os.path.getmtime(fs.path('counter/blah.foo')) time.sleep(1) _bake_assets(fs) - assert mtime == os.path.getmtime(fs.path('/counter/blah.foo')) + time.sleep(0.1) + assert mtime == os.path.getmtime(fs.path('counter/blah.foo')) - time.sleep(1) MultiRecord.RECORD_VERSION += 1 try: + time.sleep(1) _bake_assets(fs) - assert mtime < os.path.getmtime(fs.path('/counter/blah.foo')) + time.sleep(0.1) + assert mtime < os.path.getmtime(fs.path('counter/blah.foo')) finally: MultiRecord.RECORD_VERSION -= 1
--- a/tests/test_pipelines_page.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_pipelines_page.py Sun Oct 29 22:51:57 2017 -0700 @@ -3,7 +3,7 @@ import urllib.parse import pytest from piecrust.pipelines.records import MultiRecord -from piecrust.pipelines._pagebaker import PageBaker +from piecrust.pipelines._pagebaker import get_output_path from .mockutil import get_mock_app, mock_fs, mock_fs_scope @@ -41,17 +41,16 @@ app.config.set('site/pretty_urls', True) assert app.config.get('site/pretty_urls') == pretty + out_dir = '/destination' + for site_root in ['/', '/whatever/', '/~johndoe/']: app.config.set('site/root', urllib.parse.quote(site_root)) - baker = PageBaker(app, '/destination') - try: - path = baker.getOutputPath(urllib.parse.quote(site_root) + uri, - pretty) - expected = os.path.normpath( - os.path.join('/destination', expected)) - assert expected == path - finally: - baker.shutdown() + path = get_output_path(app, out_dir, + urllib.parse.quote(site_root) + uri, + pretty) + expected = os.path.normpath( + os.path.join('/destination', expected)) + assert expected == path def test_removed(): @@ -81,18 +80,22 @@ .withPage('pages/foo.md', {'layout': 'none', 'format': 'none'}, 'a foo page')) with mock_fs_scope(fs): - fs.runChef('bake') - mtime = os.path.getmtime(fs.path('kitchen/_counter/foo.html')) time.sleep(1) + fs.runChef('bake', '-o', fs.path('counter')) + time.sleep(0.1) + mtime = os.path.getmtime(fs.path('counter/foo.html')) - fs.runChef('bake') - assert mtime == os.path.getmtime(fs.path('kitchen/_counter/foo.html')) + time.sleep(1) + fs.runChef('bake', '-o', fs.path('counter')) + time.sleep(0.1) + assert mtime == os.path.getmtime(fs.path('counter/foo.html')) MultiRecord.RECORD_VERSION += 1 try: - fs.runChef('bake') - assert mtime < os.path.getmtime(fs.path( - 'kitchen/_counter/foo.html')) + time.sleep(1) + fs.runChef('bake', '-o', fs.path('counter')) + time.sleep(0.1) + assert mtime < os.path.getmtime(fs.path('counter/foo.html')) finally: MultiRecord.RECORD_VERSION -= 1
--- a/tests/test_routing.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_routing.py Sun Oct 29 22:51:57 2017 -0700 @@ -27,25 +27,21 @@ @pytest.mark.parametrize( - 'config, metadata, params, expected', + 'config, params, uri_params, expected', [ - ({'url': '/%foo%'}, - {'foo': 'bar'}, ['foo'], True), - ({'url': '/%foo%'}, - {'zoo': 'zar', 'foo': 'bar'}, ['foo'], True), - ({'url': '/%foo%'}, - {'zoo': 'zar'}, ['foo'], False), - ({'url': '/%foo%/%zoo%'}, - {'zoo': 'zar'}, ['foo', 'zoo'], False) + ({'url': '/%foo%'}, ['foo'], {'foo': 'bar'}, True), + ({'url': '/%foo%'}, ['foo'], {'zoo': 'zar', 'foo': 'bar'}, True), + ({'url': '/%foo%'}, ['foo'], {'zoo': 'zar'}, False), + ({'url': '/%foo%/%zoo%'}, ['foo', 'zoo'], {'zoo': 'zar'}, False) ]) -def test_matches_metadata(config, metadata, params, expected): +def test_matches_parameters(config, params, uri_params, expected): app = get_mock_app() app.config.set('site/root', '/') app.sources = [_getMockSource('blah', params)] config.setdefault('source', 'blah') route = Route(app, config) - m = route.matchesMetadata(metadata) + m = route.matchesParameters(uri_params) assert m == expected
--- a/tests/test_sources_autoconfig.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_sources_autoconfig.py Sun Oct 29 22:51:57 2017 -0700 @@ -5,50 +5,43 @@ @pytest.mark.parametrize( - 'fs_fac, src_config, expected_paths, expected_metadata', + 'fs_fac, src_config, expected_path, expected_slug, expected_foos', [ - (lambda: mock_fs(), {}, [], []), + (lambda: mock_fs(), + {}, + None, '', []), (lambda: mock_fs().withPage('test/_index.md'), {}, - ['_index.md'], - [{'slug': '', 'config': {'foo': []}}]), + '_index.md', '', []), (lambda: mock_fs().withPage('test/something.md'), {}, - ['something.md'], - [{'slug': 'something', 'config': {'foo': []}}]), + 'something.md', 'something', []), (lambda: mock_fs().withPage('test/bar/something.md'), {}, - ['bar/something.md'], - [{'slug': 'something', 'config': {'foo': ['bar']}}]), + 'bar/something.md', 'something', ['bar']), (lambda: mock_fs().withPage('test/bar1/bar2/something.md'), {}, - ['bar1/bar2/something.md'], - [{'slug': 'something', 'config': {'foo': ['bar1', 'bar2']}}]), + 'bar1/bar2/something.md', 'something', ['bar1', 'bar2']), (lambda: mock_fs().withPage('test/something.md'), {'collapse_single_values': True}, - ['something.md'], - [{'slug': 'something', 'config': {'foo': None}}]), + 'something.md', 'something', None), (lambda: mock_fs().withPage('test/bar/something.md'), {'collapse_single_values': True}, - ['bar/something.md'], - [{'slug': 'something', 'config': {'foo': 'bar'}}]), + 'bar/something.md', 'something', 'bar'), (lambda: mock_fs().withPage('test/bar1/bar2/something.md'), {'collapse_single_values': True}, - ['bar1/bar2/something.md'], - [{'slug': 'something', 'config': {'foo': ['bar1', 'bar2']}}]), + 'bar1/bar2/something.md', 'something', ['bar1', 'bar2']), (lambda: mock_fs().withPage('test/something.md'), {'only_single_values': True}, - ['something.md'], - [{'slug': 'something', 'config': {'foo': None}}]), + 'something.md', 'something', None), (lambda: mock_fs().withPage('test/bar/something.md'), {'only_single_values': True}, - ['bar/something.md'], - [{'slug': 'something', 'config': {'foo': 'bar'}}]), + 'bar/something.md', 'something', 'bar') ]) -def test_autoconfig_source_factories(fs_fac, src_config, expected_paths, - expected_metadata): +def test_autoconfig_source_items( + fs_fac, src_config, expected_path, expected_slug, expected_foos): site_config = { 'sources': { 'test': {'type': 'autoconfig', @@ -65,10 +58,17 @@ app = fs.getApp() s = app.getSource('test') items = list(s.getAllContents()) - paths = [os.path.relpath(i.spec, s.fs_endpoint_path) for i in items] - assert paths == slashfix(expected_paths) - metadata = [i.metadata['route_params'] for i in items] - assert metadata == expected_metadata + + if expected_path is None: + assert len(items) == 0 + else: + assert len(items) == 1 + path = os.path.relpath(items[0].spec, s.fs_endpoint_path) + assert path == slashfix(expected_path) + slug = items[0].metadata['route_params']['slug'] + assert slug == expected_slug + foos = items[0].metadata['config']['foo'] + assert foos == expected_foos def test_autoconfig_fails_if_multiple_folders(): @@ -89,27 +89,28 @@ @pytest.mark.parametrize( - 'fs_fac, expected_paths, expected_metadata', + 'fs_fac, expected_paths, expected_route_params, expected_configs', [ - (lambda: mock_fs(), [], []), + (lambda: mock_fs(), [], [], []), (lambda: mock_fs().withPage('test/_index.md'), ['_index.md'], - [{'slug': '', - 'config': {'foo': 0, 'foo_trail': [0]}}]), + [{'slug': ''}], + [{'foo': 0, 'foo_trail': [0]}]), (lambda: mock_fs().withPage('test/something.md'), ['something.md'], - [{'slug': 'something', - 'config': {'foo': 0, 'foo_trail': [0]}}]), + [{'slug': 'something'}], + [{'foo': 0, 'foo_trail': [0]}]), (lambda: mock_fs().withPage('test/08_something.md'), ['08_something.md'], - [{'slug': 'something', - 'config': {'foo': 8, 'foo_trail': [8]}}]), + [{'slug': 'something'}], + [{'foo': 8, 'foo_trail': [8]}]), (lambda: mock_fs().withPage('test/02_there/08_something.md'), ['02_there/08_something.md'], - [{'slug': 'there/something', - 'config': {'foo': 8, 'foo_trail': [2, 8]}}]), + [{'slug': 'there/something'}], + [{'foo': 8, 'foo_trail': [2, 8]}]), ]) -def test_ordered_source_factories(fs_fac, expected_paths, expected_metadata): +def test_ordered_source_items(fs_fac, expected_paths, expected_route_params, + expected_configs): site_config = { 'sources': { 'test': {'type': 'ordered', @@ -124,11 +125,16 @@ with mock_fs_scope(fs): app = fs.getApp() s = app.getSource('test') - facs = list(s.buildPageFactories()) - paths = [f.rel_path for f in facs] + items = list(s.getAllContents()) + + paths = [os.path.relpath(f.spec, s.fs_endpoint_path) for f in items] assert paths == slashfix(expected_paths) - metadata = [f.metadata for f in facs] - assert metadata == expected_metadata + metadata = [f.metadata['route_params'] for f in items] + assert metadata == expected_route_params + configs = [f.metadata['config'] for f in items] + for c in configs: + c.pop('format') + assert configs == expected_configs @pytest.mark.parametrize( @@ -176,10 +182,11 @@ app = fs.getApp() s = app.getSource('test') route_metadata = {'slug': route_path} - factory = s.findContent(route_metadata) - if factory is None: + item = s.findContent(route_metadata) + if item is None: assert expected_path is None and expected_metadata is None - return - assert factory.rel_path == slashfix(expected_path) - assert factory.metadata == expected_metadata + else: + assert os.path.relpath(item.spec, s.fs_endpoint_path) == \ + slashfix(expected_path) + assert item.metadata == expected_metadata
--- a/tests/test_sources_base.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_sources_base.py Sun Oct 29 22:51:57 2017 -0700 @@ -1,7 +1,7 @@ import os import pytest -from piecrust.app import PieCrust from .mockutil import mock_fs, mock_fs_scope +from .pathutil import slashfix @pytest.mark.parametrize('fs_fac, expected_paths, expected_slugs', [ @@ -19,7 +19,7 @@ (lambda: mock_fs().withPage('test/foo/bar.ext'), ['foo/bar.ext'], ['foo/bar.ext']), ]) -def test_default_source_factories(fs_fac, expected_paths, expected_slugs): +def test_default_source_items(fs_fac, expected_paths, expected_slugs): fs = fs_fac() fs.withConfig({ 'site': { @@ -31,25 +31,30 @@ }) fs.withDir('kitchen/test') with mock_fs_scope(fs): - app = PieCrust(fs.path('kitchen'), cache=False) + app = fs.getApp() s = app.getSource('test') - facs = list(s.buildPageFactories()) - paths = [f.rel_path for f in facs] - assert paths == expected_paths - slugs = [f.metadata['slug'] for f in facs] + items = list(s.getAllContents()) + paths = [os.path.relpath(f.spec, s.fs_endpoint_path) for f in items] + assert paths == slashfix(expected_paths) + slugs = [f.metadata['route_params']['slug'] for f in items] assert slugs == expected_slugs @pytest.mark.parametrize( - 'ref_path, expected_path, expected_metadata', - [ - ('foo.html', '/kitchen/test/foo.html', {'slug': 'foo'}), - ('foo/bar.html', '/kitchen/test/foo/bar.html', + 'fs_fac, ref_path, expected_path, expected_metadata', [ + (lambda: mock_fs().withPage('test/foo.html'), + 'foo.html', + 'test/foo.html', + {'slug': 'foo'}), + (lambda: mock_fs().withPage('test/foo/bar.html'), + 'foo/bar.html', + 'test/foo/bar.html', {'slug': 'foo/bar'}), + ]) -def test_default_source_resolve_ref(ref_path, expected_path, - expected_metadata): - fs = mock_fs() +def test_default_source_find_item(fs_fac, ref_path, expected_path, + expected_metadata): + fs = fs_fac() fs.withConfig({ 'site': { 'sources': { @@ -58,10 +63,11 @@ {'url': '/%path%', 'source': 'test'}] } }) - expected_path = fs.path(expected_path).replace('/', os.sep) with mock_fs_scope(fs): - app = PieCrust(fs.path('kitchen'), cache=False) + app = fs.getApp() s = app.getSource('test') - actual_path, actual_metadata = s.resolveRef(ref_path) - assert actual_path == expected_path - assert actual_metadata == expected_metadata + item = s.findContent({'slug': ref_path}) + assert item is not None + assert os.path.relpath(item.spec, app.root_dir) == \ + slashfix(expected_path) + assert item.metadata['route_params'] == expected_metadata
--- a/tests/test_sources_posts.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_sources_posts.py Sun Oct 29 22:51:57 2017 -0700 @@ -1,8 +1,11 @@ +import os.path import pytest from .mockutil import mock_fs, mock_fs_scope -@pytest.mark.parametrize('fs_fac, src_type, expected_paths, expected_metadata', [ +@pytest.mark.parametrize( + 'fs_fac, src_type, expected_paths, expected_metadata', + [ (lambda: mock_fs(), 'flat', [], []), (lambda: mock_fs().withPage('test/2014-01-01_foo.md'), 'flat', @@ -18,9 +21,9 @@ 'hierarchy', ['2014/01/01_foo.md'], [(2014, 1, 1, 'foo')]), - ]) -def test_post_source_factories(fs_fac, src_type, expected_paths, - expected_metadata): + ]) +def test_post_source_items(fs_fac, src_type, expected_paths, + expected_metadata): fs = fs_fac() fs.withConfig({ 'site': { @@ -28,18 +31,20 @@ 'test': {'type': 'posts/%s' % src_type}}, 'routes': [ {'url': '/%slug%', 'source': 'test'}] - } - }) + } + }) fs.withDir('kitchen/test') with mock_fs_scope(fs): - app = fs.getApp(cache=False) + app = fs.getApp() s = app.getSource('test') - facs = list(s.buildPageFactories()) - paths = [f.rel_path for f in facs] + items = list(s.getAllContents()) + paths = [os.path.relpath(f.spec, s.fs_endpoint_path) for f in items] assert paths == expected_paths metadata = [ - (f.metadata['year'], f.metadata['month'], - f.metadata['day'], f.metadata['slug']) - for f in facs] + (f.metadata['route_params']['year'], + f.metadata['route_params']['month'], + f.metadata['route_params']['day'], + f.metadata['route_params']['slug']) + for f in items] assert metadata == expected_metadata
--- a/tests/test_templating_jinjaengine.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_templating_jinjaengine.py Sun Oct 29 22:51:57 2017 -0700 @@ -26,11 +26,8 @@ .withConfig(app_config) .withPage('pages/foo', config=page_config, contents=contents)) with mock_fs_scope(fs, open_patches=open_patches): - app = fs.getApp() page = fs.getSimplePage('foo.md') - route = app.getSourceRoute('pages') - route_metadata = {'slug': 'foo'} - output = render_simple_page(page, route, route_metadata) + output = render_simple_page(page) assert output == expected @@ -41,30 +38,24 @@ fs = (mock_fs() .withConfig(app_config) .withAsset('templates/blah.jinja', layout) - .withPage('pages/foo', config={'layout': 'blah'}, + .withPage('pages/foo', config={'layout': 'blah.jinja'}, contents=contents)) with mock_fs_scope(fs, open_patches=open_patches): - app = fs.getApp() page = fs.getSimplePage('foo.md') - route = app.getSourceRoute('pages', None) - route_metadata = {'slug': 'foo'} - output = render_simple_page(page, route, route_metadata) + output = render_simple_page(page) assert output == expected def test_partial(): contents = "Info:\n{% include 'page_info.jinja' %}\n" - partial = "- URL: {{page.url}}\n- SLUG: {{page.slug}}\n" + partial = "- URL: {{page.url}}\n- SLUG: {{page.route.slug}}\n" expected = "Info:\n- URL: /foo.html\n- SLUG: foo" fs = (mock_fs() .withConfig(app_config) .withAsset('templates/page_info.jinja', partial) .withPage('pages/foo', config=page_config, contents=contents)) with mock_fs_scope(fs, open_patches=open_patches): - app = fs.getApp() page = fs.getSimplePage('foo.md') - route = app.getSourceRoute('pages', None) - route_metadata = {'slug': 'foo'} - output = render_simple_page(page, route, route_metadata) + output = render_simple_page(page) assert output == expected
--- a/tests/test_templating_pystacheengine.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/test_templating_pystacheengine.py Sun Oct 29 22:51:57 2017 -0700 @@ -26,11 +26,8 @@ .withConfig(app_config) .withPage('pages/foo', config=page_config, contents=contents)) with mock_fs_scope(fs, open_patches=open_patches): - app = fs.getApp() page = fs.getSimplePage('foo.md') - route = app.getSourceRoute('pages', None) - route_metadata = {'slug': 'foo'} - output = render_simple_page(page, route, route_metadata) + output = render_simple_page(page) assert output == expected @@ -41,14 +38,11 @@ fs = (mock_fs() .withConfig(app_config) .withAsset('templates/blah.mustache', layout) - .withPage('pages/foo', config={'layout': 'blah'}, + .withPage('pages/foo', config={'layout': 'blah.mustache'}, contents=contents)) with mock_fs_scope(fs, open_patches=open_patches): - app = fs.getApp() page = fs.getSimplePage('foo.md') - route = app.getSourceRoute('pages', None) - route_metadata = {'slug': 'foo'} - output = render_simple_page(page, route, route_metadata) + output = render_simple_page(page) # On Windows, pystache unexplicably adds `\r` to some newlines... wtf. output = output.replace('\r', '') assert output == expected @@ -56,18 +50,15 @@ def test_partial(): contents = "Info:\n{{#page}}\n{{> page_info}}\n{{/page}}\n" - partial = "- URL: {{url}}\n- SLUG: {{slug}}\n" + partial = "- URL: {{url}}\n- SLUG: {{route.slug}}\n" expected = "Info:\n- URL: /foo.html\n- SLUG: foo\n" fs = (mock_fs() .withConfig(app_config) .withAsset('templates/page_info.mustache', partial) .withPage('pages/foo', config=page_config, contents=contents)) with mock_fs_scope(fs, open_patches=open_patches): - app = fs.getApp() page = fs.getSimplePage('foo.md') - route = app.getSourceRoute('pages', None) - route_metadata = {'slug': 'foo'} - output = render_simple_page(page, route, route_metadata) + output = render_simple_page(page) # On Windows, pystache unexplicably adds `\r` to some newlines... wtf. output = output.replace('\r', '') assert output == expected
--- a/tests/tmpfs.py Sun Oct 29 22:46:41 2017 -0700 +++ b/tests/tmpfs.py Sun Oct 29 22:51:57 2017 -0700 @@ -18,7 +18,7 @@ p = p.lstrip('/\\') return os.path.join(self._root, p) - def getStructure(self, path=None): + def getStructure(self, path=''): path = self.path(path) if not os.path.exists(path): raise Exception("No such path: %s" % path) @@ -44,8 +44,11 @@ self._getStructureRecursive(e, full_cur, item) target[cur] = e else: - with open(full_cur, 'r', encoding='utf8') as fp: - target[cur] = fp.read() + try: + with open(full_cur, 'r', encoding='utf8') as fp: + target[cur] = fp.read() + except Exception as ex: + target[cur] = "ERROR: CAN'T READ '%s': %s" % (full_cur, ex) def _createDir(self, path): if not os.path.exists(path): @@ -69,7 +72,7 @@ def __init__(self, fs, open_patches=None, keep=False): self._fs = fs self._open = open - self._keep = keep + self._keep = keep or TestFileSystemBase._leave_mockfs @property def root(self):