Mercurial > piecrust2
changeset 711:ab5c6a8ae90a
bake: Replace hard-coded taxonomy support with "generator" system.
* Taxonomies are now implemented one or more `TaxonomyGenerator`s.
* A `BlogArchivesGenerator` stub is there but non-functional.
line wrap: on
line diff
--- a/piecrust/__init__.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/__init__.py Thu May 26 19:52:47 2016 -0700 @@ -3,6 +3,7 @@ ASSETS_DIR = 'assets' TEMPLATES_DIR = 'templates' THEME_DIR = 'theme' +THEMES_DIR = 'themes' CONFIG_PATH = 'config.yml' THEME_CONFIG_PATH = 'theme_config.yml' @@ -17,7 +18,7 @@ PIECRUST_URL = 'https://bolt80.com/piecrust/' -CACHE_VERSION = 23 +CACHE_VERSION = 24 try: from piecrust.__version__ import APP_VERSION
--- a/piecrust/app.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/app.py Thu May 26 19:52:47 2016 -0700 @@ -15,7 +15,6 @@ from piecrust.configuration import ConfigurationError, merge_dicts from piecrust.routing import Route from piecrust.sources.base import REALM_THEME -from piecrust.taxonomies import Taxonomy logger = logging.getLogger(__name__) @@ -151,12 +150,20 @@ return routes @cached_property - def taxonomies(self): - taxonomies = [] - for tn, tc in self.config.get('site/taxonomies').items(): - tax = Taxonomy(self, tn, tc) - taxonomies.append(tax) - return taxonomies + def generators(self): + defs = {} + for cls in self.plugin_loader.getPageGenerators(): + defs[cls.GENERATOR_NAME] = cls + + gens = [] + for n, g in self.config.get('site/generators').items(): + cls = defs.get(g['type']) + if cls is None: + raise ConfigurationError("No such page generator type: %s" % + g['type']) + gen = cls(self, n, g) + gens.append(gen) + return gens def getSource(self, source_name): for source in self.sources: @@ -164,33 +171,30 @@ return source return None - def getRoutes(self, source_name, *, skip_taxonomies=False): + def getGenerator(self, generator_name): + for gen in self.generators: + if gen.name == generator_name: + return gen + return None + + def getSourceRoutes(self, source_name): for route in self.routes: if route.source_name == source_name: - if not skip_taxonomies or route.taxonomy_name is None: - yield route + yield route - def getRoute(self, source_name, route_metadata, *, skip_taxonomies=False): - for route in self.getRoutes(source_name, - skip_taxonomies=skip_taxonomies): + def getSourceRoute(self, source_name, route_metadata): + for route in self.getSourceRoutes(source_name): if (route_metadata is None or route.matchesMetadata(route_metadata)): return route return None - def getTaxonomyRoute(self, tax_name, source_name): + def getGeneratorRoute(self, generator_name): for route in self.routes: - if (route.taxonomy_name == tax_name and - route.source_name == source_name): + if route.generator_name == generator_name: return route return None - def getTaxonomy(self, tax_name): - for tax in self.taxonomies: - if tax.name == tax_name: - return tax - return None - def _get_dir(self, default_rel_dir): abs_dir = os.path.join(self.root_dir, default_rel_dir) if os.path.isdir(abs_dir):
--- a/piecrust/appconfig.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/appconfig.py Thu May 26 19:52:47 2016 -0700 @@ -142,6 +142,7 @@ self._values = self._validateAll(values) except Exception as ex: + logger.exception(ex) raise Exception( "Error loading configuration from: %s" % ', '.join(paths)) from ex @@ -221,6 +222,7 @@ try: val2 = callback(val, values, cache_writer) except Exception as ex: + logger.exception(ex) raise Exception("Error raised in validator '%s'." % callback_name) from ex if val2 is None: @@ -288,7 +290,9 @@ } ], 'theme_tag_page': 'theme_pages:_tag.%ext%', - 'theme_category_page': 'theme_pages:_category.%ext%' + 'theme_category_page': 'theme_pages:_category.%ext%', + 'theme_month_page': 'theme_pages:_month.%ext%', + 'theme_year_page': 'theme_pages:_year.%ext%' }) }) @@ -332,9 +336,10 @@ 'posts_fs': DEFAULT_POSTS_FS, 'default_page_layout': 'default', 'default_post_layout': 'post', - 'post_url': '%year%/%month%/%day%/%slug%', - 'tag_url': 'tag/%tag%', - 'category_url': '%category%', + 'post_url': '/%year%/%month%/%day%/%slug%', + 'year_url': '/%year%', + 'tag_url': '/tag/%path:tag%', + 'category_url': '/%category%', 'posts_per_page': 5 }) }) @@ -362,25 +367,25 @@ 'func': 'pcurl(slug)' } ], - 'taxonomies': collections.OrderedDict({ - 'tags': { + 'taxonomies': collections.OrderedDict([ + ('tags', { 'multiple': True, 'term': 'tag' - }, - 'categories': { + }), + ('categories', { 'term': 'category' - } - }) + }) + ]) }) }) def get_default_content_model_for_blog( blog_name, is_only_blog, values, user_overrides, theme_site=False): - # Get the global values for various things we're interested in. + # Get the global (default) values for various things we're interested in. defs = {} names = ['posts_fs', 'posts_per_page', 'date_format', - 'default_post_layout', 'post_url', 'tag_url', 'category_url'] + 'default_post_layout', 'post_url', 'year_url'] for n in names: defs[n] = try_get_dict_value( user_overrides, 'site/%s' % n, @@ -389,7 +394,7 @@ # More stuff we need. if is_only_blog: url_prefix = '' - tax_page_prefix = '' + page_prefix = '' fs_endpoint = 'posts' data_endpoint = 'blog' item_name = 'post' @@ -401,7 +406,7 @@ fs_endpoint = 'sample/posts' else: url_prefix = blog_name + '/' - tax_page_prefix = blog_name + '/' + page_prefix = blog_name + '/' fs_endpoint = 'posts/%s' % blog_name data_endpoint = blog_name item_name = '%s-post' % blog_name @@ -414,28 +419,21 @@ blog_values = {} for n in names: blog_values[n] = blog_cfg.get(n, defs[n]) - if n in ['post_url', 'tag_url', 'category_url']: - blog_values[n] = url_prefix + blog_values[n] posts_fs = blog_values['posts_fs'] posts_per_page = blog_values['posts_per_page'] date_format = blog_values['date_format'] default_layout = blog_values['default_post_layout'] - post_url = '/' + blog_values['post_url'].lstrip('/') - tag_url = '/' + blog_values['tag_url'].lstrip('/') - category_url = '/' + blog_values['category_url'].lstrip('/') + post_url = '/' + url_prefix + blog_values['post_url'].lstrip('/') + year_url = '/' + url_prefix + blog_values['year_url'].lstrip('/') - tags_taxonomy = 'pages:%s_tag.%%ext%%' % tax_page_prefix - category_taxonomy = 'pages:%s_category.%%ext%%' % tax_page_prefix + year_archive = 'pages:%s_year.%%ext%%' % page_prefix if not theme_site: - theme_tag_page = values['site'].get('theme_tag_page') - if theme_tag_page: - tags_taxonomy += ';' + theme_tag_page - theme_category_page = values['site'].get('theme_category_page') - if theme_category_page: - category_taxonomy += ';' + theme_category_page + theme_year_page = values['site'].get('theme_year_page') + if theme_year_page: + year_archive += ';' + theme_year_page - return collections.OrderedDict({ + cfg = collections.OrderedDict({ 'site': collections.OrderedDict({ 'sources': collections.OrderedDict({ blog_name: collections.OrderedDict({ @@ -447,11 +445,14 @@ 'data_type': 'blog', 'items_per_page': posts_per_page, 'date_format': date_format, - 'default_layout': default_layout, - 'taxonomy_pages': collections.OrderedDict({ - 'tags': tags_taxonomy, - 'categories': category_taxonomy - }) + 'default_layout': default_layout + }) + }), + 'generators': collections.OrderedDict({ + ('%s_archives' % blog_name): collections.OrderedDict({ + 'type': 'blog_archives', + 'source': blog_name, + 'page': year_archive }) }), 'routes': [ @@ -461,21 +462,60 @@ 'func': 'pcposturl(year,month,day,slug)' }, { - 'url': tag_url, - 'source': blog_name, - 'taxonomy': 'tags', - 'func': 'pctagurl(tag)' - }, - { - 'url': category_url, - 'source': blog_name, - 'taxonomy': 'categories', - 'func': 'pccaturl(category)' + 'url': year_url, + 'generator': ('%s_archives' % blog_name), + 'func': 'pcyearurl(year)' } ] }) }) + # Add a generator and a route for each taxonomy. + taxonomies_cfg = values.get('site', {}).get('taxonomies', {}).copy() + taxonomies_cfg.update( + user_overrides.get('site', {}).get('taxonomies', {})) + for tax_name, tax_cfg in taxonomies_cfg.items(): + term = tax_cfg.get('term', tax_name) + + # Generator. + page_ref = 'pages:%s_%s.%%ext%%' % (page_prefix, term) + if not theme_site: + theme_page_ref = values['site'].get('theme_%s_page' % term) + if theme_page_ref: + page_ref += ';' + theme_page_ref + tax_gen_name = '%s_%s' % (blog_name, tax_name) + tax_gen = collections.OrderedDict({ + 'type': 'taxonomy', + 'source': blog_name, + 'taxonomy': tax_name, + 'page': page_ref + }) + cfg['site']['generators'][tax_gen_name] = tax_gen + + # Route. + tax_url_cfg_name = '%s_url' % term + tax_url = blog_cfg.get(tax_url_cfg_name, + try_get_dict_value( + user_overrides, + 'site/%s' % tax_url_cfg_name, + values['site'].get( + tax_url_cfg_name, + '%s/%%%s%%' % (term, term)))) + tax_url = '/' + url_prefix + tax_url.lstrip('/') + term_arg = term + if tax_cfg.get('multiple') is True: + term_arg = '+' + term + tax_func = 'pc%surl(%s)' % (term, term_arg) + tax_route = collections.OrderedDict({ + 'url': tax_url, + 'generator': tax_gen_name, + 'taxonomy': tax_name, + 'func': tax_func + }) + cfg['site']['routes'].append(tax_route) + + return cfg + # Configuration value validators. # @@ -490,8 +530,12 @@ taxonomies = v.get('taxonomies') if taxonomies is None: v['taxonomies'] = {} + generators = v.get('generators') + if generators is None: + v['generators'] = {} return v + # Make sure the site root starts and ends with a slash. def _validate_site_root(v, values, cache): if not v.startswith('/'): @@ -583,6 +627,14 @@ "Source '%s' is using a reserved endpoint name: %s" % (sn, endpoint)) + # Validate generators. + for gn, gc in sc.get('generators', {}).items(): + if not isinstance(gc, dict): + raise ConfigurationError( + "Generators for source '%s' should be defined in a " + "dictionary." % sn) + gc['source'] = sn + return v @@ -605,23 +657,46 @@ "have an 'url'.") if rc_url[0] != '/': raise ConfigurationError("Route URLs must start with '/'.") - if rc.get('source') is None: - raise ConfigurationError("Routes must specify a source.") - if rc['source'] not in list(values['site']['sources'].keys()): + + r_source = rc.get('source') + r_generator = rc.get('generator') + if r_source is None and r_generator is None: + raise ConfigurationError("Routes must specify a source or " + "generator.") + if (r_source and + r_source not in list(values['site']['sources'].keys())): raise ConfigurationError("Route is referencing unknown " - "source: %s" % rc['source']) - rc.setdefault('taxonomy', None) + "source: %s" % r_source) + if (r_generator and + r_generator not in list(values['site']['generators'].keys())): + raise ConfigurationError("Route is referencing unknown " + "generator: %s" % r_generator) + + rc.setdefault('generator', None) rc.setdefault('page_suffix', '/%num%') return v def _validate_site_taxonomies(v, values, cache): + if not isinstance(v, dict): + raise ConfigurationError( + "The 'site/taxonomies' setting must be a mapping.") for tn, tc in v.items(): tc.setdefault('multiple', False) tc.setdefault('term', tn) tc.setdefault('page', '_%s.%%ext%%' % tc['term']) + return v + +def _validate_site_generators(v, values, cache): + if not isinstance(v, dict): + raise ConfigurationError( + "The 'site/generators' setting must be a mapping.") + for gn, gc in v.items(): + if 'type' not in gc: + raise ConfigurationError( + "Generator '%s' doesn't specify a type." % gn) return v
--- a/piecrust/baking/baker.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/baking/baker.py Thu May 26 19:52:47 2016 -0700 @@ -3,13 +3,14 @@ import hashlib import logging from piecrust.baking.records import ( - BakeRecordEntry, TransitionalBakeRecord, TaxonomyInfo) + BakeRecordEntry, TransitionalBakeRecord) from piecrust.baking.worker import ( save_factory, JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE) from piecrust.chefutil import ( format_timed_scope, format_timed) from piecrust.environment import ExecutionStats +from piecrust.generation.base import PageGeneratorBakeContext from piecrust.routing import create_route_metadata from piecrust.sources.base import ( REALM_NAMES, REALM_USER, REALM_THEME) @@ -29,16 +30,13 @@ self.applied_config_variant = applied_config_variant self.applied_config_values = applied_config_values - # Remember what taxonomy pages we should skip - # (we'll bake them repeatedly later with each taxonomy term) - self.taxonomy_pages = [] - logger.debug("Gathering taxonomy page paths:") - for tax in self.app.taxonomies: - for src in self.app.sources: - tax_page_ref = tax.getPageRef(src) - for path in tax_page_ref.possible_paths: - self.taxonomy_pages.append(path) - logger.debug(" - %s" % path) + # Remember what generator pages we should skip. + self.generator_pages = [] + logger.debug("Gathering generator page paths:") + for gen in self.app.generators: + for path in gen.page_ref.possible_paths: + self.generator_pages.append(path) + logger.debug(" - %s" % path) # Register some timers. self.app.env.registerTimer('LoadJob', raise_if_registered=False) @@ -101,8 +99,8 @@ if srclist is not None: self._bakeRealm(record, pool, realm, srclist) - # Bake taxonomies. - self._bakeTaxonomies(record, pool) + # Call all the page generators. + self._bakePageGenerators(record, pool) # All done with the workers. Close the pool and get reports. reports = pool.close() @@ -197,7 +195,7 @@ for source in srclist: factories = source.getPageFactories() all_factories += [f for f in factories - if f.path not in self.taxonomy_pages] + if f.path not in self.generator_pages] self._loadRealmPages(record, pool, all_factories) self._renderRealmPages(record, pool, all_factories) @@ -272,8 +270,7 @@ logger.error(record_entry.errors[-1]) continue - route = self.app.getRoute(fac.source.name, fac.metadata, - skip_taxonomies=True) + route = self.app.getSourceRoute(fac.source.name, fac.metadata) if route is None: record_entry.errors.append( "Can't get route for page: %s" % fac.ref_spec) @@ -281,9 +278,14 @@ continue # All good, queue the job. + route_index = self.app.routes.index(route) job = { 'type': JOB_RENDER_FIRST, - 'job': save_factory(fac)} + 'job': { + 'factory_info': save_factory(fac), + 'route_index': route_index + } + } jobs.append(job) ar = pool.queueJobs(jobs, handler=_handler) @@ -291,7 +293,7 @@ def _bakeRealmPages(self, record, pool, realm, factories): def _handler(res): - entry = record.getCurrentEntry(res['path'], res['taxonomy_info']) + entry = record.getCurrentEntry(res['path']) entry.subs = res['sub_entries'] if res['errors']: entry.errors += res['errors'] @@ -317,158 +319,14 @@ ar = pool.queueJobs(jobs, handler=_handler) ar.wait() - def _bakeTaxonomies(self, record, pool): - logger.debug("Baking taxonomy pages...") - with format_timed_scope(logger, 'built taxonomy buckets', - level=logging.DEBUG, colored=False): - buckets = self._buildTaxonomyBuckets(record) - - start_time = time.perf_counter() - page_count = self._bakeTaxonomyBuckets(record, pool, buckets) - logger.info(format_timed(start_time, - "baked %d taxonomy pages." % page_count)) - - def _buildTaxonomyBuckets(self, record): - # Let's see all the taxonomy terms for which we must bake a - # listing page... first, pre-populate our big map of used terms. - # For each source name, we have a list of taxonomies, and for each - # taxonomies, a list of terms, some being 'dirty', some used last - # time, etc. - buckets = {} - tax_names = [t.name for t in self.app.taxonomies] - source_names = [s.name for s in self.app.sources] - for sn in source_names: - source_taxonomies = {} - buckets[sn] = source_taxonomies - for tn in tax_names: - source_taxonomies[tn] = _TaxonomyTermsInfo() - - # Now see which ones are 'dirty' based on our bake record. - logger.debug("Gathering dirty taxonomy terms") - for prev_entry, cur_entry in record.transitions.values(): - # Re-bake all taxonomy pages that include new or changed - # pages. - if cur_entry and cur_entry.was_any_sub_baked: - entries = [cur_entry] - if prev_entry: - entries.append(prev_entry) - - for tax in self.app.taxonomies: - changed_terms = set() - for e in entries: - terms = e.config.get(tax.setting_name) - if terms: - if not tax.is_multiple: - terms = [terms] - changed_terms |= set(terms) - - if len(changed_terms) > 0: - tt_info = buckets[cur_entry.source_name][tax.name] - tt_info.dirty_terms |= changed_terms - - # Remember all terms used. - for tax in self.app.taxonomies: - if cur_entry and not cur_entry.was_overriden: - cur_terms = cur_entry.config.get(tax.setting_name) - if cur_terms: - if not tax.is_multiple: - cur_terms = [cur_terms] - tt_info = buckets[cur_entry.source_name][tax.name] - tt_info.all_terms |= set(cur_terms) - - # Re-bake the combination pages for terms that are 'dirty'. - known_combinations = set() - logger.debug("Gathering dirty term combinations") - for prev_entry, cur_entry in record.transitions.values(): - if not cur_entry: - continue - used_taxonomy_terms = cur_entry.getAllUsedTaxonomyTerms() - for sn, tn, terms in used_taxonomy_terms: - if isinstance(terms, tuple): - known_combinations.add((sn, tn, terms)) - for sn, tn, terms in known_combinations: - tt_info = buckets[sn][tn] - tt_info.all_terms.add(terms) - if not tt_info.dirty_terms.isdisjoint(set(terms)): - tt_info.dirty_terms.add(terms) + def _bakePageGenerators(self, record, pool): + for gen in self.app.generators: + ctx = PageGeneratorBakeContext(self.app, record, pool, gen) + gen.bake(ctx) - return buckets - - def _bakeTaxonomyBuckets(self, record, pool, buckets): - def _handler(res): - entry = record.getCurrentEntry(res['path'], res['taxonomy_info']) - entry.subs = res['sub_entries'] - if res['errors']: - entry.errors += res['errors'] - if entry.has_any_error: - record.current.success = False - - # Start baking those terms. - jobs = [] - for source_name, source_taxonomies in buckets.items(): - for tax_name, tt_info in source_taxonomies.items(): - terms = tt_info.dirty_terms - if len(terms) == 0: - continue - - logger.debug( - "Baking '%s' for source '%s': %s" % - (tax_name, source_name, terms)) - tax = self.app.getTaxonomy(tax_name) - source = self.app.getSource(source_name) - tax_page_ref = tax.getPageRef(source) - if not tax_page_ref.exists: - logger.debug( - "No taxonomy page found at '%s', skipping." % - tax.page_ref) - continue - - logger.debug( - "Using taxonomy page: %s:%s" % - (tax_page_ref.source_name, tax_page_ref.rel_path)) - fac = tax_page_ref.getFactory() - - for term in terms: - logger.debug( - "Queuing: %s [%s=%s]" % - (fac.ref_spec, tax_name, term)) - tax_info = TaxonomyInfo(tax_name, source_name, term) - - cur_entry = BakeRecordEntry( - fac.source.name, fac.path, tax_info) - record.addEntry(cur_entry) - - job = self._makeBakeJob(record, fac, tax_info) - if job is not None: - jobs.append(job) - - ar = pool.queueJobs(jobs, handler=_handler) - ar.wait() - - # Now we create bake entries for all the terms that were *not* dirty. - # This is because otherwise, on the next incremental bake, we wouldn't - # find any entry for those things, and figure that we need to delete - # their outputs. - for prev_entry, cur_entry in record.transitions.values(): - # Only consider taxonomy-related entries that don't have any - # current version. - if (prev_entry and prev_entry.taxonomy_info and - not cur_entry): - ti = prev_entry.taxonomy_info - tt_info = buckets[ti.source_name][ti.taxonomy_name] - if ti.term in tt_info.all_terms: - logger.debug("Creating unbaked entry for taxonomy " - "term '%s:%s'." % (ti.taxonomy_name, ti.term)) - record.collapseEntry(prev_entry) - else: - logger.debug("Taxonomy term '%s:%s' isn't used anymore." % - (ti.taxonomy_name, ti.term)) - - return len(jobs) - - def _makeBakeJob(self, record, fac, tax_info=None): + def _makeBakeJob(self, record, fac): # Get the previous (if any) and current entry for this page. - pair = record.getPreviousAndCurrentEntries(fac.path, tax_info) + pair = record.getPreviousAndCurrentEntries(fac.path) assert pair is not None prev_entry, cur_entry = pair assert cur_entry is not None @@ -482,16 +340,7 @@ # Build the route metadata and find the appropriate route. page = fac.buildPage() route_metadata = create_route_metadata(page) - if tax_info is not None: - tax = self.app.getTaxonomy(tax_info.taxonomy_name) - route = self.app.getTaxonomyRoute(tax_info.taxonomy_name, - tax_info.source_name) - - slugified_term = route.slugifyTaxonomyTerm(tax_info.term) - route_metadata[tax.term_name] = slugified_term - else: - route = self.app.getRoute(fac.source.name, route_metadata, - skip_taxonomies=True) + route = self.app.getSourceRoute(fac.source.name, route_metadata) assert route is not None # Figure out if this page is overriden by another previously @@ -511,11 +360,14 @@ cur_entry.flags |= BakeRecordEntry.FLAG_OVERRIDEN return None + route_index = self.app.routes.index(route) job = { 'type': JOB_BAKE, 'job': { 'factory_info': save_factory(fac), - 'taxonomy_info': tax_info, + 'generator_name': None, + 'generator_record_key': None, + 'route_index': route_index, 'route_metadata': route_metadata, 'dirty_source_names': record.dirty_source_names } @@ -569,15 +421,3 @@ initargs=(ctx,)) return pool - -class _TaxonomyTermsInfo(object): - def __init__(self): - self.dirty_terms = set() - self.all_terms = set() - - def __str__(self): - return 'dirty:%s, all:%s' % (self.dirty_terms, self.all_terms) - - def __repr__(self): - return 'dirty:%s, all:%s' % (self.dirty_terms, self.all_terms) -
--- a/piecrust/baking/records.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/baking/records.py Thu May 26 19:52:47 2016 -0700 @@ -8,15 +8,10 @@ logger = logging.getLogger(__name__) -def _get_transition_key(path, taxonomy_info=None): +def _get_transition_key(path, extra_key=None): key = path - if taxonomy_info: - key += '+%s:%s=' % (taxonomy_info.source_name, - taxonomy_info.taxonomy_name) - if isinstance(taxonomy_info.term, tuple): - key += '/'.join(taxonomy_info.term) - else: - key += taxonomy_info.term + if extra_key: + key += '+%s' % extra_key return hashlib.md5(key.encode('utf8')).hexdigest() @@ -69,28 +64,18 @@ return copy.deepcopy(self.render_info) -class TaxonomyInfo(object): - def __init__(self, taxonomy_name, source_name, term): - self.taxonomy_name = taxonomy_name - self.source_name = source_name - self.term = term - - class BakeRecordEntry(object): """ An entry in the bake record. - - The `taxonomy_info` attribute should be a tuple of the form: - (taxonomy name, term, source name) """ FLAG_NONE = 0 FLAG_NEW = 2**0 FLAG_SOURCE_MODIFIED = 2**1 FLAG_OVERRIDEN = 2**2 - def __init__(self, source_name, path, taxonomy_info=None): + def __init__(self, source_name, path, extra_key=None): self.source_name = source_name self.path = path - self.taxonomy_info = taxonomy_info + self.extra_key = extra_key self.flags = self.FLAG_NONE self.config = None self.errors = [] @@ -145,14 +130,6 @@ res |= pinfo.used_source_names return res - def getAllUsedTaxonomyTerms(self): - res = set() - for o in self.subs: - for pinfo in o.render_info: - if pinfo: - res |= pinfo.used_taxonomy_terms - return res - class TransitionalBakeRecord(TransitionalRecord): def __init__(self, previous_path=None): @@ -168,10 +145,10 @@ super(TransitionalBakeRecord, self).addEntry(entry) def getTransitionKey(self, entry): - return _get_transition_key(entry.path, entry.taxonomy_info) + return _get_transition_key(entry.path, entry.extra_key) - def getPreviousAndCurrentEntries(self, path, taxonomy_info=None): - key = _get_transition_key(path, taxonomy_info) + def getPreviousAndCurrentEntries(self, path, extra_key=None): + key = _get_transition_key(path, extra_key) pair = self.transitions.get(key) return pair @@ -184,14 +161,14 @@ return cur return None - def getPreviousEntry(self, path, taxonomy_info=None): - pair = self.getPreviousAndCurrentEntries(path, taxonomy_info) + def getPreviousEntry(self, path, extra_key=None): + pair = self.getPreviousAndCurrentEntries(path, extra_key) if pair is not None: return pair[0] return None - def getCurrentEntry(self, path, taxonomy_info=None): - pair = self.getPreviousAndCurrentEntries(path, taxonomy_info) + def getCurrentEntry(self, path, extra_key=None): + pair = self.getPreviousAndCurrentEntries(path, extra_key) if pair is not None: return pair[1] return None
--- a/piecrust/baking/single.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/baking/single.py Thu May 26 19:52:47 2016 -0700 @@ -72,7 +72,7 @@ return os.path.normpath(os.path.join(*bake_path)) def bake(self, qualified_page, prev_entry, dirty_source_names, - tax_info=None): + generator_name=None): # Start baking the sub-pages. cur_sub = 1 has_more_subs = True @@ -140,8 +140,9 @@ logger.debug(" p%d -> %s" % (cur_sub, out_path)) rp = self._bakeSingle(qualified_page, cur_sub, out_path, - tax_info) + generator_name) except Exception as ex: + logger.exception(ex) page_rel_path = os.path.relpath(qualified_page.path, self.app.root_dir) raise BakingError("%s: error baking '%s'." % @@ -183,10 +184,11 @@ return sub_entries - def _bakeSingle(self, qualified_page, num, out_path, tax_info=None): - ctx = PageRenderingContext(qualified_page, page_num=num) - if tax_info: - ctx.setTaxonomyFilter(tax_info.term) + def _bakeSingle(self, qp, num, out_path, + generator_name=None): + ctx = PageRenderingContext(qp, page_num=num) + if qp.route.is_generator_route: + qp.route.generator.prepareRenderContext(ctx) with self.app.env.timerScope("PageRender"): rp = render_page(ctx)
--- a/piecrust/baking/worker.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/baking/worker.py Thu May 26 19:52:47 2016 -0700 @@ -53,7 +53,7 @@ self.ctx.previous_record_path) self.ctx.previous_record_index = {} for e in self.ctx.previous_record.entries: - key = _get_transition_key(e.path, e.taxonomy_info) + key = _get_transition_key(e.path, e.extra_key) self.ctx.previous_record_index[key] = e # Create the job handlers. @@ -150,13 +150,11 @@ class RenderFirstSubJobHandler(JobHandler): def handleJob(self, job): # Render the segments for the first sub-page of this page. - fac = load_factory(self.app, job) + fac = load_factory(self.app, job['factory_info']) self.app.env.addManifestEntry('RenderJobs', fac.ref_spec) - # These things should be OK as they're checked upstream by the baker. - route = self.app.getRoute(fac.source.name, fac.metadata, - skip_taxonomies=True) - assert route is not None + route_index = job['route_index'] + route = self.app.routes[route_index] page = fac.buildPage() route_metadata = create_route_metadata(page) @@ -198,35 +196,37 @@ fac = load_factory(self.app, job['factory_info']) self.app.env.addManifestEntry('BakeJobs', fac.ref_spec) + route_index = job['route_index'] route_metadata = job['route_metadata'] - tax_info = job['taxonomy_info'] - if tax_info is not None: - route = self.app.getTaxonomyRoute(tax_info.taxonomy_name, - tax_info.source_name) - else: - route = self.app.getRoute(fac.source.name, route_metadata, - skip_taxonomies=True) - assert route is not None + route = self.app.routes[route_index] + + gen_name = job['generator_name'] + gen_key = job['generator_record_key'] + dirty_source_names = job['dirty_source_names'] page = fac.buildPage() qp = QualifiedPage(page, route, route_metadata) result = { 'path': fac.path, - 'taxonomy_info': tax_info, + 'generator_name': gen_name, + 'generator_record_key': gen_key, 'sub_entries': None, 'errors': None} - dirty_source_names = job['dirty_source_names'] + + if job.get('needs_config', False): + result['config'] = page.config.getAll() previous_entry = None if self.ctx.previous_record_index is not None: - key = _get_transition_key(fac.path, tax_info) + key = _get_transition_key(fac.path, gen_key) previous_entry = self.ctx.previous_record_index.get(key) logger.debug("Baking page: %s" % fac.ref_spec) + logger.debug("With route metadata: %s" % route_metadata) try: sub_entries = self.page_baker.bake( - qp, previous_entry, dirty_source_names, tax_info) + qp, previous_entry, dirty_source_names, gen_name) result['sub_entries'] = sub_entries except BakingError as ex:
--- a/piecrust/commands/builtin/baking.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/commands/builtin/baking.py Thu May 26 19:52:47 2016 -0700 @@ -274,12 +274,8 @@ rel_path = os.path.relpath(entry.path, ctx.app.root_dir) logging.info(" path: %s" % rel_path) logging.info(" source: %s" % entry.source_name) - if entry.taxonomy_info: - ti = entry.taxonomy_info - logging.info(" taxonomy: %s = %s (in %s)" % - (ti.taxonomy_name, ti.term, ti.source_name)) - else: - logging.info(" taxonomy: <none>") + if entry.extra_key: + logging.info(" extra key: %s" % entry.extra_key) logging.info(" flags: %s" % _join(flags)) logging.info(" config: %s" % entry.config) @@ -326,11 +322,9 @@ logging.info(" used pagination: %s", pgn_info) logging.info(" used assets: %s", 'yes' if ri.used_assets else 'no') - logging.info(" used terms: %s" % - _join( - ['%s=%s (%s)' % (tn, t, sn) - for sn, tn, t in - ri.used_taxonomy_terms])) + logging.info(" other info:") + for k, v in ri._custom_info.items(): + logging.info(" - %s: %s" % (k, v)) if sub.errors: logging.error(" errors: %s" % sub.errors)
--- a/piecrust/commands/builtin/info.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/commands/builtin/info.py Thu May 26 19:52:47 2016 -0700 @@ -81,8 +81,8 @@ def run(self, ctx): for route in ctx.app.routes: logger.info("%s:" % route.uri_pattern) - logger.info(" source: %s" % route.source_name) - logger.info(" taxonomy: %s" % (route.taxonomy_name or '')) + logger.info(" source: %s" % (route.source_name or '')) + logger.info(" generator: %s" % (route.generator_name or '')) logger.info(" regex: %s" % route.uri_re.pattern)
--- a/piecrust/data/pagedata.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/data/pagedata.py Thu May 26 19:52:47 2016 -0700 @@ -1,7 +1,11 @@ import time +import logging import collections.abc +logger = logging.getLogger(__name__) + + class LazyPageConfigLoaderHasNoValue(Exception): """ An exception that can be returned when a loader for `LazyPageConfig` can't return any value. @@ -29,12 +33,14 @@ try: return self._getValue(name) except LazyPageConfigLoaderHasNoValue as ex: + logger.exception(ex) raise AttributeError("No such attribute: %s" % name) from ex def __getitem__(self, name): try: return self._getValue(name) except LazyPageConfigLoaderHasNoValue as ex: + logger.exception(ex) raise KeyError("No such key: %s" % name) from ex def __iter__(self): @@ -69,6 +75,7 @@ except LazyPageConfigLoaderHasNoValue: raise except Exception as ex: + logger.exception(ex) raise Exception( "Error while loading attribute '%s' for: %s" % (name, self._page.rel_path)) from ex @@ -88,6 +95,7 @@ except LazyPageConfigLoaderHasNoValue: raise except Exception as ex: + logger.exception(ex) raise Exception( "Error while loading attribute '%s' for: %s" % (name, self._page.rel_path)) from ex @@ -125,6 +133,7 @@ try: self._load() except Exception as ex: + logger.exception(ex) raise Exception( "Error while loading data for: %s" % self._page.rel_path) from ex
--- a/piecrust/data/paginationdata.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/data/paginationdata.py Thu May 26 19:52:47 2016 -0700 @@ -17,7 +17,7 @@ # TODO: this is not quite correct, as we're missing parts of the # route metadata if the current page is a taxonomy page. route_metadata = create_route_metadata(page) - self._route = page.app.getRoute(page.source.name, route_metadata) + self._route = page.app.getSourceRoute(page.source.name, route_metadata) self._route_metadata = route_metadata if self._route is None: raise Exception("Can't get route for page: %s" % page.path) @@ -66,9 +66,10 @@ ctx = PageRenderingContext(qp) render_result = render_page_segments(ctx) segs = render_result.segments - except Exception as e: + except Exception as ex: + logger.exception(ex) raise Exception( - "Error rendering segments for '%s'" % uri) from e + "Error rendering segments for '%s'" % uri) from ex else: segs = {} for name in self._page.config.get('segments'):
--- a/piecrust/data/provider.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/data/provider.py Thu May 26 19:52:47 2016 -0700 @@ -1,6 +1,7 @@ import time import collections.abc from piecrust.data.iterators import PageIterator +from piecrust.generation.taxonomy import Taxonomy from piecrust.sources.array import ArraySource @@ -72,27 +73,33 @@ self._taxonomies = {} self._ctx_set = False + @property + def posts(self): + return self._posts() + + @property + def years(self): + return self._buildYearlyArchive() + + @property + def months(self): + return self._buildMonthlyArchive() + def __getitem__(self, name): - if name == 'posts': - return self._posts() - elif name == 'years': - return self._buildYearlyArchive() - elif name == 'months': - return self._buildMonthlyArchive() - elif self._source.app.getTaxonomy(name) is not None: + if self._source.app.config.get('site/taxonomies/' + name) is not None: return self._buildTaxonomy(name) raise KeyError("No such item: %s" % name) def __iter__(self): keys = ['posts', 'years', 'months'] - keys += [t.name for t in self._source.app.taxonomies] + keys += list(self._source.app.config.get('site/taxonomies').keys()) return iter(keys) def __len__(self): - return 3 + len(self._source.app.taxonomies) + return 3 + len(self._source.app.config.get('site/taxonomies')) def _debugRenderTaxonomies(self): - return [t.name for t in self._source.app.taxonomies] + return list(self._source.app.config.get('site/taxonomies').keys()) def _posts(self): it = PageIterator(self._source, current_page=self._page) @@ -152,19 +159,19 @@ if tax_name in self._taxonomies: return self._taxonomies[tax_name] - tax_info = self._page.app.getTaxonomy(tax_name) - setting_name = tax_info.setting_name + tax_cfg = self._page.app.config.get('site/taxonomies/' + tax_name) + tax = Taxonomy(tax_name, tax_cfg) posts_by_tax_value = {} for post in self._source.getPages(): - tax_values = post.config.get(setting_name) + tax_values = post.config.get(tax.setting_name) if tax_values is None: continue if not isinstance(tax_values, list): tax_values = [tax_values] for val in tax_values: - posts_by_tax_value.setdefault(val, []) - posts_by_tax_value[val].append(post) + posts = posts_by_tax_value.setdefault(val, []) + posts.append(post) entries = [] for value, ds in posts_by_tax_value.items():
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/generation/base.py Thu May 26 19:52:47 2016 -0700 @@ -0,0 +1,137 @@ +from werkzeug.utils import cached_property +from piecrust.baking.records import BakeRecordEntry +from piecrust.baking.worker import save_factory, JOB_BAKE +from piecrust.configuration import ConfigurationError +from piecrust.routing import create_route_metadata +from piecrust.sources.pageref import PageRef + + +class InvalidRecordExtraKey(Exception): + pass + + +class PageGeneratorBakeContext(object): + def __init__(self, app, record, pool, generator): + self._app = app + self._record = record + self._pool = pool + self._generator = generator + self._job_queue = [] + self._is_running = False + + def getRecordExtraKey(self, seed): + return '%s:%s' % (self._generator.name, seed) + + def matchesRecordExtraKey(self, extra_key): + return (extra_key is not None and + extra_key.startswith(self._generator.name + ':')) + + def getSeedFromRecordExtraKey(self, extra_key): + if not self.matchesRecordExtraKey(extra_key): + raise InvalidRecordExtraKey("Invalid extra key: %s" % extra_key) + return extra_key[len(self._generator.name) + 1:] + + def getAllPageRecords(self): + return self._record.transitions.values() + + def getBakedPageRecords(self): + for prev, cur in self.getAllPageRecords(): + if cur and cur.was_any_sub_baked: + yield (prev, cur) + + def collapseRecord(self, entry): + self._record.collapseEntry(entry) + + def queueBakeJob(self, page_fac, route, extra_route_metadata, seed): + if self._is_running: + raise Exception("The job queue is running.") + + extra_key = self.getRecordExtraKey(seed) + entry = BakeRecordEntry( + page_fac.source.name, + page_fac.path, + extra_key) + self._record.addEntry(entry) + + page = page_fac.buildPage() + route_metadata = create_route_metadata(page) + route_metadata.update(extra_route_metadata) + uri = route.getUri(route_metadata) + override_entry = self._record.getOverrideEntry(page.path, uri) + if override_entry is not None: + override_source = self.app.getSource( + override_entry.source_name) + if override_source.realm == fac.source.realm: + cur_entry.errors.append( + "Page '%s' maps to URL '%s' but is overriden " + "by page '%s'." % + (fac.ref_spec, uri, override_entry.path)) + logger.error(cur_entry.errors[-1]) + cur_entry.flags |= BakeRecordEntry.FLAG_OVERRIDEN + return + + route_index = self._app.routes.index(route) + job = { + 'type': JOB_BAKE, + 'job': { + 'factory_info': save_factory(page_fac), + 'generator_name': self._generator.name, + 'generator_record_key': extra_key, + 'route_index': route_index, + 'route_metadata': route_metadata, + 'dirty_source_names': self._record.dirty_source_names, + 'needs_config': True + } + } + self._job_queue.append(job) + + def runJobQueue(self): + def _handler(res): + entry = self._record.getCurrentEntry( + res['path'], res['generator_record_key']) + entry.config = res['config'] + entry.subs = res['sub_entries'] + if res['errors']: + entry.errors += res['errors'] + if entry.has_any_error: + self._record.current.success = False + + self._is_running = True + try: + ar = self._pool.queueJobs(self._job_queue, handler=_handler) + ar.wait() + finally: + self._is_running = False + + +class PageGenerator(object): + def __init__(self, app, name, config): + self.app = app + self.name = name + self.config = config or {} + + self.source_name = config.get('source') + if self.source_name is None: + raise ConfigurationError( + "Generator '%s' requires a source name" % name) + + page_ref = config.get('page') + if page_ref is None: + raise ConfigurationError( + "Generator '%s' requires a listing page ref." % name) + self.page_ref = PageRef(app, page_ref) + + @cached_property + def source(self): + for src in self.app.sources: + if src.name == self.source_name: + return src + raise Exception("Can't find source '%s' for generator '%s'." % ( + self.source_name, self.name)) + + def bake(self, ctx): + raise NotImplementedError() + + def onRouteFunctionUsed(self, route, route_metadata): + pass +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/generation/blogarchives.py Thu May 26 19:52:47 2016 -0700 @@ -0,0 +1,9 @@ +from piecrust.generation.base import PageGenerator + + +class BlogArchivesPageGenerator(PageGenerator): + GENERATOR_NAME = 'blog_archives' + + def bake(self, ctx): + pass +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/generation/taxonomy.py Thu May 26 19:52:47 2016 -0700 @@ -0,0 +1,335 @@ +import re +import time +import logging +import unidecode +from piecrust.chefutil import format_timed, format_timed_scope +from piecrust.configuration import ConfigurationError +from piecrust.data.filters import ( + PaginationFilter, SettingFilterClause, + page_value_accessor) +from piecrust.generation.base import PageGenerator, InvalidRecordExtraKey +from piecrust.sources.pageref import PageRef, PageNotFoundError + + +logger = logging.getLogger(__name__) + + +SLUGIFY_ENCODE = 1 +SLUGIFY_TRANSLITERATE = 2 +SLUGIFY_LOWERCASE = 4 +SLUGIFY_DOT_TO_DASH = 8 +SLUGIFY_SPACE_TO_DASH = 16 + + +re_first_dot_to_dash = re.compile(r'^\.+') +re_dot_to_dash = re.compile(r'\.+') +re_space_to_dash = re.compile(r'\s+') + + +class Taxonomy(object): + def __init__(self, name, config): + self.name = name + self.config = config + self.term_name = config.get('term', name) + self.is_multiple = bool(config.get('multiple', False)) + self.separator = config.get('separator', '/') + self.page_ref = config.get('page') + self._source_page_refs = {} + + @property + def setting_name(self): + if self.is_multiple: + return self.name + return self.term_name + + +class TaxonomyPageGenerator(PageGenerator): + GENERATOR_NAME = 'taxonomy' + + def __init__(self, app, name, config): + super(TaxonomyPageGenerator, self).__init__(app, name, config) + + tax_name = config.get('taxonomy') + if tax_name is None: + raise ConfigurationError( + "Generator '%s' requires a taxonomy name." % name) + tax_config = app.config.get('site/taxonomies/' + tax_name) + if tax_config is None: + raise ConfigurationError( + "Error initializing generator '%s', no such taxonomy: %s", + (name, tax_name)) + self.taxonomy = Taxonomy(tax_name, tax_config) + + sm = config.get('slugify_mode') + if not sm: + sm = app.config.get('site/slugify_mode', 'encode') + self.slugify_mode = _parse_slugify_mode(sm) + + @property + def page_ref_path(self): + try: + return self.page_ref.path + except PageNotFoundError: + return None + + def getPageFactory(self, route_metadata): + # This will raise `PageNotFoundError` naturally if not found. + return self.page_ref.getFactory() + + def prepareRenderContext(self, ctx): + tax_terms, is_combination = self._getTaxonomyTerms( + ctx.page.route_metadata) + self._setTaxonomyFilter(ctx, tax_terms, is_combination) + + ctx.custom_data = { + self.taxonomy.term_name: tax_terms, + 'is_multiple_%s' % self.taxonomy.term_name: is_combination} + logger.debug("Prepared render context with: %s" % ctx.custom_data) + + def _getTaxonomyTerms(self, route_metadata): + all_values = route_metadata.get(self.taxonomy.term_name) + if all_values is None: + raise Exception("'%s' values couldn't be found in route metadata" % + self.taxonomy.term_name) + + if self.taxonomy.is_multiple: + sep = self.taxonomy.separator + if sep in all_values: + return tuple(all_values.split(sep)), True + return all_values, False + + def _setTaxonomyFilter(self, ctx, term_value, is_combination): + flt = PaginationFilter(value_accessor=page_value_accessor) + flt.addClause(HasTaxonomyTermsFilterClause( + self.taxonomy, self.slugify_mode, term_value, is_combination)) + ctx.pagination_filter = flt + + def onRouteFunctionUsed(self, route, route_metadata): + # Get the values. + values = route_metadata[self.taxonomy.term_name] + if self.taxonomy.is_multiple: + #TODO: here we assume the route has been properly configured. + values = tuple([str(v) for v in values]) + else: + values = (str(values),) + + # We need to register this use of a taxonomy term. + eis = self.app.env.exec_info_stack + cpi = eis.current_page_info.render_ctx.current_pass_info + if cpi: + utt = cpi.getCustomInfo('used_taxonomy_terms', [], True) + utt.append(values) + + # We need to slugify the terms before they get transformed + # into URL-bits. + s = _Slugifier(self.taxonomy, self.slugify_mode) + str_values = s.slugify(values) + route_metadata[self.taxonomy.term_name] = str_values + logger.debug("Changed route metadata to: %s" % route_metadata) + + def bake(self, ctx): + logger.debug("Baking taxonomy pages...") + with format_timed_scope(logger, 'gathered taxonomy terms', + level=logging.DEBUG, colored=False): + all_terms, dirty_terms = self._buildDirtyTaxonomyTerms(ctx) + + start_time = time.perf_counter() + page_count = self._bakeTaxonomyTerms(ctx, all_terms, dirty_terms) + logger.info(format_timed(start_time, + "baked %d taxonomy pages." % page_count)) + + def _buildDirtyTaxonomyTerms(self, ctx): + # Build the list of terms for our taxonomy, and figure out which ones + # are 'dirty' for the current bake. + logger.debug("Gathering dirty taxonomy terms") + all_terms = set() + dirty_terms = set() + + # Re-bake all taxonomy terms that include new or changed pages. + for prev_entry, cur_entry in ctx.getBakedPageRecords(): + entries = [cur_entry] + if prev_entry: + entries.append(prev_entry) + + terms = [] + for e in entries: + entry_terms = e.config.get(self.taxonomy.setting_name) + if entry_terms: + if not self.taxonomy.is_multiple: + terms.append(entry_terms) + else: + terms += entry_terms + if terms: + dirty_terms.update([(t,) for t in terms]) + + # Remember all terms used. + for _, cur_entry in ctx.getAllPageRecords(): + if cur_entry and not cur_entry.was_overriden: + cur_terms = cur_entry.config.get(self.taxonomy.setting_name) + if cur_terms: + if not self.taxonomy.is_multiple: + all_terms.add(cur_terms) + else: + all_terms |= set(cur_terms) + + # Re-bake the combination pages for terms that are 'dirty'. + if self.taxonomy.is_multiple: + known_combinations = set() + logger.debug("Gathering dirty term combinations") + for _, cur_entry in ctx.getAllPageRecords(): + if cur_entry: + used_terms = _get_all_entry_taxonomy_terms(cur_entry) + for terms in used_terms: + if len(terms) > 1: + known_combinations.add(terms) + + for terms in known_combinations: + if not dirty_terms.isdisjoint(set(terms)): + dirty_terms.add(terms) + + return all_terms, dirty_terms + + def _bakeTaxonomyTerms(self, ctx, all_terms, dirty_terms): + # Start baking those terms. + logger.debug( + "Baking '%s' for source '%s': %s" % + (self.taxonomy.name, self.source_name, dirty_terms)) + + if not self.page_ref.exists: + logger.debug( + "No taxonomy page found at '%s', skipping." % + self.page_ref) + return 0 + + route = self.app.getGeneratorRoute(self.name) + if route is None: + raise Exception("No routes have been defined for generator: %s" % + self.name) + + logger.debug("Using taxonomy page: %s" % self.page_ref) + fac = self.page_ref.getFactory() + + job_count = 0 + s = _Slugifier(self.taxonomy, self.slugify_mode) + for term in dirty_terms: + if not self.taxonomy.is_multiple: + term = term[0] + slugified_term = s.slugify(term) + + logger.debug( + "Queuing: %s [%s=%s]" % + (fac.ref_spec, self.taxonomy.name, slugified_term)) + + extra_route_metadata = {self.taxonomy.term_name: slugified_term} + ctx.queueBakeJob(fac, route, extra_route_metadata, slugified_term) + job_count += 1 + ctx.runJobQueue() + + # Now we create bake entries for all the terms that were *not* dirty. + # This is because otherwise, on the next incremental bake, we wouldn't + # find any entry for those things, and figure that we need to delete + # their outputs. + for prev_entry, cur_entry in ctx.getAllPageRecords(): + # Only consider taxonomy-related entries that don't have any + # current version (i.e. they weren't baked just now). + if (prev_entry and not cur_entry): + try: + t = ctx.getSeedFromRecordExtraKey(prev_entry.extra_key) + except InvalidRecordExtraKey: + continue + + if t in all_terms: + logger.debug("Creating unbaked entry for %s term: %s" % + (self.name, t)) + ctx.collapseRecord(prev_entry) + else: + logger.debug("Term %s in %s isn't used anymore." % + (self.name, t)) + + return job_count + + +def _get_all_entry_taxonomy_terms(entry): + res = set() + for o in entry.subs: + for pinfo in o.render_info: + if pinfo: + res |= set(pinfo.getCustomInfo('used_taxonomy_terms', [])) + return res + + +class HasTaxonomyTermsFilterClause(SettingFilterClause): + def __init__(self, taxonomy, slugify_mode, value, is_combination): + super(HasTaxonomyTermsFilterClause, self).__init__( + taxonomy.setting_name, value) + self._taxonomy = taxonomy + self._is_combination = is_combination + self._slugifier = _Slugifier(taxonomy, slugify_mode) + + def pageMatches(self, fil, page): + if self._taxonomy.is_multiple: + # Multiple taxonomy, i.e. it supports multiple terms, like tags. + page_values = fil.value_accessor(page, self.name) + if page_values is None or not isinstance(page_values, list): + return False + + page_set = set(map(self._slugifier.slugify, page_values)) + if self._is_combination: + # Multiple taxonomy, and multiple terms to match. Check that + # the ones to match are all in the page's terms. + value_set = set(self.value) + return value_set.issubset(page_set) + else: + # Multiple taxonomy, one term to match. + return self.value in page_set + else: + # Single taxonomy. Just compare the values. + page_value = fil.value_accessor(page, self.name) + if page_value is None: + return False + page_value = self._slugifier.slugify(page_value) + return page_value == self.value + + +class _Slugifier(object): + def __init__(self, taxonomy, mode): + self.taxonomy = taxonomy + self.mode = mode + + def slugify(self, term): + if isinstance(term, tuple): + return self.taxonomy.separator.join( + map(self._slugifyOne, term)) + return self._slugifyOne(term) + + def _slugifyOne(self, term): + if self.mode & SLUGIFY_TRANSLITERATE: + term = unidecode.unidecode(term) + if self.mode & SLUGIFY_LOWERCASE: + term = term.lower() + if self.mode & SLUGIFY_DOT_TO_DASH: + term = re_first_dot_to_dash.sub('', term) + term = re_dot_to_dash.sub('-', term) + if self.mode & SLUGIFY_SPACE_TO_DASH: + term = re_space_to_dash.sub('-', term) + return term + + +def _parse_slugify_mode(value): + mapping = { + 'encode': SLUGIFY_ENCODE, + 'transliterate': SLUGIFY_TRANSLITERATE, + 'lowercase': SLUGIFY_LOWERCASE, + 'dot_to_dash': SLUGIFY_DOT_TO_DASH, + 'space_to_dash': SLUGIFY_SPACE_TO_DASH} + mode = 0 + for v in value.split(','): + f = mapping.get(v.strip()) + if f is None: + if v == 'iconv': + raise Exception("'iconv' is not supported as a slugify mode " + "in PieCrust2. Use 'transliterate'.") + raise Exception("Unknown slugify flag: %s" % v) + mode |= f + return mode +
--- a/piecrust/page.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/page.py Thu May 26 19:52:47 2016 -0700 @@ -113,6 +113,7 @@ # No idea what the date/time for this page is. self._datetime = datetime.datetime.fromtimestamp(0) except Exception as ex: + logger.exception(ex) raise Exception( "Error computing time for page: %s" % self.path) from ex @@ -155,6 +156,7 @@ try: parsed_d = dateutil.parser.parse(page_date) except Exception as ex: + logger.exception(ex) raise ConfigurationError("Invalid date: %s" % page_date) from ex return datetime.date( year=parsed_d.year, @@ -175,6 +177,7 @@ try: parsed_t = dateutil.parser.parse(page_time) except Exception as ex: + logger.exception(ex) raise ConfigurationError("Invalid time: %s" % page_time) from ex return datetime.timedelta( hours=parsed_t.hour,
--- a/piecrust/plugins/base.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/plugins/base.py Thu May 26 19:52:47 2016 -0700 @@ -33,6 +33,9 @@ def getSources(self): return [] + def getPageGenerators(self): + return [] + def getPublishers(self): return [] @@ -86,6 +89,9 @@ def getSources(self): return self._getPluginComponents('getSources') + def getPageGenerators(self): + return self._getPluginComponents('getPageGenerators') + def getPublishers(self): return self._getPluginComponents('getPublishers')
--- a/piecrust/plugins/builtin.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/plugins/builtin.py Thu May 26 19:52:47 2016 -0700 @@ -21,6 +21,8 @@ from piecrust.formatting.markdownformatter import MarkdownFormatter from piecrust.formatting.textileformatter import TextileFormatter from piecrust.formatting.smartypantsformatter import SmartyPantsFormatter +from piecrust.generation.blogarchives import BlogArchivesPageGenerator +from piecrust.generation.taxonomy import TaxonomyPageGenerator from piecrust.importing.jekyll import JekyllImporter from piecrust.importing.piecrust import PieCrust1Importer from piecrust.importing.wordpress import WordpressXmlImporter @@ -87,6 +89,11 @@ OrderedPageSource, ProseSource] + def getPageGenerators(self): + return [ + TaxonomyPageGenerator, + BlogArchivesPageGenerator] + def getDataProviders(self): return [ IteratorDataProvider,
--- a/piecrust/rendering.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/rendering.py Thu May 26 19:52:47 2016 -0700 @@ -79,26 +79,27 @@ class RenderPassInfo(object): def __init__(self): self.used_source_names = set() - self.used_taxonomy_terms = set() self.used_pagination = False self.pagination_has_more = False self.used_assets = False + self._custom_info = {} - def merge(self, other): - self.used_source_names |= other.used_source_names - self.used_taxonomy_terms |= other.used_taxonomy_terms - self.used_pagination = self.used_pagination or other.used_pagination - self.pagination_has_more = (self.pagination_has_more or - other.pagination_has_more) - self.used_assets = self.used_assets or other.used_assets + def setCustomInfo(self, key, info): + self._custom_info[key] = info + + def getCustomInfo(self, key, default=None, create_if_missing=False): + if create_if_missing: + return self._custom_info.setdefault(key, default) + return self._custom_info.get(key, default) + def _toJson(self): data = { 'used_source_names': list(self.used_source_names), - 'used_taxonomy_terms': list(self.used_taxonomy_terms), 'used_pagination': self.used_pagination, 'pagination_has_more': self.pagination_has_more, - 'used_assets': self.used_assets} + 'used_assets': self.used_assets, + 'custom_info': self._custom_info} return data @staticmethod @@ -106,29 +107,26 @@ assert data is not None rpi = RenderPassInfo() rpi.used_source_names = set(data['used_source_names']) - for i in data['used_taxonomy_terms']: - terms = i[2] - if isinstance(terms, list): - terms = tuple(terms) - rpi.used_taxonomy_terms.add((i[0], i[1], terms)) rpi.used_pagination = data['used_pagination'] rpi.pagination_has_more = data['pagination_has_more'] rpi.used_assets = data['used_assets'] + rpi._custom_info = data['custom_info'] return rpi class PageRenderingContext(object): - def __init__(self, qualified_page, page_num=1, force_render=False): + def __init__(self, qualified_page, page_num=1, + force_render=False, is_from_request=False): self.page = qualified_page self.page_num = page_num self.force_render = force_render + self.is_from_request = is_from_request self.pagination_source = None self.pagination_filter = None self.custom_data = None + self.render_passes = [None, None] # Same length as RENDER_PASSES self._current_pass = PASS_NONE - self.render_passes = [None, None] # Same length as RENDER_PASSES - @property def app(self): return self.page.app @@ -168,69 +166,11 @@ pass_info = self.current_pass_info pass_info.used_source_names.add(source.name) - def setTaxonomyFilter(self, term_value, *, needs_slugifier=False): - if not self.page.route.is_taxonomy_route: - raise Exception("The page for this context is not tied to a " - "taxonomy route: %s" % self.uri) - - slugifier = None - if needs_slugifier: - slugifier = self.page.route.slugifyTaxonomyTerm - taxonomy = self.app.getTaxonomy(self.page.route.taxonomy_name) - - flt = PaginationFilter(value_accessor=page_value_accessor) - flt.addClause(HasTaxonomyTermsFilterClause( - taxonomy, term_value, slugifier)) - self.pagination_filter = flt - - is_combination = isinstance(term_value, tuple) - self.custom_data = { - taxonomy.term_name: term_value, - 'is_multiple_%s' % taxonomy.term_name: is_combination} - def _raiseIfNoCurrentPass(self): if self._current_pass == PASS_NONE: raise Exception("No rendering pass is currently active.") -class HasTaxonomyTermsFilterClause(SettingFilterClause): - def __init__(self, taxonomy, value, slugifier): - super(HasTaxonomyTermsFilterClause, self).__init__( - taxonomy.setting_name, value) - self._taxonomy = taxonomy - self._slugifier = slugifier - self._is_combination = isinstance(self.value, tuple) - - def pageMatches(self, fil, page): - if self._taxonomy.is_multiple: - # Multiple taxonomy, i.e. it supports multiple terms, like tags. - page_values = fil.value_accessor(page, self.name) - if page_values is None or not isinstance(page_values, list): - return False - - if self._slugifier is not None: - page_set = set(map(self._slugifier, page_values)) - else: - page_set = set(page_values) - - if self._is_combination: - # Multiple taxonomy, and multiple terms to match. Check that - # the ones to match are all in the page's terms. - value_set = set(self.value) - return value_set.issubset(page_set) - else: - # Multiple taxonomy, one term to match. - return self.value in page_set - - # Single taxonomy. Just compare the values. - page_value = fil.value_accessor(page, self.name) - if page_value is None: - return False - if self._slugifier is not None: - page_value = self._slugifier(page_value) - return page_value == self.value - - def render_page(ctx): eis = ctx.app.env.exec_info_stack eis.pushPage(ctx.page, ctx) @@ -285,6 +225,7 @@ layout_result['pass_info']) return rp except Exception as ex: + logger.exception(ex) page_rel_path = os.path.relpath(ctx.page.path, ctx.app.root_dir) raise Exception("Error rendering page: %s" % page_rel_path) from ex finally: @@ -402,6 +343,7 @@ try: output = engine.renderFile(full_names, layout_data) except TemplateNotFoundError as ex: + logger.exception(ex) msg = "Can't find template for page: %s\n" % page.path msg += "Looked for: %s" % ', '.join(full_names) raise Exception(msg) from ex
--- a/piecrust/routing.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/routing.py Thu May 26 19:52:47 2016 -0700 @@ -3,7 +3,7 @@ import copy import logging import urllib.parse -import unidecode +from werkzeug.utils import cached_property logger = logging.getLogger(__name__) @@ -11,9 +11,8 @@ route_re = re.compile(r'%((?P<qual>path):)?(?P<name>\w+)%') route_esc_re = re.compile(r'\\%((?P<qual>path)\\:)?(?P<name>\w+)\\%') -template_func_re = re.compile(r'^(?P<name>\w+)\((?P<first_arg>\w+)' - r'(?P<other_args>.*)\)\s*$') -template_func_arg_re = re.compile(r',\s*(?P<arg>\w+)') +template_func_re = re.compile(r'^(?P<name>\w+)\((?P<args>.*)\)\s*$') +template_func_arg_re = re.compile(r'(?P<arg>\+?\w+)') ugly_url_cleaner = re.compile(r'\.html$') @@ -21,6 +20,10 @@ pass +class InvalidRouteError(Exception): + pass + + def create_route_metadata(page): route_metadata = copy.deepcopy(page.source_metadata) route_metadata.update(page.getRouteMetadata()) @@ -38,53 +41,23 @@ raise NotImplementedError() -SLUGIFY_ENCODE = 1 -SLUGIFY_TRANSLITERATE = 2 -SLUGIFY_LOWERCASE = 4 -SLUGIFY_DOT_TO_DASH = 8 -SLUGIFY_SPACE_TO_DASH = 16 - - -re_first_dot_to_dash = re.compile(r'^\.+') -re_dot_to_dash = re.compile(r'\.+') -re_space_to_dash = re.compile(r'\s+') - - -def _parse_slugify_mode(value): - mapping = { - 'encode': SLUGIFY_ENCODE, - 'transliterate': SLUGIFY_TRANSLITERATE, - 'lowercase': SLUGIFY_LOWERCASE, - 'dot_to_dash': SLUGIFY_DOT_TO_DASH, - 'space_to_dash': SLUGIFY_SPACE_TO_DASH} - mode = 0 - for v in value.split(','): - f = mapping.get(v.strip()) - if f is None: - if v == 'iconv': - raise Exception("'iconv' is not supported as a slugify mode " - "in PieCrust2. Use 'transliterate'.") - raise Exception("Unknown slugify flag: %s" % v) - mode |= f - return mode +ROUTE_TYPE_SOURCE = 0 +ROUTE_TYPE_GENERATOR = 1 class Route(object): """ Information about a route for a PieCrust application. Each route defines the "shape" of an URL and how it maps to - sources and taxonomies. + sources and generators. """ def __init__(self, app, cfg): self.app = app - self.source_name = cfg['source'] - self.taxonomy_name = cfg.get('taxonomy') - self.taxonomy_term_sep = cfg.get('term_separator', '/') - - sm = cfg.get('slugify_mode') - if not sm: - sm = app.config.get('site/slugify_mode', 'encode') - self.slugify_mode = _parse_slugify_mode(sm) + self.source_name = cfg.get('source') + self.generator_name = cfg.get('generator') + if not self.source_name and not self.generator_name: + raise InvalidRouteError( + "Both `source` and `generator` are specified.") self.pretty_urls = app.config.get('site/pretty_urls') self.trailing_slash = app.config.get('site/trailing_slash') @@ -127,23 +100,45 @@ self.template_func = None self.template_func_name = None self.template_func_args = [] + self.template_func_vararg = None self._createTemplateFunc(cfg.get('func')) @property - def is_taxonomy_route(self): - return self.taxonomy_name is not None + def route_type(self): + if self.source_name: + return ROUTE_TYPE_SOURCE + elif self.generator_name: + return ROUTE_TYPE_GENERATOR + else: + raise InvalidRouteError() @property + def is_source_route(self): + return self.route_type == ROUTE_TYPE_SOURCE + + @property + def is_generator_route(self): + return self.route_type == ROUTE_TYPE_GENERATOR + + @cached_property def source(self): + if not self.is_source_route: + return InvalidRouteError("This is not a source route.") for src in self.app.sources: if src.name == self.source_name: return src - raise Exception("Can't find source '%s' for route '%'." % ( + raise Exception("Can't find source '%s' for route '%s'." % ( self.source_name, self.uri)) - @property - def source_realm(self): - return self.source.realm + @cached_property + def generator(self): + if not self.is_generator_route: + return InvalidRouteError("This is not a generator route.") + for gen in self.app.generators: + if gen.name == self.generator_name: + return gen + raise Exception("Can't find generator '%s' for route '%s'." % ( + self.generator_name, self.uri)) def matchesMetadata(self, route_metadata): return self.required_route_metadata.issubset(route_metadata.keys()) @@ -234,38 +229,6 @@ return uri - def getTaxonomyTerms(self, route_metadata): - if not self.is_taxonomy_route: - raise Exception("This route isn't a taxonomy route.") - - tax = self.app.getTaxonomy(self.taxonomy_name) - all_values = route_metadata.get(tax.term_name) - if all_values is None: - raise Exception("'%s' values couldn't be found in route metadata" % - tax.term_name) - - if self.taxonomy_term_sep in all_values: - return tuple(all_values.split(self.taxonomy_term_sep)) - return all_values - - def slugifyTaxonomyTerm(self, term): - if isinstance(term, tuple): - return self.taxonomy_term_sep.join( - map(self._slugifyOne, term)) - return self._slugifyOne(term) - - def _slugifyOne(self, term): - if self.slugify_mode & SLUGIFY_TRANSLITERATE: - term = unidecode.unidecode(term) - if self.slugify_mode & SLUGIFY_LOWERCASE: - term = term.lower() - if self.slugify_mode & SLUGIFY_DOT_TO_DASH: - term = re_first_dot_to_dash.sub('', term) - term = re_dot_to_dash.sub('-', term) - if self.slugify_mode & SLUGIFY_SPACE_TO_DASH: - term = re_space_to_dash.sub('-', term) - return term - def _uriFormatRepl(self, m): name = m.group('name') #TODO: fix this hard-coded shit @@ -280,7 +243,7 @@ def _uriPatternRepl(self, m): name = m.group('name') qualifier = m.group('qual') - if qualifier == 'path' or self.taxonomy_name: + if qualifier == 'path': return r'(?P<%s>[^\?]*)' % name return r'(?P<%s>[^/\?]+)' % name @@ -302,66 +265,54 @@ (self.uri_pattern, func_def)) self.template_func_name = m.group('name') - self.template_func_args.append(m.group('first_arg')) - arg_list = m.group('other_args') + self.template_func_args = [] + arg_list = m.group('args') if arg_list: - self.template_func_args += template_func_arg_re.findall(arg_list) + self.template_func_args = template_func_arg_re.findall(arg_list) + for i in range(len(self.template_func_args) - 1): + if self.template_func_args[i][0] == '+': + raise Exception("Only the last route parameter can be a " + "variable argument (prefixed with `+`)") - if self.taxonomy_name: - # This will be a taxonomy route function... this means we can - # have a variable number of parameters, but only one parameter - # definition, which is the value. - if len(self.template_func_args) != 1: - raise Exception("Route '%s' is a taxonomy route and must have " - "only one argument, which is the term value." % - self.uri_pattern) - - def template_func(*args): - if len(args) == 0: - raise Exception( - "Route function '%s' expected at least one " - "argument." % func_def) - - # Term combinations can be passed as an array, or as multiple - # arguments. - values = args - if len(args) == 1 and isinstance(args[0], list): - values = args[0] + if (self.template_func_args and + self.template_func_args[-1][0] == '+'): + self.template_func_vararg = self.template_func_args[-1][1:] - # We need to register this use of a taxonomy term. - if len(values) == 1: - registered_values = str(values[0]) - else: - registered_values = tuple([str(v) for v in values]) - eis = self.app.env.exec_info_stack - cpi = eis.current_page_info.render_ctx.current_pass_info - if cpi: - cpi.used_taxonomy_terms.add( - (self.source_name, self.taxonomy_name, - registered_values)) - - str_values = self.slugifyTaxonomyTerm(registered_values) - term_name = self.template_func_args[0] - metadata = {term_name: str_values} + def template_func(*args): + is_variable = (self.template_func_vararg is not None) + if not is_variable and len(args) != len(self.template_func_args): + raise Exception( + "Route function '%s' expected %d arguments, " + "got %d." % + (func_def, len(self.template_func_args), + len(args))) + elif is_variable and len(args) < len(self.template_func_args): + raise Exception( + "Route function '%s' expected at least %d arguments, " + "got %d." % + (func_def, len(self.template_func_args), + len(args))) - return self.getUri(metadata) + metadata = {} + non_var_args = list(self.template_func_args) + if is_variable: + del non_var_args[-1] - else: - # Normal route function. - def template_func(*args): - if len(args) != len(self.template_func_args): - raise Exception( - "Route function '%s' expected %d arguments, " - "got %d." % - (func_def, len(self.template_func_args), - len(args))) - metadata = {} - for arg_name, arg_val in zip(self.template_func_args, args): - #TODO: fix this hard-coded shit. - if arg_name in ['year', 'month', 'day']: - arg_val = int(arg_val) - metadata[arg_name] = arg_val - return self.getUri(metadata) + for arg_name, arg_val in zip(non_var_args, args): + #TODO: fix this hard-coded shit. + if arg_name in ['year', 'month', 'day']: + arg_val = int(arg_val) + metadata[arg_name] = arg_val + + if is_variable: + metadata[self.template_func_vararg] = [] + for i in range(len(non_var_args), len(args)): + metadata[self.template_func_vararg].append(args[i]) + + if self.is_generator_route: + self.generator.onRouteFunctionUsed(self, metadata) + + return self.getUri(metadata) self.template_func = template_func
--- a/piecrust/serving/server.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/serving/server.py Thu May 26 19:52:47 2016 -0700 @@ -183,18 +183,17 @@ # We have a page, let's try to render it. render_ctx = PageRenderingContext(qp, page_num=req_page.page_num, - force_render=True) - if qp.route.taxonomy_name is not None: - taxonomy = app.getTaxonomy(qp.route.taxonomy_name) - tax_terms = qp.route.getTaxonomyTerms(qp.route_metadata) - render_ctx.setTaxonomyFilter(tax_terms, needs_slugifier=True) + force_render=True, + is_from_request=True) + if qp.route.is_generator_route: + qp.route.generator.prepareRenderContext(render_ctx) # See if this page is known to use sources. If that's the case, # just don't use cached rendered segments for that page (but still # use them for pages that are included in it). uri = qp.getUri() entry = self._page_record.getEntry(uri, req_page.page_num) - if (qp.route.taxonomy_name is not None or entry is None or + if (qp.route.is_generator_route or entry is None or entry.used_source_names): cache_key = '%s:%s' % (uri, req_page.page_num) app.env.rendered_segments_repository.invalidate(cache_key) @@ -202,18 +201,6 @@ # Render the page. rendered_page = render_page(render_ctx) - # Check if this page is a taxonomy page that actually doesn't match - # anything. - if qp.route.taxonomy_name is not None: - paginator = rendered_page.data.get('pagination') - if (paginator and paginator.is_loaded and - len(paginator.items) == 0): - taxonomy = app.getTaxonomy(qp.route.taxonomy_name) - message = ("This URL matched a route for taxonomy '%s' but " - "no pages have been found to have it. This page " - "won't be generated by a bake." % taxonomy.name) - raise NotFound(message) - # Remember stuff for next time. if entry is None: entry = ServeRecordPageEntry(req_page.req_path, req_page.page_num)
--- a/piecrust/serving/util.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/serving/util.py Thu May 26 19:52:47 2016 -0700 @@ -32,16 +32,19 @@ def find_routes(routes, uri): + """ Returns routes matching the given URL, but puts generator routes + at the end. + """ res = [] - tax_res = [] + gen_res = [] for route in routes: metadata = route.matchUri(uri) if metadata is not None: - if route.is_taxonomy_route: - tax_res.append((route, metadata)) + if route.is_source_route: + res.append((route, metadata)) else: - res.append((route, metadata)) - return res + tax_res + gen_res.append((route, metadata)) + return res + gen_res def get_requested_page(app, req_path): @@ -71,19 +74,19 @@ def _get_requested_page_for_route(app, route, route_metadata, req_path): - taxonomy = None - source = app.getSource(route.source_name) - if route.taxonomy_name is None: + if not route.is_generator_route: + source = app.getSource(route.source_name) factory = source.findPageFactory(route_metadata, MODE_PARSING) if factory is None: - raise PageNotFoundError("No path found for '%s' in source '%s'." % - (req_path, source.name)) + raise PageNotFoundError( + "No path found for '%s' in source '%s'." % + (req_path, source.name)) else: - taxonomy = app.getTaxonomy(route.taxonomy_name) - - # This will raise `PageNotFoundError` naturally if not found. - tax_page_ref = taxonomy.getPageRef(source) - factory = tax_page_ref.getFactory() + factory = route.generator.getPageFactory(route_metadata) + if factory is None: + raise PageNotFoundError( + "No path found for '%s' in generator '%s'." % + (req_path, route.generator.name)) # Build the page. page = factory.buildPage()
--- a/piecrust/sources/array.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/sources/array.py Thu May 26 19:52:47 2016 -0700 @@ -42,6 +42,3 @@ for p in self.inner_source: yield CachedPageFactory(p) - def getTaxonomyPageRef(self, tax_name): - return None -
--- a/piecrust/sources/base.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/sources/base.py Thu May 26 19:52:47 2016 -0700 @@ -133,9 +133,3 @@ return self._provider_type(self, page, override) - def getTaxonomyPageRef(self, tax_name): - tax_pages = self.config.get('taxonomy_pages') - if tax_pages is None: - return None - return tax_pages.get(tax_name) -
--- a/piecrust/sources/mixins.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/sources/mixins.py Thu May 26 19:52:47 2016 -0700 @@ -23,33 +23,31 @@ return self.source.getPages() -class SourceFactoryWithoutTaxonomiesIterator(object): +class SourceFactoryWithoutGeneratorsIterator(object): def __init__(self, source): self.source = source - self._taxonomy_pages = None + self._generator_pages = None # See comment above. self.it = None def __iter__(self): - self._cacheTaxonomyPages() + self._cacheGeneratorPages() for p in self.source.getPages(): - if p.rel_path in self._taxonomy_pages: + if p.rel_path in self._generator_pages: continue yield p - def _cacheTaxonomyPages(self): - if self._taxonomy_pages is not None: + def _cacheGeneratorPages(self): + if self._generator_pages is not None: return app = self.source.app - self._taxonomy_pages = set() + self._generator_pages = set() for src in app.sources: - for tax in app.taxonomies: - ref_spec = src.getTaxonomyPageRef(tax.name) - page_ref = PageRef(app, ref_spec) - for sn, rp in page_ref.possible_split_ref_specs: + for gen in app.generators: + for sn, rp in gen.page_ref.possible_split_ref_specs: if sn == self.source.name: - self._taxonomy_pages.add(rp) + self._generator_pages.add(rp) class DateSortIterator(object): @@ -82,9 +80,9 @@ return self.config['items_per_page'] def getSourceIterator(self): - if self.config.get('iteration_includes_taxonomies', False): + if self.config.get('iteration_includes_generator_pages', False): return SourceFactoryIterator(self) - return SourceFactoryWithoutTaxonomiesIterator(self) + return SourceFactoryWithoutGeneratorsIterator(self) def getSorterIterator(self, it): return DateSortIterator(it)
--- a/piecrust/sources/pageref.py Thu May 26 19:46:28 2016 -0700 +++ b/piecrust/sources/pageref.py Thu May 26 19:52:47 2016 -0700 @@ -32,6 +32,9 @@ self._first_valid_hit_index = self._INDEX_NEEDS_LOADING self._exts = list(app.config.get('site/auto_formats').keys()) + def __str__(self): + return self._page_ref + @property def exists(self): try:
--- a/piecrust/taxonomies.py Thu May 26 19:46:28 2016 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -from piecrust.sources.pageref import PageRef, PageNotFoundError - - -class Taxonomy(object): - def __init__(self, app, name, config): - self.app = app - self.name = name - self.term_name = config.get('term', name) - self.is_multiple = config.get('multiple', False) - self.page_ref = config.get('page') - self._source_page_refs = {} - - @property - def setting_name(self): - if self.is_multiple: - return self.name - return self.term_name - - def resolvePagePath(self, source): - pr = self.getPageRef(source) - try: - return pr.path - except PageNotFoundError: - return None - - def getPageRef(self, source): - if source.name in self._source_page_refs: - return self._source_page_refs[source.name] - - ref_path = source.getTaxonomyPageRef(self.name) - page_ref = PageRef(self.app, ref_path) - self._source_page_refs[source.name] = page_ref - return page_ref -
--- a/tests/bakes/test_unicode_tags.yaml Thu May 26 19:46:28 2016 -0700 +++ b/tests/bakes/test_unicode_tags.yaml Thu May 26 19:52:47 2016 -0700 @@ -11,18 +11,18 @@ tags: [étrange, sévère] --- pages/_tag.md: | - Pages in {{pctagurl(tag)}} + Pages in {{pctagurl(tag)}} with {{tag}} {% for p in pagination.posts -%} {{p.title}} {% endfor %} pages/_index.md: '' outfiles: tag/étrange.html: | - Pages in /tag/%C3%A9trange.html + Pages in /tag/%C3%A9trange.html with étrange Post 02 Post 01 tag/sévère.html: | - Pages in /tag/s%C3%A9v%C3%A8re.html + Pages in /tag/s%C3%A9v%C3%A8re.html with sévère Post 02 --- in:
--- a/tests/test_appconfig.py Thu May 26 19:46:28 2016 -0700 +++ b/tests/test_appconfig.py Thu May 26 19:52:47 2016 -0700 @@ -49,12 +49,11 @@ with mock_fs_scope(fs): app = fs.getApp() # The order of routes is important. Sources, not so much. - # `posts` shows up 3 times in routes (posts, tags, categories) assert (list( map( - lambda v: v['source'], + lambda v: v.get('generator') or v['source'], app.config.get('site/routes'))) == - ['notes', 'posts', 'posts', 'posts', 'pages', 'theme_pages']) + ['notes', 'posts', 'posts_archives', 'posts_tags', 'posts_categories', 'pages', 'theme_pages']) assert list(app.config.get('site/sources').keys()) == [ 'theme_pages', 'pages', 'posts', 'notes'] @@ -77,9 +76,9 @@ # `posts` shows up 3 times in routes (posts, tags, categories) assert (list( map( - lambda v: v['source'], + lambda v: v.get('generator') or v['source'], app.config.get('site/routes'))) == - ['notes', 'posts', 'posts', 'posts', 'pages', 'theme_notes', 'theme_pages']) + ['notes', 'posts', 'posts_archives', 'posts_tags', 'posts_categories', 'pages', 'theme_notes', 'theme_pages']) assert list(app.config.get('site/sources').keys()) == [ 'theme_pages', 'theme_notes', 'pages', 'posts', 'notes']
--- a/tests/test_data_provider.py Thu May 26 19:46:28 2016 -0700 +++ b/tests/test_data_provider.py Thu May 26 19:52:47 2016 -0700 @@ -6,21 +6,21 @@ fs = (mock_fs() .withConfig() .withPage('posts/2015-03-01_one.md', - {'title': 'One', 'category': 'Foo'}) + {'title': 'One', 'tags': ['Foo']}) .withPage('posts/2015-03-02_two.md', - {'title': 'Two', 'category': 'Foo'}) + {'title': 'Two', 'tags': ['Foo']}) .withPage('posts/2015-03-03_three.md', - {'title': 'Three', 'category': 'Bar'}) - .withPage('pages/categories.md', + {'title': 'Three', 'tags': ['Bar']}) + .withPage('pages/tags.md', {'format': 'none', 'layout': 'none'}, - "{%for c in blog.categories%}\n" + "{%for c in blog.tags%}\n" "{{c.name}} ({{c.post_count}})\n" "{%endfor%}\n")) with mock_fs_scope(fs): app = fs.getApp() - page = app.getSource('pages').getPage({'slug': 'categories'}) - route = app.getRoute('pages', None) - route_metadata = {'slug': 'categories'} + page = app.getSource('pages').getPage({'slug': 'tags'}) + route = app.getSourceRoute('pages', None) + route_metadata = {'slug': 'tags'} qp = QualifiedPage(page, route, route_metadata) ctx = PageRenderingContext(qp) rp = render_page(ctx)
--- a/tests/test_serving.py Thu May 26 19:46:28 2016 -0700 +++ b/tests/test_serving.py Thu May 26 19:52:47 2016 -0700 @@ -76,12 +76,13 @@ with mock_fs_scope(fs): app = fs.getApp() page = app.getSource('pages').getPage({'slug': '_tag', 'tag': tag}) - route = app.getTaxonomyRoute('tags', 'posts') + route = app.getGeneratorRoute('posts_tags') + assert route is not None + route_metadata = {'slug': '_tag', 'tag': tag} - qp = QualifiedPage(page, route, route_metadata) ctx = PageRenderingContext(qp) - ctx.setTaxonomyFilter(tag) + route.generator.prepareRenderContext(ctx) rp = render_page(ctx) expected = "Pages in %s\n" % tag @@ -109,7 +110,13 @@ return c fs = (mock_fs() - .withConfig() + .withConfig({ + 'site': { + 'taxonomies': { + 'categories': {'term': 'category'} + } + } + }) .withPages(6, 'posts/2015-03-{idx1:02}_post{idx1:02}.md', config_factory) .withPage('pages/_category.md', {'layout': 'none', 'format': 'none'}, @@ -121,12 +128,13 @@ app = fs.getApp() page = app.getSource('pages').getPage({'slug': '_category', 'category': category}) - route = app.getTaxonomyRoute('categories', 'posts') + route = app.getGeneratorRoute('posts_categories') + assert route is not None + route_metadata = {'slug': '_category', 'category': category} - qp = QualifiedPage(page, route, route_metadata) ctx = PageRenderingContext(qp) - ctx.setTaxonomyFilter(category) + route.generator.prepareRenderContext(ctx) rp = render_page(ctx) expected = "Pages in %s\n" % category
--- a/tests/test_templating_jinjaengine.py Thu May 26 19:46:28 2016 -0700 +++ b/tests/test_templating_jinjaengine.py Thu May 26 19:52:47 2016 -0700 @@ -28,7 +28,7 @@ with mock_fs_scope(fs, open_patches=open_patches): app = fs.getApp() page = get_simple_page(app, 'foo.md') - route = app.getRoute('pages', None) + route = app.getSourceRoute('pages', None) route_metadata = {'slug': 'foo'} output = render_simple_page(page, route, route_metadata) assert output == expected @@ -46,7 +46,7 @@ with mock_fs_scope(fs, open_patches=open_patches): app = fs.getApp() page = get_simple_page(app, 'foo.md') - route = app.getRoute('pages', None) + route = app.getSourceRoute('pages', None) route_metadata = {'slug': 'foo'} output = render_simple_page(page, route, route_metadata) assert output == expected @@ -63,7 +63,7 @@ with mock_fs_scope(fs, open_patches=open_patches): app = fs.getApp() page = get_simple_page(app, 'foo.md') - route = app.getRoute('pages', None) + route = app.getSourceRoute('pages', None) route_metadata = {'slug': 'foo'} output = render_simple_page(page, route, route_metadata) assert output == expected
--- a/tests/test_templating_pystacheengine.py Thu May 26 19:46:28 2016 -0700 +++ b/tests/test_templating_pystacheengine.py Thu May 26 19:52:47 2016 -0700 @@ -28,7 +28,7 @@ with mock_fs_scope(fs, open_patches=open_patches): app = fs.getApp() page = get_simple_page(app, 'foo.md') - route = app.getRoute('pages', None) + route = app.getSourceRoute('pages', None) route_metadata = {'slug': 'foo'} output = render_simple_page(page, route, route_metadata) assert output == expected @@ -46,7 +46,7 @@ with mock_fs_scope(fs, open_patches=open_patches): app = fs.getApp() page = get_simple_page(app, 'foo.md') - route = app.getRoute('pages', None) + route = app.getSourceRoute('pages', None) route_metadata = {'slug': 'foo'} output = render_simple_page(page, route, route_metadata) # On Windows, pystache unexplicably adds `\r` to some newlines... wtf. @@ -65,7 +65,7 @@ with mock_fs_scope(fs, open_patches=open_patches): app = fs.getApp() page = get_simple_page(app, 'foo.md') - route = app.getRoute('pages', None) + route = app.getSourceRoute('pages', None) route_metadata = {'slug': 'foo'} output = render_simple_page(page, route, route_metadata) # On Windows, pystache unexplicably adds `\r` to some newlines... wtf.