piecrust2: piecrust/baking/baker.py comparison

comparison piecrust/baking/baker.py @ 334:b034f6f15e22

bake: Several bug taxonomy-related fixes for incorrect incremental bakes. * Improve how the baker processes taxonomy terms and figures out what needs to be re-baked or not. * Create bake entries for clean taxnomy terms so they're not deleted by an incremental bake. * Add more information to bake records. * Slugify taxonomy terms is now done by the route in one place. * Fix a bug where the cache key for invalidating rendered segments was not computed the same way as when the caching was done. * Fix how term combinations are passed around, rendered, printed, parsed, etc. (TODO: more word needed in the routing functions) * Expose to the template whether a taxonomy term is a combination or not. * Display term combinations better in the built-in theme. * Rename `route.taxonomy` to `route.taxonomy_name` to prevent confusion. * Add options to show bake records for previous bakes.

author	Ludovic Chabant <ludovic@chabant.com>
date	Fri, 03 Apr 2015 10:59:50 -0700
parents	91b07f9efdc1
children	938be93215cb

comparison

equal deleted inserted replaced

-:91b07f9efdc1
+:b034f6f15e22
 logger.debug(
 "Skipping taxonomy page: %s:%s" %
 (source.name, fac.ref_spec))
 continue
-entry = BakeRecordPageEntry(fac)
+entry = BakeRecordPageEntry(fac.source.name, fac.rel_path,
+fac.path)
 record.addEntry(entry)
 route = self.app.getRoute(source.name, fac.metadata)
 if route is None:
 entry.errors.append(
 record.current.success &= success
 def _bakeTaxonomies(self, record):
 logger.debug("Baking taxonomies")
+class _TaxonomyTermsInfo(object):
+def __init__(self):
+self.dirty_terms = set()
+self.all_terms = set()
+def __str__(self):
+return 'dirty:%s, all:%s' % (self.dirty_terms, self.all_terms)
+def __repr__(self):
+return 'dirty:%s, all:%s' % (self.dirty_terms, self.all_terms)
 # Let's see all the taxonomy terms for which we must bake a
 # listing page... first, pre-populate our big map of used terms.
+# For each source name, we have a list of taxonomies, and for each
+# taxonomies, a list of terms, some being 'dirty', some used last
+# time, etc.
 buckets = {}
 tax_names = [t.name for t in self.app.taxonomies]
 source_names = [s.name for s in self.app.sources]
 for sn in source_names:
 source_taxonomies = {}
 buckets[sn] = source_taxonomies
 for tn in tax_names:
-source_taxonomies[tn] = set()
+source_taxonomies[tn] = _TaxonomyTermsInfo()
 # Now see which ones are 'dirty' based on our bake record.
 logger.debug("Gathering dirty taxonomy terms")
 for prev_entry, cur_entry in record.transitions.values():
 for tax in self.app.taxonomies:
-changed_terms = None
 # Re-bake all taxonomy pages that include new or changed
 # pages.
-if (not prev_entry and cur_entry and
+if cur_entry and cur_entry.was_baked_successfully:
-cur_entry.was_baked_successfully):
+if prev_entry and prev_entry.was_baked_successfully:
-changed_terms = cur_entry.config.get(tax.setting_name)
+# Entry was re-baked this time. Mark as dirty both the
-elif (prev_entry and cur_entry and
+# old and new terms.
-cur_entry.was_baked_successfully):
+changed_terms = []
-changed_terms = []
+prev_terms = prev_entry.config.get(tax.setting_name)
+cur_terms = cur_entry.config.get(tax.setting_name)
+if tax.is_multiple:
+if prev_terms is not None:
+changed_terms += prev_terms
+if cur_terms is not None:
+changed_terms += cur_terms
+else:
+if prev_terms is not None:
+changed_terms.append(prev_terms)
+if cur_terms is not None:
+changed_terms.append(cur_terms)
+else:
+# Entry was not baked last time. Just mark as dirty
+# all the new terms.
+changed_terms = cur_entry.config.get(tax.setting_name)
+if changed_terms is not None:
+if not isinstance(changed_terms, list):
+changed_terms = [changed_terms]
+tt_info = buckets[cur_entry.source_name][tax.name]
+tt_info.dirty_terms |= set(changed_terms)
+# Remember all terms used.
+if cur_entry and cur_entry.was_baked_successfully:
+cur_terms = cur_entry.config.get(tax.setting_name)
+if cur_terms is not None:
+if not isinstance(cur_terms, list):
+cur_terms = [cur_terms]
+tt_info = buckets[cur_entry.source_name][tax.name]
+tt_info.all_terms |= set(cur_terms)
+elif (prev_entry and prev_entry.was_baked_successfully and
+cur_entry and not cur_entry.was_baked):
 prev_terms = prev_entry.config.get(tax.setting_name)
-cur_terms = cur_entry.config.get(tax.setting_name)
+if prev_terms is not None:
-if tax.is_multiple:
+if not isinstance(prev_terms, list):
-if prev_terms is not None:
+prev_terms = [prev_terms]
-changed_terms += prev_terms
+tt_info = buckets[prev_entry.source_name][tax.name]
-if cur_terms is not None:
+tt_info.all_terms |= set(prev_terms)
-changed_terms += cur_terms
-else:
-if prev_terms is not None:
-changed_terms.append(prev_terms)
-if cur_terms is not None:
-changed_terms.append(cur_terms)
-if changed_terms is not None:
-if not isinstance(changed_terms, list):
-changed_terms = [changed_terms]
-buckets[cur_entry.source_name][tax.name] |= (
-set(changed_terms))
 # Re-bake the combination pages for terms that are 'dirty'.
 known_combinations = set()
 logger.debug("Gathering dirty term combinations")
 for prev_entry, cur_entry in record.transitions.values():
-if cur_entry:
+if cur_entry and cur_entry.was_baked_successfully:
 known_combinations |= cur_entry.used_taxonomy_terms
 elif prev_entry:
 known_combinations |= prev_entry.used_taxonomy_terms
 for sn, tn, terms in known_combinations:
-changed_terms = buckets[sn][tn]
+tt_info = buckets[sn][tn]
-if not changed_terms.isdisjoint(set(terms)):
+tt_info.all_terms.add(terms)
-changed_terms.add(terms)
+if not tt_info.dirty_terms.isdisjoint(set(terms)):
+tt_info.dirty_terms.add(terms)
 # Start baking those terms.
 pool, queue, abort = self._createWorkerPool(record, self.num_workers)
 for source_name, source_taxonomies in buckets.items():
-for tax_name, terms in source_taxonomies.items():
+for tax_name, tt_info in source_taxonomies.items():
+terms = tt_info.dirty_terms
 if len(terms) == 0:
 continue
 logger.debug(
 "Baking '%s' for source '%s': %s" %
 tax_page_source, tax_page_rel_path,
 {tax.term_name: term})
 logger.debug(
 "Queuing: %s [%s, %s]" %
 (fac.ref_spec, tax_name, term))
-entry = BakeRecordPageEntry(fac, tax_name, term)
+entry = BakeRecordPageEntry(
+fac.source.name, fac.rel_path, fac.path,
+(tax_name, term, source_name))
 record.addEntry(entry)
-queue.addJob(
+queue.addJob(BakeWorkerJob(fac, route, entry))
-BakeWorkerJob(fac, route, entry, tax_name, term))
 success = self._waitOnWorkerPool(pool, abort)
 record.current.success &= success
+# Now we create bake entries for all the terms that were *not* dirty.
+# This is because otherwise, on the next incremental bake, we wouldn't
+# find any entry for those things, and figure that we need to delete
+# their outputs.
+for prev_entry, cur_entry in record.transitions.values():
+# Only consider taxonomy-related entries that don't have any
+# current version.
+if (prev_entry and prev_entry.taxonomy_info and
+not cur_entry):
+sn = prev_entry.source_name
+tn, tt, tsn = prev_entry.taxonomy_info
+tt_info = buckets[tsn][tn]
+if tt in tt_info.all_terms:
+logger.debug("Creating unbaked entry for taxonomy "
+"term '%s:%s'." % (tn, tt))
+entry = BakeRecordPageEntry(
+prev_entry.source_name, prev_entry.rel_path,
+prev_entry.path, prev_entry.taxonomy_info)
+record.addEntry(entry)
+else:
+logger.debug("Taxonomy term '%s:%s' isn't used anymore." %
+(tn, tt))
 def _handleDeletetions(self, record):
 for path, reason in record.getDeletions():
 logger.debug("Removing '%s': %s" % (path, reason))
 try:
 self.work_queue = work_queue
 self.abort_event = abort_event
 class BakeWorkerJob(object):
-def __init__(self, factory, route, record_entry,
+def __init__(self, factory, route, record_entry):
-taxonomy_name=None, taxonomy_term=None):
 self.factory = factory
 self.route = route
 self.record_entry = record_entry
-self.taxonomy_name = taxonomy_name
-self.taxonomy_term = taxonomy_term
 @property
 def source(self):
 return self.factory.source
 def _unsafeRun(self, job):
 start_time = time.clock()
 entry = job.record_entry
 try:
-self._page_baker.bake(
+self._page_baker.bake(job.factory, job.route, entry)
-job.factory, job.route, entry,
-taxonomy_name=job.taxonomy_name,
-taxonomy_term=job.taxonomy_term)
 except BakingError as ex:
 logger.debug("Got baking error. Adding it to the record.")
 while ex:
 entry.errors.append(str(ex))
 ex = ex.__cause__

Mercurial > piecrust2

comparison piecrust/baking/baker.py @ 334:b034f6f15e22