piecrust2: piecrust/baking/baker.py comparison

comparison piecrust/baking/baker.py @ 39:2f717f961996

Better error reporting and cache validation. Fix the processor pipeline in the preview server. Move the `pages` route to first position.

author	Ludovic Chabant <ludovic@chabant.com>
date	Thu, 21 Aug 2014 22:28:22 -0700
parents	485682a6de50
children	9e058e221108

comparison

equal deleted inserted replaced

-:091f99bfbe44
+:2f717f961996
 import logging
 import threading
 import urllib.request, urllib.error, urllib.parse
 from queue import Queue, Empty
 from piecrust.baking.records import TransitionalBakeRecord, BakeRecordPageEntry
-from piecrust.chefutil import format_timed
+from piecrust.chefutil import format_timed, log_friendly_exception
 from piecrust.data.filters import (PaginationFilter, HasFilterClause,
 IsFilterClause, AndBooleanClause)
 from piecrust.processing.base import ProcessorPipeline
 from piecrust.rendering import PageRenderingContext, render_page
 from piecrust.sources.base import (PageFactory,
 try:
 logger.debug("  p%d -> %s" % (cur_sub, out_path))
 ctx, rp = self._bakeSingle(page, sub_uri, cur_sub, out_path,
 pagination_filter, custom_data)
 except Exception as ex:
-raise Exception("Error baking page '%s' for URI '%s'." %
+raise Exception("Error baking page '%s' for URL '%s'." %
 (page.ref_spec, uri)) from ex
 # Copy page assets.
 if (cur_sub == 1 and self.copy_assets and
 ctx.used_assets is not None):
 t = time.clock()
 record.loadPrevious(record_cache.getCachePath(record_name))
 logger.debug(format_timed(t, 'loaded previous bake record',
 colored=False));
+# Figure out if we need to clean the cache because important things
+# have changed.
+self._handleCacheValidity(record)
 # Gather all sources by realm -- we're going to bake each realm
 # separately so we can handle "overlaying" (i.e. one realm overrides
 # another realm's pages).
 sources_by_realm = {}
 for source in self.app.sources:
 if not self.no_assets:
 self._bakeAssets(record)
 # Save the bake record.
 t = time.clock()
+record.current.bake_time = time.time()
 record.collapseRecords()
 record.saveCurrent(record_cache.getCachePath(record_name))
 logger.debug(format_timed(t, 'saved bake record', colored=False))
 # All done.
 self.app.config.set('baker/is_baking', False)
 logger.info('-------------------------');
 logger.info(format_timed(start_time, 'done baking'));
+def _handleCacheValidity(self, record):
+start_time = time.clock()
+reason = None
+if self.force:
+reason = "ordered to"
+elif not self.app.config.get('__cache_valid'):
+# The configuration file was changed, or we're running a new
+# version of the app.
+reason = "not valid anymore"
+elif not record.previous.bake_time:
+# We have no valid previous bake record.
+reason = "need bake record regeneration"
+else:
+# Check if any template has changed since the last bake. Since
+# there could be some advanced conditional logic going on, we'd
+# better just force a bake from scratch if that's the case.
+max_time = 0
+for d in self.app.templates_dirs:
+for _, __, filenames in os.walk(d):
+for fn in filenames:
+max_time = max(max_time, os.path.getmtime(fn))
+if max_time >= record.previous.bake_time:
+reason = "templates modified"
+if reason is not None:
+cache_dir = self.app.cache_dir
+if os.path.isdir(cache_dir):
+logger.debug("Cleaning cache: %s" % cache_dir)
+shutil.rmtree(cache_dir)
+self.force = True
+logger.info(format_timed(start_time,
+"cleaned cache (reason: %s)" % reason))
+else:
+logger.debug(format_timed(start_time, "cache is assumed valid",
+colored=False))
 def _bakeRealm(self, record, realm, srclist):
 # Gather all page factories from the sources and queue them
 # for the workers to pick up. Just skip taxonomy pages for now.
 logger.debug("Baking realm %s" % REALM_NAMES[realm])
 def _waitOnWorkerPool(self, pool, abort):
 for w in pool:
 w.join()
 if abort.is_set():
-raise Exception("Worker pool was aborted.")
+excs = [w.abort_exception for w in pool
+if w.abort_exception is not None]
+logger.error("%s errors" % len(excs))
+if self.app.debug:
+for e in excs:
+logger.exception(e)
+else:
+for e in excs:
+log_friendly_exception(logger, e)
+raise Exception("Baking was aborted due to errors.")
 class BakeWorkerContext(object):
 def __init__(self, app, out_dir, force, record, work_queue,
 abort_event):
 return self.factory.source
 class BakeWorker(threading.Thread):
 def __init__(self, wid, ctx):
-super(BakeWorker, self).__init__()
+super(BakeWorker, self).__init__(name=('worker%d' % wid))
 self.wid = wid
 self.ctx = ctx
-self.num_bakes = 0
+self.abort_exception = None
 self._page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force,
 ctx.record)
 def run(self):
 while(not self.ctx.abort_event.is_set()):
 self._unsafeRun(job)
 logger.debug("[%d] Done with page." % self.wid)
 self.ctx.work_queue.task_done()
 except Exception as ex:
 self.ctx.abort_event.set()
-logger.error("[%d] Critical error, aborting." % self.wid)
+self.abort_exception = ex
-logger.exception(ex)
+logger.debug("[%d] Critical error, aborting." % self.wid)
 break
 def _unsafeRun(self, job):
 start_time = time.clock()
 friendly_count = ''
 if bake_res.num_subs > 1:
 friendly_count = ' (%d pages)' % bake_res.num_subs
 logger.info(format_timed(start_time, '[%d] %s%s' %
 (self.wid, friendly_uri, friendly_count)))
-self.num_bakes += 1

Mercurial > piecrust2

comparison piecrust/baking/baker.py @ 39:2f717f961996