diff piecrust/baking/baker.py @ 39:2f717f961996

Better error reporting and cache validation. Fix the processor pipeline in the preview server. Move the `pages` route to first position.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 21 Aug 2014 22:28:22 -0700
parents 485682a6de50
children 9e058e221108
line wrap: on
line diff
--- a/piecrust/baking/baker.py	Thu Aug 21 10:56:17 2014 -0700
+++ b/piecrust/baking/baker.py	Thu Aug 21 22:28:22 2014 -0700
@@ -8,7 +8,7 @@
 import urllib.request, urllib.error, urllib.parse
 from queue import Queue, Empty
 from piecrust.baking.records import TransitionalBakeRecord, BakeRecordPageEntry
-from piecrust.chefutil import format_timed
+from piecrust.chefutil import format_timed, log_friendly_exception
 from piecrust.data.filters import (PaginationFilter, HasFilterClause,
         IsFilterClause, AndBooleanClause)
 from piecrust.processing.base import ProcessorPipeline
@@ -157,7 +157,7 @@
                 ctx, rp = self._bakeSingle(page, sub_uri, cur_sub, out_path,
                         pagination_filter, custom_data)
             except Exception as ex:
-                raise Exception("Error baking page '%s' for URI '%s'." %
+                raise Exception("Error baking page '%s' for URL '%s'." %
                         (page.ref_spec, uri)) from ex
 
             # Copy page assets.
@@ -263,6 +263,10 @@
             logger.debug(format_timed(t, 'loaded previous bake record',
                 colored=False));
 
+        # Figure out if we need to clean the cache because important things
+        # have changed.
+        self._handleCacheValidity(record)
+
         # Gather all sources by realm -- we're going to bake each realm
         # separately so we can handle "overlaying" (i.e. one realm overrides
         # another realm's pages).
@@ -287,6 +291,7 @@
 
         # Save the bake record.
         t = time.clock()
+        record.current.bake_time = time.time()
         record.collapseRecords()
         record.saveCurrent(record_cache.getCachePath(record_name))
         logger.debug(format_timed(t, 'saved bake record', colored=False))
@@ -296,6 +301,43 @@
         logger.info('-------------------------');
         logger.info(format_timed(start_time, 'done baking'));
 
+    def _handleCacheValidity(self, record):
+        start_time = time.clock()
+
+        reason = None
+        if self.force:
+            reason = "ordered to"
+        elif not self.app.config.get('__cache_valid'):
+            # The configuration file was changed, or we're running a new
+            # version of the app.
+            reason = "not valid anymore"
+        elif not record.previous.bake_time:
+            # We have no valid previous bake record.
+            reason = "need bake record regeneration"
+        else:
+            # Check if any template has changed since the last bake. Since
+            # there could be some advanced conditional logic going on, we'd
+            # better just force a bake from scratch if that's the case.
+            max_time = 0
+            for d in self.app.templates_dirs:
+                for _, __, filenames in os.walk(d):
+                    for fn in filenames:
+                        max_time = max(max_time, os.path.getmtime(fn))
+            if max_time >= record.previous.bake_time:
+                reason = "templates modified"
+
+        if reason is not None:
+            cache_dir = self.app.cache_dir
+            if os.path.isdir(cache_dir):
+                logger.debug("Cleaning cache: %s" % cache_dir)
+                shutil.rmtree(cache_dir)
+                self.force = True
+            logger.info(format_timed(start_time,
+                "cleaned cache (reason: %s)" % reason))
+        else:
+            logger.debug(format_timed(start_time, "cache is assumed valid",
+                colored=False))
+
     def _bakeRealm(self, record, realm, srclist):
         # Gather all page factories from the sources and queue them
         # for the workers to pick up. Just skip taxonomy pages for now.
@@ -435,7 +477,16 @@
         for w in pool:
             w.join()
         if abort.is_set():
-            raise Exception("Worker pool was aborted.")
+            excs = [w.abort_exception for w in pool
+                    if w.abort_exception is not None]
+            logger.error("%s errors" % len(excs))
+            if self.app.debug:
+                for e in excs:
+                    logger.exception(e)
+            else:
+                for e in excs:
+                    log_friendly_exception(logger, e)
+            raise Exception("Baking was aborted due to errors.")
 
 
 class BakeWorkerContext(object):
@@ -463,10 +514,10 @@
 
 class BakeWorker(threading.Thread):
     def __init__(self, wid, ctx):
-        super(BakeWorker, self).__init__()
+        super(BakeWorker, self).__init__(name=('worker%d' % wid))
         self.wid = wid
         self.ctx = ctx
-        self.num_bakes = 0
+        self.abort_exception = None
         self._page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force,
                 ctx.record)
 
@@ -484,8 +535,8 @@
                 self.ctx.work_queue.task_done()
             except Exception as ex:
                 self.ctx.abort_event.set()
-                logger.error("[%d] Critical error, aborting." % self.wid)
-                logger.exception(ex)
+                self.abort_exception = ex
+                logger.debug("[%d] Critical error, aborting." % self.wid)
                 break
 
     def _unsafeRun(self, job):
@@ -503,5 +554,4 @@
                 friendly_count = ' (%d pages)' % bake_res.num_subs
             logger.info(format_timed(start_time, '[%d] %s%s' %
                     (self.wid, friendly_uri, friendly_count)))
-            self.num_bakes += 1