Mercurial > piecrust2

diff piecrust/processing/base.py @ 215:a47580a0955b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Better error handling for the processing pipeline. Pipeline jobs now keep track of whether they've seen any errors. This is aggregated into an overall "success" flag for the processing record. Also, jobs keep going as long as there's no critical (i.e. internal) failure happening. Errors raised by processors are also better tracked: the actual processor that failed, along with the input file, are tracks in the processing record. The `bake` command returns a failure exit code if processing saw any error.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 31 Jan 2015 17:08:02 -0800
parents e34a6826a3d4
children f82262f59600
line wrap: on
line diff
--- a/piecrust/processing/base.py	Sat Jan 31 16:28:16 2015 -0800
+++ b/piecrust/processing/base.py	Sat Jan 31 17:08:02 2015 -0800
@@ -11,7 +11,8 @@
         ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord,
         FLAG_PROCESSED, FLAG_OVERRIDEN, FLAG_BYPASSED_STRUCTURED_PROCESSING)
 from piecrust.processing.tree import (
-        ProcessingTreeBuilder, ProcessingTreeRunner, ProcessingTreeError,
+        ProcessingTreeBuilder, ProcessingTreeRunner,
+        ProcessingTreeError, ProcessorError,
         STATE_DIRTY,
         print_node, get_node_name_tree)
 
@@ -239,8 +240,10 @@
                 self.processDirectory(ctx, path)
 
         # Wait on all workers.
+        record.current.success = True
         for w in pool:
             w.join()
+            record.current.success &= w.success
         if abort.is_set():
             raise Exception("Worker pool was aborted.")
 
@@ -310,6 +313,7 @@
         super(ProcessingWorker, self).__init__()
         self.wid = wid
         self.ctx = ctx
+        self.success = True
 
     def run(self):
         while(not self.ctx.abort_event.is_set()):
@@ -320,11 +324,13 @@
                 break
 
             try:
-                self._unsafeRun(job)
+                success = self._unsafeRun(job)
                 logger.debug("[%d] Done with file." % self.wid)
                 self.ctx.work_queue.task_done()
+                self.success &= success
             except Exception as ex:
                 self.ctx.abort_event.set()
+                self.success = False
                 logger.error("[%d] Critical error, aborting." % self.wid)
                 logger.exception(ex)
                 break
@@ -347,7 +353,7 @@
             record_entry.flags |= FLAG_OVERRIDEN
             logger.info(format_timed(start_time,
                     '%s [not baked, overridden]' % rel_path))
-            return
+            return True
 
         processors = pipeline.getFilteredProcessors(
                 job.mount_info['processors'])
@@ -355,9 +361,12 @@
             builder = ProcessingTreeBuilder(processors)
             tree_root = builder.build(rel_path)
         except ProcessingTreeError as ex:
-            record_entry.errors.append(str(ex))
-            logger.error("Error processing %s: %s" % (rel_path, ex))
-            return
+            msg = str(ex)
+            logger.error("Error processing %s: %s" % (rel_path, msg))
+            while ex:
+                record_entry.errors.append(str(ex))
+                ex = ex.__cause__
+            return False
 
         print_node(tree_root, recursive=True)
         leaves = tree_root.getLeaves()
@@ -380,10 +389,18 @@
                     pipeline.out_dir, self.ctx.pipeline_lock)
             if runner.processSubTree(tree_root):
                 record_entry.flags |= FLAG_PROCESSED
-                logger.info(format_timed(start_time, "[%d] %s" % (self.wid, rel_path)))
+                logger.info(format_timed(
+                    start_time, "[%d] %s" % (self.wid, rel_path)))
+            return True
         except ProcessingTreeError as ex:
-            record_entry.errors.append(str(ex))
-            logger.error("Error processing %s: %s" % (rel_path, ex))
+            msg = str(ex)
+            if isinstance(ex, ProcessorError):
+                msg = str(ex.__cause__)
+            logger.error("Error processing %s: %s" % (rel_path, msg))
+            while ex:
+                record_entry.errors.append(str(ex))
+                ex = ex.__cause__
+            return False
 
 
 def make_mount_infos(mounts, root_dir):