changeset 221:f82262f59600

bake: Fix processing record bugs and error logging for external processes. Fix problems with processing records not being collapsed correctly. Make it possible to capture external processes' `stderr` output.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 03 Feb 2015 08:21:43 -0800
parents 84e2bc2d16cb
children 1446dbc42d39
files piecrust/commands/builtin/baking.py piecrust/processing/base.py piecrust/processing/less.py piecrust/processing/records.py
diffstat 4 files changed, 51 insertions(+), 17 deletions(-) [+]
line wrap: on
line diff
--- a/piecrust/commands/builtin/baking.py	Tue Feb 03 08:20:30 2015 -0800
+++ b/piecrust/commands/builtin/baking.py	Tue Feb 03 08:21:43 2015 -0800
@@ -10,7 +10,9 @@
 from piecrust.commands.base import ChefCommand
 from piecrust.processing.base import ProcessorPipeline
 from piecrust.processing.records import (
-        ProcessorPipelineRecord, FLAG_OVERRIDEN)
+        ProcessorPipelineRecord,
+        FLAG_PREPARED, FLAG_PROCESSED, FLAG_OVERRIDEN,
+        FLAG_BYPASSED_STRUCTURED_PROCESSING)
 
 
 logger = logging.getLogger(__name__)
@@ -161,9 +163,15 @@
             if pattern:
                 if not fnmatch.fnmatch(entry.rel_input, pattern):
                     continue
-            flags = ''
+            flags = []
+            if entry.flags & FLAG_PREPARED:
+                flags.append('prepared')
+            if entry.flags & FLAG_PROCESSED:
+                flags.append('processed')
             if entry.flags & FLAG_OVERRIDEN:
-                flags += 'overriden'
+                flags.append('overriden')
+            if entry.flags & FLAG_BYPASSED_STRUCTURED_PROCESSING:
+                flags.append('external')
             logger.info(" - ")
             logger.info("   path:      %s" % entry.rel_input)
             logger.info("   out paths: %s" % entry.rel_outputs)
@@ -176,7 +184,7 @@
 
 def format_proc_tree(tree, margin='', level=0):
     name, children = tree
-    res = '%s%s%s' % (margin if level > 0 else '', level * '  ', name)
+    res = '%s%s+ %s\n' % (margin if level > 0 else '', level * '  ', name)
     if children:
         for c in children:
             res += format_proc_tree(c, margin, level + 1)
--- a/piecrust/processing/base.py	Tue Feb 03 08:20:30 2015 -0800
+++ b/piecrust/processing/base.py	Tue Feb 03 08:21:43 2015 -0800
@@ -9,7 +9,8 @@
 from piecrust.chefutil import format_timed
 from piecrust.processing.records import (
         ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord,
-        FLAG_PROCESSED, FLAG_OVERRIDEN, FLAG_BYPASSED_STRUCTURED_PROCESSING)
+        FLAG_PREPARED, FLAG_PROCESSED, FLAG_OVERRIDEN,
+        FLAG_BYPASSED_STRUCTURED_PROCESSING)
 from piecrust.processing.tree import (
         ProcessingTreeBuilder, ProcessingTreeRunner,
         ProcessingTreeError, ProcessorError,
@@ -20,6 +21,9 @@
 logger = logging.getLogger(__name__)
 
 
+re_ansicolors = re.compile('\033\\[\d+m')
+
+
 PRIORITY_FIRST = -1
 PRIORITY_NORMAL = 0
 PRIORITY_LAST = 1
@@ -105,6 +109,14 @@
         raise NotImplementedError()
 
 
+class ExternalProcessException(Exception):
+    def __init__(self, stderr_data):
+        self.stderr_data = stderr_data
+
+    def __str__(self):
+        return self.stderr_data
+
+
 class ProcessingContext(object):
     def __init__(self, base_dir, mount_info, job_queue, record=None):
         self.base_dir = base_dir
@@ -360,9 +372,10 @@
         try:
             builder = ProcessingTreeBuilder(processors)
             tree_root = builder.build(rel_path)
+            record_entry.flags |= FLAG_PREPARED
         except ProcessingTreeError as ex:
             msg = str(ex)
-            logger.error("Error processing %s: %s" % (rel_path, msg))
+            logger.error("Error preparing %s:\n%s" % (rel_path, msg))
             while ex:
                 record_entry.errors.append(str(ex))
                 ex = ex.__cause__
@@ -396,9 +409,10 @@
             msg = str(ex)
             if isinstance(ex, ProcessorError):
                 msg = str(ex.__cause__)
-            logger.error("Error processing %s: %s" % (rel_path, msg))
+            logger.error("Error processing %s:\n%s" % (rel_path, msg))
             while ex:
-                record_entry.errors.append(str(ex))
+                msg = re_ansicolors.sub('', str(ex))
+                record_entry.errors.append(msg)
                 ex = ex.__cause__
             return False
 
--- a/piecrust/processing/less.py	Tue Feb 03 08:20:30 2015 -0800
+++ b/piecrust/processing/less.py	Tue Feb 03 08:21:43 2015 -0800
@@ -1,11 +1,13 @@
 import os
 import os.path
+import sys
 import json
 import hashlib
 import logging
 import platform
 import subprocess
-from piecrust.processing.base import SimpleFileProcessor
+from piecrust.processing.base import (
+        SimpleFileProcessor, ExternalProcessException)
 from piecrust.processing.tree import FORCE_BUILD
 
 
@@ -65,15 +67,19 @@
         # otherwise it looks like `PATH` isn't taken into account.
         shell = (platform.system() == 'Windows')
         try:
-            retcode = subprocess.call(args, shell=shell)
+            proc = subprocess.Popen(
+                    args, shell=shell,
+                    stderr=subprocess.PIPE)
+            stdout_data, stderr_data = proc.communicate()
         except FileNotFoundError as ex:
             logger.error("Tried running LESS processor with command: %s" %
                          args)
             raise Exception("Error running LESS processor. "
                             "Did you install it?") from ex
-        if retcode != 0:
-            raise Exception("Error occured in LESS compiler. Please check "
-                            "log messages above for more information.")
+        if proc.returncode != 0:
+            raise ExternalProcessException(
+                    stderr_data.decode(sys.stderr.encoding))
+
         return True
 
     def _ensureInitialized(self):
--- a/piecrust/processing/records.py	Tue Feb 03 08:20:30 2015 -0800
+++ b/piecrust/processing/records.py	Tue Feb 03 08:21:43 2015 -0800
@@ -33,9 +33,10 @@
 
 
 FLAG_NONE = 0
-FLAG_PROCESSED = 2**0
-FLAG_OVERRIDEN = 2**1
-FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**2
+FLAG_PREPARED = 2**0
+FLAG_PROCESSED = 2**1
+FLAG_OVERRIDEN = 2**2
+FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**3
 
 
 class ProcessorPipelineRecordEntry(object):
@@ -53,8 +54,13 @@
         return os.path.join(self.base_dir, self.rel_input)
 
     @property
+    def was_prepared(self):
+        return bool(self.flags & FLAG_PREPARED)
+
+    @property
     def was_processed(self):
-        return bool(self.flags & FLAG_PROCESSED)
+        return (self.was_prepared and
+                (bool(self.flags & FLAG_PROCESSED) or len(self.errors) > 0))
 
     @property
     def was_processed_successfully(self):