Mercurial > piecrust2
changeset 221:f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Fix problems with processing records not being collapsed correctly.
Make it possible to capture external processes' `stderr` output.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Tue, 03 Feb 2015 08:21:43 -0800 |
parents | 84e2bc2d16cb |
children | 1446dbc42d39 |
files | piecrust/commands/builtin/baking.py piecrust/processing/base.py piecrust/processing/less.py piecrust/processing/records.py |
diffstat | 4 files changed, 51 insertions(+), 17 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/commands/builtin/baking.py Tue Feb 03 08:20:30 2015 -0800 +++ b/piecrust/commands/builtin/baking.py Tue Feb 03 08:21:43 2015 -0800 @@ -10,7 +10,9 @@ from piecrust.commands.base import ChefCommand from piecrust.processing.base import ProcessorPipeline from piecrust.processing.records import ( - ProcessorPipelineRecord, FLAG_OVERRIDEN) + ProcessorPipelineRecord, + FLAG_PREPARED, FLAG_PROCESSED, FLAG_OVERRIDEN, + FLAG_BYPASSED_STRUCTURED_PROCESSING) logger = logging.getLogger(__name__) @@ -161,9 +163,15 @@ if pattern: if not fnmatch.fnmatch(entry.rel_input, pattern): continue - flags = '' + flags = [] + if entry.flags & FLAG_PREPARED: + flags.append('prepared') + if entry.flags & FLAG_PROCESSED: + flags.append('processed') if entry.flags & FLAG_OVERRIDEN: - flags += 'overriden' + flags.append('overriden') + if entry.flags & FLAG_BYPASSED_STRUCTURED_PROCESSING: + flags.append('external') logger.info(" - ") logger.info(" path: %s" % entry.rel_input) logger.info(" out paths: %s" % entry.rel_outputs) @@ -176,7 +184,7 @@ def format_proc_tree(tree, margin='', level=0): name, children = tree - res = '%s%s%s' % (margin if level > 0 else '', level * ' ', name) + res = '%s%s+ %s\n' % (margin if level > 0 else '', level * ' ', name) if children: for c in children: res += format_proc_tree(c, margin, level + 1)
--- a/piecrust/processing/base.py Tue Feb 03 08:20:30 2015 -0800 +++ b/piecrust/processing/base.py Tue Feb 03 08:21:43 2015 -0800 @@ -9,7 +9,8 @@ from piecrust.chefutil import format_timed from piecrust.processing.records import ( ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord, - FLAG_PROCESSED, FLAG_OVERRIDEN, FLAG_BYPASSED_STRUCTURED_PROCESSING) + FLAG_PREPARED, FLAG_PROCESSED, FLAG_OVERRIDEN, + FLAG_BYPASSED_STRUCTURED_PROCESSING) from piecrust.processing.tree import ( ProcessingTreeBuilder, ProcessingTreeRunner, ProcessingTreeError, ProcessorError, @@ -20,6 +21,9 @@ logger = logging.getLogger(__name__) +re_ansicolors = re.compile('\033\\[\d+m') + + PRIORITY_FIRST = -1 PRIORITY_NORMAL = 0 PRIORITY_LAST = 1 @@ -105,6 +109,14 @@ raise NotImplementedError() +class ExternalProcessException(Exception): + def __init__(self, stderr_data): + self.stderr_data = stderr_data + + def __str__(self): + return self.stderr_data + + class ProcessingContext(object): def __init__(self, base_dir, mount_info, job_queue, record=None): self.base_dir = base_dir @@ -360,9 +372,10 @@ try: builder = ProcessingTreeBuilder(processors) tree_root = builder.build(rel_path) + record_entry.flags |= FLAG_PREPARED except ProcessingTreeError as ex: msg = str(ex) - logger.error("Error processing %s: %s" % (rel_path, msg)) + logger.error("Error preparing %s:\n%s" % (rel_path, msg)) while ex: record_entry.errors.append(str(ex)) ex = ex.__cause__ @@ -396,9 +409,10 @@ msg = str(ex) if isinstance(ex, ProcessorError): msg = str(ex.__cause__) - logger.error("Error processing %s: %s" % (rel_path, msg)) + logger.error("Error processing %s:\n%s" % (rel_path, msg)) while ex: - record_entry.errors.append(str(ex)) + msg = re_ansicolors.sub('', str(ex)) + record_entry.errors.append(msg) ex = ex.__cause__ return False
--- a/piecrust/processing/less.py Tue Feb 03 08:20:30 2015 -0800 +++ b/piecrust/processing/less.py Tue Feb 03 08:21:43 2015 -0800 @@ -1,11 +1,13 @@ import os import os.path +import sys import json import hashlib import logging import platform import subprocess -from piecrust.processing.base import SimpleFileProcessor +from piecrust.processing.base import ( + SimpleFileProcessor, ExternalProcessException) from piecrust.processing.tree import FORCE_BUILD @@ -65,15 +67,19 @@ # otherwise it looks like `PATH` isn't taken into account. shell = (platform.system() == 'Windows') try: - retcode = subprocess.call(args, shell=shell) + proc = subprocess.Popen( + args, shell=shell, + stderr=subprocess.PIPE) + stdout_data, stderr_data = proc.communicate() except FileNotFoundError as ex: logger.error("Tried running LESS processor with command: %s" % args) raise Exception("Error running LESS processor. " "Did you install it?") from ex - if retcode != 0: - raise Exception("Error occured in LESS compiler. Please check " - "log messages above for more information.") + if proc.returncode != 0: + raise ExternalProcessException( + stderr_data.decode(sys.stderr.encoding)) + return True def _ensureInitialized(self):
--- a/piecrust/processing/records.py Tue Feb 03 08:20:30 2015 -0800 +++ b/piecrust/processing/records.py Tue Feb 03 08:21:43 2015 -0800 @@ -33,9 +33,10 @@ FLAG_NONE = 0 -FLAG_PROCESSED = 2**0 -FLAG_OVERRIDEN = 2**1 -FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**2 +FLAG_PREPARED = 2**0 +FLAG_PROCESSED = 2**1 +FLAG_OVERRIDEN = 2**2 +FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**3 class ProcessorPipelineRecordEntry(object): @@ -53,8 +54,13 @@ return os.path.join(self.base_dir, self.rel_input) @property + def was_prepared(self): + return bool(self.flags & FLAG_PREPARED) + + @property def was_processed(self): - return bool(self.flags & FLAG_PROCESSED) + return (self.was_prepared and + (bool(self.flags & FLAG_PROCESSED) or len(self.errors) > 0)) @property def was_processed_successfully(self):