Mercurial > piecrust2

comparison piecrust/processing/base.py @ 221:f82262f59600

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Fix processing record bugs and error logging for external processes. Fix problems with processing records not being collapsed correctly. Make it possible to capture external processes' `stderr` output.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 03 Feb 2015 08:21:43 -0800
parents a47580a0955b
children c2ca72fb7f0b
comparison
equal deleted inserted replaced
220:84e2bc2d16cb 221:f82262f59600
7 import threading 7 import threading
8 from queue import Queue, Empty 8 from queue import Queue, Empty
9 from piecrust.chefutil import format_timed 9 from piecrust.chefutil import format_timed
10 from piecrust.processing.records import ( 10 from piecrust.processing.records import (
11 ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord, 11 ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord,
12 FLAG_PROCESSED, FLAG_OVERRIDEN, FLAG_BYPASSED_STRUCTURED_PROCESSING) 12 FLAG_PREPARED, FLAG_PROCESSED, FLAG_OVERRIDEN,
13 FLAG_BYPASSED_STRUCTURED_PROCESSING)
13 from piecrust.processing.tree import ( 14 from piecrust.processing.tree import (
14 ProcessingTreeBuilder, ProcessingTreeRunner, 15 ProcessingTreeBuilder, ProcessingTreeRunner,
15 ProcessingTreeError, ProcessorError, 16 ProcessingTreeError, ProcessorError,
16 STATE_DIRTY, 17 STATE_DIRTY,
17 print_node, get_node_name_tree) 18 print_node, get_node_name_tree)
18 19
19 20
20 logger = logging.getLogger(__name__) 21 logger = logging.getLogger(__name__)
21 22
22 23
24 re_ansicolors = re.compile('\033\\[\d+m')
25
26
23 PRIORITY_FIRST = -1 27 PRIORITY_FIRST = -1
24 PRIORITY_NORMAL = 0 28 PRIORITY_NORMAL = 0
25 PRIORITY_LAST = 1 29 PRIORITY_LAST = 1
26 30
27 31
101 out_path = os.path.join(out_dir, out_name) 105 out_path = os.path.join(out_dir, out_name)
102 return self._doProcess(path, out_path) 106 return self._doProcess(path, out_path)
103 107
104 def _doProcess(self, in_path, out_path): 108 def _doProcess(self, in_path, out_path):
105 raise NotImplementedError() 109 raise NotImplementedError()
110
111
112 class ExternalProcessException(Exception):
113 def __init__(self, stderr_data):
114 self.stderr_data = stderr_data
115
116 def __str__(self):
117 return self.stderr_data
106 118
107 119
108 class ProcessingContext(object): 120 class ProcessingContext(object):
109 def __init__(self, base_dir, mount_info, job_queue, record=None): 121 def __init__(self, base_dir, mount_info, job_queue, record=None):
110 self.base_dir = base_dir 122 self.base_dir = base_dir
358 processors = pipeline.getFilteredProcessors( 370 processors = pipeline.getFilteredProcessors(
359 job.mount_info['processors']) 371 job.mount_info['processors'])
360 try: 372 try:
361 builder = ProcessingTreeBuilder(processors) 373 builder = ProcessingTreeBuilder(processors)
362 tree_root = builder.build(rel_path) 374 tree_root = builder.build(rel_path)
375 record_entry.flags |= FLAG_PREPARED
363 except ProcessingTreeError as ex: 376 except ProcessingTreeError as ex:
364 msg = str(ex) 377 msg = str(ex)
365 logger.error("Error processing %s: %s" % (rel_path, msg)) 378 logger.error("Error preparing %s:\n%s" % (rel_path, msg))
366 while ex: 379 while ex:
367 record_entry.errors.append(str(ex)) 380 record_entry.errors.append(str(ex))
368 ex = ex.__cause__ 381 ex = ex.__cause__
369 return False 382 return False
370 383
394 return True 407 return True
395 except ProcessingTreeError as ex: 408 except ProcessingTreeError as ex:
396 msg = str(ex) 409 msg = str(ex)
397 if isinstance(ex, ProcessorError): 410 if isinstance(ex, ProcessorError):
398 msg = str(ex.__cause__) 411 msg = str(ex.__cause__)
399 logger.error("Error processing %s: %s" % (rel_path, msg)) 412 logger.error("Error processing %s:\n%s" % (rel_path, msg))
400 while ex: 413 while ex:
401 record_entry.errors.append(str(ex)) 414 msg = re_ansicolors.sub('', str(ex))
415 record_entry.errors.append(msg)
402 ex = ex.__cause__ 416 ex = ex.__cause__
403 return False 417 return False
404 418
405 419
406 def make_mount_infos(mounts, root_dir): 420 def make_mount_infos(mounts, root_dir):