Mercurial > piecrust2

annotate piecrust/processing/worker.py @ 764:f7ddd730c08d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugins: Abort the command if there's no site.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 02 Jul 2016 01:16:50 -0700
parents 9ae9390192da
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
1 import re
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import time
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
5 from piecrust.app import PieCrust, apply_variant_and_values
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 from piecrust.processing.base import PipelineContext
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from piecrust.processing.records import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 FLAG_NONE, FLAG_PREPARED, FLAG_PROCESSED,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 FLAG_BYPASSED_STRUCTURED_PROCESSING)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 from piecrust.processing.tree import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 ProcessingTreeBuilder, ProcessingTreeRunner,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 ProcessingTreeError, ProcessorError,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 get_node_name_tree, print_node,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 STATE_DIRTY)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
15 from piecrust.workerpool import IWorker
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 logger = logging.getLogger(__name__)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 split_processor_names_re = re.compile(r'[ ,]+')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 re_ansicolors = re.compile('\033\\[\d+m')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 class ProcessingWorkerContext(object):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
26 def __init__(self, appfactory, out_dir, tmp_dir, *,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
27 force=False):
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
28 self.appfactory = appfactory
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 self.out_dir = out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 self.tmp_dir = tmp_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 self.force = force
442
171dde4f61dc performance: Add profiling to the asset pipeline workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 435
diff changeset
32 self.is_profiling = False
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 self.enabled_processors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 self.additional_processors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 class ProcessingWorkerJob(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 def __init__(self, base_dir, mount_info, path, *, force=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 self.base_dir = base_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 self.mount_info = mount_info
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 self.path = path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 self.force = force
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 class ProcessingWorkerResult(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 def __init__(self, path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 self.path = path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 self.flags = FLAG_NONE
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 self.proc_tree = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 self.rel_outputs = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 self.errors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
54 class ProcessingWorker(IWorker):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
55 def __init__(self, ctx):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 self.ctx = ctx
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
57 self.work_start_time = time.perf_counter()
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
59 def initialize(self):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 # Create the app local to this worker.
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
61 app = self.ctx.appfactory.create()
421
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
62 app.env.registerTimer("PipelineWorker_%d_Total" % self.wid)
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
63 app.env.registerTimer("PipelineWorkerInit")
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 app.env.registerTimer("JobReceive")
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 app.env.registerTimer('BuildProcessingTree')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 app.env.registerTimer('RunProcessingTree')
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
67 self.app = app
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 processors = app.plugin_loader.getProcessors()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 if self.ctx.enabled_processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 logger.debug("Filtering processors to: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 self.ctx.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 processors = get_filtered_processors(processors,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 self.ctx.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 if self.ctx.additional_processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 logger.debug("Adding %s additional processors." %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 len(self.ctx.additional_processors))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 for proc in self.ctx.additional_processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 app.env.registerTimer(proc.__class__.__name__)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 proc.initialize(app)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 processors.append(proc)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
82 self.processors = processors
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 # Invoke pre-processors.
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
85 pipeline_ctx = PipelineContext(self.wid, self.app, self.ctx.out_dir,
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 self.ctx.tmp_dir, self.ctx.force)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 proc.onPipelineStart(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 # Sort our processors again in case the pre-process step involved
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 # patching the processors with some new ones.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92 processors.sort(key=lambda p: p.priority)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
94 app.env.stepTimerSince("PipelineWorkerInit", self.work_start_time)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
96 def process(self, job):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 result = ProcessingWorkerResult(job.path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 processors = get_filtered_processors(
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
100 self.processors, job.mount_info['processors'])
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 # Build the processing tree for this job.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 rel_path = os.path.relpath(job.path, job.base_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104 try:
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
105 with self.app.env.timerScope('BuildProcessingTree'):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106 builder = ProcessingTreeBuilder(processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 tree_root = builder.build(rel_path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 result.flags |= FLAG_PREPARED
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109 except ProcessingTreeError as ex:
435
5ceb86818dc5 bug: Fix a crash when errors occur while processing an asset.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
110 result.errors = _get_errors(ex)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 return result
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 # Prepare and run the tree.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 print_node(tree_root, recursive=True)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 leaves = tree_root.getLeaves()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116 result.rel_outputs = [l.path for l in leaves]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 result.proc_tree = get_node_name_tree(tree_root)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 if tree_root.getProcessor().is_bypassing_structured_processing:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119 result.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 if job.force:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 tree_root.setState(STATE_DIRTY, True)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 try:
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
125 with self.app.env.timerScope('RunProcessingTree'):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 runner = ProcessingTreeRunner(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 job.base_dir, self.ctx.tmp_dir, self.ctx.out_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 if runner.processSubTree(tree_root):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129 result.flags |= FLAG_PROCESSED
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130 except ProcessingTreeError as ex:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 if isinstance(ex, ProcessorError):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
132 ex = ex.__cause__
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 # Need to strip out colored errors from external processes.
435
5ceb86818dc5 bug: Fix a crash when errors occur while processing an asset.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
134 result.errors = _get_errors(ex, strip_colors=True)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 return result
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
137
691
9ae9390192da bake: Use standard pickle and queue for now to fix some small issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
138 def getReport(self, pool_reports):
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
139 # Invoke post-processors.
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
140 pipeline_ctx = PipelineContext(self.wid, self.app, self.ctx.out_dir,
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
141 self.ctx.tmp_dir, self.ctx.force)
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
142 for proc in self.processors:
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
143 proc.onPipelineEnd(pipeline_ctx)
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
144
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
145 self.app.env.stepTimerSince("PipelineWorker_%d_Total" % self.wid,
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
146 self.work_start_time)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
147 data = self.app.env.getStats()
691
9ae9390192da bake: Use standard pickle and queue for now to fix some small issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
148 data.timers.update(pool_reports)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
149 return {
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
150 'type': 'stats',
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
151 'data': data}
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
152
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
153
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
154 def get_filtered_processors(processors, authorized_names):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
155 if not authorized_names or authorized_names == 'all':
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
156 return processors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
157
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
158 if isinstance(authorized_names, str):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159 authorized_names = split_processor_names_re.split(authorized_names)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161 procs = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
162 has_star = 'all' in authorized_names
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163 for p in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164 for name in authorized_names:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
165 if name == p.PROCESSOR_NAME:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166 procs.append(p)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
167 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
168 if name == ('-%s' % p.PROCESSOR_NAME):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171 if has_star:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
172 procs.append(p)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173 return procs
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
175
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176 def _get_errors(ex, strip_colors=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
177 errors = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178 while ex is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179 msg = str(ex)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
180 if strip_colors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
181 msg = re_ansicolors.sub('', msg)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
182 errors.append(msg)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
183 ex = ex.__cause__
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
184 return errors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
185