Mercurial > piecrust2
annotate piecrust/processing/worker.py @ 415:0e9a94b7fdfa
bake: Improve bake record information.
* Store things in the bake record that require less interaction between the
master process and the workers. For instance, don't store the paginator
object in the render pass info -- instead, just store whether pagination
was used, and whether it had more items.
* Simplify information passing between workers and bake passes by saving the
rendering info to the JSON cache. This means the "render first sub" job
doesn't have to return anything except errors now.
* Add more performance counter info.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 20 Jun 2015 19:23:16 -0700 |
parents | c4b3a7fd2f87 |
children | 4a43d7015b75 |
rev | line source |
---|---|
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import os.path |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import re |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 import time |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 import queue |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 import logging |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 from piecrust.app import PieCrust |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 from piecrust.processing.base import PipelineContext |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 from piecrust.processing.records import ( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 FLAG_NONE, FLAG_PREPARED, FLAG_PROCESSED, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 FLAG_BYPASSED_STRUCTURED_PROCESSING) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 from piecrust.processing.tree import ( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 ProcessingTreeBuilder, ProcessingTreeRunner, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
13 ProcessingTreeError, ProcessorError, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
14 get_node_name_tree, print_node, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
15 STATE_DIRTY) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
16 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 logger = logging.getLogger(__name__) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 split_processor_names_re = re.compile(r'[ ,]+') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 re_ansicolors = re.compile('\033\\[\d+m') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 def worker_func(wid, ctx): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 logger.debug("Worker %d booting up..." % wid) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 w = ProcessingWorker(wid, ctx) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 w.run() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
29 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 class ProcessingWorkerContext(object): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 def __init__(self, root_dir, out_dir, tmp_dir, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 work_queue, results, abort_event, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 force=False, debug=False): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 self.root_dir = root_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 self.out_dir = out_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 self.tmp_dir = tmp_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 self.work_queue = work_queue |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
39 self.results = results |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
40 self.abort_event = abort_event |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
41 self.force = force |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 self.debug = debug |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
43 self.enabled_processors = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
44 self.additional_processors = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 class ProcessingWorkerJob(object): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 def __init__(self, base_dir, mount_info, path, *, force=False): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 self.base_dir = base_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 self.mount_info = mount_info |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 self.path = path |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 self.force = force |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 class ProcessingWorkerResult(object): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 def __init__(self, path): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
57 self.path = path |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 self.flags = FLAG_NONE |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
59 self.proc_tree = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
60 self.rel_outputs = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
61 self.errors = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
62 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
63 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
64 class ProcessingWorker(object): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
65 def __init__(self, wid, ctx): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
66 self.wid = wid |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
67 self.ctx = ctx |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
68 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 def run(self): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 logger.debug("Worker %d initializing..." % self.wid) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 work_start_time = time.perf_counter() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 # Create the app local to this worker. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 app = PieCrust(self.ctx.root_dir, debug=self.ctx.debug) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 app.env.fs_cache_only_for_main_page = True |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 app.env.registerTimer("Worker_%d" % self.wid) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 app.env.registerTimer("JobReceive") |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
78 app.env.registerTimer('BuildProcessingTree') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
79 app.env.registerTimer('RunProcessingTree') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
80 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
81 processors = app.plugin_loader.getProcessors() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 if self.ctx.enabled_processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
83 logger.debug("Filtering processors to: %s" % |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
84 self.ctx.enabled_processors) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
85 processors = get_filtered_processors(processors, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
86 self.ctx.enabled_processors) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
87 if self.ctx.additional_processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
88 logger.debug("Adding %s additional processors." % |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
89 len(self.ctx.additional_processors)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
90 for proc in self.ctx.additional_processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
91 app.env.registerTimer(proc.__class__.__name__) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
92 proc.initialize(app) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
93 processors.append(proc) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
94 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
95 # Invoke pre-processors. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
96 pipeline_ctx = PipelineContext(self.wid, app, self.ctx.out_dir, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
97 self.ctx.tmp_dir, self.ctx.force) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
98 for proc in processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
99 proc.onPipelineStart(pipeline_ctx) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
100 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
101 # Sort our processors again in case the pre-process step involved |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
102 # patching the processors with some new ones. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
103 processors.sort(key=lambda p: p.priority) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
104 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
105 aborted_with_exception = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
106 while not self.ctx.abort_event.is_set(): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
107 try: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
108 with app.env.timerScope('JobReceive'): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
109 job = self.ctx.work_queue.get(True, 0.01) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
110 except queue.Empty: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
111 continue |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
112 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
113 try: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
114 result = self._unsafeRun(app, processors, job) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
115 self.ctx.results.put_nowait(result) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
116 except Exception as ex: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
117 self.ctx.abort_event.set() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
118 aborted_with_exception = ex |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
119 logger.error("[%d] Critical error, aborting." % self.wid) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
120 if self.ctx.debug: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
121 logger.exception(ex) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
122 break |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
123 finally: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
124 self.ctx.work_queue.task_done() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
125 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
126 if aborted_with_exception is not None: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
127 msgs = _get_errors(aborted_with_exception) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
128 self.ctx.results.put_nowait({'type': 'error', 'messages': msgs}) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
129 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
130 # Invoke post-processors. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
131 for proc in processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
132 proc.onPipelineEnd(pipeline_ctx) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
133 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
134 app.env.stepTimer("Worker_%d" % self.wid, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
135 time.perf_counter() - work_start_time) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
136 self.ctx.results.put_nowait({ |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
137 'type': 'timers', 'data': app.env._timers}) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
138 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
139 def _unsafeRun(self, app, processors, job): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
140 result = ProcessingWorkerResult(job.path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
141 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
142 processors = get_filtered_processors( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
143 processors, job.mount_info['processors']) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
144 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
145 # Build the processing tree for this job. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
146 rel_path = os.path.relpath(job.path, job.base_dir) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
147 try: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
148 with app.env.timerScope('BuildProcessingTree'): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
149 builder = ProcessingTreeBuilder(processors) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
150 tree_root = builder.build(rel_path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
151 result.flags |= FLAG_PREPARED |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
152 except ProcessingTreeError as ex: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
153 result.errors += _get_errors(ex) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
154 return result |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
155 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
156 # Prepare and run the tree. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
157 print_node(tree_root, recursive=True) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
158 leaves = tree_root.getLeaves() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
159 result.rel_outputs = [l.path for l in leaves] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
160 result.proc_tree = get_node_name_tree(tree_root) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
161 if tree_root.getProcessor().is_bypassing_structured_processing: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
162 result.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
163 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
164 if job.force: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
165 tree_root.setState(STATE_DIRTY, True) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
166 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
167 try: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
168 with app.env.timerScope('RunProcessingTree'): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
169 runner = ProcessingTreeRunner( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
170 job.base_dir, self.ctx.tmp_dir, self.ctx.out_dir) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
171 if runner.processSubTree(tree_root): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
172 result.flags |= FLAG_PROCESSED |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
173 except ProcessingTreeError as ex: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
174 if isinstance(ex, ProcessorError): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
175 ex = ex.__cause__ |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
176 # Need to strip out colored errors from external processes. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
177 result.errors += _get_errors(ex, strip_colors=True) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
178 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
179 return result |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
180 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
181 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
182 def get_filtered_processors(processors, authorized_names): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
183 if not authorized_names or authorized_names == 'all': |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
184 return processors |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
185 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
186 if isinstance(authorized_names, str): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
187 authorized_names = split_processor_names_re.split(authorized_names) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
188 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
189 procs = [] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
190 has_star = 'all' in authorized_names |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
191 for p in processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
192 for name in authorized_names: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
193 if name == p.PROCESSOR_NAME: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
194 procs.append(p) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
195 break |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
196 if name == ('-%s' % p.PROCESSOR_NAME): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
197 break |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
198 else: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
199 if has_star: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
200 procs.append(p) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
201 return procs |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
202 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
203 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
204 def _get_errors(ex, strip_colors=False): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
205 errors = [] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
206 while ex is not None: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
207 msg = str(ex) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
208 if strip_colors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
209 msg = re_ansicolors.sub('', msg) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
210 errors.append(msg) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
211 ex = ex.__cause__ |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
212 return errors |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
213 |