Mercurial > piecrust2

annotate piecrust/processing/worker.py @ 414:c4b3a7fd2f87

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Make pipeline processing multi-process. Not many changes here, as it's pretty straightforward, but an API change for processors so they know if they're being initialized/disposed from the main process or from one of the workers. This makes it possible to do global stuff that has side-effects (e.g. create a directory) vs. doing in-memory stuff.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:20:30 -0700
parents
children 4a43d7015b75
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import re
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import time
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import queue
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import logging
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 from piecrust.app import PieCrust
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from piecrust.processing.base import PipelineContext
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 from piecrust.processing.records import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 FLAG_NONE, FLAG_PREPARED, FLAG_PROCESSED,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 FLAG_BYPASSED_STRUCTURED_PROCESSING)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 from piecrust.processing.tree import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 ProcessingTreeBuilder, ProcessingTreeRunner,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 ProcessingTreeError, ProcessorError,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 get_node_name_tree, print_node,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 STATE_DIRTY)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 logger = logging.getLogger(__name__)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 split_processor_names_re = re.compile(r'[ ,]+')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 re_ansicolors = re.compile('\033\\[\d+m')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 def worker_func(wid, ctx):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 logger.debug("Worker %d booting up..." % wid)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 w = ProcessingWorker(wid, ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 w.run()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 class ProcessingWorkerContext(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 def __init__(self, root_dir, out_dir, tmp_dir,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 work_queue, results, abort_event,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 force=False, debug=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 self.root_dir = root_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self.out_dir = out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 self.tmp_dir = tmp_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 self.work_queue = work_queue
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 self.results = results
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 self.abort_event = abort_event
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 self.force = force
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 self.debug = debug
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 self.enabled_processors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 self.additional_processors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 class ProcessingWorkerJob(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 def __init__(self, base_dir, mount_info, path, *, force=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 self.base_dir = base_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 self.mount_info = mount_info
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 self.path = path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 self.force = force
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 class ProcessingWorkerResult(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 def __init__(self, path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 self.path = path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 self.flags = FLAG_NONE
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 self.proc_tree = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 self.rel_outputs = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 self.errors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 class ProcessingWorker(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 def __init__(self, wid, ctx):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 self.wid = wid
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 self.ctx = ctx
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 def run(self):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 logger.debug("Worker %d initializing..." % self.wid)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 work_start_time = time.perf_counter()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 # Create the app local to this worker.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 app = PieCrust(self.ctx.root_dir, debug=self.ctx.debug)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 app.env.fs_cache_only_for_main_page = True
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 app.env.registerTimer("Worker_%d" % self.wid)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 app.env.registerTimer("JobReceive")
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 app.env.registerTimer('BuildProcessingTree')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 app.env.registerTimer('RunProcessingTree')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 processors = app.plugin_loader.getProcessors()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 if self.ctx.enabled_processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 logger.debug("Filtering processors to: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 self.ctx.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 processors = get_filtered_processors(processors,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 self.ctx.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 if self.ctx.additional_processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 logger.debug("Adding %s additional processors." %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 len(self.ctx.additional_processors))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 for proc in self.ctx.additional_processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 app.env.registerTimer(proc.__class__.__name__)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92 proc.initialize(app)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 processors.append(proc)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 # Invoke pre-processors.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 pipeline_ctx = PipelineContext(self.wid, app, self.ctx.out_dir,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 self.ctx.tmp_dir, self.ctx.force)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 proc.onPipelineStart(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 # Sort our processors again in case the pre-process step involved
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 # patching the processors with some new ones.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 processors.sort(key=lambda p: p.priority)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105 aborted_with_exception = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106 while not self.ctx.abort_event.is_set():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 with app.env.timerScope('JobReceive'):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109 job = self.ctx.work_queue.get(True, 0.01)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 except queue.Empty:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 continue
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 result = self._unsafeRun(app, processors, job)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 self.ctx.results.put_nowait(result)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116 except Exception as ex:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 self.ctx.abort_event.set()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 aborted_with_exception = ex
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119 logger.error("[%d] Critical error, aborting." % self.wid)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120 if self.ctx.debug:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 logger.exception(ex)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 finally:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 self.ctx.work_queue.task_done()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 if aborted_with_exception is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 msgs = _get_errors(aborted_with_exception)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 self.ctx.results.put_nowait({'type': 'error', 'messages': msgs})
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130 # Invoke post-processors.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
132 proc.onPipelineEnd(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134 app.env.stepTimer("Worker_%d" % self.wid,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135 time.perf_counter() - work_start_time)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 self.ctx.results.put_nowait({
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
137 'type': 'timers', 'data': app.env._timers})
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
138
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
139 def _unsafeRun(self, app, processors, job):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140 result = ProcessingWorkerResult(job.path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142 processors = get_filtered_processors(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143 processors, job.mount_info['processors'])
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
144
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
145 # Build the processing tree for this job.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
146 rel_path = os.path.relpath(job.path, job.base_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
147 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
148 with app.env.timerScope('BuildProcessingTree'):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
149 builder = ProcessingTreeBuilder(processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
150 tree_root = builder.build(rel_path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
151 result.flags |= FLAG_PREPARED
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
152 except ProcessingTreeError as ex:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
153 result.errors += _get_errors(ex)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
154 return result
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
155
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
156 # Prepare and run the tree.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
157 print_node(tree_root, recursive=True)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
158 leaves = tree_root.getLeaves()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159 result.rel_outputs = [l.path for l in leaves]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160 result.proc_tree = get_node_name_tree(tree_root)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161 if tree_root.getProcessor().is_bypassing_structured_processing:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
162 result.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164 if job.force:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
165 tree_root.setState(STATE_DIRTY, True)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
167 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
168 with app.env.timerScope('RunProcessingTree'):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169 runner = ProcessingTreeRunner(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170 job.base_dir, self.ctx.tmp_dir, self.ctx.out_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171 if runner.processSubTree(tree_root):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
172 result.flags |= FLAG_PROCESSED
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173 except ProcessingTreeError as ex:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174 if isinstance(ex, ProcessorError):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
175 ex = ex.__cause__
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176 # Need to strip out colored errors from external processes.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
177 result.errors += _get_errors(ex, strip_colors=True)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179 return result
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
180
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
181
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
182 def get_filtered_processors(processors, authorized_names):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
183 if not authorized_names or authorized_names == 'all':
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
184 return processors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
185
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
186 if isinstance(authorized_names, str):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
187 authorized_names = split_processor_names_re.split(authorized_names)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
188
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
189 procs = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
190 has_star = 'all' in authorized_names
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
191 for p in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
192 for name in authorized_names:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
193 if name == p.PROCESSOR_NAME:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
194 procs.append(p)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
195 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196 if name == ('-%s' % p.PROCESSOR_NAME):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
197 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
198 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
199 if has_star:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
200 procs.append(p)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
201 return procs
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
202
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
203
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
204 def _get_errors(ex, strip_colors=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
205 errors = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
206 while ex is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
207 msg = str(ex)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
208 if strip_colors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
209 msg = re_ansicolors.sub('', msg)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
210 errors.append(msg)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
211 ex = ex.__cause__
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
212 return errors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
213