Mercurial > piecrust2
annotate piecrust/processing/pipeline.py @ 415:0e9a94b7fdfa
bake: Improve bake record information.
* Store things in the bake record that require less interaction between the
master process and the workers. For instance, don't store the paginator
object in the render pass info -- instead, just store whether pagination
was used, and whether it had more items.
* Simplify information passing between workers and bake passes by saving the
rendering info to the JSON cache. This means the "render first sub" job
doesn't have to return anything except errors now.
* Add more performance counter info.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 20 Jun 2015 19:23:16 -0700 |
parents | c4b3a7fd2f87 |
children | 4a43d7015b75 |
rev | line source |
---|---|
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import os |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import os.path |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 import re |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 import time |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 import queue |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 import hashlib |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 import logging |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 import multiprocessing |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 from piecrust.chefutil import format_timed, format_timed_scope |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 from piecrust.processing.base import PipelineContext |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 from piecrust.processing.records import ( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
13 FLAG_PROCESSED) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
14 from piecrust.processing.worker import ( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
15 ProcessingWorkerContext, ProcessingWorkerJob, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
16 worker_func, get_filtered_processors) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 logger = logging.getLogger(__name__) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 class _ProcessingContext(object): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 def __init__(self, pool, record, base_dir, mount_info): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 self.pool = pool |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 self.record = record |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 self.base_dir = base_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 self.mount_info = mount_info |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
29 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 class ProcessorPipeline(object): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 def __init__(self, app, out_dir, force=False): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 assert app and out_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 self.app = app |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 self.out_dir = out_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 self.force = force |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 tmp_dir = app.sub_cache_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 if not tmp_dir: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
39 import tempfile |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
40 tmp_dir = os.path.join(tempfile.gettempdir(), 'piecrust') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
41 self.tmp_dir = os.path.join(tmp_dir, 'proc') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
43 baker_params = app.config.get('baker') or {} |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
44 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 assets_dirs = baker_params.get('assets_dirs', app.assets_dirs) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 self.mounts = make_mount_infos(assets_dirs, self.app.root_dir) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 self.num_workers = baker_params.get( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 'workers', multiprocessing.cpu_count()) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 ignores = baker_params.get('ignore', []) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 ignores += [ |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 '_cache', '_counter', |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 'theme_info.yml', |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 '.DS_Store', 'Thumbs.db', |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 '.git*', '.hg*', '.svn'] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
57 self.ignore_patterns = make_re(ignores) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 self.force_patterns = make_re(baker_params.get('force', [])) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
59 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
60 # Those things are mostly for unit-testing. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
61 self.enabled_processors = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
62 self.additional_processors = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
63 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
64 def addIgnorePatterns(self, patterns): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
65 self.ignore_patterns += make_re(patterns) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
66 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
67 def run(self, src_dir_or_file=None, *, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
68 delete=True, previous_record=None, save_record=True): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 start_time = time.perf_counter() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 # Get the list of processors for this run. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 processors = self.app.plugin_loader.getProcessors() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 if self.enabled_processors is not None: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 logger.debug("Filtering processors to: %s" % |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 self.enabled_processors) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 processors = get_filtered_processors(processors, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 self.enabled_processors) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
78 if self.additional_processors is not None: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
79 logger.debug("Adding %s additional processors." % |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
80 len(self.additional_processors)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
81 for proc in self.additional_processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 self.app.env.registerTimer(proc.__class__.__name__, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
83 raise_if_registered=False) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
84 proc.initialize(self.app) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
85 processors.append(proc) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
86 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
87 # Invoke pre-processors. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
88 pipeline_ctx = PipelineContext(-1, self.app, self.out_dir, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
89 self.tmp_dir, self.force) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
90 for proc in processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
91 proc.onPipelineStart(pipeline_ctx) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
92 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
93 # Pre-processors can define additional ignore patterns. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
94 self.ignore_patterns += make_re( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
95 pipeline_ctx._additional_ignore_patterns) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
96 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
97 # Create the worker pool. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
98 pool = _WorkerPool() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
99 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
100 # Create the pipeline record. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
101 record = TransitionalProcessorPipelineRecord() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
102 record_cache = self.app.cache.getCache('proc') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
103 record_name = ( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
104 hashlib.md5(self.out_dir.encode('utf8')).hexdigest() + |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
105 '.record') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
106 if previous_record: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
107 record.setPrevious(previous_record) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
108 elif not self.force and record_cache.has(record_name): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
109 with format_timed_scope(logger, 'loaded previous bake record', |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
110 level=logging.DEBUG, colored=False): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
111 record.loadPrevious(record_cache.getCachePath(record_name)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
112 logger.debug("Got %d entries in process record." % |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
113 len(record.previous.entries)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
114 record.current.success = True |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
115 record.current.processed_count = 0 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
116 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
117 # Work! |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
118 def _handler(res): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
119 entry = record.getCurrentEntry(res.path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
120 assert entry is not None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
121 entry.flags |= res.flags |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
122 entry.proc_tree = res.proc_tree |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
123 entry.rel_outputs = res.rel_outputs |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
124 if res.errors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
125 entry.errors += res.errors |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
126 record.current.success = False |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
127 if entry.flags & FLAG_PROCESSED: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
128 record.current.processed_count += 1 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
129 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
130 pool = self._createWorkerPool() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
131 expected_result_count = self._process(src_dir_or_file, pool, record) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
132 self._waitOnWorkerPool(pool, expected_result_count, _handler) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
133 self._terminateWorkerPool(pool) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
134 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
135 # Get timing information from the workers. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
136 record.current.timers = {} |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
137 for _ in range(len(pool.workers)): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
138 try: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
139 timers = pool.results.get(True, 0.1) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
140 except queue.Empty: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
141 logger.error("Didn't get timing information from all workers.") |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
142 break |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
143 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
144 for name, val in timers['data'].items(): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
145 main_val = record.current.timers.setdefault(name, 0) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
146 record.current.timers[name] = main_val + val |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
147 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
148 # Invoke post-processors. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
149 pipeline_ctx.record = record.current |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
150 for proc in processors: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
151 proc.onPipelineEnd(pipeline_ctx) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
152 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
153 # Handle deletions. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
154 if delete: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
155 for path, reason in record.getDeletions(): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
156 logger.debug("Removing '%s': %s" % (path, reason)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
157 try: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
158 os.remove(path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
159 except FileNotFoundError: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
160 pass |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
161 logger.info('[delete] %s' % path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
162 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
163 # Finalize the process record. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
164 record.current.process_time = time.time() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
165 record.current.out_dir = self.out_dir |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
166 record.collapseRecords() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
167 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
168 # Save the process record. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
169 if save_record: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
170 with format_timed_scope(logger, 'saved bake record', |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
171 level=logging.DEBUG, colored=False): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
172 record.saveCurrent(record_cache.getCachePath(record_name)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
173 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
174 logger.info(format_timed( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
175 start_time, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
176 "processed %d assets." % record.current.processed_count)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
177 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
178 return record.detach() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
179 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
180 def _process(self, src_dir_or_file, pool, record): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
181 expected_result_count = 0 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
182 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
183 if src_dir_or_file is not None: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
184 # Process only the given path. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
185 # Find out what mount point this is in. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
186 for name, info in self.mounts.items(): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
187 path = info['path'] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
188 if src_dir_or_file[:len(path)] == path: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
189 base_dir = path |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
190 mount_info = info |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
191 break |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
192 else: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
193 known_roots = [i['path'] for i in self.mounts.values()] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
194 raise Exception("Input path '%s' is not part of any known " |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
195 "mount point: %s" % |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
196 (src_dir_or_file, known_roots)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
197 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
198 ctx = _ProcessingContext(pool, record, base_dir, mount_info) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
199 logger.debug("Initiating processing pipeline on: %s" % |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
200 src_dir_or_file) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
201 if os.path.isdir(src_dir_or_file): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
202 expected_result_count = self._processDirectory( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
203 ctx, src_dir_or_file) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
204 elif os.path.isfile(src_dir_or_file): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
205 self._processFile(ctx, src_dir_or_file) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
206 expected_result_count = 1 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
207 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
208 else: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
209 # Process everything. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
210 for name, info in self.mounts.items(): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
211 path = info['path'] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
212 ctx = _ProcessingContext(pool, record, path, info) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
213 logger.debug("Initiating processing pipeline on: %s" % path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
214 expected_result_count = self._processDirectory(ctx, path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
215 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
216 return expected_result_count |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
217 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
218 def _processDirectory(self, ctx, start_dir): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
219 queued_count = 0 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
220 for dirpath, dirnames, filenames in os.walk(start_dir): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
221 rel_dirpath = os.path.relpath(dirpath, start_dir) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
222 dirnames[:] = [d for d in dirnames |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
223 if not re_matchany( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
224 d, self.ignore_patterns, rel_dirpath)] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
225 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
226 for filename in filenames: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
227 if re_matchany(filename, self.ignore_patterns, rel_dirpath): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
228 continue |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
229 self._processFile(ctx, os.path.join(dirpath, filename)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
230 queued_count += 1 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
231 return queued_count |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
232 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
233 def _processFile(self, ctx, path): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
234 # TODO: handle overrides between mount-points. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
235 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
236 entry = ProcessorPipelineRecordEntry(path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
237 ctx.record.addEntry(entry) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
238 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
239 previous_entry = ctx.record.getPreviousEntry(path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
240 force_this = (self.force or previous_entry is None or |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
241 not previous_entry.was_processed_successfully) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
242 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
243 job = ProcessingWorkerJob(ctx.base_dir, ctx.mount_info, path, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
244 force=force_this) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
245 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
246 logger.debug("Queuing: %s" % path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
247 ctx.pool.queue.put_nowait(job) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
248 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
249 def _createWorkerPool(self): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
250 pool = _WorkerPool() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
251 for i in range(self.num_workers): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
252 ctx = ProcessingWorkerContext( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
253 self.app.root_dir, self.out_dir, self.tmp_dir, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
254 pool.queue, pool.results, pool.abort_event, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
255 self.force, self.app.debug) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
256 ctx.enabled_processors = self.enabled_processors |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
257 ctx.additional_processors = self.additional_processors |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
258 w = multiprocessing.Process( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
259 name='Worker_%d' % i, |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
260 target=worker_func, args=(i, ctx)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
261 w.start() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
262 pool.workers.append(w) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
263 return pool |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
264 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
265 def _waitOnWorkerPool(self, pool, expected_result_count, result_handler): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
266 abort_with_exception = None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
267 try: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
268 got_count = 0 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
269 while got_count < expected_result_count: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
270 try: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
271 res = pool.results.get(True, 10) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
272 except queue.Empty: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
273 logger.error( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
274 "Got %d results, expected %d, and timed-out " |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
275 "for 10 seconds. A worker might be stuck?" % |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
276 (got_count, expected_result_count)) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
277 abort_with_exception = Exception("Worker time-out.") |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
278 break |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
279 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
280 if isinstance(res, dict) and res.get('type') == 'error': |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
281 abort_with_exception = Exception( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
282 'Worker critical error:\n' + |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
283 '\n'.join(res['messages'])) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
284 break |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
285 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
286 got_count += 1 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
287 result_handler(res) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
288 except KeyboardInterrupt as kiex: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
289 logger.warning("Bake aborted by user... " |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
290 "waiting for workers to stop.") |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
291 abort_with_exception = kiex |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
292 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
293 if abort_with_exception: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
294 pool.abort_event.set() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
295 for w in pool.workers: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
296 w.join(2) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
297 raise abort_with_exception |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
298 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
299 def _terminateWorkerPool(self, pool): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
300 pool.abort_event.set() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
301 for w in pool.workers: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
302 w.join() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
303 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
304 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
305 class _WorkerPool(object): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
306 def __init__(self): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
307 self.queue = multiprocessing.JoinableQueue() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
308 self.results = multiprocessing.Queue() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
309 self.abort_event = multiprocessing.Event() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
310 self.workers = [] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
311 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
312 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
313 def make_mount_infos(mounts, root_dir): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
314 if isinstance(mounts, list): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
315 mounts = {m: {} for m in mounts} |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
316 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
317 for name, info in mounts.items(): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
318 if not isinstance(info, dict): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
319 raise Exception("Asset directory info for '%s' is not a " |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
320 "dictionary." % name) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
321 info.setdefault('processors', 'all -uglifyjs -cleancss') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
322 info['path'] = os.path.join(root_dir, name) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
323 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
324 return mounts |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
325 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
326 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
327 def make_re(patterns): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
328 re_patterns = [] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
329 for pat in patterns: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
330 if pat[0] == '/' and pat[-1] == '/' and len(pat) > 2: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
331 re_patterns.append(pat[1:-1]) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
332 else: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
333 escaped_pat = ( |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
334 re.escape(pat) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
335 .replace(r'\*', r'[^/\\]*') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
336 .replace(r'\?', r'[^/\\]')) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
337 re_patterns.append(escaped_pat) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
338 return [re.compile(p) for p in re_patterns] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
339 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
340 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
341 def re_matchany(filename, patterns, dirname=None): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
342 if dirname and dirname != '.': |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
343 filename = os.path.join(dirname, filename) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
344 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
345 # skip patterns use a forward slash regardless of the platform. |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
346 filename = filename.replace('\\', '/') |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
347 for pattern in patterns: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
348 if pattern.search(filename): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
349 return True |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
350 return False |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
351 |