annotate piecrust/processing/pipeline.py @ 415:0e9a94b7fdfa

bake: Improve bake record information. * Store things in the bake record that require less interaction between the master process and the workers. For instance, don't store the paginator object in the render pass info -- instead, just store whether pagination was used, and whether it had more items. * Simplify information passing between workers and bake passes by saving the rendering info to the JSON cache. This means the "render first sub" job doesn't have to return anything except errors now. * Add more performance counter info.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:23:16 -0700
parents c4b3a7fd2f87
children 4a43d7015b75
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import re
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import time
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import queue
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 import hashlib
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 import logging
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 import multiprocessing
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 from piecrust.chefutil import format_timed, format_timed_scope
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 from piecrust.processing.base import PipelineContext
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 from piecrust.processing.records import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 FLAG_PROCESSED)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 from piecrust.processing.worker import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 ProcessingWorkerContext, ProcessingWorkerJob,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 worker_func, get_filtered_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 logger = logging.getLogger(__name__)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 class _ProcessingContext(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 def __init__(self, pool, record, base_dir, mount_info):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 self.pool = pool
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 self.record = record
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 self.base_dir = base_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 self.mount_info = mount_info
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 class ProcessorPipeline(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 def __init__(self, app, out_dir, force=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 assert app and out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 self.app = app
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 self.out_dir = out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 self.force = force
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 tmp_dir = app.sub_cache_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 if not tmp_dir:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 import tempfile
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 tmp_dir = os.path.join(tempfile.gettempdir(), 'piecrust')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 self.tmp_dir = os.path.join(tmp_dir, 'proc')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 baker_params = app.config.get('baker') or {}
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 assets_dirs = baker_params.get('assets_dirs', app.assets_dirs)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 self.mounts = make_mount_infos(assets_dirs, self.app.root_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 self.num_workers = baker_params.get(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 'workers', multiprocessing.cpu_count())
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 ignores = baker_params.get('ignore', [])
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 ignores += [
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 '_cache', '_counter',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 'theme_info.yml',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 '.DS_Store', 'Thumbs.db',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 '.git*', '.hg*', '.svn']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 self.ignore_patterns = make_re(ignores)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 self.force_patterns = make_re(baker_params.get('force', []))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 # Those things are mostly for unit-testing.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 self.enabled_processors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 self.additional_processors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 def addIgnorePatterns(self, patterns):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 self.ignore_patterns += make_re(patterns)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 def run(self, src_dir_or_file=None, *,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 delete=True, previous_record=None, save_record=True):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 start_time = time.perf_counter()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 # Get the list of processors for this run.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 processors = self.app.plugin_loader.getProcessors()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 if self.enabled_processors is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 logger.debug("Filtering processors to: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 self.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 processors = get_filtered_processors(processors,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 self.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 if self.additional_processors is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 logger.debug("Adding %s additional processors." %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 len(self.additional_processors))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 for proc in self.additional_processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 self.app.env.registerTimer(proc.__class__.__name__,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 raise_if_registered=False)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 proc.initialize(self.app)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 processors.append(proc)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 # Invoke pre-processors.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 pipeline_ctx = PipelineContext(-1, self.app, self.out_dir,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 self.tmp_dir, self.force)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 proc.onPipelineStart(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 # Pre-processors can define additional ignore patterns.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 self.ignore_patterns += make_re(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 pipeline_ctx._additional_ignore_patterns)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 # Create the worker pool.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 pool = _WorkerPool()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 # Create the pipeline record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 record = TransitionalProcessorPipelineRecord()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 record_cache = self.app.cache.getCache('proc')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 record_name = (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104 hashlib.md5(self.out_dir.encode('utf8')).hexdigest() +
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105 '.record')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106 if previous_record:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 record.setPrevious(previous_record)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 elif not self.force and record_cache.has(record_name):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109 with format_timed_scope(logger, 'loaded previous bake record',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 level=logging.DEBUG, colored=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 record.loadPrevious(record_cache.getCachePath(record_name))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112 logger.debug("Got %d entries in process record." %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 len(record.previous.entries))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 record.current.success = True
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 record.current.processed_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 # Work!
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 def _handler(res):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119 entry = record.getCurrentEntry(res.path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120 assert entry is not None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 entry.flags |= res.flags
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 entry.proc_tree = res.proc_tree
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 entry.rel_outputs = res.rel_outputs
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 if res.errors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125 entry.errors += res.errors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 record.current.success = False
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 if entry.flags & FLAG_PROCESSED:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 record.current.processed_count += 1
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130 pool = self._createWorkerPool()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 expected_result_count = self._process(src_dir_or_file, pool, record)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
132 self._waitOnWorkerPool(pool, expected_result_count, _handler)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 self._terminateWorkerPool(pool)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135 # Get timing information from the workers.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 record.current.timers = {}
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
137 for _ in range(len(pool.workers)):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
138 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
139 timers = pool.results.get(True, 0.1)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140 except queue.Empty:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141 logger.error("Didn't get timing information from all workers.")
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
144 for name, val in timers['data'].items():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
145 main_val = record.current.timers.setdefault(name, 0)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
146 record.current.timers[name] = main_val + val
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
147
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
148 # Invoke post-processors.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
149 pipeline_ctx.record = record.current
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
150 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
151 proc.onPipelineEnd(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
152
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
153 # Handle deletions.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
154 if delete:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
155 for path, reason in record.getDeletions():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
156 logger.debug("Removing '%s': %s" % (path, reason))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
157 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
158 os.remove(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159 except FileNotFoundError:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160 pass
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161 logger.info('[delete] %s' % path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
162
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163 # Finalize the process record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164 record.current.process_time = time.time()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
165 record.current.out_dir = self.out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166 record.collapseRecords()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
167
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
168 # Save the process record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169 if save_record:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170 with format_timed_scope(logger, 'saved bake record',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171 level=logging.DEBUG, colored=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
172 record.saveCurrent(record_cache.getCachePath(record_name))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174 logger.info(format_timed(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
175 start_time,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176 "processed %d assets." % record.current.processed_count))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
177
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178 return record.detach()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
180 def _process(self, src_dir_or_file, pool, record):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
181 expected_result_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
182
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
183 if src_dir_or_file is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
184 # Process only the given path.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
185 # Find out what mount point this is in.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
186 for name, info in self.mounts.items():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
187 path = info['path']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
188 if src_dir_or_file[:len(path)] == path:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
189 base_dir = path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
190 mount_info = info
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
191 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
192 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
193 known_roots = [i['path'] for i in self.mounts.values()]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
194 raise Exception("Input path '%s' is not part of any known "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
195 "mount point: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196 (src_dir_or_file, known_roots))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
197
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
198 ctx = _ProcessingContext(pool, record, base_dir, mount_info)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
199 logger.debug("Initiating processing pipeline on: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
200 src_dir_or_file)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
201 if os.path.isdir(src_dir_or_file):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
202 expected_result_count = self._processDirectory(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
203 ctx, src_dir_or_file)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
204 elif os.path.isfile(src_dir_or_file):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
205 self._processFile(ctx, src_dir_or_file)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
206 expected_result_count = 1
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
207
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
208 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
209 # Process everything.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
210 for name, info in self.mounts.items():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
211 path = info['path']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
212 ctx = _ProcessingContext(pool, record, path, info)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
213 logger.debug("Initiating processing pipeline on: %s" % path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
214 expected_result_count = self._processDirectory(ctx, path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
215
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
216 return expected_result_count
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
217
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
218 def _processDirectory(self, ctx, start_dir):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
219 queued_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
220 for dirpath, dirnames, filenames in os.walk(start_dir):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
221 rel_dirpath = os.path.relpath(dirpath, start_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
222 dirnames[:] = [d for d in dirnames
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
223 if not re_matchany(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
224 d, self.ignore_patterns, rel_dirpath)]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
225
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
226 for filename in filenames:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
227 if re_matchany(filename, self.ignore_patterns, rel_dirpath):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
228 continue
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
229 self._processFile(ctx, os.path.join(dirpath, filename))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
230 queued_count += 1
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
231 return queued_count
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
232
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
233 def _processFile(self, ctx, path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
234 # TODO: handle overrides between mount-points.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
235
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
236 entry = ProcessorPipelineRecordEntry(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
237 ctx.record.addEntry(entry)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
238
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
239 previous_entry = ctx.record.getPreviousEntry(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
240 force_this = (self.force or previous_entry is None or
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
241 not previous_entry.was_processed_successfully)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
242
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
243 job = ProcessingWorkerJob(ctx.base_dir, ctx.mount_info, path,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
244 force=force_this)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
245
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
246 logger.debug("Queuing: %s" % path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
247 ctx.pool.queue.put_nowait(job)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
248
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
249 def _createWorkerPool(self):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
250 pool = _WorkerPool()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
251 for i in range(self.num_workers):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
252 ctx = ProcessingWorkerContext(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
253 self.app.root_dir, self.out_dir, self.tmp_dir,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
254 pool.queue, pool.results, pool.abort_event,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
255 self.force, self.app.debug)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
256 ctx.enabled_processors = self.enabled_processors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
257 ctx.additional_processors = self.additional_processors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
258 w = multiprocessing.Process(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
259 name='Worker_%d' % i,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
260 target=worker_func, args=(i, ctx))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
261 w.start()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
262 pool.workers.append(w)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
263 return pool
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
264
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
265 def _waitOnWorkerPool(self, pool, expected_result_count, result_handler):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
266 abort_with_exception = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
267 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
268 got_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
269 while got_count < expected_result_count:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
270 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
271 res = pool.results.get(True, 10)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
272 except queue.Empty:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
273 logger.error(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
274 "Got %d results, expected %d, and timed-out "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
275 "for 10 seconds. A worker might be stuck?" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
276 (got_count, expected_result_count))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
277 abort_with_exception = Exception("Worker time-out.")
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
278 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
279
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
280 if isinstance(res, dict) and res.get('type') == 'error':
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
281 abort_with_exception = Exception(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
282 'Worker critical error:\n' +
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
283 '\n'.join(res['messages']))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
284 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
285
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
286 got_count += 1
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
287 result_handler(res)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
288 except KeyboardInterrupt as kiex:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
289 logger.warning("Bake aborted by user... "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
290 "waiting for workers to stop.")
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
291 abort_with_exception = kiex
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
292
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
293 if abort_with_exception:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
294 pool.abort_event.set()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
295 for w in pool.workers:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
296 w.join(2)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
297 raise abort_with_exception
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
298
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
299 def _terminateWorkerPool(self, pool):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
300 pool.abort_event.set()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
301 for w in pool.workers:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
302 w.join()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
303
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
304
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
305 class _WorkerPool(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
306 def __init__(self):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
307 self.queue = multiprocessing.JoinableQueue()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
308 self.results = multiprocessing.Queue()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
309 self.abort_event = multiprocessing.Event()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
310 self.workers = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
311
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
312
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
313 def make_mount_infos(mounts, root_dir):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
314 if isinstance(mounts, list):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
315 mounts = {m: {} for m in mounts}
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
316
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
317 for name, info in mounts.items():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
318 if not isinstance(info, dict):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
319 raise Exception("Asset directory info for '%s' is not a "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
320 "dictionary." % name)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
321 info.setdefault('processors', 'all -uglifyjs -cleancss')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
322 info['path'] = os.path.join(root_dir, name)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
323
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
324 return mounts
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
325
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
326
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
327 def make_re(patterns):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
328 re_patterns = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
329 for pat in patterns:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
330 if pat[0] == '/' and pat[-1] == '/' and len(pat) > 2:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
331 re_patterns.append(pat[1:-1])
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
332 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
333 escaped_pat = (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
334 re.escape(pat)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
335 .replace(r'\*', r'[^/\\]*')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
336 .replace(r'\?', r'[^/\\]'))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
337 re_patterns.append(escaped_pat)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
338 return [re.compile(p) for p in re_patterns]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
339
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
340
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
341 def re_matchany(filename, patterns, dirname=None):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
342 if dirname and dirname != '.':
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
343 filename = os.path.join(dirname, filename)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
344
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
345 # skip patterns use a forward slash regardless of the platform.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
346 filename = filename.replace('\\', '/')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
347 for pattern in patterns:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
348 if pattern.search(filename):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
349 return True
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
350 return False
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
351