Mercurial > piecrust2

annotate piecrust/processing/pipeline.py @ 421:4a43d7015b75

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Improve performance timers reports. Add timers per-worker, and separate bake and pipeline workers.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 23:27:39 -0700
parents c4b3a7fd2f87
children 6238dcfc7a78
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import re
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import time
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import queue
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 import hashlib
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 import logging
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 import multiprocessing
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 from piecrust.chefutil import format_timed, format_timed_scope
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 from piecrust.processing.base import PipelineContext
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 from piecrust.processing.records import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 FLAG_PROCESSED)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 from piecrust.processing.worker import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 ProcessingWorkerContext, ProcessingWorkerJob,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 worker_func, get_filtered_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 logger = logging.getLogger(__name__)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 class _ProcessingContext(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 def __init__(self, pool, record, base_dir, mount_info):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 self.pool = pool
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 self.record = record
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 self.base_dir = base_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 self.mount_info = mount_info
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 class ProcessorPipeline(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 def __init__(self, app, out_dir, force=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 assert app and out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 self.app = app
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 self.out_dir = out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 self.force = force
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 tmp_dir = app.sub_cache_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 if not tmp_dir:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 import tempfile
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 tmp_dir = os.path.join(tempfile.gettempdir(), 'piecrust')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 self.tmp_dir = os.path.join(tmp_dir, 'proc')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 baker_params = app.config.get('baker') or {}
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 assets_dirs = baker_params.get('assets_dirs', app.assets_dirs)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 self.mounts = make_mount_infos(assets_dirs, self.app.root_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 self.num_workers = baker_params.get(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 'workers', multiprocessing.cpu_count())
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 ignores = baker_params.get('ignore', [])
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 ignores += [
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 '_cache', '_counter',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 'theme_info.yml',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 '.DS_Store', 'Thumbs.db',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 '.git*', '.hg*', '.svn']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 self.ignore_patterns = make_re(ignores)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 self.force_patterns = make_re(baker_params.get('force', []))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 # Those things are mostly for unit-testing.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 self.enabled_processors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 self.additional_processors = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 def addIgnorePatterns(self, patterns):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 self.ignore_patterns += make_re(patterns)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 def run(self, src_dir_or_file=None, *,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 delete=True, previous_record=None, save_record=True):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 start_time = time.perf_counter()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 # Get the list of processors for this run.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 processors = self.app.plugin_loader.getProcessors()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 if self.enabled_processors is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 logger.debug("Filtering processors to: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 self.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 processors = get_filtered_processors(processors,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 self.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 if self.additional_processors is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 logger.debug("Adding %s additional processors." %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 len(self.additional_processors))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 for proc in self.additional_processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 self.app.env.registerTimer(proc.__class__.__name__,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 raise_if_registered=False)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 proc.initialize(self.app)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 processors.append(proc)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 # Invoke pre-processors.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 pipeline_ctx = PipelineContext(-1, self.app, self.out_dir,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 self.tmp_dir, self.force)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 proc.onPipelineStart(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 # Pre-processors can define additional ignore patterns.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 self.ignore_patterns += make_re(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 pipeline_ctx._additional_ignore_patterns)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 # Create the worker pool.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 pool = _WorkerPool()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 # Create the pipeline record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 record = TransitionalProcessorPipelineRecord()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 record_cache = self.app.cache.getCache('proc')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 record_name = (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104 hashlib.md5(self.out_dir.encode('utf8')).hexdigest() +
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105 '.record')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106 if previous_record:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 record.setPrevious(previous_record)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 elif not self.force and record_cache.has(record_name):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109 with format_timed_scope(logger, 'loaded previous bake record',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 level=logging.DEBUG, colored=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 record.loadPrevious(record_cache.getCachePath(record_name))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112 logger.debug("Got %d entries in process record." %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 len(record.previous.entries))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 record.current.success = True
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 record.current.processed_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 # Work!
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 def _handler(res):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119 entry = record.getCurrentEntry(res.path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120 assert entry is not None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 entry.flags |= res.flags
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 entry.proc_tree = res.proc_tree
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 entry.rel_outputs = res.rel_outputs
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 if res.errors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125 entry.errors += res.errors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 record.current.success = False
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 if entry.flags & FLAG_PROCESSED:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 record.current.processed_count += 1
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130 pool = self._createWorkerPool()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 expected_result_count = self._process(src_dir_or_file, pool, record)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
132 self._waitOnWorkerPool(pool, expected_result_count, _handler)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 self._terminateWorkerPool(pool)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135 # Get timing information from the workers.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 record.current.timers = {}
421
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
137 for i in range(len(pool.workers)):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
138 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
139 timers = pool.results.get(True, 0.1)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140 except queue.Empty:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141 logger.error("Didn't get timing information from all workers.")
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143
421
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
144 worker_name = 'PipelineWorker_%d' % i
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
145 record.current.timers[worker_name] = {}
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
146 for name, val in timers['data'].items():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
147 main_val = record.current.timers.setdefault(name, 0)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
148 record.current.timers[name] = main_val + val
421
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
149 record.current.timers[worker_name][name] = val
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
150
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
151 # Invoke post-processors.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
152 pipeline_ctx.record = record.current
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
153 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
154 proc.onPipelineEnd(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
155
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
156 # Handle deletions.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
157 if delete:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
158 for path, reason in record.getDeletions():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159 logger.debug("Removing '%s': %s" % (path, reason))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161 os.remove(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
162 except FileNotFoundError:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163 pass
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164 logger.info('[delete] %s' % path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
165
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166 # Finalize the process record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
167 record.current.process_time = time.time()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
168 record.current.out_dir = self.out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169 record.collapseRecords()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171 # Save the process record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
172 if save_record:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173 with format_timed_scope(logger, 'saved bake record',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174 level=logging.DEBUG, colored=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
175 record.saveCurrent(record_cache.getCachePath(record_name))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
177 logger.info(format_timed(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178 start_time,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179 "processed %d assets." % record.current.processed_count))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
180
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
181 return record.detach()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
182
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
183 def _process(self, src_dir_or_file, pool, record):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
184 expected_result_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
185
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
186 if src_dir_or_file is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
187 # Process only the given path.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
188 # Find out what mount point this is in.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
189 for name, info in self.mounts.items():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
190 path = info['path']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
191 if src_dir_or_file[:len(path)] == path:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
192 base_dir = path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
193 mount_info = info
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
194 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
195 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196 known_roots = [i['path'] for i in self.mounts.values()]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
197 raise Exception("Input path '%s' is not part of any known "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
198 "mount point: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
199 (src_dir_or_file, known_roots))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
200
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
201 ctx = _ProcessingContext(pool, record, base_dir, mount_info)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
202 logger.debug("Initiating processing pipeline on: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
203 src_dir_or_file)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
204 if os.path.isdir(src_dir_or_file):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
205 expected_result_count = self._processDirectory(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
206 ctx, src_dir_or_file)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
207 elif os.path.isfile(src_dir_or_file):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
208 self._processFile(ctx, src_dir_or_file)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
209 expected_result_count = 1
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
210
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
211 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
212 # Process everything.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
213 for name, info in self.mounts.items():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
214 path = info['path']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
215 ctx = _ProcessingContext(pool, record, path, info)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
216 logger.debug("Initiating processing pipeline on: %s" % path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
217 expected_result_count = self._processDirectory(ctx, path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
218
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
219 return expected_result_count
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
220
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
221 def _processDirectory(self, ctx, start_dir):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
222 queued_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
223 for dirpath, dirnames, filenames in os.walk(start_dir):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
224 rel_dirpath = os.path.relpath(dirpath, start_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
225 dirnames[:] = [d for d in dirnames
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
226 if not re_matchany(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
227 d, self.ignore_patterns, rel_dirpath)]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
228
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
229 for filename in filenames:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
230 if re_matchany(filename, self.ignore_patterns, rel_dirpath):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
231 continue
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
232 self._processFile(ctx, os.path.join(dirpath, filename))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
233 queued_count += 1
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
234 return queued_count
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
235
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
236 def _processFile(self, ctx, path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
237 # TODO: handle overrides between mount-points.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
238
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
239 entry = ProcessorPipelineRecordEntry(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
240 ctx.record.addEntry(entry)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
241
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
242 previous_entry = ctx.record.getPreviousEntry(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
243 force_this = (self.force or previous_entry is None or
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
244 not previous_entry.was_processed_successfully)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
245
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
246 job = ProcessingWorkerJob(ctx.base_dir, ctx.mount_info, path,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
247 force=force_this)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
248
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
249 logger.debug("Queuing: %s" % path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
250 ctx.pool.queue.put_nowait(job)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
251
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
252 def _createWorkerPool(self):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
253 pool = _WorkerPool()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
254 for i in range(self.num_workers):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
255 ctx = ProcessingWorkerContext(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
256 self.app.root_dir, self.out_dir, self.tmp_dir,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
257 pool.queue, pool.results, pool.abort_event,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
258 self.force, self.app.debug)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
259 ctx.enabled_processors = self.enabled_processors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
260 ctx.additional_processors = self.additional_processors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
261 w = multiprocessing.Process(
421
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
262 name='PipelineWorker_%d' % i,
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
263 target=worker_func, args=(i, ctx))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
264 w.start()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
265 pool.workers.append(w)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
266 return pool
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
267
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
268 def _waitOnWorkerPool(self, pool, expected_result_count, result_handler):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
269 abort_with_exception = None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
270 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
271 got_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
272 while got_count < expected_result_count:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
273 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
274 res = pool.results.get(True, 10)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
275 except queue.Empty:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
276 logger.error(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
277 "Got %d results, expected %d, and timed-out "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
278 "for 10 seconds. A worker might be stuck?" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
279 (got_count, expected_result_count))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
280 abort_with_exception = Exception("Worker time-out.")
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
281 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
282
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
283 if isinstance(res, dict) and res.get('type') == 'error':
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
284 abort_with_exception = Exception(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
285 'Worker critical error:\n' +
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
286 '\n'.join(res['messages']))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
287 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
288
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
289 got_count += 1
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
290 result_handler(res)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
291 except KeyboardInterrupt as kiex:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
292 logger.warning("Bake aborted by user... "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
293 "waiting for workers to stop.")
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
294 abort_with_exception = kiex
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
295
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
296 if abort_with_exception:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
297 pool.abort_event.set()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
298 for w in pool.workers:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
299 w.join(2)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
300 raise abort_with_exception
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
301
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
302 def _terminateWorkerPool(self, pool):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
303 pool.abort_event.set()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
304 for w in pool.workers:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
305 w.join()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
306
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
307
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
308 class _WorkerPool(object):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
309 def __init__(self):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
310 self.queue = multiprocessing.JoinableQueue()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
311 self.results = multiprocessing.Queue()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
312 self.abort_event = multiprocessing.Event()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
313 self.workers = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
314
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
315
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
316 def make_mount_infos(mounts, root_dir):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
317 if isinstance(mounts, list):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
318 mounts = {m: {} for m in mounts}
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
319
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
320 for name, info in mounts.items():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
321 if not isinstance(info, dict):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
322 raise Exception("Asset directory info for '%s' is not a "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
323 "dictionary." % name)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
324 info.setdefault('processors', 'all -uglifyjs -cleancss')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
325 info['path'] = os.path.join(root_dir, name)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
326
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
327 return mounts
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
328
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
329
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
330 def make_re(patterns):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
331 re_patterns = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
332 for pat in patterns:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
333 if pat[0] == '/' and pat[-1] == '/' and len(pat) > 2:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
334 re_patterns.append(pat[1:-1])
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
335 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
336 escaped_pat = (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
337 re.escape(pat)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
338 .replace(r'\*', r'[^/\\]*')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
339 .replace(r'\?', r'[^/\\]'))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
340 re_patterns.append(escaped_pat)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
341 return [re.compile(p) for p in re_patterns]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
342
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
343
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
344 def re_matchany(filename, patterns, dirname=None):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
345 if dirname and dirname != '.':
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
346 filename = os.path.join(dirname, filename)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
347
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
348 # skip patterns use a forward slash regardless of the platform.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
349 filename = filename.replace('\\', '/')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
350 for pattern in patterns:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
351 if pattern.search(filename):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
352 return True
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
353 return False
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
354