Mercurial > piecrust2

annotate piecrust/processing/pipeline.py @ 828:bca06fc064c0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Use assetor provided by page source when paginating
author Ben Artin <ben@artins.org>
date Sun, 01 Jan 2017 19:32:19 -0500
parents 9a92e2804562
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import re
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import time
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import hashlib
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 import logging
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 import multiprocessing
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 from piecrust.chefutil import format_timed, format_timed_scope
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
9 from piecrust.environment import ExecutionStats
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 from piecrust.processing.base import PipelineContext
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 from piecrust.processing.records import (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 FLAG_PROCESSED)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 from piecrust.processing.worker import (
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
15 ProcessingWorkerJob,
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
16 get_filtered_processors)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 logger = logging.getLogger(__name__)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 class _ProcessingContext(object):
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
23 def __init__(self, jobs, record, base_dir, mount_info):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
24 self.jobs = jobs
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 self.record = record
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 self.base_dir = base_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 self.mount_info = mount_info
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 class ProcessorPipeline(object):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
31 def __init__(self, app, out_dir, force=False,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
32 applied_config_variant=None,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
33 applied_config_values=None):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 assert app and out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 self.app = app
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self.out_dir = out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 self.force = force
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
38 self.applied_config_variant = applied_config_variant
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
39 self.applied_config_values = applied_config_values
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
41 tmp_dir = app.cache_dir
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 if not tmp_dir:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 import tempfile
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 tmp_dir = os.path.join(tempfile.gettempdir(), 'piecrust')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 self.tmp_dir = os.path.join(tmp_dir, 'proc')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46
570
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
47 baker_params = app.config.get('baker', {})
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48
570
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
49 mount_params = baker_params.get('assets_dirs', {})
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
50 self.mounts = make_mount_infos(app, mount_params)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 self.num_workers = baker_params.get(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 'workers', multiprocessing.cpu_count())
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 ignores = baker_params.get('ignore', [])
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 ignores += [
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 '_cache', '_counter',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 '.DS_Store', 'Thumbs.db',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 '.git*', '.hg*', '.svn']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 self.ignore_patterns = make_re(ignores)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 self.force_patterns = make_re(baker_params.get('force', []))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 # Those things are mostly for unit-testing.
492
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
64 #
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
65 # Note that additiona processors can't be passed as instances.
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
66 # Instead, we need some factory functions because we need to create
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
67 # one instance right away to use during the initialization phase, and
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
68 # another instance to pass to the worker pool. The initialized one will
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
69 # be tied to the PieCrust app instance, which can't be pickled across
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
70 # processes.
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 self.enabled_processors = None
492
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
72 self.additional_processors_factories = None
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 def addIgnorePatterns(self, patterns):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 self.ignore_patterns += make_re(patterns)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 def run(self, src_dir_or_file=None, *,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 delete=True, previous_record=None, save_record=True):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 start_time = time.perf_counter()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 # Get the list of processors for this run.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 processors = self.app.plugin_loader.getProcessors()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 if self.enabled_processors is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 logger.debug("Filtering processors to: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 self.enabled_processors)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 processors = get_filtered_processors(processors,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 self.enabled_processors)
492
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
88 if self.additional_processors_factories is not None:
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 logger.debug("Adding %s additional processors." %
492
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
90 len(self.additional_processors_factories))
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
91 for proc_fac in self.additional_processors_factories:
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
92 proc = proc_fac()
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 self.app.env.registerTimer(proc.__class__.__name__,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 raise_if_registered=False)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 proc.initialize(self.app)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 processors.append(proc)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 # Invoke pre-processors.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 pipeline_ctx = PipelineContext(-1, self.app, self.out_dir,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 self.tmp_dir, self.force)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 proc.onPipelineStart(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104 # Pre-processors can define additional ignore patterns.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105 self.ignore_patterns += make_re(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106 pipeline_ctx._additional_ignore_patterns)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 # Create the pipeline record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109 record = TransitionalProcessorPipelineRecord()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 record_cache = self.app.cache.getCache('proc')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 record_name = (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112 hashlib.md5(self.out_dir.encode('utf8')).hexdigest() +
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 '.record')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 if previous_record:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 record.setPrevious(previous_record)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116 elif not self.force and record_cache.has(record_name):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 with format_timed_scope(logger, 'loaded previous bake record',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 level=logging.DEBUG, colored=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119 record.loadPrevious(record_cache.getCachePath(record_name))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120 logger.debug("Got %d entries in process record." %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 len(record.previous.entries))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 record.current.success = True
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 record.current.processed_count = 0
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125 # Work!
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 def _handler(res):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 entry = record.getCurrentEntry(res.path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 assert entry is not None
549
7453baeb0839 bake: Set the flags, don't combine.
Ludovic Chabant <ludovic@chabant.com>
parents: 492
diff changeset
129 entry.flags = res.flags
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130 entry.proc_tree = res.proc_tree
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 entry.rel_outputs = res.rel_outputs
434
6238dcfc7a78 reporting: Print errors that occured during pipeline processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
132 if entry.flags & FLAG_PROCESSED:
6238dcfc7a78 reporting: Print errors that occured during pipeline processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
133 record.current.processed_count += 1
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134 if res.errors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135 entry.errors += res.errors
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 record.current.success = False
434
6238dcfc7a78 reporting: Print errors that occured during pipeline processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
137
6238dcfc7a78 reporting: Print errors that occured during pipeline processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
138 rel_path = os.path.relpath(res.path, self.app.root_dir)
6238dcfc7a78 reporting: Print errors that occured during pipeline processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
139 logger.error("Errors found in %s:" % rel_path)
6238dcfc7a78 reporting: Print errors that occured during pipeline processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
140 for e in entry.errors:
6238dcfc7a78 reporting: Print errors that occured during pipeline processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
141 logger.error(" " + e)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
143 jobs = []
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
144 self._process(src_dir_or_file, record, jobs)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
145 pool = self._createWorkerPool()
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
146 ar = pool.queueJobs(jobs, handler=_handler)
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
147 ar.wait()
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
148
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
149 # Shutdown the workers and get timing information from them.
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
150 reports = pool.close()
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
151 total_stats = ExecutionStats()
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
152 record.current.stats['_Total'] = total_stats
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
153 for i in range(len(reports)):
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
154 worker_stats = reports[i]['data']
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
155 if worker_stats is not None:
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
156 worker_name = 'PipelineWorker_%d' % i
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
157 record.current.stats[worker_name] = worker_stats
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
158 total_stats.mergeStats(worker_stats)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160 # Invoke post-processors.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161 pipeline_ctx.record = record.current
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
162 for proc in processors:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163 proc.onPipelineEnd(pipeline_ctx)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
165 # Handle deletions.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166 if delete:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
167 for path, reason in record.getDeletions():
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
168 logger.debug("Removing '%s': %s" % (path, reason))
753
9a92e2804562 bake: Add the list of deleted files to the bake/processing records.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
169 record.current.deleted.append(path)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170 try:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171 os.remove(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
172 except FileNotFoundError:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173 pass
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174 logger.info('[delete] %s' % path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
175
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176 # Finalize the process record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
177 record.current.process_time = time.time()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178 record.current.out_dir = self.out_dir
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179 record.collapseRecords()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
180
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
181 # Save the process record.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
182 if save_record:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
183 with format_timed_scope(logger, 'saved bake record',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
184 level=logging.DEBUG, colored=False):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
185 record.saveCurrent(record_cache.getCachePath(record_name))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
186
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
187 logger.info(format_timed(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
188 start_time,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
189 "processed %d assets." % record.current.processed_count))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
190
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
191 return record.detach()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
192
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
193 def _process(self, src_dir_or_file, record, jobs):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
194 if src_dir_or_file is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
195 # Process only the given path.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196 # Find out what mount point this is in.
570
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
197 for path, info in self.mounts.items():
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
198 if src_dir_or_file[:len(path)] == path:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
199 base_dir = path
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
200 mount_info = info
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
201 break
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
202 else:
570
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
203 known_roots = list(self.mounts.keys())
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
204 raise Exception("Input path '%s' is not part of any known "
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
205 "mount point: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
206 (src_dir_or_file, known_roots))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
207
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
208 ctx = _ProcessingContext(jobs, record, base_dir, mount_info)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
209 logger.debug("Initiating processing pipeline on: %s" %
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
210 src_dir_or_file)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
211 if os.path.isdir(src_dir_or_file):
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
212 self._processDirectory(ctx, src_dir_or_file)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
213 elif os.path.isfile(src_dir_or_file):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
214 self._processFile(ctx, src_dir_or_file)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
215
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
216 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
217 # Process everything.
570
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
218 for path, info in self.mounts.items():
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
219 ctx = _ProcessingContext(jobs, record, path, info)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
220 logger.debug("Initiating processing pipeline on: %s" % path)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
221 self._processDirectory(ctx, path)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
222
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
223 def _processDirectory(self, ctx, start_dir):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
224 for dirpath, dirnames, filenames in os.walk(start_dir):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
225 rel_dirpath = os.path.relpath(dirpath, start_dir)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
226 dirnames[:] = [d for d in dirnames
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
227 if not re_matchany(
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
228 d, self.ignore_patterns, rel_dirpath)]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
229
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
230 for filename in filenames:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
231 if re_matchany(filename, self.ignore_patterns, rel_dirpath):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
232 continue
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
233 self._processFile(ctx, os.path.join(dirpath, filename))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
234
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
235 def _processFile(self, ctx, path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
236 # TODO: handle overrides between mount-points.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
237
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
238 entry = ProcessorPipelineRecordEntry(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
239 ctx.record.addEntry(entry)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
240
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
241 previous_entry = ctx.record.getPreviousEntry(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
242 force_this = (self.force or previous_entry is None or
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
243 not previous_entry.was_processed_successfully)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
244
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
245 job = ProcessingWorkerJob(ctx.base_dir, ctx.mount_info, path,
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
246 force=force_this)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
247 ctx.jobs.append(job)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
248
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
249 def _createWorkerPool(self):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
250 from piecrust.app import PieCrustFactory
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
251 from piecrust.workerpool import WorkerPool
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
252 from piecrust.processing.worker import (
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
253 ProcessingWorkerContext, ProcessingWorker)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
254
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
255 appfactory = PieCrustFactory(
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
256 self.app.root_dir,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
257 cache=self.app.cache.enabled,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
258 cache_key=self.app.cache_key,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
259 config_variant=self.applied_config_variant,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
260 config_values=self.applied_config_values,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
261 debug=self.app.debug,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
262 theme_site=self.app.theme_site)
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
263
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
264 ctx = ProcessingWorkerContext(
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
265 appfactory,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
266 self.out_dir, self.tmp_dir,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
267 force=self.force)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
268 ctx.enabled_processors = self.enabled_processors
492
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
269 if self.additional_processors_factories is not None:
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
270 ctx.additional_processors = [
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
271 proc_fac()
d90ccdf18156 tests: Fix processing tests on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
272 for proc_fac in self.additional_processors_factories]
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
273
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
274 pool = WorkerPool(
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
275 worker_class=ProcessingWorker,
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
276 initargs=(ctx,))
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 442
diff changeset
277 return pool
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
278
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
279
570
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
280 def make_mount_infos(app, mount_params):
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
281 mounts = {d: {} for d in app.assets_dirs}
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
282
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
283 for name, cfg in mount_params.items():
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
284 mdir = os.path.join(app.root_dir, name)
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
285 mounts[mdir] = cfg
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
286
570
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
287 for mdir, info in mounts.items():
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
288 mname = os.path.basename(mdir)
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
289 info_from_config = mount_params.get(mname)
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
290 if info_from_config is not None:
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
291 if not isinstance(info, dict):
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
292 raise Exception("Asset directory info for '%s' is not a "
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
293 "dictionary." % mname)
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
294 info.update(info_from_config)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
295 info.setdefault('processors', 'all -uglifyjs -cleancss')
570
7dabfdd056a1 serve: Fix corner cases where the pipeline doesn't run correctly.
Ludovic Chabant <ludovic@chabant.com>
parents: 549
diff changeset
296 info['name'] = mname
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
297
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
298 return mounts
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
299
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
300
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
301 def make_re(patterns):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
302 re_patterns = []
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
303 for pat in patterns:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
304 if pat[0] == '/' and pat[-1] == '/' and len(pat) > 2:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
305 re_patterns.append(pat[1:-1])
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
306 else:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
307 escaped_pat = (
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
308 re.escape(pat)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
309 .replace(r'\*', r'[^/\\]*')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
310 .replace(r'\?', r'[^/\\]'))
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
311 re_patterns.append(escaped_pat)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
312 return [re.compile(p) for p in re_patterns]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
313
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
314
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
315 def re_matchany(filename, patterns, dirname=None):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
316 if dirname and dirname != '.':
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
317 filename = os.path.join(dirname, filename)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
318
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
319 # skip patterns use a forward slash regardless of the platform.
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
320 filename = filename.replace('\\', '/')
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
321 for pattern in patterns:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
322 if pattern.search(filename):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
323 return True
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
324 return False
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
325