Mercurial > piecrust2

annotate piecrust/baking/worker.py @ 1165:a928ee22c20a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cm: Upgrade Jinja2 version.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 04 Oct 2019 09:00:57 -0700
parents bd544b65cfad
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import time
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import logging
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
3 from piecrust.pipelines.base import (
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
4 PipelineManager, PipelineJobRunContext,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
5 get_pipeline_name_for_source)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
6 from piecrust.pipelines.records import (
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
7 MultiRecordHistory, MultiRecord, load_records)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
8 from piecrust.workerpool import IWorker
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 logger = logging.getLogger(__name__)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 class BakeWorkerContext(object):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
15 def __init__(self, appfactory, out_dir, *,
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
16 force=False, previous_records_path=None,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
17 allowed_pipelines=None, forbidden_pipelines=None):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
18 self.appfactory = appfactory
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 self.out_dir = out_dir
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
20 self.force = force
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
21 self.previous_records_path = previous_records_path
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
22 self.allowed_pipelines = allowed_pipelines
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
23 self.forbidden_pipelines = forbidden_pipelines
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
24
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
25
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
26 class BakeWorker(IWorker):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
27 def __init__(self, ctx):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
28 self.ctx = ctx
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
29 self.app = None
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
30 self.stats = None
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
31 self.previous_records = None
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
32 self._work_start_time = time.perf_counter()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
33 self._sources = {}
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
34 self._ppctx = None
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
35
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
36 def initialize(self):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
37 # Create the app local to this worker.
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
38 app = self.ctx.appfactory.create()
471
5b57a189fd98 bug: Correctly setup the environment/app for bake workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 466
diff changeset
39 app.config.set('baker/is_baking', True)
476
27e3b3f05648 bake: Set the worker ID in the configuration. It's useful.
Ludovic Chabant <ludovic@chabant.com>
parents: 471
diff changeset
40 app.config.set('baker/worker_id', self.wid)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
41 app.config.set('site/asset_url_format', '%page_uri%/%filename%')
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
42
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
43 app.env.fs_cache_only_for_main_page = True
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
44
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
45 stats = app.env.stats
1019
bd544b65cfad bake: More detailed stats, and fix a problem with some error reporting.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
46 stats.registerTimer("Worker_%d_Total" % self.wid)
bd544b65cfad bake: More detailed stats, and fix a problem with some error reporting.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
47 stats.registerTimer("Worker_%d_Init" % self.wid)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
48
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
49 self.app = app
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
50 self.stats = stats
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
51
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
52 # Load previous record
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
53 if self.ctx.previous_records_path:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
54 previous_records = load_records(self.ctx.previous_records_path)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
55 else:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
56 previous_records = MultiRecord()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
57 self.previous_records = previous_records
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
58
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
59 # Create the pipelines.
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
60 self.ppmngr = PipelineManager(
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
61 app, self.ctx.out_dir,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
62 worker_id=self.wid, force=self.ctx.force)
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
63 ok_pp = self.ctx.allowed_pipelines
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
64 nok_pp = self.ctx.forbidden_pipelines
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
65 for src in app.sources:
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
66 pname = get_pipeline_name_for_source(src)
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
67 if ok_pp is not None and pname not in ok_pp:
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
68 continue
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
69 if nok_pp is not None and pname in nok_pp:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
70 continue
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
71
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
72 self.ppmngr.createPipeline(src)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
73
900
bf65a1a6992a bake: Add performance timers for pipeline jobs.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
74 stats.registerTimer("PipelineJobs_%s" % pname,
bf65a1a6992a bake: Add performance timers for pipeline jobs.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
75 raise_if_registered=False)
bf65a1a6992a bake: Add performance timers for pipeline jobs.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
76
1019
bd544b65cfad bake: More detailed stats, and fix a problem with some error reporting.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
77 stats.stepTimerSince(
bd544b65cfad bake: More detailed stats, and fix a problem with some error reporting.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
78 "Worker_%d_Init" % self.wid, self._work_start_time)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
79
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
80 def process(self, job):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
81 source_name, item_spec = job['job_spec']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
82 logger.debug("Received job: %s@%s" % (source_name, item_spec))
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
84 # Run the job!
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
85 job_start = time.perf_counter()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
86 pp = self.ppmngr.getPipeline(source_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
87 runctx = PipelineJobRunContext(job, pp.record_name,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
88 self.previous_records)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
89 ppres = {
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
90 'item_spec': item_spec
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
91 }
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
92 pp.run(job, runctx, ppres)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
94 # Log time spent in this pipeline.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
95 self.stats.stepTimerSince("PipelineJobs_%s" % pp.PIPELINE_NAME,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
96 job_start)
900
bf65a1a6992a bake: Add performance timers for pipeline jobs.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
97
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
98 return ppres
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
100 def getStats(self):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
101 stats = self.app.env.stats
1019
bd544b65cfad bake: More detailed stats, and fix a problem with some error reporting.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
102 stats.stepTimerSince("Worker_%d_Total" % self.wid,
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
103 self._work_start_time)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
104 return stats
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
106 def shutdown(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
107 for src, pp in self._sources.values():
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
108 pp.shutdown(self._ppctx)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109