Mercurial > piecrust2

annotate piecrust/baking/worker.py @ 1014:071f30aa04bb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Do template caching in a background job if possible.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 28 Nov 2017 21:28:15 -0800
parents 2e5c5d33d62c
children fa489c5e829e
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import time
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import logging
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
3 from piecrust.pipelines.base import (
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
4 PipelineManager, PipelineJobRunContext,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
5 get_pipeline_name_for_source)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
6 from piecrust.pipelines.records import (
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
7 MultiRecordHistory, MultiRecord, load_records)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
8 from piecrust.workerpool import IWorker
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 logger = logging.getLogger(__name__)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 class BakeWorkerContext(object):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
15 def __init__(self, appfactory, out_dir, *,
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
16 force=False, previous_records_path=None,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
17 allowed_pipelines=None, forbidden_pipelines=None):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
18 self.appfactory = appfactory
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 self.out_dir = out_dir
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
20 self.force = force
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
21 self.previous_records_path = previous_records_path
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
22 self.allowed_pipelines = allowed_pipelines
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
23 self.forbidden_pipelines = forbidden_pipelines
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
24
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
25
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
26 class BakeWorker(IWorker):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
27 def __init__(self, ctx):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
28 self.ctx = ctx
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
29 self.app = None
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
30 self.stats = None
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
31 self.previous_records = None
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
32 self._work_start_time = time.perf_counter()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
33 self._sources = {}
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
34 self._ppctx = None
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
35
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
36 def initialize(self):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
37 # Create the app local to this worker.
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
38 app = self.ctx.appfactory.create()
471
5b57a189fd98 bug: Correctly setup the environment/app for bake workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 466
diff changeset
39 app.config.set('baker/is_baking', True)
476
27e3b3f05648 bake: Set the worker ID in the configuration. It's useful.
Ludovic Chabant <ludovic@chabant.com>
parents: 471
diff changeset
40 app.config.set('baker/worker_id', self.wid)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
41 app.config.set('site/asset_url_format', '%page_uri%/%filename%')
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
42
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
43 app.env.fs_cache_only_for_main_page = True
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
44
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
45 stats = app.env.stats
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
46 stats.registerTimer("BakeWorker_%d_Total" % self.wid)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
47 stats.registerTimer("BakeWorkerInit")
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
48
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
49 self.app = app
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
50 self.stats = stats
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
51
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
52 # Load previous record
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
53 if self.ctx.previous_records_path:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
54 previous_records = load_records(self.ctx.previous_records_path)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
55 else:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
56 previous_records = MultiRecord()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
57 self.previous_records = previous_records
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
58
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
59 # Create the pipelines.
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
60 self.ppmngr = PipelineManager(
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
61 app, self.ctx.out_dir,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
62 worker_id=self.wid, force=self.ctx.force)
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
63 ok_pp = self.ctx.allowed_pipelines
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
64 nok_pp = self.ctx.forbidden_pipelines
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
65 for src in app.sources:
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
66 pname = get_pipeline_name_for_source(src)
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
67 if ok_pp is not None and pname not in ok_pp:
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
68 continue
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
69 if nok_pp is not None and pname in nok_pp:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
70 continue
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
71
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
72 self.ppmngr.createPipeline(src)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
73
900
bf65a1a6992a bake: Add performance timers for pipeline jobs.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
74 stats.registerTimer("PipelineJobs_%s" % pname,
bf65a1a6992a bake: Add performance timers for pipeline jobs.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
75 raise_if_registered=False)
bf65a1a6992a bake: Add performance timers for pipeline jobs.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
76
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
77 stats.stepTimerSince("BakeWorkerInit", self._work_start_time)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
78
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
79 def process(self, job):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
80 source_name, item_spec = job['job_spec']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
81 logger.debug("Received job: %s@%s" % (source_name, item_spec))
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82
1014
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
83 # Check for special jobs.
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
84 if source_name == '__special__':
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
85 if item_spec == 'populate_template_cache':
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
86 for eng in self.app.plugin_loader.getTemplateEngines():
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
87 eng.populateCache()
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
88 else:
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
89 raise Exception("Unknown special job: %s" % item_spec)
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
90 return {}
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1005
diff changeset
91
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
92 # Run the job!
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
93 job_start = time.perf_counter()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
94 pp = self.ppmngr.getPipeline(source_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
95 runctx = PipelineJobRunContext(job, pp.record_name,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
96 self.previous_records)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
97 ppres = {
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
98 'item_spec': item_spec
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
99 }
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
100 pp.run(job, runctx, ppres)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
102 # Log time spent in this pipeline.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
103 self.stats.stepTimerSince("PipelineJobs_%s" % pp.PIPELINE_NAME,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 900
diff changeset
104 job_start)
900
bf65a1a6992a bake: Add performance timers for pipeline jobs.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
105
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
106 return ppres
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
108 def getStats(self):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
109 stats = self.app.env.stats
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
110 stats.stepTimerSince("BakeWorker_%d_Total" % self.wid,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
111 self._work_start_time)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
112 return stats
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
114 def shutdown(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
115 for src, pp in self._sources.values():
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
116 pp.shutdown(self._ppctx)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117