Mercurial > piecrust2

annotate piecrust/baking/worker.py @ 877:d6d35b2efd04

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Rename "pass" to "step" and make the page pipeline use different steps. That pipeline is now first loading all pages, and then rendering full pages unless they trigger a sub-render.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 15 Jun 2017 22:16:23 -0700
parents d1095774bfcf
children bf65a1a6992a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import time
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import logging
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
3 from piecrust.pipelines.base import (
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
4 PipelineManager, PipelineJobRunContext, PipelineJobResult,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
5 get_pipeline_name_for_source)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
6 from piecrust.pipelines.records import (
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
7 MultiRecordHistory, MultiRecord, load_records)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
8 from piecrust.workerpool import IWorker
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 logger = logging.getLogger(__name__)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 class BakeWorkerContext(object):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
15 def __init__(self, appfactory, out_dir, *,
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
16 force=False, previous_records_path=None,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
17 allowed_pipelines=None, forbidden_pipelines=None):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
18 self.appfactory = appfactory
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 self.out_dir = out_dir
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
20 self.force = force
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
21 self.previous_records_path = previous_records_path
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
22 self.allowed_pipelines = allowed_pipelines
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
23 self.forbidden_pipelines = forbidden_pipelines
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
24
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
25
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
26 class BakeWorker(IWorker):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
27 def __init__(self, ctx):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
28 self.ctx = ctx
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
29 self.app = None
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
30 self.record_histories = None
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
31 self._work_start_time = time.perf_counter()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
32 self._sources = {}
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
33 self._ppctx = None
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
34
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
35 def initialize(self):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
36 # Create the app local to this worker.
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
37 app = self.ctx.appfactory.create()
471
5b57a189fd98 bug: Correctly setup the environment/app for bake workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 466
diff changeset
38 app.config.set('baker/is_baking', True)
476
27e3b3f05648 bake: Set the worker ID in the configuration. It's useful.
Ludovic Chabant <ludovic@chabant.com>
parents: 471
diff changeset
39 app.config.set('baker/worker_id', self.wid)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
40 app.config.set('site/asset_url_format', '%page_uri%/%filename%')
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
41
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
42 app.env.fs_cache_only_for_main_page = True
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
43
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
44 stats = app.env.stats
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
45 stats.registerTimer("BakeWorker_%d_Total" % self.wid)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
46 stats.registerTimer("BakeWorkerInit")
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
47
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
48 self.app = app
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
49
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
50 # Load previous record
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
51 if self.ctx.previous_records_path:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
52 previous_records = load_records(self.ctx.previous_records_path)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
53 else:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
54 previous_records = MultiRecord()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
55 current_records = MultiRecord()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
56 self.record_histories = MultiRecordHistory(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
57 previous_records, current_records)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
58
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
59 # Create the pipelines.
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
60 self.ppmngr = PipelineManager(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
61 app, self.ctx.out_dir, self.record_histories,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
62 worker_id=self.wid, force=self.ctx.force)
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
63 ok_pp = self.ctx.allowed_pipelines
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
64 nok_pp = self.ctx.forbidden_pipelines
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
65 for src in app.sources:
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
66 pname = get_pipeline_name_for_source(src)
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
67 if ok_pp is not None and pname not in ok_pp:
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
68 continue
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
69 if nok_pp is not None and pname in nok_pp:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
70 continue
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
71
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
72 self.ppmngr.createPipeline(src)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
73
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
74 stats.stepTimerSince("BakeWorkerInit", self._work_start_time)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
75
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
76 def process(self, job):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
77 item = job.content_item
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
78 logger.debug("Received job: %s@%s" % (job.source_name, item.spec))
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
79
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
80 ppinfo = self.ppmngr.getPipeline(job.source_name)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
81 pp = ppinfo.pipeline
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
82
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
83 runctx = PipelineJobRunContext(job, pp.record_name,
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
84 self.record_histories)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
85
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
86 ppres = PipelineJobResult()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
87 # For subsequent pass jobs, there will be a record entry given. For
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
88 # first pass jobs, there's none so we get the pipeline to create it.
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
89 ppres.record_entry = job.data.get('record_entry')
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
90 if ppres.record_entry is None:
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
91 ppres.record_entry = pp.createRecordEntry(job, runctx)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
92 pp.run(job, runctx, ppres)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
93 return ppres
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
94
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
95 def getStats(self):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
96 stats = self.app.env.stats
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
97 stats.stepTimerSince("BakeWorker_%d_Total" % self.wid,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
98 self._work_start_time)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
99 return stats
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
101 def shutdown(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
102 for src, pp in self._sources.values():
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
103 pp.shutdown(self._ppctx)
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
104