Mercurial > piecrust2

annotate piecrust/baking/worker.py @ 996:92a98c84a925

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cm: Fix crashes when page assets deployment fails. This can happen for instance if page assets are read-only files.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 May 2017 09:26:37 -0700
parents c3cb2f9df882
children 2e5c5d33d62c
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import time
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import logging
466
456db44dcc53 bake: Pass the config variants and values from the CLI to the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 455
diff changeset
3 from piecrust.app import PieCrust, apply_variant_and_values
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
4 from piecrust.baking.records import BakeRecord, _get_transition_key
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 from piecrust.baking.single import PageBaker, BakingError
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
6 from piecrust.environment import AbortedSourceUseError
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from piecrust.rendering import (
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 QualifiedPage, PageRenderingContext, render_page_segments)
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 426
diff changeset
9 from piecrust.routing import create_route_metadata
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 from piecrust.sources.base import PageFactory
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
11 from piecrust.workerpool import IWorker
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 logger = logging.getLogger(__name__)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 class BakeWorkerContext(object):
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
18 def __init__(self, appfactory, out_dir, *,
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
19 force=False, previous_record_path=None):
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
20 self.appfactory = appfactory
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 self.out_dir = out_dir
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
22 self.force = force
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
23 self.previous_record_path = previous_record_path
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
24 self.app = None
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
25 self.previous_record = None
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
26 self.previous_record_index = None
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
27
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
28
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
29 class BakeWorker(IWorker):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
30 def __init__(self, ctx):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
31 self.ctx = ctx
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
32 self.work_start_time = time.perf_counter()
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
33
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
34 def initialize(self):
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
35 # Create the app local to this worker.
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
36 app = self.ctx.appfactory.create()
471
5b57a189fd98 bug: Correctly setup the environment/app for bake workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 466
diff changeset
37 app.config.set('baker/is_baking', True)
476
27e3b3f05648 bake: Set the worker ID in the configuration. It's useful.
Ludovic Chabant <ludovic@chabant.com>
parents: 471
diff changeset
38 app.config.set('baker/worker_id', self.wid)
471
5b57a189fd98 bug: Correctly setup the environment/app for bake workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 466
diff changeset
39 app.env.base_asset_url_format = '%uri%'
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
40 app.env.fs_cache_only_for_main_page = True
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
41 app.env.registerTimer("BakeWorker_%d_Total" % self.wid)
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
42 app.env.registerTimer("BakeWorkerInit")
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
43 app.env.registerTimer("JobReceive")
696
3bc9f857eb48 bake: Add stat about aborted jobs
Ludovic Chabant <ludovic@chabant.com>
parents: 691
diff changeset
44 app.env.registerCounter("SourceUseAbortions")
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
45 app.env.registerManifest("LoadJobs")
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
46 app.env.registerManifest("RenderJobs")
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
47 app.env.registerManifest("BakeJobs")
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
48 self.ctx.app = app
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
49
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
50 # Load previous record
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
51 if self.ctx.previous_record_path:
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
52 self.ctx.previous_record = BakeRecord.load(
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
53 self.ctx.previous_record_path)
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
54 self.ctx.previous_record_index = {}
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
55 for e in self.ctx.previous_record.entries:
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
56 key = _get_transition_key(e.path, e.extra_key)
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
57 self.ctx.previous_record_index[key] = e
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
58
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
59 # Create the job handlers.
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
60 job_handlers = {
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
61 JOB_LOAD: LoadJobHandler(self.ctx),
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
62 JOB_RENDER_FIRST: RenderFirstSubJobHandler(self.ctx),
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
63 JOB_BAKE: BakeJobHandler(self.ctx)}
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
64 for jt, jh in job_handlers.items():
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
65 app.env.registerTimer(type(jh).__name__)
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
66 self.job_handlers = job_handlers
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
67
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
68 app.env.stepTimerSince("BakeWorkerInit", self.work_start_time)
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
69
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
70 def process(self, job):
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
71 handler = self.job_handlers[job['type']]
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
72 with self.ctx.app.env.timerScope(type(handler).__name__):
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
73 return handler.handleJob(job['job'])
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
74
691
9ae9390192da bake: Use standard pickle and queue for now to fix some small issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 690
diff changeset
75 def getReport(self, pool_reports):
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
76 self.ctx.app.env.stepTimerSince("BakeWorker_%d_Total" % self.wid,
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
77 self.work_start_time)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
78 data = self.ctx.app.env.getStats()
691
9ae9390192da bake: Use standard pickle and queue for now to fix some small issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 690
diff changeset
79 data.timers.update(pool_reports)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
80 return {
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
81 'type': 'stats',
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
82 'data': data}
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
84 def shutdown(self):
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
85 for jh in self.job_handlers.values():
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
86 jh.shutdown()
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
87
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE = range(0, 3)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92 class JobHandler(object):
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
93 def __init__(self, ctx):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 self.ctx = ctx
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
96 @property
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
97 def app(self):
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
98 return self.ctx.app
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
99
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 def handleJob(self, job):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 raise NotImplementedError()
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
103 def shutdown(self):
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
104 pass
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
105
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 def _get_errors(ex):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 errors = []
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109 while ex is not None:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 errors.append(str(ex))
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 ex = ex.__cause__
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112 return errors
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
115 def save_factory(fac):
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
116 return {
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
117 'source_name': fac.source.name,
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
118 'rel_path': fac.rel_path,
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
119 'metadata': fac.metadata}
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
122 def load_factory(app, info):
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
123 source = app.getSource(info['source_name'])
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
124 return PageFactory(source, info['rel_path'], info['metadata'])
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 class LoadJobHandler(JobHandler):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 def handleJob(self, job):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129 # Just make sure the page has been cached.
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
130 fac = load_factory(self.app, job)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 logger.debug("Loading page: %s" % fac.ref_spec)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
132 self.app.env.addManifestEntry('LoadJobs', fac.ref_spec)
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
133 result = {
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
134 'source_name': fac.source.name,
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
135 'path': fac.path,
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
136 'config': None,
721
234d0c7c02cf bake: Add the timestamp of the page to each record entry.
Ludovic Chabant <ludovic@chabant.com>
parents: 711
diff changeset
137 'timestamp': None,
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
138 'errors': None}
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
139 try:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140 page = fac.buildPage()
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141 page._load()
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
142 result['config'] = page.config.getAll()
721
234d0c7c02cf bake: Add the timestamp of the page to each record entry.
Ludovic Chabant <ludovic@chabant.com>
parents: 711
diff changeset
143 result['timestamp'] = page.datetime.timestamp()
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
144 except Exception as ex:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
145 logger.debug("Got loading error. Sending it to master.")
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
146 result['errors'] = _get_errors(ex)
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
147 if self.ctx.app.debug:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
148 logger.exception(ex)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
149 return result
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
150
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
151
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
152 class RenderFirstSubJobHandler(JobHandler):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
153 def handleJob(self, job):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
154 # Render the segments for the first sub-page of this page.
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
155 fac = load_factory(self.app, job['factory_info'])
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
156 self.app.env.addManifestEntry('RenderJobs', fac.ref_spec)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
157
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
158 route_index = job['route_index']
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
159 route = self.app.routes[route_index]
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161 page = fac.buildPage()
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 426
diff changeset
162 route_metadata = create_route_metadata(page)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163 qp = QualifiedPage(page, route, route_metadata)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164 ctx = PageRenderingContext(qp)
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
165 self.app.env.abort_source_use = True
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
167 result = {
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
168 'path': fac.path,
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
169 'aborted': False,
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
170 'errors': None}
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171 logger.debug("Preparing page: %s" % fac.ref_spec)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
172 try:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173 render_page_segments(ctx)
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
174 except AbortedSourceUseError:
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
175 logger.debug("Page %s was aborted." % fac.ref_spec)
696
3bc9f857eb48 bake: Add stat about aborted jobs
Ludovic Chabant <ludovic@chabant.com>
parents: 691
diff changeset
176 self.app.env.stepCounter("SourceUseAbortions")
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
177 result['aborted'] = True
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178 except Exception as ex:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179 logger.debug("Got rendering error. Sending it to master.")
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
180 result['errors'] = _get_errors(ex)
690
f7207f4dab82 bake: Fix a crash when a rendering error occurs.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
181 if self.ctx.app.debug:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
182 logger.exception(ex)
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
183 finally:
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
184 self.app.env.abort_source_use = False
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
185 return result
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
186
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
187
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
188 class BakeJobHandler(JobHandler):
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
189 def __init__(self, ctx):
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
190 super(BakeJobHandler, self).__init__(ctx)
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
191 self.page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
192
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
193 def shutdown(self):
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
194 self.page_baker.shutdown()
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 696
diff changeset
195
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196 def handleJob(self, job):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
197 # Actually bake the page and all its sub-pages to the output folder.
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
198 fac = load_factory(self.app, job['factory_info'])
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
199 self.app.env.addManifestEntry('BakeJobs', fac.ref_spec)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
200
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
201 route_index = job['route_index']
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
202 route_metadata = job['route_metadata']
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
203 route = self.app.routes[route_index]
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
204
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
205 gen_name = job['generator_name']
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
206 gen_key = job['generator_record_key']
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
207 dirty_source_names = job['dirty_source_names']
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
208
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 426
diff changeset
209 page = fac.buildPage()
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 426
diff changeset
210 qp = QualifiedPage(page, route, route_metadata)
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 426
diff changeset
211
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
212 result = {
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
213 'path': fac.path,
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
214 'generator_name': gen_name,
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
215 'generator_record_key': gen_key,
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
216 'sub_entries': None,
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
217 'errors': None}
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
218
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
219 if job.get('needs_config', False):
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
220 result['config'] = page.config.getAll()
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
221
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
222 previous_entry = None
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
223 if self.ctx.previous_record_index is not None:
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
224 key = _get_transition_key(fac.path, gen_key)
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
225 previous_entry = self.ctx.previous_record_index.get(key)
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
226
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
227 logger.debug("Baking page: %s" % fac.ref_spec)
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
228 logger.debug("With route metadata: %s" % route_metadata)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
229 try:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
230 sub_entries = self.page_baker.bake(
711
ab5c6a8ae90a bake: Replace hard-coded taxonomy support with "generator" system.
Ludovic Chabant <ludovic@chabant.com>
parents: 702
diff changeset
231 qp, previous_entry, dirty_source_names, gen_name)
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
232 result['sub_entries'] = sub_entries
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
233
813
c3cb2f9df882 bake: Don?t swallow generic errors during baking
Ben Artin <ben@artins.org>
parents: 721
diff changeset
234 except Exception as ex:
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
235 logger.debug("Got baking error. Sending it to master.")
451
838f3964f400 bake: Optimize the bake by not using custom classes for passing info.
Ludovic Chabant <ludovic@chabant.com>
parents: 447
diff changeset
236 result['errors'] = _get_errors(ex)
996
92a98c84a925 cm: Fix crashes when page assets deployment fails.
Ludovic Chabant <ludovic@chabant.com>
parents: 813
diff changeset
237 result['sub_entries'] = []
666
81d9c3a3a0b5 internal: Get rid of the whole "sub cache" business.
Ludovic Chabant <ludovic@chabant.com>
parents: 663
diff changeset
238 if self.ctx.app.debug:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
239 logger.exception(ex)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
240
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 441
diff changeset
241 return result
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
242