Mercurial > piecrust2
annotate piecrust/baking/worker.py @ 415:0e9a94b7fdfa
bake: Improve bake record information.
* Store things in the bake record that require less interaction between the
master process and the workers. For instance, don't store the paginator
object in the render pass info -- instead, just store whether pagination
was used, and whether it had more items.
* Simplify information passing between workers and bake passes by saving the
rendering info to the JSON cache. This means the "render first sub" job
doesn't have to return anything except errors now.
* Add more performance counter info.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 20 Jun 2015 19:23:16 -0700 |
parents | e7b865f8f335 |
children | 4a43d7015b75 |
rev | line source |
---|---|
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import time |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import copy |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 import queue |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 import logging |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 from piecrust.app import PieCrust |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 from piecrust.baking.single import PageBaker, BakingError |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 from piecrust.rendering import ( |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 QualifiedPage, PageRenderingContext, render_page_segments) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 from piecrust.sources.base import PageFactory |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 logger = logging.getLogger(__name__) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
13 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
14 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
15 def worker_func(wid, ctx): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
16 logger.debug("Worker %d booting up..." % wid) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 w = BakeWorker(wid, ctx) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 w.run() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 class BakeWorkerContext(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 def __init__(self, root_dir, out_dir, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 work_queue, results, abort_event, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 force=False, debug=False): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 self.root_dir = root_dir |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 self.out_dir = out_dir |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 self.work_queue = work_queue |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 self.results = results |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
29 self.abort_event = abort_event |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 self.force = force |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 self.debug = debug |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE = range(0, 3) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 class BakeWorkerJob(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 def __init__(self, job_type, payload): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
39 self.job_type = job_type |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
40 self.payload = payload |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
41 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
43 class BakeWorker(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
44 def __init__(self, wid, ctx): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 self.wid = wid |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 self.ctx = ctx |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 def run(self): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 logger.debug("Working %d initializing..." % self.wid) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 work_start_time = time.perf_counter() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 # Create the app local to this worker. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 app = PieCrust(self.ctx.root_dir, debug=self.ctx.debug) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 app.env.fs_cache_only_for_main_page = True |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 app.env.registerTimer("Worker_%d" % self.wid) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 app.env.registerTimer("JobReceive") |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
57 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 # Create the job handlers. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
59 job_handlers = { |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
60 JOB_LOAD: LoadJobHandler(app, self.ctx), |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
61 JOB_RENDER_FIRST: RenderFirstSubJobHandler(app, self.ctx), |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
62 JOB_BAKE: BakeJobHandler(app, self.ctx)} |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
63 for jt, jh in job_handlers.items(): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
64 app.env.registerTimer(type(jh).__name__) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
65 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
66 # Start working! |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
67 aborted_with_exception = None |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
68 while not self.ctx.abort_event.is_set(): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 with app.env.timerScope('JobReceive'): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 job = self.ctx.work_queue.get(True, 0.01) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 except queue.Empty: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 continue |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 handler = job_handlers[job.job_type] |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 with app.env.timerScope(type(handler).__name__): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
78 handler.handleJob(job) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
79 except Exception as ex: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
80 self.ctx.abort_event.set() |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
81 aborted_with_exception = ex |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 logger.debug("[%d] Critical error, aborting." % self.wid) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
83 if self.ctx.debug: |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
84 logger.exception(ex) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
85 break |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
86 finally: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
87 self.ctx.work_queue.task_done() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
88 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
89 if aborted_with_exception is not None: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
90 msgs = _get_errors(aborted_with_exception) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
91 self.ctx.results.put_nowait({'type': 'error', 'messages': msgs}) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
92 |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
93 # Send our timers to the main process before exiting. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
94 app.env.stepTimer("Worker_%d" % self.wid, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
95 time.perf_counter() - work_start_time) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
96 self.ctx.results.put_nowait({ |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
97 'type': 'timers', 'data': app.env._timers}) |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
98 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
99 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
100 class JobHandler(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
101 def __init__(self, app, ctx): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
102 self.app = app |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
103 self.ctx = ctx |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
104 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
105 def handleJob(self, job): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
106 raise NotImplementedError() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
107 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
108 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
109 def _get_errors(ex): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
110 errors = [] |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
111 while ex is not None: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
112 errors.append(str(ex)) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
113 ex = ex.__cause__ |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
114 return errors |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
115 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
116 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
117 class PageFactoryInfo(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
118 def __init__(self, fac): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
119 self.source_name = fac.source.name |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
120 self.rel_path = fac.rel_path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
121 self.metadata = fac.metadata |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
122 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
123 def build(self, app): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
124 source = app.getSource(self.source_name) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
125 return PageFactory(source, self.rel_path, self.metadata) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
126 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
127 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
128 class LoadJobPayload(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
129 def __init__(self, fac): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
130 self.factory_info = PageFactoryInfo(fac) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
131 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
132 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
133 class LoadJobResult(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
134 def __init__(self, source_name, path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
135 self.source_name = source_name |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
136 self.path = path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
137 self.config = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
138 self.errors = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
139 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
140 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
141 class RenderFirstSubJobPayload(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
142 def __init__(self, fac): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
143 self.factory_info = PageFactoryInfo(fac) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
144 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
145 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
146 class RenderFirstSubJobResult(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
147 def __init__(self, path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
148 self.path = path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
149 self.errors = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
150 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
151 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
152 class BakeJobPayload(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
153 def __init__(self, fac, route_metadata, previous_entry, |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
154 dirty_source_names, tax_info=None): |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
155 self.factory_info = PageFactoryInfo(fac) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
156 self.route_metadata = route_metadata |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
157 self.previous_entry = previous_entry |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
158 self.dirty_source_names = dirty_source_names |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
159 self.taxonomy_info = tax_info |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
160 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
161 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
162 class BakeJobResult(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
163 def __init__(self, path, tax_info=None): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
164 self.path = path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
165 self.taxonomy_info = tax_info |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
166 self.sub_entries = None |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
167 self.errors = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
168 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
169 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
170 class LoadJobHandler(JobHandler): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
171 def handleJob(self, job): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
172 # Just make sure the page has been cached. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
173 fac = job.payload.factory_info.build(self.app) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
174 logger.debug("Loading page: %s" % fac.ref_spec) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
175 result = LoadJobResult(fac.source.name, fac.path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
176 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
177 page = fac.buildPage() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
178 page._load() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
179 result.config = page.config.get() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
180 except Exception as ex: |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
181 logger.debug("Got loading error. Sending it to master.") |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
182 result.errors = _get_errors(ex) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
183 if self.ctx.debug: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
184 logger.exception(ex) |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
185 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
186 self.ctx.results.put_nowait(result) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
187 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
188 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
189 class RenderFirstSubJobHandler(JobHandler): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
190 def handleJob(self, job): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
191 # Render the segments for the first sub-page of this page. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
192 fac = job.payload.factory_info.build(self.app) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
193 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
194 # These things should be OK as they're checked upstream by the baker. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
195 route = self.app.getRoute(fac.source.name, fac.metadata, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
196 skip_taxonomies=True) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
197 assert route is not None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
198 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
199 page = fac.buildPage() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
200 route_metadata = copy.deepcopy(fac.metadata) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
201 qp = QualifiedPage(page, route, route_metadata) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
202 ctx = PageRenderingContext(qp) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
203 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
204 result = RenderFirstSubJobResult(fac.path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
205 logger.debug("Preparing page: %s" % fac.ref_spec) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
206 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
207 render_page_segments(ctx) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
208 except Exception as ex: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
209 logger.debug("Got rendering error. Sending it to master.") |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
210 result.errors = _get_errors(ex) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
211 if self.ctx.debug: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
212 logger.exception(ex) |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
213 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
214 self.ctx.results.put_nowait(result) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
215 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
216 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
217 class BakeJobHandler(JobHandler): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
218 def __init__(self, app, ctx): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
219 super(BakeJobHandler, self).__init__(app, ctx) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
220 self.page_baker = PageBaker(app, ctx.out_dir, ctx.force) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
221 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
222 def handleJob(self, job): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
223 # Actually bake the page and all its sub-pages to the output folder. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
224 fac = job.payload.factory_info.build(self.app) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
225 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
226 route_metadata = job.payload.route_metadata |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
227 tax_info = job.payload.taxonomy_info |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
228 if tax_info is not None: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
229 route = self.app.getTaxonomyRoute(tax_info.taxonomy_name, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
230 tax_info.source_name) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
231 else: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
232 route = self.app.getRoute(fac.source.name, route_metadata, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
233 skip_taxonomies=True) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
234 assert route is not None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
235 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
236 result = BakeJobResult(fac.path, tax_info) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
237 previous_entry = job.payload.previous_entry |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
238 dirty_source_names = job.payload.dirty_source_names |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
239 logger.debug("Baking page: %s" % fac.ref_spec) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
240 try: |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
241 sub_entries = self.page_baker.bake( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
242 fac, route, route_metadata, previous_entry, |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
243 dirty_source_names, tax_info) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
244 result.sub_entries = sub_entries |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
245 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
246 except BakingError as ex: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
247 logger.debug("Got baking error. Sending it to master.") |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
248 result.errors = _get_errors(ex) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
249 if self.ctx.debug: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
250 logger.exception(ex) |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
251 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
252 self.ctx.results.put_nowait(result) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
253 |