Mercurial > piecrust2
annotate piecrust/baking/worker.py @ 411:e7b865f8f335
bake: Enable multiprocess baking.
Baking is now done by running a worker per CPU, and sending jobs to them.
This changes several things across the codebase:
* Ability to not cache things related to pages other than the 'main' page
(i.e. the page at the bottom of the execution stack).
* Decouple the baking process from the bake records, so only the main process
keeps track (and modifies) the bake record.
* Remove the need for 'batch page getters' and loading a page directly from
the page factories.
There are various smaller changes too included here, including support for
scope performance timers that are saved with the bake record and can be
printed out to the console. Yes I got carried away.
For testing, the in-memory 'mock' file-system doesn't work anymore, since
we're spawning processes, so this is replaced by a 'tmpfs' file-system which
is saved in temporary files on disk and deleted after tests have run.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Fri, 12 Jun 2015 17:09:19 -0700 |
parents | |
children | 0e9a94b7fdfa |
rev | line source |
---|---|
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import time |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import copy |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 import queue |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 import logging |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 from piecrust.app import PieCrust |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 from piecrust.baking.single import PageBaker, BakingError |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 from piecrust.rendering import ( |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 QualifiedPage, PageRenderingContext, render_page_segments) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 from piecrust.sources.base import PageFactory |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 logger = logging.getLogger(__name__) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
13 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
14 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
15 def worker_func(wid, ctx): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
16 logger.debug("Worker %d booting up..." % wid) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 w = BakeWorker(wid, ctx) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 w.run() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 class BakeWorkerContext(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 def __init__(self, root_dir, out_dir, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 work_queue, results, abort_event, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 force=False, debug=False): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 self.root_dir = root_dir |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 self.out_dir = out_dir |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 self.work_queue = work_queue |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 self.results = results |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
29 self.abort_event = abort_event |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 self.force = force |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 self.debug = debug |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE = range(0, 3) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 class BakeWorkerJob(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 def __init__(self, job_type, payload): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
39 self.job_type = job_type |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
40 self.payload = payload |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
41 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
43 class BakeWorker(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
44 def __init__(self, wid, ctx): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 self.wid = wid |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 self.ctx = ctx |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 def run(self): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 logger.debug("Working %d initializing..." % self.wid) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 work_start_time = time.perf_counter() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 # Create the app local to this worker. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 app = PieCrust(self.ctx.root_dir, debug=self.ctx.debug) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 app.env.fs_cache_only_for_main_page = True |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 app.env.registerTimer("Worker_%d" % self.wid) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 app.env.registerTimer("JobReceive") |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
57 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 # Create the job handlers. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
59 job_handlers = { |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
60 JOB_LOAD: LoadJobHandler(app, self.ctx), |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
61 JOB_RENDER_FIRST: RenderFirstSubJobHandler(app, self.ctx), |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
62 JOB_BAKE: BakeJobHandler(app, self.ctx)} |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
63 for jt, jh in job_handlers.items(): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
64 app.env.registerTimer(type(jh).__name__) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
65 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
66 # Start working! |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
67 while not self.ctx.abort_event.is_set(): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
68 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 with app.env.timerScope('JobReceive'): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 job = self.ctx.work_queue.get(True, 0.01) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 except queue.Empty: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 continue |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 handler = job_handlers[job.job_type] |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 with app.env.timerScope(type(handler).__name__): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 handler.handleJob(job) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
78 except Exception as ex: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
79 self.ctx.abort_event.set() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
80 self.abort_exception = ex |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
81 self.success = False |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 logger.debug("[%d] Critical error, aborting." % self.wid) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
83 if self.ctx.app.debug: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
84 logger.exception(ex) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
85 break |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
86 finally: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
87 self.ctx.work_queue.task_done() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
88 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
89 # Send our timers to the main process before exiting. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
90 app.env.stepTimer("Worker_%d" % self.wid, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
91 time.perf_counter() - work_start_time) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
92 self.ctx.results.put_nowait(app.env._timers) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
93 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
94 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
95 class JobHandler(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
96 def __init__(self, app, ctx): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
97 self.app = app |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
98 self.ctx = ctx |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
99 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
100 def handleJob(self, job): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
101 raise NotImplementedError() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
102 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
103 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
104 def _get_errors(ex): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
105 errors = [] |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
106 while ex is not None: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
107 errors.append(str(ex)) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
108 ex = ex.__cause__ |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
109 return errors |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
110 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
111 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
112 class PageFactoryInfo(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
113 def __init__(self, fac): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
114 self.source_name = fac.source.name |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
115 self.rel_path = fac.rel_path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
116 self.metadata = fac.metadata |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
117 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
118 def build(self, app): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
119 source = app.getSource(self.source_name) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
120 return PageFactory(source, self.rel_path, self.metadata) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
121 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
122 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
123 class LoadJobPayload(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
124 def __init__(self, fac): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
125 self.factory_info = PageFactoryInfo(fac) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
126 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
127 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
128 class LoadJobResult(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
129 def __init__(self, source_name, path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
130 self.source_name = source_name |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
131 self.path = path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
132 self.config = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
133 self.errors = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
134 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
135 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
136 class RenderFirstSubJobPayload(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
137 def __init__(self, fac): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
138 self.factory_info = PageFactoryInfo(fac) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
139 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
140 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
141 class RenderFirstSubJobResult(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
142 def __init__(self, path): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
143 self.path = path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
144 self.used_assets = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
145 self.used_pagination = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
146 self.pagination_has_more = False |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
147 self.errors = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
148 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
149 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
150 class BakeJobPayload(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
151 def __init__(self, fac, route_metadata, previous_entry, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
152 first_render_info, dirty_source_names, tax_info=None): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
153 self.factory_info = PageFactoryInfo(fac) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
154 self.route_metadata = route_metadata |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
155 self.previous_entry = previous_entry |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
156 self.dirty_source_names = dirty_source_names |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
157 self.first_render_info = first_render_info |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
158 self.taxonomy_info = tax_info |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
159 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
160 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
161 class BakeJobResult(object): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
162 def __init__(self, path, tax_info=None): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
163 self.path = path |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
164 self.taxonomy_info = tax_info |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
165 self.bake_info = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
166 self.errors = None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
167 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
168 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
169 class LoadJobHandler(JobHandler): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
170 def handleJob(self, job): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
171 # Just make sure the page has been cached. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
172 fac = job.payload.factory_info.build(self.app) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
173 logger.debug("Loading page: %s" % fac.ref_spec) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
174 result = LoadJobResult(fac.source.name, fac.path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
175 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
176 page = fac.buildPage() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
177 page._load() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
178 result.config = page.config.get() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
179 except Exception as ex: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
180 result.errors = _get_errors(ex) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
181 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
182 self.ctx.results.put_nowait(result) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
183 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
184 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
185 class RenderFirstSubJobHandler(JobHandler): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
186 def handleJob(self, job): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
187 # Render the segments for the first sub-page of this page. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
188 fac = job.payload.factory_info.build(self.app) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
189 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
190 # These things should be OK as they're checked upstream by the baker. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
191 route = self.app.getRoute(fac.source.name, fac.metadata, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
192 skip_taxonomies=True) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
193 assert route is not None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
194 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
195 page = fac.buildPage() |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
196 route_metadata = copy.deepcopy(fac.metadata) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
197 qp = QualifiedPage(page, route, route_metadata) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
198 ctx = PageRenderingContext(qp) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
199 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
200 result = RenderFirstSubJobResult(fac.path) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
201 logger.debug("Preparing page: %s" % fac.ref_spec) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
202 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
203 render_page_segments(ctx) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
204 result.used_assets = ctx.used_assets |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
205 result.used_pagination = ctx.used_pagination is not None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
206 if result.used_pagination: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
207 result.pagination_has_more = ctx.used_pagination.has_more |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
208 except Exception as ex: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
209 logger.debug("Got rendering error. Sending it to master.") |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
210 result.errors = _get_errors(ex) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
211 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
212 self.ctx.results.put_nowait(result) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
213 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
214 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
215 class BakeJobHandler(JobHandler): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
216 def __init__(self, app, ctx): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
217 super(BakeJobHandler, self).__init__(app, ctx) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
218 self.page_baker = PageBaker(app, ctx.out_dir, ctx.force) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
219 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
220 def handleJob(self, job): |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
221 # Actually bake the page and all its sub-pages to the output folder. |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
222 fac = job.payload.factory_info.build(self.app) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
223 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
224 route_metadata = job.payload.route_metadata |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
225 tax_info = job.payload.taxonomy_info |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
226 if tax_info is not None: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
227 route = self.app.getTaxonomyRoute(tax_info.taxonomy_name, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
228 tax_info.source_name) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
229 else: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
230 route = self.app.getRoute(fac.source.name, route_metadata, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
231 skip_taxonomies=True) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
232 assert route is not None |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
233 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
234 result = BakeJobResult(fac.path, tax_info) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
235 previous_entry = job.payload.previous_entry |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
236 first_render_info = job.payload.first_render_info |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
237 dirty_source_names = job.payload.dirty_source_names |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
238 logger.debug("Baking page: %s" % fac.ref_spec) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
239 try: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
240 report = self.page_baker.bake(fac, route, route_metadata, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
241 previous_entry, first_render_info, |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
242 dirty_source_names, tax_info) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
243 result.bake_info = report |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
244 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
245 except BakingError as ex: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
246 logger.debug("Got baking error. Sending it to master.") |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
247 result.errors = _get_errors(ex) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
248 |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
249 self.ctx.results.put_nowait(result) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
250 |