Mercurial > piecrust2

comparison piecrust/baking/worker.py @ 453:8351a77e13f5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Don't pass the previous record entries to the workers. Workers now load the previous record on their own and find the previous entry in their own copy.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 07 Jul 2015 20:19:54 -0700
parents 838f3964f400
children cb3446be44b7
comparison
equal deleted inserted replaced
452:55026b7bb1bf 453:8351a77e13f5
1 import time 1 import time
2 import logging 2 import logging
3 from piecrust.app import PieCrust 3 from piecrust.app import PieCrust
4 from piecrust.baking.records import BakeRecord, _get_transition_key
4 from piecrust.baking.single import PageBaker, BakingError 5 from piecrust.baking.single import PageBaker, BakingError
5 from piecrust.rendering import ( 6 from piecrust.rendering import (
6 QualifiedPage, PageRenderingContext, render_page_segments) 7 QualifiedPage, PageRenderingContext, render_page_segments)
7 from piecrust.routing import create_route_metadata 8 from piecrust.routing import create_route_metadata
8 from piecrust.sources.base import PageFactory 9 from piecrust.sources.base import PageFactory
12 logger = logging.getLogger(__name__) 13 logger = logging.getLogger(__name__)
13 14
14 15
15 class BakeWorkerContext(object): 16 class BakeWorkerContext(object):
16 def __init__(self, root_dir, sub_cache_dir, out_dir, 17 def __init__(self, root_dir, sub_cache_dir, out_dir,
18 previous_record_path=None,
17 force=False, debug=False): 19 force=False, debug=False):
18 self.root_dir = root_dir 20 self.root_dir = root_dir
19 self.sub_cache_dir = sub_cache_dir 21 self.sub_cache_dir = sub_cache_dir
20 self.out_dir = out_dir 22 self.out_dir = out_dir
23 self.previous_record_path = previous_record_path
21 self.force = force 24 self.force = force
22 self.debug = debug 25 self.debug = debug
26 self.app = None
27 self.previous_record = None
28 self.previous_record_index = None
23 29
24 30
25 class BakeWorker(IWorker): 31 class BakeWorker(IWorker):
26 def __init__(self, ctx): 32 def __init__(self, ctx):
27 self.ctx = ctx 33 self.ctx = ctx
33 app._useSubCacheDir(self.ctx.sub_cache_dir) 39 app._useSubCacheDir(self.ctx.sub_cache_dir)
34 app.env.fs_cache_only_for_main_page = True 40 app.env.fs_cache_only_for_main_page = True
35 app.env.registerTimer("BakeWorker_%d_Total" % self.wid) 41 app.env.registerTimer("BakeWorker_%d_Total" % self.wid)
36 app.env.registerTimer("BakeWorkerInit") 42 app.env.registerTimer("BakeWorkerInit")
37 app.env.registerTimer("JobReceive") 43 app.env.registerTimer("JobReceive")
38 self.app = app 44 self.ctx.app = app
45
46 # Load previous record
47 if self.ctx.previous_record_path:
48 self.ctx.previous_record = BakeRecord.load(
49 self.ctx.previous_record_path)
50 self.ctx.previous_record_index = {}
51 for e in self.ctx.previous_record.entries:
52 key = _get_transition_key(e.path, e.taxonomy_info)
53 self.ctx.previous_record_index[key] = e
39 54
40 # Create the job handlers. 55 # Create the job handlers.
41 job_handlers = { 56 job_handlers = {
42 JOB_LOAD: LoadJobHandler(app, self.ctx), 57 JOB_LOAD: LoadJobHandler(self.ctx),
43 JOB_RENDER_FIRST: RenderFirstSubJobHandler(app, self.ctx), 58 JOB_RENDER_FIRST: RenderFirstSubJobHandler(self.ctx),
44 JOB_BAKE: BakeJobHandler(app, self.ctx)} 59 JOB_BAKE: BakeJobHandler(self.ctx)}
45 for jt, jh in job_handlers.items(): 60 for jt, jh in job_handlers.items():
46 app.env.registerTimer(type(jh).__name__) 61 app.env.registerTimer(type(jh).__name__)
47 self.job_handlers = job_handlers 62 self.job_handlers = job_handlers
48 63
49 app.env.stepTimerSince("BakeWorkerInit", self.work_start_time) 64 app.env.stepTimerSince("BakeWorkerInit", self.work_start_time)
50 65
51 def process(self, job): 66 def process(self, job):
52 handler = self.job_handlers[job['type']] 67 handler = self.job_handlers[job['type']]
53 with self.app.env.timerScope(type(handler).__name__): 68 with self.ctx.app.env.timerScope(type(handler).__name__):
54 return handler.handleJob(job['job']) 69 return handler.handleJob(job['job'])
55 70
56 def getReport(self): 71 def getReport(self):
57 self.app.env.stepTimerSince("BakeWorker_%d_Total" % self.wid, 72 self.ctx.app.env.stepTimerSince("BakeWorker_%d_Total" % self.wid,
58 self.work_start_time) 73 self.work_start_time)
59 return { 74 return {
60 'type': 'timers', 75 'type': 'timers',
61 'data': self.app.env._timers} 76 'data': self.ctx.app.env._timers}
62 77
63 78
64 JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE = range(0, 3) 79 JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE = range(0, 3)
65 80
66 81
67 class JobHandler(object): 82 class JobHandler(object):
68 def __init__(self, app, ctx): 83 def __init__(self, ctx):
69 self.app = app
70 self.ctx = ctx 84 self.ctx = ctx
85
86 @property
87 def app(self):
88 return self.ctx.app
71 89
72 def handleJob(self, job): 90 def handleJob(self, job):
73 raise NotImplementedError() 91 raise NotImplementedError()
74 92
75 93
143 logger.exception(ex) 161 logger.exception(ex)
144 return result 162 return result
145 163
146 164
147 class BakeJobHandler(JobHandler): 165 class BakeJobHandler(JobHandler):
148 def __init__(self, app, ctx): 166 def __init__(self, ctx):
149 super(BakeJobHandler, self).__init__(app, ctx) 167 super(BakeJobHandler, self).__init__(ctx)
150 self.page_baker = PageBaker(app, ctx.out_dir, ctx.force) 168 self.page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force)
151 169
152 def handleJob(self, job): 170 def handleJob(self, job):
153 # Actually bake the page and all its sub-pages to the output folder. 171 # Actually bake the page and all its sub-pages to the output folder.
154 fac = load_factory(self.app, job['factory_info']) 172 fac = load_factory(self.app, job['factory_info'])
155 173
169 result = { 187 result = {
170 'path': fac.path, 188 'path': fac.path,
171 'taxonomy_info': tax_info, 189 'taxonomy_info': tax_info,
172 'sub_entries': None, 190 'sub_entries': None,
173 'errors': None} 191 'errors': None}
174 previous_entry = job['prev_entry']
175 dirty_source_names = job['dirty_source_names'] 192 dirty_source_names = job['dirty_source_names']
193
194 previous_entry = None
195 if self.ctx.previous_record_index is not None:
196 key = _get_transition_key(fac.path, tax_info)
197 previous_entry = self.ctx.previous_record_index.get(key)
198
176 logger.debug("Baking page: %s" % fac.ref_spec) 199 logger.debug("Baking page: %s" % fac.ref_spec)
177 try: 200 try:
178 sub_entries = self.page_baker.bake( 201 sub_entries = self.page_baker.bake(
179 qp, previous_entry, dirty_source_names, tax_info) 202 qp, previous_entry, dirty_source_names, tax_info)
180 result['sub_entries'] = sub_entries 203 result['sub_entries'] = sub_entries