Mercurial > piecrust2

comparison piecrust/baking/worker.py @ 415:0e9a94b7fdfa

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Improve bake record information. * Store things in the bake record that require less interaction between the master process and the workers. For instance, don't store the paginator object in the render pass info -- instead, just store whether pagination was used, and whether it had more items. * Simplify information passing between workers and bake passes by saving the rendering info to the JSON cache. This means the "render first sub" job doesn't have to return anything except errors now. * Add more performance counter info.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:23:16 -0700
parents e7b865f8f335
children 4a43d7015b75
comparison
equal deleted inserted replaced
414:c4b3a7fd2f87 415:0e9a94b7fdfa
62 JOB_BAKE: BakeJobHandler(app, self.ctx)} 62 JOB_BAKE: BakeJobHandler(app, self.ctx)}
63 for jt, jh in job_handlers.items(): 63 for jt, jh in job_handlers.items():
64 app.env.registerTimer(type(jh).__name__) 64 app.env.registerTimer(type(jh).__name__)
65 65
66 # Start working! 66 # Start working!
67 aborted_with_exception = None
67 while not self.ctx.abort_event.is_set(): 68 while not self.ctx.abort_event.is_set():
68 try: 69 try:
69 with app.env.timerScope('JobReceive'): 70 with app.env.timerScope('JobReceive'):
70 job = self.ctx.work_queue.get(True, 0.01) 71 job = self.ctx.work_queue.get(True, 0.01)
71 except queue.Empty: 72 except queue.Empty:
75 handler = job_handlers[job.job_type] 76 handler = job_handlers[job.job_type]
76 with app.env.timerScope(type(handler).__name__): 77 with app.env.timerScope(type(handler).__name__):
77 handler.handleJob(job) 78 handler.handleJob(job)
78 except Exception as ex: 79 except Exception as ex:
79 self.ctx.abort_event.set() 80 self.ctx.abort_event.set()
80 self.abort_exception = ex 81 aborted_with_exception = ex
81 self.success = False
82 logger.debug("[%d] Critical error, aborting." % self.wid) 82 logger.debug("[%d] Critical error, aborting." % self.wid)
83 if self.ctx.app.debug: 83 if self.ctx.debug:
84 logger.exception(ex) 84 logger.exception(ex)
85 break 85 break
86 finally: 86 finally:
87 self.ctx.work_queue.task_done() 87 self.ctx.work_queue.task_done()
88 88
89 if aborted_with_exception is not None:
90 msgs = _get_errors(aborted_with_exception)
91 self.ctx.results.put_nowait({'type': 'error', 'messages': msgs})
92
89 # Send our timers to the main process before exiting. 93 # Send our timers to the main process before exiting.
90 app.env.stepTimer("Worker_%d" % self.wid, 94 app.env.stepTimer("Worker_%d" % self.wid,
91 time.perf_counter() - work_start_time) 95 time.perf_counter() - work_start_time)
92 self.ctx.results.put_nowait(app.env._timers) 96 self.ctx.results.put_nowait({
97 'type': 'timers', 'data': app.env._timers})
93 98
94 99
95 class JobHandler(object): 100 class JobHandler(object):
96 def __init__(self, app, ctx): 101 def __init__(self, app, ctx):
97 self.app = app 102 self.app = app
139 144
140 145
141 class RenderFirstSubJobResult(object): 146 class RenderFirstSubJobResult(object):
142 def __init__(self, path): 147 def __init__(self, path):
143 self.path = path 148 self.path = path
144 self.used_assets = None
145 self.used_pagination = None
146 self.pagination_has_more = False
147 self.errors = None 149 self.errors = None
148 150
149 151
150 class BakeJobPayload(object): 152 class BakeJobPayload(object):
151 def __init__(self, fac, route_metadata, previous_entry, 153 def __init__(self, fac, route_metadata, previous_entry,
152 first_render_info, dirty_source_names, tax_info=None): 154 dirty_source_names, tax_info=None):
153 self.factory_info = PageFactoryInfo(fac) 155 self.factory_info = PageFactoryInfo(fac)
154 self.route_metadata = route_metadata 156 self.route_metadata = route_metadata
155 self.previous_entry = previous_entry 157 self.previous_entry = previous_entry
156 self.dirty_source_names = dirty_source_names 158 self.dirty_source_names = dirty_source_names
157 self.first_render_info = first_render_info
158 self.taxonomy_info = tax_info 159 self.taxonomy_info = tax_info
159 160
160 161
161 class BakeJobResult(object): 162 class BakeJobResult(object):
162 def __init__(self, path, tax_info=None): 163 def __init__(self, path, tax_info=None):
163 self.path = path 164 self.path = path
164 self.taxonomy_info = tax_info 165 self.taxonomy_info = tax_info
165 self.bake_info = None 166 self.sub_entries = None
166 self.errors = None 167 self.errors = None
167 168
168 169
169 class LoadJobHandler(JobHandler): 170 class LoadJobHandler(JobHandler):
170 def handleJob(self, job): 171 def handleJob(self, job):
175 try: 176 try:
176 page = fac.buildPage() 177 page = fac.buildPage()
177 page._load() 178 page._load()
178 result.config = page.config.get() 179 result.config = page.config.get()
179 except Exception as ex: 180 except Exception as ex:
181 logger.debug("Got loading error. Sending it to master.")
180 result.errors = _get_errors(ex) 182 result.errors = _get_errors(ex)
183 if self.ctx.debug:
184 logger.exception(ex)
181 185
182 self.ctx.results.put_nowait(result) 186 self.ctx.results.put_nowait(result)
183 187
184 188
185 class RenderFirstSubJobHandler(JobHandler): 189 class RenderFirstSubJobHandler(JobHandler):
199 203
200 result = RenderFirstSubJobResult(fac.path) 204 result = RenderFirstSubJobResult(fac.path)
201 logger.debug("Preparing page: %s" % fac.ref_spec) 205 logger.debug("Preparing page: %s" % fac.ref_spec)
202 try: 206 try:
203 render_page_segments(ctx) 207 render_page_segments(ctx)
204 result.used_assets = ctx.used_assets
205 result.used_pagination = ctx.used_pagination is not None
206 if result.used_pagination:
207 result.pagination_has_more = ctx.used_pagination.has_more
208 except Exception as ex: 208 except Exception as ex:
209 logger.debug("Got rendering error. Sending it to master.") 209 logger.debug("Got rendering error. Sending it to master.")
210 result.errors = _get_errors(ex) 210 result.errors = _get_errors(ex)
211 if self.ctx.debug:
212 logger.exception(ex)
211 213
212 self.ctx.results.put_nowait(result) 214 self.ctx.results.put_nowait(result)
213 215
214 216
215 class BakeJobHandler(JobHandler): 217 class BakeJobHandler(JobHandler):
231 skip_taxonomies=True) 233 skip_taxonomies=True)
232 assert route is not None 234 assert route is not None
233 235
234 result = BakeJobResult(fac.path, tax_info) 236 result = BakeJobResult(fac.path, tax_info)
235 previous_entry = job.payload.previous_entry 237 previous_entry = job.payload.previous_entry
236 first_render_info = job.payload.first_render_info
237 dirty_source_names = job.payload.dirty_source_names 238 dirty_source_names = job.payload.dirty_source_names
238 logger.debug("Baking page: %s" % fac.ref_spec) 239 logger.debug("Baking page: %s" % fac.ref_spec)
239 try: 240 try:
240 report = self.page_baker.bake(fac, route, route_metadata, 241 sub_entries = self.page_baker.bake(
241 previous_entry, first_render_info, 242 fac, route, route_metadata, previous_entry,
242 dirty_source_names, tax_info) 243 dirty_source_names, tax_info)
243 result.bake_info = report 244 result.sub_entries = sub_entries
244 245
245 except BakingError as ex: 246 except BakingError as ex:
246 logger.debug("Got baking error. Sending it to master.") 247 logger.debug("Got baking error. Sending it to master.")
247 result.errors = _get_errors(ex) 248 result.errors = _get_errors(ex)
249 if self.ctx.debug:
250 logger.exception(ex)
248 251
249 self.ctx.results.put_nowait(result) 252 self.ctx.results.put_nowait(result)
250 253