Mercurial > piecrust2

comparison piecrust/baking/baker.py @ 217:1f4c3dae1fe8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Better error handling for site baking. The site baker now keeps track of whether any worker saw any error. It returns the current record after a bake. The `bake` command uses this return value to figure out what kind of exit code to return.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 31 Jan 2015 17:35:36 -0800
parents e725af1d48fb
children 65e6d72f3877
comparison
equal deleted inserted replaced
216:c5ada46b281a 217:1f4c3dae1fe8
59 t = time.clock() 59 t = time.clock()
60 record.loadPrevious(record_cache.getCachePath(record_name)) 60 record.loadPrevious(record_cache.getCachePath(record_name))
61 logger.debug(format_timed( 61 logger.debug(format_timed(
62 t, 'loaded previous bake record', 62 t, 'loaded previous bake record',
63 colored=False)) 63 colored=False))
64 record.current.success = True
64 65
65 # Figure out if we need to clean the cache because important things 66 # Figure out if we need to clean the cache because important things
66 # have changed. 67 # have changed.
67 self._handleCacheValidity(record) 68 self._handleCacheValidity(record)
68 69
96 logger.debug(format_timed(t, 'saved bake record', colored=False)) 97 logger.debug(format_timed(t, 'saved bake record', colored=False))
97 98
98 # All done. 99 # All done.
99 self.app.config.set('baker/is_baking', False) 100 self.app.config.set('baker/is_baking', False)
100 logger.debug(format_timed(start_time, 'done baking')) 101 logger.debug(format_timed(start_time, 'done baking'))
102
103 return record.detach()
101 104
102 def _handleCacheValidity(self, record): 105 def _handleCacheValidity(self, record):
103 start_time = time.clock() 106 start_time = time.clock()
104 107
105 reason = None 108 reason = None
171 logger.error(entry.errors[-1]) 174 logger.error(entry.errors[-1])
172 continue 175 continue
173 176
174 queue.addJob(BakeWorkerJob(fac, route, entry)) 177 queue.addJob(BakeWorkerJob(fac, route, entry))
175 178
176 self._waitOnWorkerPool(pool, abort) 179 success = self._waitOnWorkerPool(pool, abort)
180 record.current.success &= success
177 181
178 def _bakeTaxonomies(self, record): 182 def _bakeTaxonomies(self, record):
179 logger.debug("Baking taxonomies") 183 logger.debug("Baking taxonomies")
180 184
181 # Let's see all the taxonomy terms for which we must bake a 185 # Let's see all the taxonomy terms for which we must bake a
268 entry = BakeRecordPageEntry(fac, tax_name, term) 272 entry = BakeRecordPageEntry(fac, tax_name, term)
269 record.addEntry(entry) 273 record.addEntry(entry)
270 queue.addJob( 274 queue.addJob(
271 BakeWorkerJob(fac, route, entry, tax_name, term)) 275 BakeWorkerJob(fac, route, entry, tax_name, term))
272 276
273 self._waitOnWorkerPool(pool, abort) 277 success = self._waitOnWorkerPool(pool, abort)
278 record.current.success &= success
274 279
275 def _handleDeletetions(self, record): 280 def _handleDeletetions(self, record):
276 for path, reason in record.getDeletions(): 281 for path, reason in record.getDeletions():
277 logger.debug("Removing '%s': %s" % (path, reason)) 282 logger.debug("Removing '%s': %s" % (path, reason))
278 try: 283 try:
297 302
298 def _waitOnWorkerPool(self, pool, abort): 303 def _waitOnWorkerPool(self, pool, abort):
299 for w in pool: 304 for w in pool:
300 w.start() 305 w.start()
301 306
307 success = True
302 try: 308 try:
303 for w in pool: 309 for w in pool:
304 w.join() 310 w.join()
311 success &= w.success
305 except KeyboardInterrupt: 312 except KeyboardInterrupt:
306 logger.warning("Bake aborted by user... " 313 logger.warning("Bake aborted by user... "
307 "waiting for workers to stop.") 314 "waiting for workers to stop.")
308 abort.set() 315 abort.set()
309 for w in pool: 316 for w in pool:
320 else: 327 else:
321 for e in excs: 328 for e in excs:
322 log_friendly_exception(logger, e) 329 log_friendly_exception(logger, e)
323 raise BakingError("Baking was aborted due to errors.") 330 raise BakingError("Baking was aborted due to errors.")
324 331
332 return success
333
325 334
326 class BakeWorkerContext(object): 335 class BakeWorkerContext(object):
327 def __init__(self, app, out_dir, force, record, work_queue, 336 def __init__(self, app, out_dir, force, record, work_queue,
328 abort_event): 337 abort_event):
329 self.app = app 338 self.app = app
352 def __init__(self, wid, ctx): 361 def __init__(self, wid, ctx):
353 super(BakeWorker, self).__init__(name=('worker%d' % wid)) 362 super(BakeWorker, self).__init__(name=('worker%d' % wid))
354 self.wid = wid 363 self.wid = wid
355 self.ctx = ctx 364 self.ctx = ctx
356 self.abort_exception = None 365 self.abort_exception = None
366 self.success = True
357 self._page_baker = PageBaker( 367 self._page_baker = PageBaker(
358 ctx.app, ctx.out_dir, ctx.force, 368 ctx.app, ctx.out_dir, ctx.force,
359 ctx.record) 369 ctx.record)
360 370
361 def run(self): 371 def run(self):
365 if job is None: 375 if job is None:
366 logger.debug( 376 logger.debug(
367 "[%d] No more work... shutting down." % 377 "[%d] No more work... shutting down." %
368 self.wid) 378 self.wid)
369 break 379 break
370 self._unsafeRun(job) 380 success = self._unsafeRun(job)
371 logger.debug("[%d] Done with page." % self.wid) 381 logger.debug("[%d] Done with page." % self.wid)
372 self.ctx.work_queue.onJobFinished(job) 382 self.ctx.work_queue.onJobFinished(job)
383 self.success &= success
373 except Exception as ex: 384 except Exception as ex:
374 self.ctx.abort_event.set() 385 self.ctx.abort_event.set()
375 self.abort_exception = ex 386 self.abort_exception = ex
387 self.success = False
376 logger.debug("[%d] Critical error, aborting." % self.wid) 388 logger.debug("[%d] Critical error, aborting." % self.wid)
377 if self.ctx.app.debug: 389 if self.ctx.app.debug:
378 logger.exception(ex) 390 logger.exception(ex)
379 break 391 break
380 392
391 logger.debug("Got baking error. Adding it to the record.") 403 logger.debug("Got baking error. Adding it to the record.")
392 while ex: 404 while ex:
393 entry.errors.append(str(ex)) 405 entry.errors.append(str(ex))
394 ex = ex.__cause__ 406 ex = ex.__cause__
395 407
408 if entry.errors:
409 for e in entry.errors:
410 logger.error(e)
411 return False
412
396 if entry.was_baked_successfully: 413 if entry.was_baked_successfully:
397 uri = entry.out_uris[0] 414 uri = entry.out_uris[0]
398 friendly_uri = uri if uri != '' else '[main page]' 415 friendly_uri = uri if uri != '' else '[main page]'
399 friendly_count = '' 416 friendly_count = ''
400 if entry.num_subs > 1: 417 if entry.num_subs > 1:
401 friendly_count = ' (%d pages)' % entry.num_subs 418 friendly_count = ' (%d pages)' % entry.num_subs
402 logger.info(format_timed( 419 logger.info(format_timed(
403 start_time, '[%d] %s%s' % 420 start_time, '[%d] %s%s' %
404 (self.wid, friendly_uri, friendly_count))) 421 (self.wid, friendly_uri, friendly_count)))
405 elif entry.errors: 422
406 for e in entry.errors: 423 return True
407 logger.error(e) 424
408