Mercurial > piecrust2
diff piecrust/baking/baker.py @ 120:133845647083
Better error management and removal support in baking/processing.
* Baker and processor pipeline now store errors in their records.
* They also support deleting output files that are no longer valid.
* The basic transitional record class implements more boilerplate code.
* The processor pipeline is run from the `bake` command directly.
* New unit tests.
* Unit test mocking now mocks `os.remove` too.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sun, 09 Nov 2014 14:46:23 -0800 |
parents | 7d2fdf43d7ca |
children | bc63dc20baa0 |
line wrap: on
line diff
--- a/piecrust/baking/baker.py Wed Oct 29 08:19:58 2014 -0700 +++ b/piecrust/baking/baker.py Sun Nov 09 14:46:23 2014 -0800 @@ -12,7 +12,6 @@ from piecrust.chefutil import format_timed, log_friendly_exception from piecrust.data.filters import (PaginationFilter, HasFilterClause, IsFilterClause, AndBooleanClause) -from piecrust.processing.base import ProcessorPipeline from piecrust.rendering import (PageRenderingContext, render_page, PASS_FORMATTING, PASS_RENDERING) from piecrust.sources.base import (PageFactory, @@ -144,7 +143,7 @@ # see if any of those got baked, or are going to be baked for some # reason. If so, we need to bake this one too. # (this happens for instance with the main page of a blog). - if prev_record_entry: + if prev_record_entry and prev_record_entry.was_baked_successfully: invalidated_render_passes = set() used_src_names = list(prev_record_entry.used_source_names) for src_name, rdr_pass in used_src_names: @@ -266,13 +265,13 @@ class Baker(object): def __init__(self, app, out_dir=None, force=False, portable=False, - no_assets=False): + no_assets=False, num_workers=4): self.app = app self.out_dir = out_dir or os.path.join(app.root_dir, '_counter') self.force = force self.portable = portable self.no_assets = no_assets - self.num_workers = app.config.get('baker/workers') or 4 + self.num_workers = num_workers # Remember what taxonomy pages we should skip # (we'll bake them repeatedly later with each taxonomy term) @@ -301,7 +300,9 @@ # Load/create the bake record. record = TransitionalBakeRecord() record_cache = self.app.cache.getCache('baker') - record_name = (hashlib.md5(self.out_dir.encode('utf8')).hexdigest() + + record_name = ( + 'pages_' + + hashlib.md5(self.out_dir.encode('utf8')).hexdigest() + '.record') if not self.force and record_cache.has(record_name): t = time.clock() @@ -331,9 +332,8 @@ # Bake taxonomies. self._bakeTaxonomies(record) - # Bake the assets. - if not self.no_assets: - self._bakeAssets(record) + # Delete files from the output. + self._handleDeletetions(record) # Save the bake record. t = time.clock() @@ -345,8 +345,7 @@ # All done. self.app.config.set('baker/is_baking', False) - logger.info('-------------------------'); - logger.info(format_timed(start_time, 'done baking')); + logger.debug(format_timed(start_time, 'done baking')); def _handleCacheValidity(self, record): start_time = time.clock() @@ -407,13 +406,16 @@ (source.name, fac.ref_spec)) continue + entry = BakeRecordPageEntry(fac) + record.addEntry(entry) + route = self.app.getRoute(source.name, fac.metadata) if route is None: - logger.error("Can't get route for page: %s" % fac.ref_spec) + entry.errors.append("Can't get route for page: %s" % + fac.ref_spec) + logger.error(entry.errors[-1]) continue - entry = BakeRecordPageEntry(fac) - record.addEntry(entry) queue.addJob(BakeWorkerJob(fac, route, entry)) self._waitOnWorkerPool(pool, abort) @@ -439,9 +441,11 @@ changed_terms = None # Re-bake all taxonomy pages that include new or changed # pages. - if not prev_entry and cur_entry and cur_entry.was_baked: + if (not prev_entry and cur_entry and + cur_entry.was_baked_successfully): changed_terms = cur_entry.config.get(tax.name) - elif prev_entry and cur_entry and cur_entry.was_baked: + elif (prev_entry and cur_entry and + cur_entry.was_baked_successfully): changed_terms = [] prev_terms = prev_entry.config.get(tax.name) cur_terms = cur_entry.config.get(tax.name) @@ -508,16 +512,11 @@ self._waitOnWorkerPool(pool, abort) - def _bakeAssets(self, record): - mounts = self.app.assets_dirs - baker_params = self.app.config.get('baker') or {} - skip_patterns = baker_params.get('skip_patterns') - force_patterns = baker_params.get('force_patterns') - proc = ProcessorPipeline( - self.app, mounts, self.out_dir, force=self.force, - skip_patterns=skip_patterns, force_patterns=force_patterns, - num_workers=self.num_workers) - proc.run() + def _handleDeletetions(self, record): + for path, reason in record.getDeletions(): + logger.debug("Removing '%s': %s" % (path, reason)) + os.remove(path) + logger.info('[delete] %s' % path) def _createWorkerPool(self, record, pool_size=4): pool = [] @@ -697,11 +696,17 @@ start_time = time.clock() entry = job.record_entry - self._page_baker.bake(job.factory, job.route, entry, - taxonomy_name=job.taxonomy_name, - taxonomy_term=job.taxonomy_term) + try: + self._page_baker.bake(job.factory, job.route, entry, + taxonomy_name=job.taxonomy_name, + taxonomy_term=job.taxonomy_term) + except BakingError as ex: + logger.debug("Got baking error. Adding it to the record.") + while ex: + entry.errors.append(str(ex)) + ex = ex.__cause__ - if entry.was_baked: + if entry.was_baked_successfully: uri = entry.out_uris[0] friendly_uri = uri if uri != '' else '[main page]' friendly_count = '' @@ -709,4 +714,7 @@ friendly_count = ' (%d pages)' % entry.num_subs logger.info(format_timed(start_time, '[%d] %s%s' % (self.wid, friendly_uri, friendly_count))) + elif entry.errors: + for e in entry.errors: + logger.error(e)