Mercurial > piecrust2
diff piecrust/pipelines/_pagerecords.py @ 989:8adc27285d93
bake: Big pass on bake performance.
- Reduce the amount of data passed between processes.
- Make inter-process data simple objects to make it easier to test with
alternatives to pickle.
- Make sources have the basic requirement to be able to find a content item
from an item spec (path).
- Make Hoedown the default Markdown formatter.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sun, 19 Nov 2017 14:29:17 -0800 |
parents | 45ad976712ec |
children | 1857dbd4580f |
line wrap: on
line diff
--- a/piecrust/pipelines/_pagerecords.py Fri Nov 03 23:14:56 2017 -0700 +++ b/piecrust/pipelines/_pagerecords.py Sun Nov 19 14:29:17 2017 -0800 @@ -2,42 +2,37 @@ from piecrust.pipelines.records import RecordEntry, get_flag_descriptions -class SubPagePipelineRecordEntry: +class SubPageFlags: FLAG_NONE = 0 FLAG_BAKED = 2**0 FLAG_FORCED_BY_SOURCE = 2**1 FLAG_FORCED_BY_NO_PREVIOUS = 2**2 FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3 FLAG_FORCED_BY_GENERAL_FORCE = 2**4 - FLAG_FORMATTING_INVALIDATED = 2**5 + FLAG_RENDER_CACHE_INVALIDATED = 2**5 + - def __init__(self, out_uri, out_path): - self.out_uri = out_uri - self.out_path = out_path - self.flags = self.FLAG_NONE - self.errors = [] - self.render_info = [None, None] # Same length as RENDER_PASSES - - @property - def was_clean(self): - return (self.flags & self.FLAG_BAKED) == 0 and len(self.errors) == 0 +def create_subpage_job_result(out_uri, out_path): + return { + 'out_uri': out_uri, + 'out_path': out_path, + 'flags': SubPageFlags.FLAG_NONE, + 'errors': [], + 'render_info': None + } - @property - def was_baked(self): - return (self.flags & self.FLAG_BAKED) != 0 - @property - def was_baked_successfully(self): - return self.was_baked and len(self.errors) == 0 +def was_subpage_clean(sub): + return ((sub['flags'] & SubPageFlags.FLAG_BAKED) == 0 and + len(sub['errors']) == 0) + - def anyPass(self, func): - for pinfo in self.render_info: - if pinfo and func(pinfo): - return True - return False +def was_subpage_baked(sub): + return (sub['flags'] & SubPageFlags.FLAG_BAKED) != 0 - def copyRenderInfo(self): - return copy.deepcopy(self.render_info) + +def was_subpage_baked_successfully(sub): + return was_subpage_baked(sub) and len(sub['errors']) == 0 class PagePipelineRecordEntry(RecordEntry): @@ -47,11 +42,13 @@ FLAG_OVERRIDEN = 2**2 FLAG_COLLAPSED_FROM_LAST_RUN = 2**3 FLAG_IS_DRAFT = 2**4 + FLAG_ABORTED_FOR_SOURCE_USE = 2**5 def __init__(self): super().__init__() self.flags = self.FLAG_NONE self.config = None + self.route_params = None self.timestamp = None self.subs = [] @@ -70,7 +67,7 @@ @property def was_any_sub_baked(self): for o in self.subs: - if o.was_baked: + if was_subpage_baked(o): return True return False @@ -79,7 +76,7 @@ if len(self.errors) > 0: return True for o in self.subs: - if len(o.errors) > 0: + if len(o['errors']) > 0: return True return False @@ -89,55 +86,64 @@ def getAllErrors(self): yield from self.errors for o in self.subs: - yield from o.errors + yield from o['errors'] def getAllUsedSourceNames(self): res = set() for o in self.subs: - for pinfo in o.render_info: - if pinfo: - res |= pinfo.used_source_names + pinfo = o.get('render_info') + if pinfo: + res |= pinfo['used_source_names'] return res def getAllOutputPaths(self): for o in self.subs: - yield o.out_path + yield o['out_path'] def describe(self): d = super().describe() d['Flags'] = get_flag_descriptions(self.flags, flag_descriptions) for i, sub in enumerate(self.subs): d['Sub%02d' % i] = { - 'URI': sub.out_uri, - 'Path': sub.out_path, + 'URI': sub['out_uri'], + 'Path': sub['out_path'], 'Flags': get_flag_descriptions( - sub.flags, sub_flag_descriptions), - 'RenderInfo': [ - _describe_render_info(sub.render_info[0]), - _describe_render_info(sub.render_info[1]) - ] + sub['flags'], sub_flag_descriptions), + 'RenderInfo': _describe_render_info(sub['render_info']) } return d +def add_page_job_result(result): + result.update({ + 'flags': PagePipelineRecordEntry.FLAG_NONE, + 'errors': [], + 'subs': [] + }) + + +def merge_job_result_into_record_entry(record_entry, result): + record_entry.flags |= result['flags'] + record_entry.errors += result['errors'] + record_entry.subs += result['subs'] + + flag_descriptions = { PagePipelineRecordEntry.FLAG_NEW: 'new', PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED: 'touched', PagePipelineRecordEntry.FLAG_OVERRIDEN: 'overriden', PagePipelineRecordEntry.FLAG_COLLAPSED_FROM_LAST_RUN: 'from last run', - PagePipelineRecordEntry.FLAG_IS_DRAFT: 'draft'} + PagePipelineRecordEntry.FLAG_IS_DRAFT: 'draft', + PagePipelineRecordEntry.FLAG_ABORTED_FOR_SOURCE_USE: 'aborted for source use'} sub_flag_descriptions = { - SubPagePipelineRecordEntry.FLAG_BAKED: 'baked', - SubPagePipelineRecordEntry.FLAG_FORCED_BY_SOURCE: 'forced by source', - SubPagePipelineRecordEntry.FLAG_FORCED_BY_NO_PREVIOUS: 'forced b/c new', - SubPagePipelineRecordEntry.FLAG_FORCED_BY_PREVIOUS_ERRORS: - 'forced by errors', - SubPagePipelineRecordEntry.FLAG_FORCED_BY_GENERAL_FORCE: - 'manually forced', - SubPagePipelineRecordEntry.FLAG_FORMATTING_INVALIDATED: - 'formatting invalidated' + SubPageFlags.FLAG_BAKED: 'baked', + SubPageFlags.FLAG_FORCED_BY_SOURCE: 'forced by source', + SubPageFlags.FLAG_FORCED_BY_NO_PREVIOUS: 'forced b/c new', + SubPageFlags.FLAG_FORCED_BY_PREVIOUS_ERRORS: 'forced by errors', + SubPageFlags.FLAG_FORCED_BY_GENERAL_FORCE: 'manually forced', + SubPageFlags.FLAG_RENDER_CACHE_INVALIDATED: 'cache invalidated' } @@ -145,8 +151,8 @@ if ri is None: return '<null>' return { - 'UsedPagination': ri.used_pagination, - 'PaginationHasMore': ri.pagination_has_more, - 'UsedAssets': ri.used_assets, - 'UsedSourceNames': ri.used_source_names + 'UsedPagination': ri['used_pagination'], + 'PaginationHasMore': ri['pagination_has_more'], + 'UsedAssets': ri['used_assets'], + 'UsedSourceNames': ri['used_source_names'] }