diff piecrust/pipelines/_pagerecords.py @ 989:8adc27285d93

bake: Big pass on bake performance. - Reduce the amount of data passed between processes. - Make inter-process data simple objects to make it easier to test with alternatives to pickle. - Make sources have the basic requirement to be able to find a content item from an item spec (path). - Make Hoedown the default Markdown formatter.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 19 Nov 2017 14:29:17 -0800
parents 45ad976712ec
children 1857dbd4580f
line wrap: on
line diff
--- a/piecrust/pipelines/_pagerecords.py	Fri Nov 03 23:14:56 2017 -0700
+++ b/piecrust/pipelines/_pagerecords.py	Sun Nov 19 14:29:17 2017 -0800
@@ -2,42 +2,37 @@
 from piecrust.pipelines.records import RecordEntry, get_flag_descriptions
 
 
-class SubPagePipelineRecordEntry:
+class SubPageFlags:
     FLAG_NONE = 0
     FLAG_BAKED = 2**0
     FLAG_FORCED_BY_SOURCE = 2**1
     FLAG_FORCED_BY_NO_PREVIOUS = 2**2
     FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3
     FLAG_FORCED_BY_GENERAL_FORCE = 2**4
-    FLAG_FORMATTING_INVALIDATED = 2**5
+    FLAG_RENDER_CACHE_INVALIDATED = 2**5
+
 
-    def __init__(self, out_uri, out_path):
-        self.out_uri = out_uri
-        self.out_path = out_path
-        self.flags = self.FLAG_NONE
-        self.errors = []
-        self.render_info = [None, None]  # Same length as RENDER_PASSES
-
-    @property
-    def was_clean(self):
-        return (self.flags & self.FLAG_BAKED) == 0 and len(self.errors) == 0
+def create_subpage_job_result(out_uri, out_path):
+    return {
+        'out_uri': out_uri,
+        'out_path': out_path,
+        'flags': SubPageFlags.FLAG_NONE,
+        'errors': [],
+        'render_info': None
+    }
 
-    @property
-    def was_baked(self):
-        return (self.flags & self.FLAG_BAKED) != 0
 
-    @property
-    def was_baked_successfully(self):
-        return self.was_baked and len(self.errors) == 0
+def was_subpage_clean(sub):
+    return ((sub['flags'] & SubPageFlags.FLAG_BAKED) == 0 and
+            len(sub['errors']) == 0)
+
 
-    def anyPass(self, func):
-        for pinfo in self.render_info:
-            if pinfo and func(pinfo):
-                return True
-        return False
+def was_subpage_baked(sub):
+    return (sub['flags'] & SubPageFlags.FLAG_BAKED) != 0
 
-    def copyRenderInfo(self):
-        return copy.deepcopy(self.render_info)
+
+def was_subpage_baked_successfully(sub):
+    return was_subpage_baked(sub) and len(sub['errors']) == 0
 
 
 class PagePipelineRecordEntry(RecordEntry):
@@ -47,11 +42,13 @@
     FLAG_OVERRIDEN = 2**2
     FLAG_COLLAPSED_FROM_LAST_RUN = 2**3
     FLAG_IS_DRAFT = 2**4
+    FLAG_ABORTED_FOR_SOURCE_USE = 2**5
 
     def __init__(self):
         super().__init__()
         self.flags = self.FLAG_NONE
         self.config = None
+        self.route_params = None
         self.timestamp = None
         self.subs = []
 
@@ -70,7 +67,7 @@
     @property
     def was_any_sub_baked(self):
         for o in self.subs:
-            if o.was_baked:
+            if was_subpage_baked(o):
                 return True
         return False
 
@@ -79,7 +76,7 @@
         if len(self.errors) > 0:
             return True
         for o in self.subs:
-            if len(o.errors) > 0:
+            if len(o['errors']) > 0:
                 return True
         return False
 
@@ -89,55 +86,64 @@
     def getAllErrors(self):
         yield from self.errors
         for o in self.subs:
-            yield from o.errors
+            yield from o['errors']
 
     def getAllUsedSourceNames(self):
         res = set()
         for o in self.subs:
-            for pinfo in o.render_info:
-                if pinfo:
-                    res |= pinfo.used_source_names
+            pinfo = o.get('render_info')
+            if pinfo:
+                res |= pinfo['used_source_names']
         return res
 
     def getAllOutputPaths(self):
         for o in self.subs:
-            yield o.out_path
+            yield o['out_path']
 
     def describe(self):
         d = super().describe()
         d['Flags'] = get_flag_descriptions(self.flags, flag_descriptions)
         for i, sub in enumerate(self.subs):
             d['Sub%02d' % i] = {
-                'URI': sub.out_uri,
-                'Path': sub.out_path,
+                'URI': sub['out_uri'],
+                'Path': sub['out_path'],
                 'Flags': get_flag_descriptions(
-                    sub.flags, sub_flag_descriptions),
-                'RenderInfo': [
-                    _describe_render_info(sub.render_info[0]),
-                    _describe_render_info(sub.render_info[1])
-                ]
+                    sub['flags'], sub_flag_descriptions),
+                'RenderInfo': _describe_render_info(sub['render_info'])
             }
         return d
 
 
+def add_page_job_result(result):
+    result.update({
+        'flags': PagePipelineRecordEntry.FLAG_NONE,
+        'errors': [],
+        'subs': []
+    })
+
+
+def merge_job_result_into_record_entry(record_entry, result):
+    record_entry.flags |= result['flags']
+    record_entry.errors += result['errors']
+    record_entry.subs += result['subs']
+
+
 flag_descriptions = {
     PagePipelineRecordEntry.FLAG_NEW: 'new',
     PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED: 'touched',
     PagePipelineRecordEntry.FLAG_OVERRIDEN: 'overriden',
     PagePipelineRecordEntry.FLAG_COLLAPSED_FROM_LAST_RUN: 'from last run',
-    PagePipelineRecordEntry.FLAG_IS_DRAFT: 'draft'}
+    PagePipelineRecordEntry.FLAG_IS_DRAFT: 'draft',
+    PagePipelineRecordEntry.FLAG_ABORTED_FOR_SOURCE_USE: 'aborted for source use'}
 
 
 sub_flag_descriptions = {
-    SubPagePipelineRecordEntry.FLAG_BAKED: 'baked',
-    SubPagePipelineRecordEntry.FLAG_FORCED_BY_SOURCE: 'forced by source',
-    SubPagePipelineRecordEntry.FLAG_FORCED_BY_NO_PREVIOUS: 'forced b/c new',
-    SubPagePipelineRecordEntry.FLAG_FORCED_BY_PREVIOUS_ERRORS:
-    'forced by errors',
-    SubPagePipelineRecordEntry.FLAG_FORCED_BY_GENERAL_FORCE:
-    'manually forced',
-    SubPagePipelineRecordEntry.FLAG_FORMATTING_INVALIDATED:
-    'formatting invalidated'
+    SubPageFlags.FLAG_BAKED: 'baked',
+    SubPageFlags.FLAG_FORCED_BY_SOURCE: 'forced by source',
+    SubPageFlags.FLAG_FORCED_BY_NO_PREVIOUS: 'forced b/c new',
+    SubPageFlags.FLAG_FORCED_BY_PREVIOUS_ERRORS: 'forced by errors',
+    SubPageFlags.FLAG_FORCED_BY_GENERAL_FORCE: 'manually forced',
+    SubPageFlags.FLAG_RENDER_CACHE_INVALIDATED: 'cache invalidated'
 }
 
 
@@ -145,8 +151,8 @@
     if ri is None:
         return '<null>'
     return {
-        'UsedPagination': ri.used_pagination,
-        'PaginationHasMore': ri.pagination_has_more,
-        'UsedAssets': ri.used_assets,
-        'UsedSourceNames': ri.used_source_names
+        'UsedPagination': ri['used_pagination'],
+        'PaginationHasMore': ri['pagination_has_more'],
+        'UsedAssets': ri['used_assets'],
+        'UsedSourceNames': ri['used_source_names']
     }