Mercurial > piecrust2

comparison piecrust/pipelines/_pagerecords.py @ 989:8adc27285d93

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Big pass on bake performance. - Reduce the amount of data passed between processes. - Make inter-process data simple objects to make it easier to test with alternatives to pickle. - Make sources have the basic requirement to be able to find a content item from an item spec (path). - Make Hoedown the default Markdown formatter.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 19 Nov 2017 14:29:17 -0800
parents 45ad976712ec
children 1857dbd4580f
comparison
equal deleted inserted replaced
988:f83ae0a5d793 989:8adc27285d93
1 import copy 1 import copy
2 from piecrust.pipelines.records import RecordEntry, get_flag_descriptions 2 from piecrust.pipelines.records import RecordEntry, get_flag_descriptions
3 3
4 4
5 class SubPagePipelineRecordEntry: 5 class SubPageFlags:
6 FLAG_NONE = 0 6 FLAG_NONE = 0
7 FLAG_BAKED = 2**0 7 FLAG_BAKED = 2**0
8 FLAG_FORCED_BY_SOURCE = 2**1 8 FLAG_FORCED_BY_SOURCE = 2**1
9 FLAG_FORCED_BY_NO_PREVIOUS = 2**2 9 FLAG_FORCED_BY_NO_PREVIOUS = 2**2
10 FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3 10 FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3
11 FLAG_FORCED_BY_GENERAL_FORCE = 2**4 11 FLAG_FORCED_BY_GENERAL_FORCE = 2**4
12 FLAG_FORMATTING_INVALIDATED = 2**5 12 FLAG_RENDER_CACHE_INVALIDATED = 2**5
13 13
14 def __init__(self, out_uri, out_path):
15 self.out_uri = out_uri
16 self.out_path = out_path
17 self.flags = self.FLAG_NONE
18 self.errors = []
19 self.render_info = [None, None] # Same length as RENDER_PASSES
20 14
21 @property 15 def create_subpage_job_result(out_uri, out_path):
22 def was_clean(self): 16 return {
23 return (self.flags & self.FLAG_BAKED) == 0 and len(self.errors) == 0 17 'out_uri': out_uri,
18 'out_path': out_path,
19 'flags': SubPageFlags.FLAG_NONE,
20 'errors': [],
21 'render_info': None
22 }
24 23
25 @property
26 def was_baked(self):
27 return (self.flags & self.FLAG_BAKED) != 0
28 24
29 @property 25 def was_subpage_clean(sub):
30 def was_baked_successfully(self): 26 return ((sub['flags'] & SubPageFlags.FLAG_BAKED) == 0 and
31 return self.was_baked and len(self.errors) == 0 27 len(sub['errors']) == 0)
32 28
33 def anyPass(self, func):
34 for pinfo in self.render_info:
35 if pinfo and func(pinfo):
36 return True
37 return False
38 29
39 def copyRenderInfo(self): 30 def was_subpage_baked(sub):
40 return copy.deepcopy(self.render_info) 31 return (sub['flags'] & SubPageFlags.FLAG_BAKED) != 0
32
33
34 def was_subpage_baked_successfully(sub):
35 return was_subpage_baked(sub) and len(sub['errors']) == 0
41 36
42 37
43 class PagePipelineRecordEntry(RecordEntry): 38 class PagePipelineRecordEntry(RecordEntry):
44 FLAG_NONE = 0 39 FLAG_NONE = 0
45 FLAG_NEW = 2**0 40 FLAG_NEW = 2**0
46 FLAG_SOURCE_MODIFIED = 2**1 41 FLAG_SOURCE_MODIFIED = 2**1
47 FLAG_OVERRIDEN = 2**2 42 FLAG_OVERRIDEN = 2**2
48 FLAG_COLLAPSED_FROM_LAST_RUN = 2**3 43 FLAG_COLLAPSED_FROM_LAST_RUN = 2**3
49 FLAG_IS_DRAFT = 2**4 44 FLAG_IS_DRAFT = 2**4
45 FLAG_ABORTED_FOR_SOURCE_USE = 2**5
50 46
51 def __init__(self): 47 def __init__(self):
52 super().__init__() 48 super().__init__()
53 self.flags = self.FLAG_NONE 49 self.flags = self.FLAG_NONE
54 self.config = None 50 self.config = None
51 self.route_params = None
55 self.timestamp = None 52 self.timestamp = None
56 self.subs = [] 53 self.subs = []
57 54
58 @property 55 @property
59 def was_touched(self): 56 def was_touched(self):
68 return len(self.subs) 65 return len(self.subs)
69 66
70 @property 67 @property
71 def was_any_sub_baked(self): 68 def was_any_sub_baked(self):
72 for o in self.subs: 69 for o in self.subs:
73 if o.was_baked: 70 if was_subpage_baked(o):
74 return True 71 return True
75 return False 72 return False
76 73
77 @property 74 @property
78 def has_any_error(self): 75 def has_any_error(self):
79 if len(self.errors) > 0: 76 if len(self.errors) > 0:
80 return True 77 return True
81 for o in self.subs: 78 for o in self.subs:
82 if len(o.errors) > 0: 79 if len(o['errors']) > 0:
83 return True 80 return True
84 return False 81 return False
85 82
86 def getSub(self, page_num): 83 def getSub(self, page_num):
87 return self.subs[page_num - 1] 84 return self.subs[page_num - 1]
88 85
89 def getAllErrors(self): 86 def getAllErrors(self):
90 yield from self.errors 87 yield from self.errors
91 for o in self.subs: 88 for o in self.subs:
92 yield from o.errors 89 yield from o['errors']
93 90
94 def getAllUsedSourceNames(self): 91 def getAllUsedSourceNames(self):
95 res = set() 92 res = set()
96 for o in self.subs: 93 for o in self.subs:
97 for pinfo in o.render_info: 94 pinfo = o.get('render_info')
98 if pinfo: 95 if pinfo:
99 res |= pinfo.used_source_names 96 res |= pinfo['used_source_names']
100 return res 97 return res
101 98
102 def getAllOutputPaths(self): 99 def getAllOutputPaths(self):
103 for o in self.subs: 100 for o in self.subs:
104 yield o.out_path 101 yield o['out_path']
105 102
106 def describe(self): 103 def describe(self):
107 d = super().describe() 104 d = super().describe()
108 d['Flags'] = get_flag_descriptions(self.flags, flag_descriptions) 105 d['Flags'] = get_flag_descriptions(self.flags, flag_descriptions)
109 for i, sub in enumerate(self.subs): 106 for i, sub in enumerate(self.subs):
110 d['Sub%02d' % i] = { 107 d['Sub%02d' % i] = {
111 'URI': sub.out_uri, 108 'URI': sub['out_uri'],
112 'Path': sub.out_path, 109 'Path': sub['out_path'],
113 'Flags': get_flag_descriptions( 110 'Flags': get_flag_descriptions(
114 sub.flags, sub_flag_descriptions), 111 sub['flags'], sub_flag_descriptions),
115 'RenderInfo': [ 112 'RenderInfo': _describe_render_info(sub['render_info'])
116 _describe_render_info(sub.render_info[0]),
117 _describe_render_info(sub.render_info[1])
118 ]
119 } 113 }
120 return d 114 return d
115
116
117 def add_page_job_result(result):
118 result.update({
119 'flags': PagePipelineRecordEntry.FLAG_NONE,
120 'errors': [],
121 'subs': []
122 })
123
124
125 def merge_job_result_into_record_entry(record_entry, result):
126 record_entry.flags |= result['flags']
127 record_entry.errors += result['errors']
128 record_entry.subs += result['subs']
121 129
122 130
123 flag_descriptions = { 131 flag_descriptions = {
124 PagePipelineRecordEntry.FLAG_NEW: 'new', 132 PagePipelineRecordEntry.FLAG_NEW: 'new',
125 PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED: 'touched', 133 PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED: 'touched',
126 PagePipelineRecordEntry.FLAG_OVERRIDEN: 'overriden', 134 PagePipelineRecordEntry.FLAG_OVERRIDEN: 'overriden',
127 PagePipelineRecordEntry.FLAG_COLLAPSED_FROM_LAST_RUN: 'from last run', 135 PagePipelineRecordEntry.FLAG_COLLAPSED_FROM_LAST_RUN: 'from last run',
128 PagePipelineRecordEntry.FLAG_IS_DRAFT: 'draft'} 136 PagePipelineRecordEntry.FLAG_IS_DRAFT: 'draft',
137 PagePipelineRecordEntry.FLAG_ABORTED_FOR_SOURCE_USE: 'aborted for source use'}
129 138
130 139
131 sub_flag_descriptions = { 140 sub_flag_descriptions = {
132 SubPagePipelineRecordEntry.FLAG_BAKED: 'baked', 141 SubPageFlags.FLAG_BAKED: 'baked',
133 SubPagePipelineRecordEntry.FLAG_FORCED_BY_SOURCE: 'forced by source', 142 SubPageFlags.FLAG_FORCED_BY_SOURCE: 'forced by source',
134 SubPagePipelineRecordEntry.FLAG_FORCED_BY_NO_PREVIOUS: 'forced b/c new', 143 SubPageFlags.FLAG_FORCED_BY_NO_PREVIOUS: 'forced b/c new',
135 SubPagePipelineRecordEntry.FLAG_FORCED_BY_PREVIOUS_ERRORS: 144 SubPageFlags.FLAG_FORCED_BY_PREVIOUS_ERRORS: 'forced by errors',
136 'forced by errors', 145 SubPageFlags.FLAG_FORCED_BY_GENERAL_FORCE: 'manually forced',
137 SubPagePipelineRecordEntry.FLAG_FORCED_BY_GENERAL_FORCE: 146 SubPageFlags.FLAG_RENDER_CACHE_INVALIDATED: 'cache invalidated'
138 'manually forced',
139 SubPagePipelineRecordEntry.FLAG_FORMATTING_INVALIDATED:
140 'formatting invalidated'
141 } 147 }
142 148
143 149
144 def _describe_render_info(ri): 150 def _describe_render_info(ri):
145 if ri is None: 151 if ri is None:
146 return '<null>' 152 return '<null>'
147 return { 153 return {
148 'UsedPagination': ri.used_pagination, 154 'UsedPagination': ri['used_pagination'],
149 'PaginationHasMore': ri.pagination_has_more, 155 'PaginationHasMore': ri['pagination_has_more'],
150 'UsedAssets': ri.used_assets, 156 'UsedAssets': ri['used_assets'],
151 'UsedSourceNames': ri.used_source_names 157 'UsedSourceNames': ri['used_source_names']
152 } 158 }