Mercurial > piecrust2

comparison piecrust/baking/records.py @ 411:e7b865f8f335

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Enable multiprocess baking. Baking is now done by running a worker per CPU, and sending jobs to them. This changes several things across the codebase: * Ability to not cache things related to pages other than the 'main' page (i.e. the page at the bottom of the execution stack). * Decouple the baking process from the bake records, so only the main process keeps track (and modifies) the bake record. * Remove the need for 'batch page getters' and loading a page directly from the page factories. There are various smaller changes too included here, including support for scope performance timers that are saved with the bake record and can be printed out to the console. Yes I got carried away. For testing, the in-memory 'mock' file-system doesn't work anymore, since we're spawning processes, so this is replaced by a 'tmpfs' file-system which is saved in temporary files on disk and deleted after tests have run.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 Jun 2015 17:09:19 -0700
parents c12ee6936b8c
children 0e9a94b7fdfa
comparison
equal deleted inserted replaced
410:d1a472464e57 411:e7b865f8f335
1 import copy 1 import copy
2 import os.path 2 import os.path
3 import hashlib
3 import logging 4 import logging
4 from piecrust.records import Record, TransitionalRecord 5 from piecrust.records import Record, TransitionalRecord
5 6
6 7
7 logger = logging.getLogger(__name__) 8 logger = logging.getLogger(__name__)
8 9
9 10
10 def _get_transition_key(source_name, rel_path, taxonomy_info=None): 11 def _get_transition_key(path, taxonomy_info=None):
11 key = '%s:%s' % (source_name, rel_path) 12 key = path
12 if taxonomy_info: 13 if taxonomy_info:
13 taxonomy_name, taxonomy_term, taxonomy_source_name = taxonomy_info 14 key += '+%s:%s=' % (taxonomy_info.source_name,
14 key += ';%s:%s=' % (taxonomy_source_name, taxonomy_name) 15 taxonomy_info.taxonomy_name)
15 if isinstance(taxonomy_term, tuple): 16 if isinstance(taxonomy_info.term, tuple):
16 key += '/'.join(taxonomy_term) 17 key += '/'.join(taxonomy_info.term)
17 else: 18 else:
18 key += taxonomy_term 19 key += taxonomy_info.term
19 return key 20 return hashlib.md5(key.encode('utf8')).hexdigest()
20 21
21 22
22 class BakeRecord(Record): 23 class BakeRecord(Record):
23 RECORD_VERSION = 12 24 RECORD_VERSION = 14
24 25
25 def __init__(self): 26 def __init__(self):
26 super(BakeRecord, self).__init__() 27 super(BakeRecord, self).__init__()
27 self.out_dir = None 28 self.out_dir = None
28 self.bake_time = None 29 self.bake_time = None
30 self.timers = None
29 self.success = True 31 self.success = True
30 32
31 33
32 class BakeRecordPassInfo(object): 34 class BakePassInfo(object):
33 def __init__(self): 35 def __init__(self):
34 self.used_source_names = set() 36 self.used_source_names = set()
35 self.used_taxonomy_terms = set() 37 self.used_taxonomy_terms = set()
36 38
37 39
38 class BakeRecordSubPageEntry(object): 40 class SubPageBakeInfo(object):
39 FLAG_NONE = 0 41 FLAG_NONE = 0
40 FLAG_BAKED = 2**0 42 FLAG_BAKED = 2**0
41 FLAG_FORCED_BY_SOURCE = 2**1 43 FLAG_FORCED_BY_SOURCE = 2**1
42 FLAG_FORCED_BY_NO_PREVIOUS = 2**2 44 FLAG_FORCED_BY_NO_PREVIOUS = 2**2
43 FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3 45 FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3
66 for p, pinfo in self.render_passes.items(): 68 for p, pinfo in self.render_passes.items():
67 if p not in other.render_passes: 69 if p not in other.render_passes:
68 other.render_passes[p] = copy.deepcopy(pinfo) 70 other.render_passes[p] = copy.deepcopy(pinfo)
69 71
70 72
71 class BakeRecordPageEntry(object): 73 class PageBakeInfo(object):
74 def __init__(self):
75 self.subs = []
76 self.assets = []
77
78
79 class FirstRenderInfo(object):
80 def __init__(self):
81 self.assets = []
82 self.used_pagination = False
83 self.pagination_has_more = False
84
85
86 class TaxonomyInfo(object):
87 def __init__(self, taxonomy_name, source_name, term):
88 self.taxonomy_name = taxonomy_name
89 self.source_name = source_name
90 self.term = term
91
92
93 class BakeRecordEntry(object):
72 """ An entry in the bake record. 94 """ An entry in the bake record.
73 95
74 The `taxonomy_info` attribute should be a tuple of the form: 96 The `taxonomy_info` attribute should be a tuple of the form:
75 (taxonomy name, term, source name) 97 (taxonomy name, term, source name)
76 """ 98 """
77 FLAG_NONE = 0 99 FLAG_NONE = 0
78 FLAG_NEW = 2**0 100 FLAG_NEW = 2**0
79 FLAG_SOURCE_MODIFIED = 2**1 101 FLAG_SOURCE_MODIFIED = 2**1
80 FLAG_OVERRIDEN = 2**2 102 FLAG_OVERRIDEN = 2**2
81 103
82 def __init__(self, source_name, rel_path, path, taxonomy_info=None): 104 def __init__(self, source_name, path, taxonomy_info=None):
83 self.source_name = source_name 105 self.source_name = source_name
84 self.rel_path = rel_path
85 self.path = path 106 self.path = path
86 self.taxonomy_info = taxonomy_info 107 self.taxonomy_info = taxonomy_info
87 self.flags = self.FLAG_NONE 108 self.flags = self.FLAG_NONE
88 self.config = None 109 self.config = None
89 self.subs = []
90 self.assets = []
91 self.errors = [] 110 self.errors = []
111 self.bake_info = None
112 self.first_render_info = None
92 113
93 @property 114 @property
94 def path_mtime(self): 115 def path_mtime(self):
95 return os.path.getmtime(self.path) 116 return os.path.getmtime(self.path)
96 117
98 def was_overriden(self): 119 def was_overriden(self):
99 return (self.flags & self.FLAG_OVERRIDEN) != 0 120 return (self.flags & self.FLAG_OVERRIDEN) != 0
100 121
101 @property 122 @property
102 def num_subs(self): 123 def num_subs(self):
103 return len(self.subs) 124 if self.bake_info is None:
125 return 0
126 return len(self.bake_info.subs)
104 127
105 @property 128 @property
106 def was_any_sub_baked(self): 129 def was_any_sub_baked(self):
107 for o in self.subs: 130 if self.bake_info is not None:
108 if o.was_baked: 131 for o in self.bake_info.subs:
109 return True 132 if o.was_baked:
133 return True
110 return False 134 return False
111 135
136 @property
137 def subs(self):
138 if self.bake_info is not None:
139 return self.bake_info.subs
140 return []
141
142 @property
143 def has_any_error(self):
144 if len(self.errors) > 0:
145 return True
146 if self.bake_info is not None:
147 for o in self.bake_info.subs:
148 if len(o.errors) > 0:
149 return True
150 return False
151
112 def getSub(self, sub_index): 152 def getSub(self, sub_index):
113 return self.subs[sub_index - 1] 153 if self.bake_info is None:
154 raise Exception("No bake info available on this entry.")
155 return self.bake_info.subs[sub_index - 1]
114 156
115 def getAllErrors(self): 157 def getAllErrors(self):
116 yield from self.errors 158 yield from self.errors
117 for o in self.subs: 159 if self.bake_info is not None:
118 yield from o.errors 160 for o in self.bake_info.subs:
161 yield from o.errors
119 162
120 def getAllUsedSourceNames(self): 163 def getAllUsedSourceNames(self):
121 res = set() 164 res = set()
122 for o in self.subs: 165 if self.bake_info is not None:
123 for p, pinfo in o.render_passes.items(): 166 for o in self.bake_info.subs:
124 res |= pinfo.used_source_names 167 for p, pinfo in o.render_passes.items():
168 res |= pinfo.used_source_names
125 return res 169 return res
126 170
127 def getAllUsedTaxonomyTerms(self): 171 def getAllUsedTaxonomyTerms(self):
128 res = set() 172 res = set()
129 for o in self.subs: 173 if self.bake_info is not None:
130 for p, pinfo in o.render_passes.items(): 174 for o in self.bake_info.subs:
131 res |= pinfo.used_taxonomy_terms 175 for p, pinfo in o.render_passes.items():
176 res |= pinfo.used_taxonomy_terms
132 return res 177 return res
133 178
134 179
135 class TransitionalBakeRecord(TransitionalRecord): 180 class TransitionalBakeRecord(TransitionalRecord):
136 def __init__(self, previous_path=None): 181 def __init__(self, previous_path=None):
139 self.dirty_source_names = set() 184 self.dirty_source_names = set()
140 185
141 def addEntry(self, entry): 186 def addEntry(self, entry):
142 if (self.previous.bake_time and 187 if (self.previous.bake_time and
143 entry.path_mtime >= self.previous.bake_time): 188 entry.path_mtime >= self.previous.bake_time):
144 entry.flags |= BakeRecordPageEntry.FLAG_SOURCE_MODIFIED 189 entry.flags |= BakeRecordEntry.FLAG_SOURCE_MODIFIED
145 self.dirty_source_names.add(entry.source_name) 190 self.dirty_source_names.add(entry.source_name)
146 super(TransitionalBakeRecord, self).addEntry(entry) 191 super(TransitionalBakeRecord, self).addEntry(entry)
147 192
148 def getTransitionKey(self, entry): 193 def getTransitionKey(self, entry):
149 return _get_transition_key(entry.source_name, entry.rel_path, 194 return _get_transition_key(entry.path, entry.taxonomy_info)
150 entry.taxonomy_info) 195
151 196 def getPreviousAndCurrentEntries(self, path, taxonomy_info=None):
152 def getOverrideEntry(self, factory, uri): 197 key = _get_transition_key(path, taxonomy_info)
198 pair = self.transitions.get(key)
199 return pair
200
201 def getOverrideEntry(self, path, uri):
153 for pair in self.transitions.values(): 202 for pair in self.transitions.values():
154 cur = pair[1] 203 cur = pair[1]
155 if (cur and 204 if cur and cur.path != path:
156 (cur.source_name != factory.source.name or 205 for o in cur.subs:
157 cur.rel_path != factory.rel_path)): 206 if o.out_uri == uri:
158 for o in cur.subs: 207 return cur
159 if o.out_uri == uri:
160 return cur
161 return None 208 return None
162 209
163 def getPreviousEntry(self, source_name, rel_path, taxonomy_info=None): 210 def getPreviousEntry(self, path, taxonomy_info=None):
164 key = _get_transition_key(source_name, rel_path, taxonomy_info) 211 pair = self.getPreviousAndCurrentEntries(path, taxonomy_info)
165 pair = self.transitions.get(key)
166 if pair is not None: 212 if pair is not None:
167 return pair[0] 213 return pair[0]
168 return None 214 return None
169 215
216 def getCurrentEntry(self, path, taxonomy_info=None):
217 pair = self.getPreviousAndCurrentEntries(path, taxonomy_info)
218 if pair is not None:
219 return pair[1]
220 return None
221
170 def collapseEntry(self, prev_entry): 222 def collapseEntry(self, prev_entry):
171 cur_entry = copy.deepcopy(prev_entry) 223 cur_entry = copy.deepcopy(prev_entry)
172 cur_entry.flags = BakeRecordPageEntry.FLAG_NONE 224 cur_entry.flags = BakeRecordEntry.FLAG_NONE
173 for o in cur_entry.subs: 225 for o in cur_entry.subs:
174 o.flags = BakeRecordSubPageEntry.FLAG_NONE 226 o.flags = SubPageBakeInfo.FLAG_NONE
175 self.addEntry(cur_entry) 227 self.addEntry(cur_entry)
176 228
177 def getDeletions(self): 229 def getDeletions(self):
178 for prev, cur in self.transitions.values(): 230 for prev, cur in self.transitions.values():
179 if prev and not cur: 231 if prev and not cur:
185 diff = set(prev_out_paths) - set(cur_out_paths) 237 diff = set(prev_out_paths) - set(cur_out_paths)
186 for p in diff: 238 for p in diff:
187 yield (p, 'source file changed outputs') 239 yield (p, 'source file changed outputs')
188 240
189 def _onNewEntryAdded(self, entry): 241 def _onNewEntryAdded(self, entry):
190 entry.flags |= BakeRecordPageEntry.FLAG_NEW 242 entry.flags |= BakeRecordEntry.FLAG_NEW
191 243