Mercurial > piecrust2

comparison piecrust/baking/baker.py @ 972:bbf5a96b56db

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
internal: Clean up baker code.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 17 Oct 2017 01:03:07 -0700
parents 7f1da7e7b154
children 45ad976712ec
comparison
equal deleted inserted replaced
971:5485a11591ec 972:bbf5a96b56db
81 81
82 # Pre-create all caches. 82 # Pre-create all caches.
83 for cache_name in ['app', 'baker', 'pages', 'renders']: 83 for cache_name in ['app', 'baker', 'pages', 'renders']:
84 self.app.cache.getCache(cache_name) 84 self.app.cache.getCache(cache_name)
85 85
86 # Create the pipelines.
87 ppmngr = self._createPipelineManager(record_histories)
88
89 # Create the worker processes.
90 pool_userdata = _PoolUserData(self, ppmngr)
91 pool = self._createWorkerPool(records_path, pool_userdata)
92
93 # Bake the realms.
94 self._bakeRealms(pool, ppmngr, record_histories)
95
96 # Handle deletions, collapse records, etc.
97 ppmngr.postJobRun()
98 ppmngr.deleteStaleOutputs()
99 ppmngr.collapseRecords()
100
101 # All done with the workers. Close the pool and get reports.
102 pool_stats = pool.close()
103 current_records.stats = _merge_execution_stats(stats, *pool_stats)
104
105 # Shutdown the pipelines.
106 ppmngr.shutdownPipelines()
107
108 # Backup previous records, save the current ones.
109 current_records.bake_time = time.time()
110 current_records.out_dir = self.out_dir
111 _save_bake_records(current_records, records_path,
112 rotate_previous=self.rotate_bake_records)
113
114 # All done.
115 self.app.config.set('baker/is_baking', False)
116 logger.debug(format_timed(start_time, 'done baking'))
117
118 return current_records
119
120 def _handleCacheValidity(self, previous_records, current_records):
121 start_time = time.perf_counter()
122
123 reason = None
124 if self.force:
125 reason = "ordered to"
126 elif not self.app.config.get('__cache_valid'):
127 # The configuration file was changed, or we're running a new
128 # version of the app.
129 reason = "not valid anymore"
130 elif previous_records.invalidated:
131 # We have no valid previous bake records.
132 reason = "need bake records regeneration"
133 else:
134 # Check if any template has changed since the last bake. Since
135 # there could be some advanced conditional logic going on, we'd
136 # better just force a bake from scratch if that's the case.
137 max_time = 0
138 for d in self.app.templates_dirs:
139 for dpath, _, filenames in os.walk(d):
140 for fn in filenames:
141 full_fn = os.path.join(dpath, fn)
142 max_time = max(max_time, os.path.getmtime(full_fn))
143 if max_time >= previous_records.bake_time:
144 reason = "templates modified"
145
146 if reason is not None:
147 # We have to bake everything from scratch.
148 self.app.cache.clearCaches(except_names=['app', 'baker'])
149 self.force = True
150 current_records.incremental_count = 0
151 previous_records = MultiRecord()
152 logger.info(format_timed(
153 start_time, "cleaned cache (reason: %s)" % reason))
154 return False
155 else:
156 current_records.incremental_count += 1
157 logger.debug(format_timed(
158 start_time, "cache is assumed valid", colored=False))
159 return True
160
161 def _createPipelineManager(self, record_histories):
86 # Gather all sources by realm -- we're going to bake each realm 162 # Gather all sources by realm -- we're going to bake each realm
87 # separately so we can handle "overriding" (i.e. one realm overrides 163 # separately so we can handle "overriding" (i.e. one realm overrides
88 # another realm's pages, like the user realm overriding the theme 164 # another realm's pages, like the user realm overriding the theme
89 # realm). 165 # realm).
90 # 166 #
112 has_any_pp = True 188 has_any_pp = True
113 if not has_any_pp: 189 if not has_any_pp:
114 raise Exception("The website has no content sources, or the bake " 190 raise Exception("The website has no content sources, or the bake "
115 "command was invoked with all pipelines filtered " 191 "command was invoked with all pipelines filtered "
116 "out. There's nothing to do.") 192 "out. There's nothing to do.")
117 193 return ppmngr
118 # Create the worker processes. 194
119 pool_userdata = _PoolUserData(self, ppmngr) 195 def _bakeRealms(self, pool, ppmngr, record_histories):
120 pool = self._createWorkerPool(records_path, pool_userdata)
121 realm_list = [REALM_USER, REALM_THEME]
122
123 # Bake the realms -- user first, theme second, so that a user item 196 # Bake the realms -- user first, theme second, so that a user item
124 # can override a theme item. 197 # can override a theme item.
125 # Do this for as many times as we have pipeline passes left to do. 198 # Do this for as many times as we have pipeline passes left to do.
199 realm_list = [REALM_USER, REALM_THEME]
126 pp_by_pass_and_realm = _get_pipeline_infos_by_pass_and_realm( 200 pp_by_pass_and_realm = _get_pipeline_infos_by_pass_and_realm(
127 ppmngr.getPipelines()) 201 ppmngr.getPipelines())
128 202
129 for pp_pass_num in sorted(pp_by_pass_and_realm.keys()): 203 for pp_pass_num in sorted(pp_by_pass_and_realm.keys()):
130 logger.debug("Pipelines pass %d" % pp_pass_num) 204 logger.debug("Pipelines pass %d" % pp_pass_num)
132 for realm in realm_list: 206 for realm in realm_list:
133 pplist = pp_by_realm.get(realm) 207 pplist = pp_by_realm.get(realm)
134 if pplist is not None: 208 if pplist is not None:
135 self._bakeRealm( 209 self._bakeRealm(
136 pool, record_histories, pp_pass_num, realm, pplist) 210 pool, record_histories, pp_pass_num, realm, pplist)
137
138 # Handle deletions, collapse records, etc.
139 ppmngr.postJobRun()
140 ppmngr.deleteStaleOutputs()
141 ppmngr.collapseRecords()
142
143 # All done with the workers. Close the pool and get reports.
144 pool_stats = pool.close()
145 total_stats = ExecutionStats()
146 total_stats.mergeStats(stats)
147 for ps in pool_stats:
148 if ps is not None:
149 total_stats.mergeStats(ps)
150 current_records.stats = total_stats
151
152 # Shutdown the pipelines.
153 ppmngr.shutdownPipelines()
154
155 # Backup previous records.
156 if self.rotate_bake_records:
157 records_dir, records_fn = os.path.split(records_path)
158 records_id, _ = os.path.splitext(records_fn)
159 for i in range(8, -1, -1):
160 suffix = '' if i == 0 else '.%d' % i
161 records_path_i = os.path.join(
162 records_dir,
163 '%s%s.records' % (records_id, suffix))
164 if os.path.exists(records_path_i):
165 records_path_next = os.path.join(
166 records_dir,
167 '%s.%s.records' % (records_id, i + 1))
168 if os.path.exists(records_path_next):
169 os.remove(records_path_next)
170 os.rename(records_path_i, records_path_next)
171
172 # Save the bake records.
173 with format_timed_scope(logger, "saved bake records.",
174 level=logging.DEBUG, colored=False):
175 current_records.bake_time = time.time()
176 current_records.out_dir = self.out_dir
177 current_records.save(records_path)
178
179 # All done.
180 self.app.config.set('baker/is_baking', False)
181 logger.debug(format_timed(start_time, 'done baking'))
182
183 return current_records
184
185 def _handleCacheValidity(self, previous_records, current_records):
186 start_time = time.perf_counter()
187
188 reason = None
189 if self.force:
190 reason = "ordered to"
191 elif not self.app.config.get('__cache_valid'):
192 # The configuration file was changed, or we're running a new
193 # version of the app.
194 reason = "not valid anymore"
195 elif previous_records.invalidated:
196 # We have no valid previous bake records.
197 reason = "need bake records regeneration"
198 else:
199 # Check if any template has changed since the last bake. Since
200 # there could be some advanced conditional logic going on, we'd
201 # better just force a bake from scratch if that's the case.
202 max_time = 0
203 for d in self.app.templates_dirs:
204 for dpath, _, filenames in os.walk(d):
205 for fn in filenames:
206 full_fn = os.path.join(dpath, fn)
207 max_time = max(max_time, os.path.getmtime(full_fn))
208 if max_time >= previous_records.bake_time:
209 reason = "templates modified"
210
211 if reason is not None:
212 # We have to bake everything from scratch.
213 self.app.cache.clearCaches(except_names=['app', 'baker'])
214 self.force = True
215 current_records.incremental_count = 0
216 previous_records = MultiRecord()
217 logger.info(format_timed(
218 start_time, "cleaned cache (reason: %s)" % reason))
219 return False
220 else:
221 current_records.incremental_count += 1
222 logger.debug(format_timed(
223 start_time, "cache is assumed valid", colored=False))
224 return True
225 211
226 def _bakeRealm(self, pool, record_histories, pp_pass_num, realm, pplist): 212 def _bakeRealm(self, pool, record_histories, pp_pass_num, realm, pplist):
227 # Start with the first step, where we iterate on the content sources' 213 # Start with the first step, where we iterate on the content sources'
228 # items and run jobs on those. 214 # items and run jobs on those.
229 pool.userdata.cur_step = 0 215 pool.userdata.cur_step = 0
403 pp_by_realm = pp_by_pass_and_realm.setdefault(pp_pass_num, {}) 389 pp_by_realm = pp_by_pass_and_realm.setdefault(pp_pass_num, {})
404 pplist = pp_by_realm.setdefault( 390 pplist = pp_by_realm.setdefault(
405 pp_info.pipeline.source.config['realm'], []) 391 pp_info.pipeline.source.config['realm'], [])
406 pplist.append(pp_info) 392 pplist.append(pp_info)
407 393
394
395 def _merge_execution_stats(base_stats, *other_stats):
396 total_stats = ExecutionStats()
397 total_stats.mergeStats(base_stats)
398 for ps in other_stats:
399 if ps is not None:
400 total_stats.mergeStats(ps)
401 return total_stats
402
403
404 def _save_bake_records(records, records_path, *, rotate_previous):
405 if rotate_previous:
406 records_dir, records_fn = os.path.split(records_path)
407 records_id, _ = os.path.splitext(records_fn)
408 for i in range(8, -1, -1):
409 suffix = '' if i == 0 else '.%d' % i
410 records_path_i = os.path.join(
411 records_dir,
412 '%s%s.records' % (records_id, suffix))
413 if os.path.exists(records_path_i):
414 records_path_next = os.path.join(
415 records_dir,
416 '%s.%s.records' % (records_id, i + 1))
417 if os.path.exists(records_path_next):
418 os.remove(records_path_next)
419 os.rename(records_path_i, records_path_next)
420
421 with format_timed_scope(logger, "saved bake records.",
422 level=logging.DEBUG, colored=False):
423 records.save(records_path)