Mercurial > piecrust2

comparison piecrust/baking/baker.py @ 876:d1095774bfcf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactor: Fix some issues with record/cache entry collisions, add counters.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 15 Jun 2017 07:33:40 -0700
parents 504ddb370df8
children d6d35b2efd04
comparison
equal deleted inserted replaced
875:7169bf42ec60 876:d1095774bfcf
41 logger.debug(" Root URL: %s" % self.app.config.get('site/root')) 41 logger.debug(" Root URL: %s" % self.app.config.get('site/root'))
42 42
43 # Get into bake mode. 43 # Get into bake mode.
44 self.app.config.set('baker/is_baking', True) 44 self.app.config.set('baker/is_baking', True)
45 self.app.config.set('site/asset_url_format', '%page_uri%/%filename%') 45 self.app.config.set('site/asset_url_format', '%page_uri%/%filename%')
46
47 stats = self.app.env.stats
48 stats.registerTimer('WorkerTastPut')
46 49
47 # Make sure the output directory exists. 50 # Make sure the output directory exists.
48 if not os.path.isdir(self.out_dir): 51 if not os.path.isdir(self.out_dir):
49 os.makedirs(self.out_dir, 0o755) 52 os.makedirs(self.out_dir, 0o755)
50 53
109 realm_list = [REALM_USER, REALM_THEME] 112 realm_list = [REALM_USER, REALM_THEME]
110 113
111 # Bake the realms -- user first, theme second, so that a user item 114 # Bake the realms -- user first, theme second, so that a user item
112 # can override a theme item. 115 # can override a theme item.
113 # Do this for as many times as we have pipeline passes left to do. 116 # Do this for as many times as we have pipeline passes left to do.
114 pp_by_pass_and_realm = {} 117 pp_by_pass_and_realm = _get_pipeline_infos_by_pass_and_realm(
115 for ppinfo in ppmngr.getPipelines(): 118 ppmngr.getPipelines())
116 pp_by_realm = pp_by_pass_and_realm.setdefault(
117 ppinfo.pipeline.PASS_NUM, {})
118 pplist = pp_by_realm.setdefault(
119 ppinfo.pipeline.source.config['realm'], [])
120 pplist.append(ppinfo)
121 119
122 for pp_pass_num in sorted(pp_by_pass_and_realm.keys()): 120 for pp_pass_num in sorted(pp_by_pass_and_realm.keys()):
123 logger.debug("Pipelines pass %d" % pp_pass_num) 121 logger.debug("Pipelines pass %d" % pp_pass_num)
124 pp_by_realm = pp_by_pass_and_realm[pp_pass_num] 122 pp_by_realm = pp_by_pass_and_realm[pp_pass_num]
125 for realm in realm_list: 123 for realm in realm_list:
134 ppmngr.collapseRecords() 132 ppmngr.collapseRecords()
135 133
136 # All done with the workers. Close the pool and get reports. 134 # All done with the workers. Close the pool and get reports.
137 pool_stats = pool.close() 135 pool_stats = pool.close()
138 total_stats = ExecutionStats() 136 total_stats = ExecutionStats()
137 total_stats.mergeStats(stats)
139 for ps in pool_stats: 138 for ps in pool_stats:
140 if ps is not None: 139 if ps is not None:
141 total_stats.mergeStats(ps) 140 total_stats.mergeStats(ps)
142 current_records.stats = total_stats 141 current_records.stats = total_stats
143 142
222 pool.userdata.next_pass_jobs = next_pass_jobs 221 pool.userdata.next_pass_jobs = next_pass_jobs
223 222
224 start_time = time.perf_counter() 223 start_time = time.perf_counter()
225 job_count = 0 224 job_count = 0
226 realm_name = REALM_NAMES[realm].lower() 225 realm_name = REALM_NAMES[realm].lower()
226 stats = self.app.env.stats
227 227
228 for ppinfo in pplist: 228 for ppinfo in pplist:
229 src = ppinfo.source 229 src = ppinfo.source
230 pp = ppinfo.pipeline 230 pp = ppinfo.pipeline
231 231
238 jcctx = PipelineJobCreateContext(pp_pass_num, record_histories) 238 jcctx = PipelineJobCreateContext(pp_pass_num, record_histories)
239 jobs = pp.createJobs(jcctx) 239 jobs = pp.createJobs(jcctx)
240 if jobs is not None: 240 if jobs is not None:
241 job_count += len(jobs) 241 job_count += len(jobs)
242 pool.queueJobs(jobs) 242 pool.queueJobs(jobs)
243
244 stats.stepTimer('WorkerTastPut', time.perf_counter() - start_time)
243 245
244 if job_count == 0: 246 if job_count == 0:
245 logger.debug("No jobs queued! Bailing out of this bake pass.") 247 logger.debug("No jobs queued! Bailing out of this bake pass.")
246 return 248 return
247 249
270 "Queuing jobs for source '%s' (%s, pass %d)." % 272 "Queuing jobs for source '%s' (%s, pass %d)." %
271 (sn, realm_name, pool.userdata.cur_pass)) 273 (sn, realm_name, pool.userdata.cur_pass))
272 job_count += len(jobs) 274 job_count += len(jobs)
273 pool.userdata.next_pass_jobs[sn] = [] 275 pool.userdata.next_pass_jobs[sn] = []
274 pool.queueJobs(jobs) 276 pool.queueJobs(jobs)
277
278 stats.stepTimer('WorkerTastPut', time.perf_counter() - start_time)
275 279
276 if job_count == 0: 280 if job_count == 0:
277 break 281 break
278 282
279 pool.wait() 283 pool.wait()
346 e = entry_class() 350 e = entry_class()
347 e.item_spec = job.content_item.spec 351 e.item_spec = job.content_item.spec
348 e.errors.append(str(exc_data)) 352 e.errors.append(str(exc_data))
349 record.addEntry(e) 353 record.addEntry(e)
350 else: 354 else:
351 e = record.getEntry(job.content_item.spec) 355 e = record.getEntry(job.record_entry_spec)
352 e.errors.append(str(exc_data)) 356 e.errors.append(str(exc_data))
353 357
354 record.success = False 358 record.success = False
355 userdata.records.success = False 359 userdata.records.success = False
356 360
364 self.baker = baker 368 self.baker = baker
365 self.ppmngr = ppmngr 369 self.ppmngr = ppmngr
366 self.records = ppmngr.record_histories.current 370 self.records = ppmngr.record_histories.current
367 self.cur_pass = 0 371 self.cur_pass = 0
368 self.next_pass_jobs = {} 372 self.next_pass_jobs = {}
373
374
375 def _get_pipeline_infos_by_pass_and_realm(pp_infos):
376 pp_by_pass_and_realm = {}
377 for pp_info in pp_infos:
378 pp_pass_num = pp_info.pipeline.PASS_NUM
379 if isinstance(pp_pass_num, list):
380 for ppn in pp_pass_num:
381 _add_pipeline_info_to_pass_and_realm_dict(
382 ppn, pp_info, pp_by_pass_and_realm)
383 else:
384 _add_pipeline_info_to_pass_and_realm_dict(
385 pp_pass_num, pp_info, pp_by_pass_and_realm)
386 return pp_by_pass_and_realm
387
388
389 def _add_pipeline_info_to_pass_and_realm_dict(pp_pass_num, pp_info,
390 pp_by_pass_and_realm):
391 pp_by_realm = pp_by_pass_and_realm.setdefault(pp_pass_num, {})
392 pplist = pp_by_realm.setdefault(
393 pp_info.pipeline.source.config['realm'], [])
394 pplist.append(pp_info)
395