Mercurial > piecrust2

comparison piecrust/baking/baker.py @ 1007:09dc0240f08a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Simplify output.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 25 Nov 2017 23:01:33 -0800
parents 8adc27285d93
children c4cf3cfe2726
comparison
equal deleted inserted replaced
1006:58ef814cc83e 1007:09dc0240f08a
247 247
248 start_time = time.perf_counter() 248 start_time = time.perf_counter()
249 job_count = 0 249 job_count = 0
250 stats = self.app.env.stats 250 stats = self.app.env.stats
251 realm_name = REALM_NAMES[realm].lower() 251 realm_name = REALM_NAMES[realm].lower()
252 participating_source_names = []
252 253
253 for ppinfo in pplist: 254 for ppinfo in pplist:
254 src = ppinfo.source 255 src = ppinfo.source
255 pp = ppinfo.pipeline 256 pp = ppinfo.pipeline
256 jcctx = PipelineJobCreateContext(pp_pass_num, pp.record_name, 257 jcctx = PipelineJobCreateContext(pp_pass_num, pp.record_name,
260 jobs = pp.createJobs(jcctx) 261 jobs = pp.createJobs(jcctx)
261 if jobs is not None: 262 if jobs is not None:
262 new_job_count = len(jobs) 263 new_job_count = len(jobs)
263 job_count += new_job_count 264 job_count += new_job_count
264 pool.queueJobs(jobs) 265 pool.queueJobs(jobs)
266 participating_source_names.append(src.name)
265 else: 267 else:
266 new_job_count = 0 268 new_job_count = 0
267 269
268 logger.debug( 270 logger.debug(
269 "Queued %d jobs for source '%s' using pipeline '%s' " 271 "Queued %d jobs for source '%s' using pipeline '%s' "
277 return 279 return
278 280
279 pool.wait() 281 pool.wait()
280 282
281 logger.info(format_timed( 283 logger.info(format_timed(
282 start_time, "%d pipeline jobs completed (%s, step 0)." % 284 start_time, "%d jobs completed (%s)." %
283 (job_count, realm_name))) 285 (job_count, ', '.join(participating_source_names))))
284 286
285 # Now let's see if any job created a follow-up job. Let's keep 287 # Now let's see if any job created a follow-up job. Let's keep
286 # processing those jobs as long as they create new ones. 288 # processing those jobs as long as they create new ones.
287 pool.userdata.cur_step = 1 289 pool.userdata.cur_step = 1
288 while True: 290 while True:
292 next_step_jobs = pool.userdata.next_step_jobs 294 next_step_jobs = pool.userdata.next_step_jobs
293 pool.userdata.next_step_jobs = {} 295 pool.userdata.next_step_jobs = {}
294 296
295 start_time = time.perf_counter() 297 start_time = time.perf_counter()
296 job_count = 0 298 job_count = 0
299 participating_source_names = []
297 300
298 for sn, jobs in next_step_jobs.items(): 301 for sn, jobs in next_step_jobs.items():
299 if jobs: 302 if jobs:
300 logger.debug( 303 logger.debug(
301 "Queuing jobs for source '%s' (%s, step %d)." % 304 "Queuing jobs for source '%s' (%s, step %d)." %
308 pp.validateNextStepJobs(jobs, valctx) 311 pp.validateNextStepJobs(jobs, valctx)
309 312
310 job_count += len(jobs) 313 job_count += len(jobs)
311 pool.userdata.next_step_jobs[sn] = [] 314 pool.userdata.next_step_jobs[sn] = []
312 pool.queueJobs(jobs) 315 pool.queueJobs(jobs)
316 participating_source_names.append(sn)
313 317
314 stats.stepTimer('MasterTaskPut_2+', time.perf_counter() - start_time) 318 stats.stepTimer('MasterTaskPut_2+', time.perf_counter() - start_time)
315 319
316 if job_count == 0: 320 if job_count == 0:
317 break 321 break
318 322
319 pool.wait() 323 pool.wait()
320 324
321 logger.info(format_timed( 325 logger.info(format_timed(
322 start_time, 326 start_time,
323 "%d pipeline jobs completed (%s, step %d)." % 327 "%d jobs completed (%s)." %
324 (job_count, realm_name, pool.userdata.cur_step))) 328 (job_count, ', '.join(participating_source_names))))
325 329
326 pool.userdata.cur_step += 1 330 pool.userdata.cur_step += 1
327 331
328 def _logErrors(self, item_spec, errors): 332 def _logErrors(self, item_spec, errors):
329 logger.error("Errors found in %s:" % item_spec) 333 logger.error("Errors found in %s:" % item_spec)