Mercurial > piecrust2

comparison piecrust/processing/pipeline.py @ 687:61d606fbc313

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Change `show-timers` to `show-stats`, add stats.
author Ludovic Chabant <ludovic@chabant.com>
date Mon, 14 Mar 2016 08:26:14 -0700
parents 81d9c3a3a0b5
children 9a92e2804562
comparison
equal deleted inserted replaced
686:1a6c4c2683fd 687:61d606fbc313
4 import time 4 import time
5 import hashlib 5 import hashlib
6 import logging 6 import logging
7 import multiprocessing 7 import multiprocessing
8 from piecrust.chefutil import format_timed, format_timed_scope 8 from piecrust.chefutil import format_timed, format_timed_scope
9 from piecrust.environment import ExecutionStats
9 from piecrust.processing.base import PipelineContext 10 from piecrust.processing.base import PipelineContext
10 from piecrust.processing.records import ( 11 from piecrust.processing.records import (
11 ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord, 12 ProcessorPipelineRecordEntry, TransitionalProcessorPipelineRecord,
12 FLAG_PROCESSED) 13 FLAG_PROCESSED)
13 from piecrust.processing.worker import ( 14 from piecrust.processing.worker import (
52 'workers', multiprocessing.cpu_count()) 53 'workers', multiprocessing.cpu_count())
53 54
54 ignores = baker_params.get('ignore', []) 55 ignores = baker_params.get('ignore', [])
55 ignores += [ 56 ignores += [
56 '_cache', '_counter', 57 '_cache', '_counter',
57 'theme_info.yml',
58 '.DS_Store', 'Thumbs.db', 58 '.DS_Store', 'Thumbs.db',
59 '.git*', '.hg*', '.svn'] 59 '.git*', '.hg*', '.svn']
60 self.ignore_patterns = make_re(ignores) 60 self.ignore_patterns = make_re(ignores)
61 self.force_patterns = make_re(baker_params.get('force', [])) 61 self.force_patterns = make_re(baker_params.get('force', []))
62 62
146 ar = pool.queueJobs(jobs, handler=_handler) 146 ar = pool.queueJobs(jobs, handler=_handler)
147 ar.wait() 147 ar.wait()
148 148
149 # Shutdown the workers and get timing information from them. 149 # Shutdown the workers and get timing information from them.
150 reports = pool.close() 150 reports = pool.close()
151 record.current.timers = {} 151 total_stats = ExecutionStats()
152 record.current.stats['_Total'] = total_stats
152 for i in range(len(reports)): 153 for i in range(len(reports)):
153 timers = reports[i] 154 worker_stats = reports[i]['data']
154 if timers is None: 155 if worker_stats is not None:
155 continue 156 worker_name = 'PipelineWorker_%d' % i
156 157 record.current.stats[worker_name] = worker_stats
157 worker_name = 'PipelineWorker_%d' % i 158 total_stats.mergeStats(worker_stats)
158 record.current.timers[worker_name] = {}
159 for name, val in timers['data'].items():
160 main_val = record.current.timers.setdefault(name, 0)
161 record.current.timers[name] = main_val + val
162 record.current.timers[worker_name][name] = val
163 159
164 # Invoke post-processors. 160 # Invoke post-processors.
165 pipeline_ctx.record = record.current 161 pipeline_ctx.record = record.current
166 for proc in processors: 162 for proc in processors:
167 proc.onPipelineEnd(pipeline_ctx) 163 proc.onPipelineEnd(pipeline_ctx)