Mercurial > piecrust2

comparison piecrust/commands/builtin/baking.py @ 411:e7b865f8f335

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Enable multiprocess baking. Baking is now done by running a worker per CPU, and sending jobs to them. This changes several things across the codebase: * Ability to not cache things related to pages other than the 'main' page (i.e. the page at the bottom of the execution stack). * Decouple the baking process from the bake records, so only the main process keeps track (and modifies) the bake record. * Remove the need for 'batch page getters' and loading a page directly from the page factories. There are various smaller changes too included here, including support for scope performance timers that are saved with the bake record and can be printed out to the console. Yes I got carried away. For testing, the in-memory 'mock' file-system doesn't work anymore, since we're spawning processes, so this is replaced by a 'tmpfs' file-system which is saved in temporary files on disk and deleted after tests have run.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 Jun 2015 17:09:19 -0700
parents 2cd2b5d07129
children 0e9a94b7fdfa
comparison
equal deleted inserted replaced
410:d1a472464e57 411:e7b865f8f335
4 import hashlib 4 import hashlib
5 import fnmatch 5 import fnmatch
6 import datetime 6 import datetime
7 from piecrust.baking.baker import Baker 7 from piecrust.baking.baker import Baker
8 from piecrust.baking.records import ( 8 from piecrust.baking.records import (
9 BakeRecord, BakeRecordPageEntry, BakeRecordSubPageEntry) 9 BakeRecord, BakeRecordEntry, SubPageBakeInfo)
10 from piecrust.chefutil import format_timed 10 from piecrust.chefutil import format_timed
11 from piecrust.commands.base import ChefCommand 11 from piecrust.commands.base import ChefCommand
12 from piecrust.processing.base import ProcessorPipeline 12 from piecrust.processing.base import ProcessorPipeline
13 from piecrust.processing.records import ( 13 from piecrust.processing.records import (
14 ProcessorPipelineRecord, 14 ProcessorPipelineRecord,
34 parser.add_argument( 34 parser.add_argument(
35 '-f', '--force', 35 '-f', '--force',
36 help="Force re-baking the entire website.", 36 help="Force re-baking the entire website.",
37 action='store_true') 37 action='store_true')
38 parser.add_argument( 38 parser.add_argument(
39 '-w', '--workers',
40 help="The number of worker processes to spawn.",
41 type=int, default=-1)
42 parser.add_argument(
39 '--assets-only', 43 '--assets-only',
40 help="Only bake the assets (don't bake the web pages).", 44 help="Only bake the assets (don't bake the web pages).",
41 action='store_true') 45 action='store_true')
42 parser.add_argument( 46 parser.add_argument(
43 '--html-only', 47 '--html-only',
44 help="Only bake HTML files (don't run the asset pipeline).", 48 help="Only bake HTML files (don't run the asset pipeline).",
49 action='store_true')
50 parser.add_argument(
51 '--show-timers',
52 help="Show detailed timing information.",
45 action='store_true') 53 action='store_true')
46 54
47 def run(self, ctx): 55 def run(self, ctx):
48 out_dir = (ctx.args.output or 56 out_dir = (ctx.args.output or
49 os.path.join(ctx.app.root_dir, '_counter')) 57 os.path.join(ctx.app.root_dir, '_counter'))
50 58
51 success = True 59 success = True
52 start_time = time.clock() 60 start_time = time.perf_counter()
53 try: 61 try:
54 # Bake the site sources. 62 # Bake the site sources.
55 if not ctx.args.assets_only: 63 if not ctx.args.assets_only:
56 success = success & self._bakeSources(ctx, out_dir) 64 success = success & self._bakeSources(ctx, out_dir)
57 65
69 else: 77 else:
70 logger.error(str(ex)) 78 logger.error(str(ex))
71 return 1 79 return 1
72 80
73 def _bakeSources(self, ctx, out_dir): 81 def _bakeSources(self, ctx, out_dir):
82 if ctx.args.workers > 0:
83 ctx.app.config.set('baker/workers', ctx.args.workers)
74 baker = Baker( 84 baker = Baker(
75 ctx.app, out_dir, 85 ctx.app, out_dir,
76 force=ctx.args.force) 86 force=ctx.args.force)
77 record = baker.bake() 87 record = baker.bake()
88
89 if ctx.args.show_timers:
90 if record.timers:
91 from colorama import Fore
92 logger.info("-------------------")
93 logger.info("Timing information:")
94 for name in sorted(record.timers.keys()):
95 val_str = '%8.1f s' % record.timers[name]
96 logger.info(
97 "[%s%s%s] %s" %
98 (Fore.GREEN, val_str, Fore.RESET, name))
99 else:
100 logger.warning("Timing information is not available.")
101
78 return record.success 102 return record.success
79 103
80 def _bakeAssets(self, ctx, out_dir): 104 def _bakeAssets(self, ctx, out_dir):
81 proc = ProcessorPipeline( 105 proc = ProcessorPipeline(
82 ctx.app, out_dir, 106 ctx.app, out_dir,
149 any([o for o in entry.out_paths 173 any([o for o in entry.out_paths
150 if fnmatch.fnmatch(o, out_pattern)])): 174 if fnmatch.fnmatch(o, out_pattern)])):
151 continue 175 continue
152 176
153 flags = [] 177 flags = []
154 if entry.flags & BakeRecordPageEntry.FLAG_OVERRIDEN: 178 if entry.flags & BakeRecordEntry.FLAG_OVERRIDEN:
155 flags.append('overriden') 179 flags.append('overriden')
156 180
157 passes = {PASS_RENDERING: 'render', PASS_FORMATTING: 'format'} 181 passes = {PASS_RENDERING: 'render', PASS_FORMATTING: 'format'}
158 182
159 logging.info(" - ") 183 logging.info(" - ")
160 logging.info(" path: %s" % entry.rel_path) 184 logging.info(" path: %s" % entry.rel_path)
161 logging.info(" spec: %s:%s" % (entry.source_name, 185 logging.info(" spec: %s:%s" % (entry.source_name,
162 entry.rel_path)) 186 entry.rel_path))
163 if entry.taxonomy_info: 187 if entry.taxonomy_info:
164 tn, t, sn = entry.taxonomy_info 188 tax_name, term, source_name = entry.taxonomy_info
165 logging.info(" taxonomy: %s (%s:%s)" % 189 logging.info(" taxonomy: %s (%s:%s)" %
166 (t, sn, tn)) 190 (term, source_name, tax_name))
167 else: 191 else:
168 logging.info(" taxonomy: <none>") 192 logging.info(" taxonomy: <none>")
169 logging.info(" flags: %s" % ', '.join(flags)) 193 logging.info(" flags: %s" % ', '.join(flags))
170 logging.info(" config: %s" % entry.config) 194 logging.info(" config: %s" % entry.config)
171 195
176 logging.info(" path: %s" % os.path.relpath(sub.out_path, 200 logging.info(" path: %s" % os.path.relpath(sub.out_path,
177 out_dir)) 201 out_dir))
178 logging.info(" baked?: %s" % sub.was_baked) 202 logging.info(" baked?: %s" % sub.was_baked)
179 203
180 sub_flags = [] 204 sub_flags = []
181 if sub.flags & BakeRecordSubPageEntry.FLAG_FORCED_BY_SOURCE: 205 if sub.flags & SubPageBakeInfo.FLAG_FORCED_BY_SOURCE:
182 sub_flags.append('forced by source') 206 sub_flags.append('forced by source')
183 if sub.flags & BakeRecordSubPageEntry.FLAG_FORCED_BY_NO_PREVIOUS: 207 if sub.flags & SubPageBakeInfo.FLAG_FORCED_BY_NO_PREVIOUS:
184 sub_flags.append('forced by missing previous record entry') 208 sub_flags.append('forced by missing previous record entry')
185 if sub.flags & BakeRecordSubPageEntry.FLAG_FORCED_BY_PREVIOUS_ERRORS: 209 if sub.flags & SubPageBakeInfo.FLAG_FORCED_BY_PREVIOUS_ERRORS:
186 sub_flags.append('forced by previous errors') 210 sub_flags.append('forced by previous errors')
187 logging.info(" flags: %s" % ', '.join(sub_flags)) 211 logging.info(" flags: %s" % ', '.join(sub_flags))
188 212
189 for p, pi in sub.render_passes.items(): 213 for p, pi in sub.render_passes.items():
190 logging.info(" %s pass:" % passes[p]) 214 logging.info(" %s pass:" % passes[p])
191 logging.info(" used srcs: %s" % 215 logging.info(" used srcs: %s" %
192 ', '.join(pi.used_source_names)) 216 ', '.join(pi.used_source_names))
193 logging.info(" used terms: %s" % 217 logging.info(" used terms: %s" %
194 ', '.join( 218 ', '.join(
195 ['%s (%s:%s)' % (t, sn, tn) 219 ['%s (%s:%s)' % (t, sn, tn)
196 for sn, tn, t in pi.used_taxonomy_terms])) 220 for sn, tn, t in
221 pi.used_taxonomy_terms]))
197 222
198 if sub.errors: 223 if sub.errors:
199 logging.error(" errors: %s" % sub.errors) 224 logging.error(" errors: %s" % sub.errors)
200 225
201 logging.info(" assets: %s" % ', '.join(entry.assets)) 226 logging.info(" assets: %s" % ', '.join(entry.assets))