Mercurial > piecrust2

comparison piecrust/commands/builtin/baking.py @ 853:f070a4fc033c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
core: Continue PieCrust3 refactor, simplify pages. The asset pipeline is still the only function pipeline at this point. * No more `QualifiedPage`, and several other pieces of code deleted. * Data providers are simpler and more focused. For instance, the page iterator doesn't try to support other types of items. * Route parameters are proper known source metadata to remove the confusion between the two. * Make the baker and pipeline more correctly manage records and record histories. * Add support for record collapsing and deleting stale outputs in the asset pipeline.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 21 May 2017 00:06:59 -0700
parents 4850f8c21b6e
children 08e02c2a2a1a
comparison
equal deleted inserted replaced
852:4850f8c21b6e 853:f070a4fc033c
1 import time 1 import time
2 import os.path 2 import os.path
3 import logging 3 import logging
4 import hashlib
5 import fnmatch 4 import fnmatch
6 import datetime 5 import datetime
7 from colorama import Fore 6 from colorama import Fore
8 from piecrust.commands.base import ChefCommand 7 from piecrust.commands.base import ChefCommand
9 8
101 records = baker.bake() 100 records = baker.bake()
102 101
103 return records 102 return records
104 103
105 104
105 class ShowRecordCommand(ChefCommand):
106 def __init__(self):
107 super(ShowRecordCommand, self).__init__()
108 self.name = 'showrecord'
109 self.description = ("Shows the bake record for a given output "
110 "directory.")
111
112 def setupParser(self, parser, app):
113 parser.add_argument(
114 '-o', '--output',
115 help="The output directory for which to show the bake record "
116 "(defaults to `_counter`)",
117 nargs='?')
118 parser.add_argument(
119 '-i', '--in-path',
120 help="A pattern that will be used to filter the relative path "
121 "of entries to show.")
122 parser.add_argument(
123 '-t', '--out-path',
124 help="A pattern that will be used to filter the output path "
125 "of entries to show.")
126 parser.add_argument(
127 '--fails',
128 action='store_true',
129 help="Only show record entries for failures.")
130 parser.add_argument(
131 '--last',
132 type=int,
133 default=0,
134 help="Show the last Nth bake record.")
135 parser.add_argument(
136 '--html-only',
137 action='store_true',
138 help="Only show records for pages (not from the asset "
139 "pipeline).")
140 parser.add_argument(
141 '--assets-only',
142 action='store_true',
143 help="Only show records for assets (not from pages).")
144 parser.add_argument(
145 '-p', '--pipelines',
146 nargs='*',
147 help="Only show records for the given pipeline(s).")
148 parser.add_argument(
149 '--show-stats',
150 action='store_true',
151 help="Show stats from the record.")
152 parser.add_argument(
153 '--show-manifest',
154 help="Show manifest entries from the record.")
155
156 def run(self, ctx):
157 from piecrust.baking.baker import get_bake_records_path
158 from piecrust.pipelines.records import load_records
159
160 out_dir = ctx.args.output or os.path.join(ctx.app.root_dir, '_counter')
161 suffix = '' if ctx.args.last == 0 else '.%d' % ctx.args.last
162 records_path = get_bake_records_path(ctx.app, out_dir, suffix=suffix)
163 records = load_records(records_path)
164 if records.invalidated:
165 raise Exception(
166 "The bake record was saved by a previous version of "
167 "PieCrust and can't be shown.")
168
169 in_pattern = None
170 if ctx.args.in_path:
171 in_pattern = '*%s*' % ctx.args.in_path.strip('*')
172
173 out_pattern = None
174 if ctx.args.out_path:
175 out_pattern = '*%s*' % ctx.args.out.strip('*')
176
177 pipelines = ctx.args.pipelines
178 if not pipelines:
179 pipelines = [p.PIPELINE_NAME
180 for p in ctx.app.plugin_loader.getPipelines()]
181 if ctx.args.assets_only:
182 pipelines = ['asset']
183 if ctx.args.html_only:
184 pipelines = ['page']
185
186 logger.info("Bake record for: %s" % out_dir)
187 logger.info("Status: %s" % ('SUCCESS' if records.success
188 else 'FAILURE'))
189 logger.info("Date/time: %s" %
190 datetime.datetime.fromtimestamp(records.bake_time))
191 logger.info("Incremental count: %d" % records.incremental_count)
192 logger.info("Versions: %s/%s" % (records._app_version,
193 records._record_version))
194 logger.info("")
195
196 for rec in records.records:
197 if ctx.args.fails and rec.success:
198 continue
199
200 logger.info("Record: %s" % rec.name)
201 logger.info("Status: %s" % ('SUCCESS' if rec.success
202 else 'FAILURE'))
203 for e in rec.entries:
204 if ctx.args.fails and e.success:
205 continue
206 if in_pattern and not fnmatch.fnmatch(e.item_spec, in_pattern):
207 continue
208 if out_pattern and not any(
209 [fnmatch.fnmatch(op, out_pattern)
210 for op in e.out_paths]):
211 continue
212 _print_record_entry(e)
213
214 logger.info("")
215
216 stats = records.stats
217 if ctx.args.show_stats:
218 _show_stats(stats)
219
220 if ctx.args.show_manifest:
221 for name in sorted(stats.manifests.keys()):
222 if ctx.args.show_manifest.lower() in name.lower():
223 val = stats.manifests[name]
224 logger.info(
225 " [%s%s%s] [%d entries]" %
226 (Fore.CYAN, name, Fore.RESET, len(val)))
227 for v in val:
228 logger.info(" - %s" % v)
229
230
106 def _show_stats(stats, *, full=False): 231 def _show_stats(stats, *, full=False):
107 indent = ' ' 232 indent = ' '
108 233
109 logger.info(' Timers:') 234 logger.info(' Timers:')
110 for name, val in sorted(stats.timers.items(), key=lambda i: i[1], 235 for name, val in sorted(stats.timers.items(), key=lambda i: i[1],
130 if full: 255 if full:
131 for v in val: 256 for v in val:
132 logger.info("%s - %s" % (indent, v)) 257 logger.info("%s - %s" % (indent, v))
133 258
134 259
135 class ShowRecordCommand(ChefCommand): 260 def _print_record_entry(e):
136 def __init__(self): 261 logger.info(" - %s" % e.item_spec)
137 super(ShowRecordCommand, self).__init__() 262 logger.info(" Outputs:")
138 self.name = 'showrecord' 263 if e.out_paths:
139 self.description = ("Shows the bake record for a given output " 264 for op in e.out_paths:
140 "directory.") 265 logger.info(" - %s" % op)
141 266 else:
142 def setupParser(self, parser, app): 267 logger.info(" <none>")
143 parser.add_argument( 268
144 '-o', '--output', 269 e_desc = e.describe()
145 help="The output directory for which to show the bake record " 270 for k in sorted(e_desc.keys()):
146 "(defaults to `_counter`)", 271 logger.info(" %s: %s" % (k, e_desc[k]))
147 nargs='?') 272
148 parser.add_argument( 273 if e.errors:
149 '-p', '--path', 274 logger.error(" Errors:")
150 help="A pattern that will be used to filter the relative path " 275 for err in e.errors:
151 "of entries to show.") 276 logger.error(" - %s" % err)
152 parser.add_argument(
153 '-t', '--out',
154 help="A pattern that will be used to filter the output path "
155 "of entries to show.")
156 parser.add_argument(
157 '--last',
158 type=int,
159 default=0,
160 help="Show the last Nth bake record.")
161 parser.add_argument(
162 '--html-only',
163 action='store_true',
164 help="Only show records for pages (not from the asset "
165 "pipeline).")
166 parser.add_argument(
167 '--assets-only',
168 action='store_true',
169 help="Only show records for assets (not from pages).")
170 parser.add_argument(
171 '--show-stats',
172 action='store_true',
173 help="Show stats from the record.")
174 parser.add_argument(
175 '--show-manifest',
176 help="Show manifest entries from the record.")
177
178 def run(self, ctx):
179 from piecrust.processing.records import (
180 FLAG_PREPARED, FLAG_PROCESSED, FLAG_BYPASSED_STRUCTURED_PROCESSING,
181 FLAG_COLLAPSED_FROM_LAST_RUN)
182 from piecrust.rendering import (
183 PASS_FORMATTING, PASS_RENDERING)
184
185 out_dir = ctx.args.output or os.path.join(ctx.app.root_dir, '_counter')
186 record_id = hashlib.md5(out_dir.encode('utf8')).hexdigest()
187 suffix = '' if ctx.args.last == 0 else '.%d' % ctx.args.last
188 record_name = '%s%s.record' % (record_id, suffix)
189
190 pattern = None
191 if ctx.args.path:
192 pattern = '*%s*' % ctx.args.path.strip('*')
193
194 out_pattern = None
195 if ctx.args.out:
196 out_pattern = '*%s*' % ctx.args.out.strip('*')
197
198 if not ctx.args.show_stats and not ctx.args.show_manifest:
199 if not ctx.args.assets_only:
200 self._showBakeRecord(
201 ctx, record_name, pattern, out_pattern)
202 if not ctx.args.html_only:
203 self._showProcessingRecord(
204 ctx, record_name, pattern, out_pattern)
205 return
206
207 stats = {}
208 bake_rec = self._getBakeRecord(ctx, record_name)
209 if bake_rec:
210 _merge_stats(bake_rec.stats, stats)
211 proc_rec = self._getProcessingRecord(ctx, record_name)
212 if proc_rec:
213 _merge_stats(proc_rec.stats, stats)
214
215 if ctx.args.show_stats:
216 _show_stats(stats, full=False)
217
218 if ctx.args.show_manifest:
219 for name in sorted(stats.keys()):
220 logger.info('%s:' % name)
221 s = stats[name]
222 for name in sorted(s.manifests.keys()):
223 if ctx.args.show_manifest.lower() in name.lower():
224 val = s.manifests[name]
225 logger.info(
226 " [%s%s%s] [%d entries]" %
227 (Fore.CYAN, name, Fore.RESET, len(val)))
228 for v in val:
229 logger.info(" - %s" % v)
230
231 def _getBakeRecord(self, ctx, record_name):
232 record_cache = ctx.app.cache.getCache('baker')
233 if not record_cache.has(record_name):
234 logger.warning(
235 "No page bake record has been created for this output "
236 "path.")
237 return None
238
239 record = BakeRecord.load(record_cache.getCachePath(record_name))
240 return record
241
242 def _showBakeRecord(self, ctx, record_name, pattern, out_pattern):
243 record = self._getBakeRecord(ctx, record_name)
244 if record is None:
245 return
246
247 logging.info("Bake record for: %s" % record.out_dir)
248 logging.info("From: %s" % record_name)
249 logging.info("Last baked: %s" %
250 datetime.datetime.fromtimestamp(record.bake_time))
251 if record.success:
252 logging.info("Status: success")
253 else:
254 logging.error("Status: failed")
255 logging.info("Entries:")
256 for entry in record.entries:
257 if pattern and not fnmatch.fnmatch(entry.path, pattern):
258 continue
259 if out_pattern and not (
260 any([o for o in entry.all_out_paths
261 if fnmatch.fnmatch(o, out_pattern)])):
262 continue
263
264 flags = _get_flag_descriptions(
265 entry.flags,
266 {
267 BakeRecordEntry.FLAG_NEW: 'new',
268 BakeRecordEntry.FLAG_SOURCE_MODIFIED: 'modified',
269 BakeRecordEntry.FLAG_OVERRIDEN: 'overriden'})
270
271 logging.info(" - ")
272
273 rel_path = os.path.relpath(entry.path, ctx.app.root_dir)
274 logging.info(" path: %s" % rel_path)
275 logging.info(" source: %s" % entry.source_name)
276 if entry.extra_key:
277 logging.info(" extra key: %s" % entry.extra_key)
278 logging.info(" flags: %s" % _join(flags))
279 logging.info(" config: %s" % entry.config)
280
281 if entry.errors:
282 logging.error(" errors: %s" % entry.errors)
283
284 logging.info(" %d sub-pages:" % len(entry.subs))
285 for sub in entry.subs:
286 sub_flags = _get_flag_descriptions(
287 sub.flags,
288 {
289 SubPageBakeInfo.FLAG_BAKED: 'baked',
290 SubPageBakeInfo.FLAG_FORCED_BY_SOURCE:
291 'forced by source',
292 SubPageBakeInfo.FLAG_FORCED_BY_NO_PREVIOUS:
293 'forced by missing previous record entry',
294 SubPageBakeInfo.FLAG_FORCED_BY_PREVIOUS_ERRORS:
295 'forced by previous errors',
296 SubPageBakeInfo.FLAG_FORMATTING_INVALIDATED:
297 'formatting invalidated'})
298
299 logging.info(" - ")
300 logging.info(" URL: %s" % sub.out_uri)
301 logging.info(" path: %s" % os.path.relpath(
302 sub.out_path, record.out_dir))
303 logging.info(" flags: %s" % _join(sub_flags))
304
305 pass_names = {
306 PASS_FORMATTING: 'formatting pass',
307 PASS_RENDERING: 'rendering pass'}
308 for p, ri in enumerate(sub.render_info):
309 logging.info(" - %s" % pass_names[p])
310 if not ri:
311 logging.info(" no info")
312 continue
313
314 logging.info(" used sources: %s" %
315 _join(ri.used_source_names))
316 pgn_info = 'no'
317 if ri.used_pagination:
318 pgn_info = 'yes'
319 if ri.pagination_has_more:
320 pgn_info += ', has more'
321 logging.info(" used pagination: %s", pgn_info)
322 logging.info(" used assets: %s",
323 'yes' if ri.used_assets else 'no')
324 logging.info(" other info:")
325 for k, v in ri._custom_info.items():
326 logging.info(" - %s: %s" % (k, v))
327
328 if sub.errors:
329 logging.error(" errors: %s" % sub.errors)
330
331 def _getProcessingRecord(self, ctx, record_name):
332 record_cache = ctx.app.cache.getCache('proc')
333 if not record_cache.has(record_name):
334 logger.warning(
335 "No asset processing record has been created for this "
336 "output path.")
337 return None
338
339 record = ProcessorPipelineRecord.load(
340 record_cache.getCachePath(record_name))
341 return record
342
343 def _showProcessingRecord(self, ctx, record_name, pattern, out_pattern):
344 record = self._getProcessingRecord(ctx, record_name)
345 if record is None:
346 return
347
348 logging.info("")
349 logging.info("Processing record for: %s" % record.out_dir)
350 logging.info("Last baked: %s" %
351 datetime.datetime.fromtimestamp(record.process_time))
352 if record.success:
353 logging.info("Status: success")
354 else:
355 logging.error("Status: failed")
356 logging.info("Entries:")
357 for entry in record.entries:
358 rel_path = os.path.relpath(entry.path, ctx.app.root_dir)
359 if pattern and not fnmatch.fnmatch(rel_path, pattern):
360 continue
361 if out_pattern and not (
362 any([o for o in entry.rel_outputs
363 if fnmatch.fnmatch(o, out_pattern)])):
364 continue
365
366 flags = _get_flag_descriptions(
367 entry.flags,
368 {
369 FLAG_PREPARED: 'prepared',
370 FLAG_PROCESSED: 'processed',
371 FLAG_BYPASSED_STRUCTURED_PROCESSING: 'external',
372 FLAG_COLLAPSED_FROM_LAST_RUN: 'from last run'})
373
374 logger.info(" - ")
375 logger.info(" path: %s" % rel_path)
376 logger.info(" out paths: %s" % entry.rel_outputs)
377 logger.info(" flags: %s" % _join(flags))
378 logger.info(" proc tree: %s" % _format_proc_tree(
379 entry.proc_tree, 14*' '))
380
381 if entry.errors:
382 logger.error(" errors: %s" % entry.errors)
383
384
385 def _join(items, sep=', ', text_if_none='none'):
386 if items:
387 return sep.join(items)
388 return text_if_none
389
390
391 def _get_flag_descriptions(flags, descriptions):
392 res = []
393 for k, v in descriptions.items():
394 if flags & k:
395 res.append(v)
396 return res
397
398
399 def _format_proc_tree(tree, margin='', level=0):
400 name, children = tree
401 res = '%s%s+ %s\n' % (margin if level > 0 else '', level * ' ', name)
402 if children:
403 for c in children:
404 res += _format_proc_tree(c, margin, level + 1)
405 return res
406