annotate piecrust/commands/builtin/baking.py @ 415:0e9a94b7fdfa

bake: Improve bake record information. * Store things in the bake record that require less interaction between the master process and the workers. For instance, don't store the paginator object in the render pass info -- instead, just store whether pagination was used, and whether it had more items. * Simplify information passing between workers and bake passes by saving the rendering info to the JSON cache. This means the "render first sub" job doesn't have to return anything except errors now. * Add more performance counter info.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:23:16 -0700
parents e7b865f8f335
children 4a43d7015b75
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
121
300eb1c2cb14 Fix stupid bug.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
1 import time
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import logging
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import hashlib
92
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
5 import fnmatch
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
6 import datetime
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from piecrust.baking.baker import Baker
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
8 from piecrust.baking.records import (
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
9 BakeRecord, BakeRecordEntry, SubPageBakeInfo)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
10 from piecrust.chefutil import format_timed
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 from piecrust.commands.base import ChefCommand
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
12 from piecrust.processing.pipeline import ProcessorPipeline
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
13 from piecrust.processing.records import (
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 220
diff changeset
14 ProcessorPipelineRecord,
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
15 FLAG_PREPARED, FLAG_PROCESSED,
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 220
diff changeset
16 FLAG_BYPASSED_STRUCTURED_PROCESSING)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
17 from piecrust.rendering import (
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
18 PASS_FORMATTING, PASS_RENDERING)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 class BakeCommand(ChefCommand):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 def __init__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 super(BakeCommand, self).__init__()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 self.name = 'bake'
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 self.description = "Bakes your website into static HTML files."
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 def setupParser(self, parser, app):
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
31 parser.add_argument(
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
32 '-o', '--output',
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 help="The directory to put all the baked HTML files into "
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 "(defaults to `_counter`)")
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
35 parser.add_argument(
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
36 '-f', '--force',
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 help="Force re-baking the entire website.",
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 action='store_true')
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
39 parser.add_argument(
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
40 '-w', '--workers',
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
41 help="The number of worker processes to spawn.",
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
42 type=int, default=-1)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
43 parser.add_argument(
220
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
44 '--assets-only',
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
45 help="Only bake the assets (don't bake the web pages).",
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
46 action='store_true')
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
47 parser.add_argument(
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
48 '--html-only',
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
49 help="Only bake the pages (don't run the asset pipeline).",
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 action='store_true')
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
51 parser.add_argument(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
52 '--show-timers',
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
53 help="Show detailed timing information.",
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
54 action='store_true')
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 def run(self, ctx):
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
57 out_dir = (ctx.args.output or
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
58 os.path.join(ctx.app.root_dir, '_counter'))
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
59
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
60 success = True
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
61 ctx.timers = {}
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
62 start_time = time.perf_counter()
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
63 try:
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
64 # Bake the site sources.
220
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
65 if not ctx.args.assets_only:
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
66 success = success & self._bakeSources(ctx, out_dir)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
67
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
68 # Bake the assets.
220
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
69 if not ctx.args.html_only:
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
70 success = success & self._bakeAssets(ctx, out_dir)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
71
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
72 # Show merged timers.
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
73 if ctx.args.show_timers:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
74 from colorama import Fore
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
75 logger.info("-------------------")
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
76 logger.info("Timing information:")
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
77 for name in sorted(ctx.timers.keys()):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
78 val_str = '%8.1f s' % ctx.timers[name]
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
79 logger.info(
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
80 "[%s%s%s] %s" %
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
81 (Fore.GREEN, val_str, Fore.RESET, name))
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
82
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
83 # All done.
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
84 logger.info('-------------------------')
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
85 logger.info(format_timed(start_time, 'done baking'))
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
86 return 0 if success else 1
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
87 except Exception as ex:
43
f14889d6b067 Proper debug logging.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
88 if ctx.app.debug:
f14889d6b067 Proper debug logging.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
89 logger.exception(ex)
f14889d6b067 Proper debug logging.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
90 else:
f14889d6b067 Proper debug logging.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
91 logger.error(str(ex))
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
92 return 1
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
94 def _bakeSources(self, ctx, out_dir):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
95 if ctx.args.workers > 0:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
96 ctx.app.config.set('baker/workers', ctx.args.workers)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
97 baker = Baker(
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
98 ctx.app, out_dir,
205
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 201
diff changeset
99 force=ctx.args.force)
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 215
diff changeset
100 record = baker.bake()
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
101 _merge_timers(record.timers, ctx.timers)
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 215
diff changeset
102 return record.success
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
103
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
104 def _bakeAssets(self, ctx, out_dir):
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
105 proc = ProcessorPipeline(
205
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 201
diff changeset
106 ctx.app, out_dir,
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 201
diff changeset
107 force=ctx.args.force)
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
108 record = proc.run()
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
109 _merge_timers(record.timers, ctx.timers)
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
110 return record.success
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
111
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
112
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
113 def _merge_timers(source, target):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
114 if source is None:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
115 return
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
116
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
117 for name, val in source.items():
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
118 if name not in target:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
119 target[name] = 0
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
120 target[name] += val
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
121
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
122
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 class ShowRecordCommand(ChefCommand):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 def __init__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125 super(ShowRecordCommand, self).__init__()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 self.name = 'showrecord'
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
127 self.description = ("Shows the bake record for a given output "
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
128 "directory.")
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130 def setupParser(self, parser, app):
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
131 parser.add_argument(
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
132 '-o', '--output',
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 help="The output directory for which to show the bake record "
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134 "(defaults to `_counter`)",
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135 nargs='?')
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
136 parser.add_argument(
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
137 '-p', '--path',
92
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
138 help="A pattern that will be used to filter the relative path "
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
139 "of entries to show.")
331
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
140 parser.add_argument(
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
141 '-t', '--out',
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
142 help="A pattern that will be used to filter the output path "
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
143 "of entries to show.")
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
144 parser.add_argument(
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
145 '--last',
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
146 type=int,
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
147 default=0,
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
148 help="Show the last Nth bake record.")
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
149 parser.add_argument(
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
150 '--html-only',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
151 action='store_true',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
152 help="Only show records for pages (not from the asset "
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
153 "pipeline).")
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
154 parser.add_argument(
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
155 '--assets-only',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
156 action='store_true',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
157 help="Only show records for assets (not from pages).")
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
158
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159 def run(self, ctx):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160 out_dir = ctx.args.output or os.path.join(ctx.app.root_dir, '_counter')
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
161 record_id = hashlib.md5(out_dir.encode('utf8')).hexdigest()
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
162 suffix = '' if ctx.args.last == 0 else '.%d' % ctx.args.last
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
163 record_name = '%s%s.record' % (record_id, suffix)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164
92
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
165 pattern = None
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
166 if ctx.args.path:
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
167 pattern = '*%s*' % ctx.args.path.strip('*')
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
168
331
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
169 out_pattern = None
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
170 if ctx.args.out:
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
171 out_pattern = '*%s*' % ctx.args.out.strip('*')
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
172
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
173 if not ctx.args.assets_only:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
174 self._showBakeRecord(ctx, record_name, pattern, out_pattern)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
175 if not ctx.args.html_only:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
176 self._showProcessingRecord(ctx, record_name, pattern, out_pattern)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
177
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
178 def _showBakeRecord(self, ctx, record_name, pattern, out_pattern):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
179 # Show the bake record.
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
180 record_cache = ctx.app.cache.getCache('baker')
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
181 if not record_cache.has(record_name):
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
182 raise Exception("No record has been created for this output path. "
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
183 "Did you bake there yet?")
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
184
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
185 record = BakeRecord.load(record_cache.getCachePath(record_name))
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
186 logging.info("Bake record for: %s" % record.out_dir)
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
187 logging.info("From: %s" % record_name)
92
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
188 logging.info("Last baked: %s" %
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
189 datetime.datetime.fromtimestamp(record.bake_time))
218
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
190 if record.success:
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
191 logging.info("Status: success")
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
192 else:
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
193 logging.error("Status: failed")
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
194 logging.info("Entries:")
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
195 for entry in record.entries:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
196 if pattern and not fnmatch.fnmatch(entry.path, pattern):
331
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
197 continue
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
198 if out_pattern and not (
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
199 any([o for o in entry.out_paths
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
200 if fnmatch.fnmatch(o, out_pattern)])):
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
201 continue
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
202
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
203 flags = _get_flag_descriptions(
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
204 entry.flags,
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
205 {
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
206 BakeRecordEntry.FLAG_NEW: 'new',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
207 BakeRecordEntry.FLAG_SOURCE_MODIFIED: 'modified',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
208 BakeRecordEntry.FLAG_OVERRIDEN: 'overriden'})
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
209
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
210 logging.info(" - ")
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
211
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
212 rel_path = os.path.relpath(entry.path, ctx.app.root_dir)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
213 logging.info(" path: %s" % rel_path)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
214 logging.info(" source: %s" % entry.source_name)
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
215 if entry.taxonomy_info:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
216 ti = entry.taxonomy_info
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
217 logging.info(" taxonomy: %s = %s (in %s)" %
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
218 (ti.taxonomy_name, ti.term, ti.source_name))
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
219 else:
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
220 logging.info(" taxonomy: <none>")
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
221 logging.info(" flags: %s" % _join(flags))
92
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
222 logging.info(" config: %s" % entry.config)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
223
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
224 if entry.errors:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
225 logging.error(" errors: %s" % entry.errors)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
226
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
227 logging.info(" %d sub-pages:" % len(entry.subs))
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
228 for sub in entry.subs:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
229 sub_flags = _get_flag_descriptions(
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
230 sub.flags,
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
231 {
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
232 SubPageBakeInfo.FLAG_BAKED: 'baked',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
233 SubPageBakeInfo.FLAG_FORCED_BY_SOURCE:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
234 'forced by source',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
235 SubPageBakeInfo.FLAG_FORCED_BY_NO_PREVIOUS:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
236 'forced by missing previous record entry',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
237 SubPageBakeInfo.FLAG_FORCED_BY_PREVIOUS_ERRORS:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
238 'forced by previous errors',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
239 SubPageBakeInfo.FLAG_FORMATTING_INVALIDATED:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
240 'formatting invalidated'})
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
241
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
242 logging.info(" - ")
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
243 logging.info(" URL: %s" % sub.out_uri)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
244 logging.info(" path: %s" % os.path.relpath(
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
245 sub.out_path, record.out_dir))
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
246 logging.info(" flags: %s" % _join(sub_flags))
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
247
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
248 if sub.render_info:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
249 pass_names = {
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
250 PASS_FORMATTING: 'formatting pass',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
251 PASS_RENDERING: 'rendering pass'}
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
252 for p, ri in sub.render_info.items():
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
253 logging.info(" - %s" % pass_names[p])
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
254 logging.info(" used sources: %s" %
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
255 _join(ri.used_source_names))
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
256 pgn_info = 'no'
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
257 if ri.used_pagination:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
258 pgn_info = 'yes'
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
259 if ri.pagination_has_more:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
260 pgn_info += ', has more'
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
261 logging.info(" used pagination: %s", pgn_info)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
262 logging.info(" used assets: %s",
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
263 'yes' if ri.used_assets else 'no')
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
264 logging.info(" used terms: %s" %
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
265 _join(
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
266 ['%s=%s (%s)' % (tn, t, sn)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
267 for sn, tn, t in
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
268 ri.used_taxonomy_terms]))
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
269 else:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
270 logging.info(" no render info")
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
271
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
272 if sub.errors:
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
273 logging.error(" errors: %s" % sub.errors)
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
274
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
275 def _showProcessingRecord(self, ctx, record_name, pattern, out_pattern):
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
276 record_cache = ctx.app.cache.getCache('proc')
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
277 if not record_cache.has(record_name):
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
278 raise Exception("No record has been created for this output path. "
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
279 "Did you bake there yet?")
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
280
218
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
281 # Show the pipeline record.
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
282 record = ProcessorPipelineRecord.load(
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
283 record_cache.getCachePath(record_name))
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
284 logging.info("")
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
285 logging.info("Processing record for: %s" % record.out_dir)
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
286 logging.info("Last baked: %s" %
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
287 datetime.datetime.fromtimestamp(record.process_time))
218
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
288 if record.success:
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
289 logging.info("Status: success")
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
290 else:
10f24c62b05b showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 217
diff changeset
291 logging.error("Status: failed")
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
292 logging.info("Entries:")
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
293 for entry in record.entries:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
294 rel_path = os.path.relpath(entry.path, ctx.app.root_dir)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
295 if pattern and not fnmatch.fnmatch(rel_path, pattern):
331
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
296 continue
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
297 if out_pattern and not (
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
298 any([o for o in entry.rel_outputs
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
299 if fnmatch.fnmatch(o, out_pattern)])):
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
300 continue
f0fc2a9d3191 showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
301
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
302 flags = _get_flag_descriptions(
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
303 entry.flags,
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
304 {
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
305 FLAG_PREPARED: 'prepared',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
306 FLAG_PROCESSED: 'processed',
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
307 FLAG_BYPASSED_STRUCTURED_PROCESSING: 'external'})
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
308
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
309 logger.info(" - ")
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
310 logger.info(" path: %s" % rel_path)
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
311 logger.info(" out paths: %s" % entry.rel_outputs)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
312 logger.info(" flags: %s" % _join(flags))
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
313 logger.info(" proc tree: %s" % _format_proc_tree(
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
314 entry.proc_tree, 14*' '))
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
315
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
316 if entry.errors:
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
317 logger.error(" errors: %s" % entry.errors)
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
318
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
319
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
320 def _join(items, sep=', ', text_if_none='none'):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
321 if items:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
322 return sep.join(items)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
323 return text_if_none
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
324
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
325
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
326 def _get_flag_descriptions(flags, descriptions):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
327 res = []
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
328 for k, v in descriptions.items():
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
329 if flags & k:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
330 res.append(v)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
331 return res
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
332
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
333
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
334 def _format_proc_tree(tree, margin='', level=0):
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
335 name, children = tree
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 220
diff changeset
336 res = '%s%s+ %s\n' % (margin if level > 0 else '', level * ' ', name)
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
337 if children:
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
338 for c in children:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
339 res += _format_proc_tree(c, margin, level + 1)
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
340 return res
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
341