Mercurial > piecrust2
annotate piecrust/commands/builtin/baking.py @ 415:0e9a94b7fdfa
bake: Improve bake record information.
* Store things in the bake record that require less interaction between the
master process and the workers. For instance, don't store the paginator
object in the render pass info -- instead, just store whether pagination
was used, and whether it had more items.
* Simplify information passing between workers and bake passes by saving the
rendering info to the JSON cache. This means the "render first sub" job
doesn't have to return anything except errors now.
* Add more performance counter info.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 20 Jun 2015 19:23:16 -0700 |
parents | e7b865f8f335 |
children | 4a43d7015b75 |
rev | line source |
---|---|
121 | 1 import time |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import os.path |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 import logging |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 import hashlib |
92
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
5 import fnmatch |
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
6 import datetime |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 from piecrust.baking.baker import Baker |
334
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
8 from piecrust.baking.records import ( |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
9 BakeRecord, BakeRecordEntry, SubPageBakeInfo) |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
10 from piecrust.chefutil import format_timed |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 from piecrust.commands.base import ChefCommand |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
12 from piecrust.processing.pipeline import ProcessorPipeline |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
13 from piecrust.processing.records import ( |
221
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
220
diff
changeset
|
14 ProcessorPipelineRecord, |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
15 FLAG_PREPARED, FLAG_PROCESSED, |
221
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
220
diff
changeset
|
16 FLAG_BYPASSED_STRUCTURED_PROCESSING) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
17 from piecrust.rendering import ( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
18 PASS_FORMATTING, PASS_RENDERING) |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 logger = logging.getLogger(__name__) |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 class BakeCommand(ChefCommand): |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 def __init__(self): |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 super(BakeCommand, self).__init__() |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 self.name = 'bake' |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 self.description = "Bakes your website into static HTML files." |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
29 |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 def setupParser(self, parser, app): |
192
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
31 parser.add_argument( |
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
32 '-o', '--output', |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 help="The directory to put all the baked HTML files into " |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 "(defaults to `_counter`)") |
192
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
35 parser.add_argument( |
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
36 '-f', '--force', |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 help="Force re-baking the entire website.", |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 action='store_true') |
192
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
39 parser.add_argument( |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
40 '-w', '--workers', |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
41 help="The number of worker processes to spawn.", |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
42 type=int, default=-1) |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
43 parser.add_argument( |
220
84e2bc2d16cb
bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents:
218
diff
changeset
|
44 '--assets-only', |
84e2bc2d16cb
bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents:
218
diff
changeset
|
45 help="Only bake the assets (don't bake the web pages).", |
84e2bc2d16cb
bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents:
218
diff
changeset
|
46 action='store_true') |
84e2bc2d16cb
bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents:
218
diff
changeset
|
47 parser.add_argument( |
84e2bc2d16cb
bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents:
218
diff
changeset
|
48 '--html-only', |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
49 help="Only bake the pages (don't run the asset pipeline).", |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 action='store_true') |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
51 parser.add_argument( |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
52 '--show-timers', |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
53 help="Show detailed timing information.", |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
54 action='store_true') |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 def run(self, ctx): |
127
bc63dc20baa0
Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents:
121
diff
changeset
|
57 out_dir = (ctx.args.output or |
bc63dc20baa0
Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents:
121
diff
changeset
|
58 os.path.join(ctx.app.root_dir, '_counter')) |
bc63dc20baa0
Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents:
121
diff
changeset
|
59 |
215
a47580a0955b
bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents:
205
diff
changeset
|
60 success = True |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
61 ctx.timers = {} |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
62 start_time = time.perf_counter() |
39
2f717f961996
Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents:
3
diff
changeset
|
63 try: |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
64 # Bake the site sources. |
220
84e2bc2d16cb
bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents:
218
diff
changeset
|
65 if not ctx.args.assets_only: |
84e2bc2d16cb
bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents:
218
diff
changeset
|
66 success = success & self._bakeSources(ctx, out_dir) |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
67 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
68 # Bake the assets. |
220
84e2bc2d16cb
bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents:
218
diff
changeset
|
69 if not ctx.args.html_only: |
215
a47580a0955b
bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents:
205
diff
changeset
|
70 success = success & self._bakeAssets(ctx, out_dir) |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
71 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
72 # Show merged timers. |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
73 if ctx.args.show_timers: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
74 from colorama import Fore |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
75 logger.info("-------------------") |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
76 logger.info("Timing information:") |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
77 for name in sorted(ctx.timers.keys()): |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
78 val_str = '%8.1f s' % ctx.timers[name] |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
79 logger.info( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
80 "[%s%s%s] %s" % |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
81 (Fore.GREEN, val_str, Fore.RESET, name)) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
82 |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
83 # All done. |
192
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
84 logger.info('-------------------------') |
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
85 logger.info(format_timed(start_time, 'done baking')) |
215
a47580a0955b
bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents:
205
diff
changeset
|
86 return 0 if success else 1 |
39
2f717f961996
Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents:
3
diff
changeset
|
87 except Exception as ex: |
43 | 88 if ctx.app.debug: |
89 logger.exception(ex) | |
90 else: | |
91 logger.error(str(ex)) | |
39
2f717f961996
Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents:
3
diff
changeset
|
92 return 1 |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
93 |
127
bc63dc20baa0
Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents:
121
diff
changeset
|
94 def _bakeSources(self, ctx, out_dir): |
411
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
95 if ctx.args.workers > 0: |
e7b865f8f335
bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
359
diff
changeset
|
96 ctx.app.config.set('baker/workers', ctx.args.workers) |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
97 baker = Baker( |
127
bc63dc20baa0
Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents:
121
diff
changeset
|
98 ctx.app, out_dir, |
205
e725af1d48fb
bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents:
201
diff
changeset
|
99 force=ctx.args.force) |
217
1f4c3dae1fe8
bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
215
diff
changeset
|
100 record = baker.bake() |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
101 _merge_timers(record.timers, ctx.timers) |
217
1f4c3dae1fe8
bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents:
215
diff
changeset
|
102 return record.success |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
103 |
127
bc63dc20baa0
Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents:
121
diff
changeset
|
104 def _bakeAssets(self, ctx, out_dir): |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
105 proc = ProcessorPipeline( |
205
e725af1d48fb
bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents:
201
diff
changeset
|
106 ctx.app, out_dir, |
e725af1d48fb
bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents:
201
diff
changeset
|
107 force=ctx.args.force) |
215
a47580a0955b
bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents:
205
diff
changeset
|
108 record = proc.run() |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
109 _merge_timers(record.timers, ctx.timers) |
215
a47580a0955b
bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents:
205
diff
changeset
|
110 return record.success |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
111 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
92
diff
changeset
|
112 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
113 def _merge_timers(source, target): |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
114 if source is None: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
115 return |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
116 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
117 for name, val in source.items(): |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
118 if name not in target: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
119 target[name] = 0 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
120 target[name] += val |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
121 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
122 |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
123 class ShowRecordCommand(ChefCommand): |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
124 def __init__(self): |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
125 super(ShowRecordCommand, self).__init__() |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
126 self.name = 'showrecord' |
192
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
127 self.description = ("Shows the bake record for a given output " |
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
128 "directory.") |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
129 |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
130 def setupParser(self, parser, app): |
192
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
131 parser.add_argument( |
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
132 '-o', '--output', |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
133 help="The output directory for which to show the bake record " |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
134 "(defaults to `_counter`)", |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
135 nargs='?') |
192
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
136 parser.add_argument( |
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
137 '-p', '--path', |
92
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
138 help="A pattern that will be used to filter the relative path " |
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
139 "of entries to show.") |
331
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
140 parser.add_argument( |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
141 '-t', '--out', |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
142 help="A pattern that will be used to filter the output path " |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
143 "of entries to show.") |
334
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
144 parser.add_argument( |
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
145 '--last', |
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
146 type=int, |
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
147 default=0, |
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
148 help="Show the last Nth bake record.") |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
149 parser.add_argument( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
150 '--html-only', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
151 action='store_true', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
152 help="Only show records for pages (not from the asset " |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
153 "pipeline).") |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
154 parser.add_argument( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
155 '--assets-only', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
156 action='store_true', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
157 help="Only show records for assets (not from pages).") |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
158 |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
159 def run(self, ctx): |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
160 out_dir = ctx.args.output or os.path.join(ctx.app.root_dir, '_counter') |
334
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
161 record_id = hashlib.md5(out_dir.encode('utf8')).hexdigest() |
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
162 suffix = '' if ctx.args.last == 0 else '.%d' % ctx.args.last |
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
163 record_name = '%s%s.record' % (record_id, suffix) |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
164 |
92
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
165 pattern = None |
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
166 if ctx.args.path: |
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
167 pattern = '*%s*' % ctx.args.path.strip('*') |
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
168 |
331
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
169 out_pattern = None |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
170 if ctx.args.out: |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
171 out_pattern = '*%s*' % ctx.args.out.strip('*') |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
172 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
173 if not ctx.args.assets_only: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
174 self._showBakeRecord(ctx, record_name, pattern, out_pattern) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
175 if not ctx.args.html_only: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
176 self._showProcessingRecord(ctx, record_name, pattern, out_pattern) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
177 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
178 def _showBakeRecord(self, ctx, record_name, pattern, out_pattern): |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
179 # Show the bake record. |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
180 record_cache = ctx.app.cache.getCache('baker') |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
181 if not record_cache.has(record_name): |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
182 raise Exception("No record has been created for this output path. " |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
183 "Did you bake there yet?") |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
184 |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
185 record = BakeRecord.load(record_cache.getCachePath(record_name)) |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
186 logging.info("Bake record for: %s" % record.out_dir) |
334
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
187 logging.info("From: %s" % record_name) |
92
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
188 logging.info("Last baked: %s" % |
192
4c0ab0b044fe
cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents:
127
diff
changeset
|
189 datetime.datetime.fromtimestamp(record.bake_time)) |
218
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
190 if record.success: |
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
191 logging.info("Status: success") |
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
192 else: |
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
193 logging.error("Status: failed") |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
194 logging.info("Entries:") |
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
195 for entry in record.entries: |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
196 if pattern and not fnmatch.fnmatch(entry.path, pattern): |
331
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
197 continue |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
198 if out_pattern and not ( |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
199 any([o for o in entry.out_paths |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
200 if fnmatch.fnmatch(o, out_pattern)])): |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
201 continue |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
202 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
203 flags = _get_flag_descriptions( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
204 entry.flags, |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
205 { |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
206 BakeRecordEntry.FLAG_NEW: 'new', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
207 BakeRecordEntry.FLAG_SOURCE_MODIFIED: 'modified', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
208 BakeRecordEntry.FLAG_OVERRIDEN: 'overriden'}) |
334
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
209 |
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
210 logging.info(" - ") |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
211 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
212 rel_path = os.path.relpath(entry.path, ctx.app.root_dir) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
213 logging.info(" path: %s" % rel_path) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
214 logging.info(" source: %s" % entry.source_name) |
334
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
215 if entry.taxonomy_info: |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
216 ti = entry.taxonomy_info |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
217 logging.info(" taxonomy: %s = %s (in %s)" % |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
218 (ti.taxonomy_name, ti.term, ti.source_name)) |
334
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
219 else: |
b034f6f15e22
bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents:
331
diff
changeset
|
220 logging.info(" taxonomy: <none>") |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
221 logging.info(" flags: %s" % _join(flags)) |
92
0dd43c5f5484
More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
45
diff
changeset
|
222 logging.info(" config: %s" % entry.config) |
338
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
223 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
224 if entry.errors: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
225 logging.error(" errors: %s" % entry.errors) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
226 |
338
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
227 logging.info(" %d sub-pages:" % len(entry.subs)) |
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
228 for sub in entry.subs: |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
229 sub_flags = _get_flag_descriptions( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
230 sub.flags, |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
231 { |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
232 SubPageBakeInfo.FLAG_BAKED: 'baked', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
233 SubPageBakeInfo.FLAG_FORCED_BY_SOURCE: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
234 'forced by source', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
235 SubPageBakeInfo.FLAG_FORCED_BY_NO_PREVIOUS: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
236 'forced by missing previous record entry', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
237 SubPageBakeInfo.FLAG_FORCED_BY_PREVIOUS_ERRORS: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
238 'forced by previous errors', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
239 SubPageBakeInfo.FLAG_FORMATTING_INVALIDATED: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
240 'formatting invalidated'}) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
241 |
338
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
242 logging.info(" - ") |
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
243 logging.info(" URL: %s" % sub.out_uri) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
244 logging.info(" path: %s" % os.path.relpath( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
245 sub.out_path, record.out_dir)) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
246 logging.info(" flags: %s" % _join(sub_flags)) |
338
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
247 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
248 if sub.render_info: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
249 pass_names = { |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
250 PASS_FORMATTING: 'formatting pass', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
251 PASS_RENDERING: 'rendering pass'} |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
252 for p, ri in sub.render_info.items(): |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
253 logging.info(" - %s" % pass_names[p]) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
254 logging.info(" used sources: %s" % |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
255 _join(ri.used_source_names)) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
256 pgn_info = 'no' |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
257 if ri.used_pagination: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
258 pgn_info = 'yes' |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
259 if ri.pagination_has_more: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
260 pgn_info += ', has more' |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
261 logging.info(" used pagination: %s", pgn_info) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
262 logging.info(" used assets: %s", |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
263 'yes' if ri.used_assets else 'no') |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
264 logging.info(" used terms: %s" % |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
265 _join( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
266 ['%s=%s (%s)' % (tn, t, sn) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
267 for sn, tn, t in |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
268 ri.used_taxonomy_terms])) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
269 else: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
270 logging.info(" no render info") |
338
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
271 |
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
272 if sub.errors: |
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
273 logging.error(" errors: %s" % sub.errors) |
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
334
diff
changeset
|
274 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
275 def _showProcessingRecord(self, ctx, record_name, pattern, out_pattern): |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
276 record_cache = ctx.app.cache.getCache('proc') |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
277 if not record_cache.has(record_name): |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
278 raise Exception("No record has been created for this output path. " |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
279 "Did you bake there yet?") |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
280 |
218
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
281 # Show the pipeline record. |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
282 record = ProcessorPipelineRecord.load( |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
283 record_cache.getCachePath(record_name)) |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
284 logging.info("") |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
285 logging.info("Processing record for: %s" % record.out_dir) |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
286 logging.info("Last baked: %s" % |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
287 datetime.datetime.fromtimestamp(record.process_time)) |
218
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
288 if record.success: |
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
289 logging.info("Status: success") |
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
290 else: |
10f24c62b05b
showrecord: Show the overall status (success/failed) of the bake.
Ludovic Chabant <ludovic@chabant.com>
parents:
217
diff
changeset
|
291 logging.error("Status: failed") |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
292 logging.info("Entries:") |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
293 for entry in record.entries: |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
294 rel_path = os.path.relpath(entry.path, ctx.app.root_dir) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
295 if pattern and not fnmatch.fnmatch(rel_path, pattern): |
331
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
296 continue |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
297 if out_pattern and not ( |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
298 any([o for o in entry.rel_outputs |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
299 if fnmatch.fnmatch(o, out_pattern)])): |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
300 continue |
f0fc2a9d3191
showrecord: Add ability to filter on the output path.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
301 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
302 flags = _get_flag_descriptions( |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
303 entry.flags, |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
304 { |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
305 FLAG_PREPARED: 'prepared', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
306 FLAG_PROCESSED: 'processed', |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
307 FLAG_BYPASSED_STRUCTURED_PROCESSING: 'external'}) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
308 |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
309 logger.info(" - ") |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
310 logger.info(" path: %s" % rel_path) |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
311 logger.info(" out paths: %s" % entry.rel_outputs) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
312 logger.info(" flags: %s" % _join(flags)) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
313 logger.info(" proc tree: %s" % _format_proc_tree( |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
314 entry.proc_tree, 14*' ')) |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
315 |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
316 if entry.errors: |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
317 logger.error(" errors: %s" % entry.errors) |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
318 |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
319 |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
320 def _join(items, sep=', ', text_if_none='none'): |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
321 if items: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
322 return sep.join(items) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
323 return text_if_none |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
324 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
325 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
326 def _get_flag_descriptions(flags, descriptions): |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
327 res = [] |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
328 for k, v in descriptions.items(): |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
329 if flags & k: |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
330 res.append(v) |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
331 return res |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
332 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
333 |
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
334 def _format_proc_tree(tree, margin='', level=0): |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
335 name, children = tree |
221
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
220
diff
changeset
|
336 res = '%s%s+ %s\n' % (margin if level > 0 else '', level * ' ', name) |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
337 if children: |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
338 for c in children: |
415
0e9a94b7fdfa
bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents:
411
diff
changeset
|
339 res += _format_proc_tree(c, margin, level + 1) |
194
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
340 return res |
5d8351cb32d8
showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
193
diff
changeset
|
341 |