Mercurial > piecrust2

annotate piecrust/commands/builtin/baking.py @ 1152:74c0c7483986

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
copyasset: Add `copyasset` command.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 19 Jan 2019 17:40:13 -0800
parents fe0af94ca757
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
1 import os.path
121
300eb1c2cb14 Fix stupid bug.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
2 import time
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import logging
92
0dd43c5f5484 More options for the `showrecord` command.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
4 import datetime
421
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
5 from colorama import Fore
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 from piecrust.commands.base import ChefCommand
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 class BakeCommand(ChefCommand):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 def __init__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 super(BakeCommand, self).__init__()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 self.name = 'bake'
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 self.description = "Bakes your website into static HTML files."
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 def setupParser(self, parser, app):
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
19 parser.add_argument(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
20 '-o', '--output',
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
21 help="The directory to put all the baked HTML files into "
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
22 "(defaults to `_counter`)")
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
23 parser.add_argument(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
24 '-f', '--force',
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
25 help="Force re-baking the entire website.",
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
26 action='store_true')
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
27 parser.add_argument(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
28 '-p', '--pipelines',
1032
54159951d97a bake: Add `--sources` argument.
Ludovic Chabant <ludovic@chabant.com>
parents: 1017
diff changeset
29 help="Specifies the pipelines to run.",
54159951d97a bake: Add `--sources` argument.
Ludovic Chabant <ludovic@chabant.com>
parents: 1017
diff changeset
30 action='append')
54159951d97a bake: Add `--sources` argument.
Ludovic Chabant <ludovic@chabant.com>
parents: 1017
diff changeset
31 parser.add_argument(
54159951d97a bake: Add `--sources` argument.
Ludovic Chabant <ludovic@chabant.com>
parents: 1017
diff changeset
32 '-s', '--sources',
54159951d97a bake: Add `--sources` argument.
Ludovic Chabant <ludovic@chabant.com>
parents: 1017
diff changeset
33 help="Specifies the content sources to run.",
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
34 action='append')
192
4c0ab0b044fe cosmetic: Fix some PEP8 issues.
Ludovic Chabant <ludovic@chabant.com>
parents: 127
diff changeset
35 parser.add_argument(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
36 '-w', '--workers',
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
37 help="The number of worker processes to spawn.",
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
38 type=int, default=-1)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
39 parser.add_argument(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
40 '--batch-size',
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
41 help="The number of jobs per batch.",
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
42 type=int, default=-1)
462
04abc97dd3b6 bake: Add CLI argument to specify job batch size.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
43 parser.add_argument(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
44 '--assets-only',
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
45 help="Only bake the assets (don't bake the web pages).",
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
46 action='store_true')
220
84e2bc2d16cb bake: Change arguments to selectively bake to make them symmetrical.
Ludovic Chabant <ludovic@chabant.com>
parents: 218
diff changeset
47 parser.add_argument(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
48 '--html-only',
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
49 help="Only bake the pages (don't run the asset pipeline).",
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
50 action='store_true')
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
51 parser.add_argument(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
52 '--show-stats',
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
53 help="Show detailed information about the bake.",
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
54 action='store_true')
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
55 parser.add_argument(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
56 '--profile',
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
57 help="Run the bake several times, for profiling.",
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
58 type=int, default=-1)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 def run(self, ctx):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
61 from piecrust.chefutil import format_timed
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
62 from piecrust.environment import ExecutionStats
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
63
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
64 out_dir = (ctx.args.output or
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
65 os.path.join(ctx.app.root_dir, '_counter'))
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 121
diff changeset
66
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
67 success = True
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
68 avg_stats = ExecutionStats()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
69 avg_stats.registerTimer('Total')
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
70 start_time = time.perf_counter()
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
71
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
72 num_iter = 1
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
73 if ctx.args.profile > 0:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
74 num_iter = ctx.args.profile
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
75
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
76 for i in range(num_iter):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
77 iter_start_time = time.perf_counter()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
78 if num_iter > 1:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
79 import gc
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
80 gc.collect()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
81 logger.info("---- %d/%d ----" % (i + 1, num_iter))
1017
3c669bb9498e bake: Don't cheat when profiling.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
82 # Don't cheat -- the app instance caches a bunch of stuff
3c669bb9498e bake: Don't cheat when profiling.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
83 # so we need to create a fresh one.
3c669bb9498e bake: Don't cheat when profiling.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
84 ctx.app = ctx.appfactory.create()
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
85
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
86 try:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
87 records = self._doBake(ctx, out_dir)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
88 except Exception as ex:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
89 if ctx.app.debug:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
90 logger.exception(ex)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
91 else:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
92 logger.error(str(ex))
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
93 return 1
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
94
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
95 success = success and records.success
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
96 avg_stats.mergeStats(records.stats)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
97 avg_stats.stepTimerSince('Total', iter_start_time)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
98
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
99 # Show merged stats.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
100 if ctx.args.show_stats:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
101 if num_iter > 1:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
102 _average_stats(avg_stats, num_iter)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
103
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
104 logger.info("-------------------")
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
105 logger.info("Timing information:")
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
106 _show_stats(avg_stats)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
107
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
108 # All done.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
109 logger.info('-------------------------')
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
110 logger.info(format_timed(start_time, 'done baking'))
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
111 return 0 if success else 1
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
113 def _doBake(self, ctx, out_dir):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
114 from piecrust.baking.baker import Baker
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
115
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
116 if ctx.args.workers > 0:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 359
diff changeset
117 ctx.app.config.set('baker/workers', ctx.args.workers)
462
04abc97dd3b6 bake: Add CLI argument to specify job batch size.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
118 if ctx.args.batch_size > 0:
04abc97dd3b6 bake: Add CLI argument to specify job batch size.
Ludovic Chabant <ludovic@chabant.com>
parents: 421
diff changeset
119 ctx.app.config.set('baker/batch_size', ctx.args.batch_size)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
120
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
121 allowed_pipelines = None
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
122 forbidden_pipelines = None
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
123 if ctx.args.html_only:
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
124 forbidden_pipelines = ['asset']
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
125 elif ctx.args.assets_only:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
126 allowed_pipelines = ['asset']
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
127 elif ctx.args.pipelines:
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
128 if allowed_pipelines or forbidden_pipelines:
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
129 raise Exception(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
130 "Can't specify `--html-only` or `--assets-only` with "
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
131 "`--pipelines`.")
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
132 allowed_pipelines = []
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
133 forbidden_pipelines = []
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
134 for p in ctx.args.pipelines:
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
135 if p[0] == '-':
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
136 forbidden_pipelines.append(p)
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
137 else:
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
138 allowed_pipelines.append(p)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
139 if not allowed_pipelines:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
140 allowed_pipelines = None
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
141 if not forbidden_pipelines:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
142 forbidden_pipelines = None
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 92
diff changeset
143
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
144 baker = Baker(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
145 ctx.appfactory, ctx.app, out_dir,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
146 force=ctx.args.force,
1032
54159951d97a bake: Add `--sources` argument.
Ludovic Chabant <ludovic@chabant.com>
parents: 1017
diff changeset
147 allowed_sources=ctx.args.sources,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
148 allowed_pipelines=allowed_pipelines,
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
149 forbidden_pipelines=forbidden_pipelines)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
150 records = baker.bake()
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
151
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
152 return records
421
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
153
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
154
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
155 class ShowRecordCommand(ChefCommand):
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
156 def __init__(self):
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
157 super(ShowRecordCommand, self).__init__()
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
158 self.name = 'showrecords'
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
159 self.description = ("Shows the bake records for a given output "
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
160 "directory.")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
161
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
162 def setupParser(self, parser, app):
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
163 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
164 '-o', '--output',
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
165 help="The output directory for which to show the bake records "
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
166 "(defaults to `_counter`)",
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
167 nargs='?')
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
168 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
169 '-i', '--in-path',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
170 help="A pattern that will be used to filter the relative path "
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
171 "of entries to show.")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
172 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
173 '-t', '--out-path',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
174 help="A pattern that will be used to filter the output path "
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
175 "of entries to show.")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
176 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
177 '--fails',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
178 action='store_true',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
179 help="Only show record entries for failures.")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
180 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
181 '--last',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
182 type=int,
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
183 default=0,
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
184 help="Show the last Nth bake records.")
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
185 parser.add_argument(
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
186 '--records',
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
187 help="Load the specified records file.")
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
188 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
189 '--html-only',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
190 action='store_true',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
191 help="Only show records for pages (not from the asset "
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
192 "pipeline).")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
193 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
194 '--assets-only',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
195 action='store_true',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
196 help="Only show records for assets (not from pages).")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
197 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
198 '-p', '--pipelines',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
199 nargs='*',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
200 help="Only show records for the given pipeline(s).")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
201 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
202 '--show-stats',
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
203 action='store_true',
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
204 help="Show stats from the records.")
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
205 parser.add_argument(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
206 '--show-manifest',
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
207 help="Show manifest entries from the records.")
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
208
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
209 def run(self, ctx):
879
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 873
diff changeset
210 import fnmatch
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
211 from piecrust.baking.baker import get_bake_records_path
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
212 from piecrust.pipelines.records import load_records
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
213
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
214 records_path = ctx.args.records
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
215 if records_path is None:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
216 out_dir = ctx.args.output or os.path.join(ctx.app.root_dir,
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
217 '_counter')
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
218 suffix = '' if ctx.args.last == 0 else '.%d' % ctx.args.last
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
219 records_path = get_bake_records_path(ctx.app, out_dir,
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
220 suffix=suffix)
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
221 logger.info("Bake records for output: %s" % out_dir)
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
222 else:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
223 logger.info("Bake records from: %s" % records_path)
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
224
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
225 records = load_records(records_path, True)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
226 if records.invalidated:
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
227 raise Exception(
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
228 "The bake records were saved by a previous version of "
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
229 "PieCrust and can't be shown.")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
230
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
231 in_pattern = None
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
232 if ctx.args.in_path:
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
233 in_pattern = '*%s*' % ctx.args.in_path.strip('*')
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
234
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
235 out_pattern = None
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
236 if ctx.args.out_path:
883
803c5c28a71f showrecord: Fix bug when filtering output paths.
Ludovic Chabant <ludovic@chabant.com>
parents: 879
diff changeset
237 out_pattern = '*%s*' % ctx.args.out_path.strip('*')
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
238
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
239 pipelines = ctx.args.pipelines
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
240 if pipelines is None:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
241 if ctx.args.assets_only:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
242 pipelines = ['asset']
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
243 if ctx.args.html_only:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
244 pipelines = ['page']
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
245
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
246 logger.info("Status: %s" % ('SUCCESS' if records.success
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
247 else 'FAILURE'))
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
248 logger.info("Date/time: %s" %
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
249 datetime.datetime.fromtimestamp(records.bake_time))
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
250 logger.info("Incremental count: %d" % records.incremental_count)
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
251 logger.info("Versions: %s/%s" % (records._app_version,
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
252 records._record_version))
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
253 logger.info("")
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
254
898
98becbc75ccc showrecord: Don't show the records if we just want to see a manifest.
Ludovic Chabant <ludovic@chabant.com>
parents: 883
diff changeset
255 if not ctx.args.show_stats and not ctx.args.show_manifest:
988
f83ae0a5d793 showrecords: Sort records by name.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
256 for rec in sorted(records.records, key=lambda r: r.name):
873
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
257 if ctx.args.fails and rec.success:
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
258 logger.debug(
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
259 "Ignoring record '%s' because it was successful, "
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
260 "and `--fail` was passed." % rec.name)
873
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
261 continue
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
262
873
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
263 ppname = rec.name[rec.name.index('@') + 1:]
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
264 if pipelines is not None and ppname not in pipelines:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
265 logging.debug(
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
266 "Ignoring record '%s' because it was created by "
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
267 "pipeline '%s', which isn't listed in "
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 898
diff changeset
268 "`--pipelines`." % (rec.name, ppname))
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
269 continue
873
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
270
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
271 entries_to_show = []
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
272
873
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
273 for e in rec.getEntries():
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
274 if ctx.args.fails and e.success:
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
275 continue
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
276 if in_pattern and not fnmatch.fnmatch(e.item_spec,
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
277 in_pattern):
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
278 continue
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
279 if out_pattern and not any(
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
280 [fnmatch.fnmatch(op, out_pattern)
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
281 for op in e.getAllOutputPaths()]):
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
282 continue
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
283 entries_to_show.append(e)
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
284
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
285 if entries_to_show:
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
286 logger.info("Record: %s" % rec.name)
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
287 logger.info("Status: %s" % ('SUCCESS' if rec.success
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
288 else 'FAILURE'))
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
289 logger.info("User Data:")
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
290 if not rec.user_data:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
291 logger.info(" <empty>")
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
292 else:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
293 for k, v in rec.user_data.items():
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
294 logger.info(" %s: %s" % (k, v))
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
295
1133
fe0af94ca757 showrecords: Tiny cosmetic change for the output.
Ludovic Chabant <ludovic@chabant.com>
parents: 1032
diff changeset
296 logger.info("Entries:")
873
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
297 for e in entries_to_show:
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
298 _print_record_entry(e)
93ea115027fc showrecord: Don't show empty records when filtering.
Ludovic Chabant <ludovic@chabant.com>
parents: 872
diff changeset
299 logger.info("")
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
300
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
301 stats = records.stats
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
302 if ctx.args.show_stats:
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
303 _show_stats(stats)
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
304
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
305 if ctx.args.show_manifest:
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
306 for name in sorted(stats.manifests.keys()):
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
307 if ctx.args.show_manifest.lower() in name.lower():
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
308 val = stats.manifests[name]
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
309 logger.info(
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
310 " [%s%s%s] [%d entries]" %
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
311 (Fore.CYAN, name, Fore.RESET, len(val)))
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
312 for v in val:
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
313 logger.info(" - %s" % v)
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
314
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
315
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
316 def _average_stats(stats, cnt):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
317 for name in stats.timers:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
318 stats.timers[name] /= cnt
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
319 for name in stats.counters:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
320 stats.counters[name] /= cnt
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
321
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 988
diff changeset
322
692
c11a4339fccb bake: Show more stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
323 def _show_stats(stats, *, full=False):
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
324 indent = ' '
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
325
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
326 logger.info(' Timers:')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
327 for name, val in sorted(stats.timers.items(), key=lambda i: i[1],
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
328 reverse=True):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
329 val_str = '%8.1f s' % val
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
330 logger.info(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
331 "%s[%s%s%s] %s" %
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
332 (indent, Fore.GREEN, val_str, Fore.RESET, name))
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
333
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
334 logger.info(' Counters:')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
335 for name in sorted(stats.counters.keys()):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
336 val_str = '%8d ' % stats.counters[name]
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
337 logger.info(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
338 "%s[%s%s%s] %s" %
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
339 (indent, Fore.GREEN, val_str, Fore.RESET, name))
421
4a43d7015b75 bake: Improve performance timers reports.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
340
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
341 logger.info(' Manifests:')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
342 for name in sorted(stats.manifests.keys()):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
343 val = stats.manifests[name]
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
344 logger.info(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
345 "%s[%s%s%s] [%d entries]" %
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
346 (indent, Fore.CYAN, name, Fore.RESET, len(val)))
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
347 if full:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
348 for v in val:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 804
diff changeset
349 logger.info("%s - %s" % (indent, v))
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
350
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
351
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
352 def _print_record_entry(e):
879
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 873
diff changeset
353 import pprint
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 873
diff changeset
354 import textwrap
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 873
diff changeset
355
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
356 logger.info(" - %s" % e.item_spec)
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
357 logger.info(" Outputs:")
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
358 out_paths = list(e.getAllOutputPaths())
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
359 if out_paths:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
360 for op in out_paths:
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
361 logger.info(" - %s" % op)
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
362 else:
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
363 logger.info(" <none>")
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 331
diff changeset
364
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
365 e_desc = e.describe()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
366 for k, v in e_desc.items():
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
367 if isinstance(v, dict):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
368 text = pprint.pformat(v, indent=2)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
369 logger.info(" %s:" % k)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
370 logger.info(textwrap.indent(text, ' '))
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
371 else:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
372 logger.info(" %s: %s" % (k, v))
194
5d8351cb32d8 showrecord: Also show the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 193
diff changeset
373
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
374 errors = list(e.getAllErrors())
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
375 if errors:
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
376 logger.error(" Errors:")
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
377 for err in errors:
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
378 logger.error(" - %s" % err)