Mercurial > piecrust2

annotate piecrust/baking/baker.py @ 1070:a013a3bea22a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cm: Upgrade release script.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 13 Feb 2018 23:37:59 -0800
parents 971b4d67e82a
children 5f97b5b59dfe
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import time
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import hashlib
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
5 from piecrust.chefutil import (
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
6 format_timed_scope, format_timed)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
7 from piecrust.environment import ExecutionStats
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
8 from piecrust.pipelines.base import (
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
9 PipelineJobCreateContext, PipelineJobResultHandleContext,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
10 PipelineJobValidateContext, PipelineManager,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
11 get_pipeline_name_for_source)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
12 from piecrust.pipelines.records import (
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
13 MultiRecordHistory, MultiRecord, RecordEntry,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
14 load_records)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
15 from piecrust.sources.base import REALM_USER, REALM_THEME, REALM_NAMES
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
21 def get_bake_records_path(app, out_dir, *, suffix=''):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
22 records_cache = app.cache.getCache('baker')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
23 records_id = hashlib.md5(out_dir.encode('utf8')).hexdigest()
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
24 records_name = '%s%s.records' % (records_id, suffix)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
25 return records_cache.getCachePath(records_name)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
26
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
27
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 class Baker(object):
918
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
29 def __init__(self, appfactory, app, out_dir, *,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
30 force=False,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
31 allowed_pipelines=None,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
32 forbidden_pipelines=None,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
33 allowed_sources=None,
1051
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1050
diff changeset
34 rotate_bake_records=True,
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1050
diff changeset
35 keep_unused_records=False):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
36 self.appfactory = appfactory
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 self.app = app
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
38 self.out_dir = out_dir
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 self.force = force
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
40 self.allowed_pipelines = allowed_pipelines
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
41 self.forbidden_pipelines = forbidden_pipelines
918
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
42 self.allowed_sources = allowed_sources
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
43 self.rotate_bake_records = rotate_bake_records
1051
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1050
diff changeset
44 self.keep_unused_records = keep_unused_records
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 def bake(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
47 start_time = time.perf_counter()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
48
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
49 # Setup baker.
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 logger.debug(" Bake Output: %s" % self.out_dir)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 logger.debug(" Root URL: %s" % self.app.config.get('site/root'))
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 # Get into bake mode.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 self.app.config.set('baker/is_baking', True)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
55 self.app.config.set('site/asset_url_format', '%page_uri%/%filename%')
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
57 stats = self.app.env.stats
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
58 stats.registerTimer('LoadSourceContents', raise_if_registered=False)
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
59 stats.registerTimer('CacheTemplates', raise_if_registered=False)
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
60
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 # Make sure the output directory exists.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 if not os.path.isdir(self.out_dir):
5
474c9882decf Upgrade to Python 3.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
63 os.makedirs(self.out_dir, 0o755)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
65 # Load/create the bake records.
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
66 records_path = get_bake_records_path(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
67 self.app, self.out_dir)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
68 if not self.force and os.path.isfile(records_path):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
69 with format_timed_scope(logger, "loaded previous bake records",
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
70 level=logging.DEBUG, colored=False):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
71 previous_records = load_records(records_path)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
72 else:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
73 previous_records = MultiRecord()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
74 current_records = MultiRecord()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
76 # Figure out if we need to clean the cache because important things
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
77 # have changed.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
78 is_cache_valid = self._handleCacheValidity(previous_records,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
79 current_records)
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
80 if not is_cache_valid:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
81 previous_records = MultiRecord()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
82
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
83 # Create the bake records history which tracks what's up-to-date
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
84 # or not since last time we baked to the given output folder.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
85 record_histories = MultiRecordHistory(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
86 previous_records, current_records)
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
87
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
88 # Pre-create all caches.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
89 for cache_name in ['app', 'baker', 'pages', 'renders']:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
90 self.app.cache.getCache(cache_name)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
91
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
92 # Create the pipelines.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
93 ppmngr = self._createPipelineManager(record_histories)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
95 # Done with all the setup, let's start the actual work.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
96 logger.info(format_timed(start_time, "setup baker"))
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
97
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
98 # Load all sources, pre-cache templates.
1014
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1011
diff changeset
99 load_start_time = time.perf_counter()
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
100 self._populateTemplateCaches()
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
101 logger.info(format_timed(load_start_time, "cache templates"))
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
102
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
103 # Create the worker processes.
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
104 pool_userdata = _PoolUserData(self, ppmngr)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
105 pool = self._createWorkerPool(records_path, pool_userdata)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
106
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
107 # Bake the realms.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
108 self._bakeRealms(pool, ppmngr, record_histories)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
109
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
110 # Handle deletions, collapse records, etc.
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
111 ppmngr.postJobRun()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
112 ppmngr.deleteStaleOutputs()
1051
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1050
diff changeset
113 ppmngr.collapseRecords(self.keep_unused_records)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
114
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
115 # All done with the workers. Close the pool and get reports.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
116 pool_stats = pool.close()
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
117 current_records.stats = _merge_execution_stats(stats, *pool_stats)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
119 # Shutdown the pipelines.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
120 ppmngr.shutdownPipelines()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
122 # Backup previous records, save the current ones.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
123 current_records.bake_time = time.time()
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
124 current_records.out_dir = self.out_dir
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
125 _save_bake_records(current_records, records_path,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
126 rotate_previous=self.rotate_bake_records)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 # All done.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129 self.app.config.set('baker/is_baking', False)
151
fd146f54bdaa Forgot this wasn't C++.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
130 logger.debug(format_timed(start_time, 'done baking'))
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
132 return current_records
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
133
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
134 def _handleCacheValidity(self, previous_records, current_records):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
135 start_time = time.perf_counter()
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
136
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
137 reason = None
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
138 if self.force:
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
139 reason = "ordered to"
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
140 elif not self.app.config.get('__cache_valid'):
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
141 # The configuration file was changed, or we're running a new
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
142 # version of the app.
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
143 reason = "not valid anymore"
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
144 elif previous_records.invalidated:
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
145 # We have no valid previous bake records.
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
146 reason = "need bake records regeneration"
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
147 else:
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
148 # Check if any template has changed since the last bake. Since
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
149 # there could be some advanced conditional logic going on, we'd
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
150 # better just force a bake from scratch if that's the case.
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
151 max_time = 0
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
152 for d in self.app.templates_dirs:
42
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
153 for dpath, _, filenames in os.walk(d):
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
154 for fn in filenames:
42
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
155 full_fn = os.path.join(dpath, fn)
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
156 max_time = max(max_time, os.path.getmtime(full_fn))
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
157 if max_time >= previous_records.bake_time:
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
158 reason = "templates modified"
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
159
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
160 if reason is not None:
85
3471ffa059b2 Add a `BakeScheduler` to handle build dependencies. Add unit-tests.
Ludovic Chabant <ludovic@chabant.com>
parents: 50
diff changeset
161 # We have to bake everything from scratch.
707
5f552aedd918 bake: Don't clean the `baker` cache on a force bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 691
diff changeset
162 self.app.cache.clearCaches(except_names=['app', 'baker'])
45
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 42
diff changeset
163 self.force = True
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
164 current_records.incremental_count = 0
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
165 previous_records = MultiRecord()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
166 logger.debug(format_timed(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
167 start_time, "cleaned cache (reason: %s)" % reason,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
168 colored=False))
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
169 return False
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
170 else:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
171 current_records.incremental_count += 1
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 151
diff changeset
172 logger.debug(format_timed(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
173 start_time, "cache is assumed valid", colored=False))
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
174 return True
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
175
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
176 def _createPipelineManager(self, record_histories):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
177 # Gather all sources by realm -- we're going to bake each realm
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
178 # separately so we can handle "overriding" (i.e. one realm overrides
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
179 # another realm's pages, like the user realm overriding the theme
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
180 # realm).
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
181 #
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
182 # Also, create and initialize each pipeline for each source.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
183 has_any_pp = False
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
184 ppmngr = PipelineManager(
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
185 self.app, self.out_dir,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
186 record_histories=record_histories)
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
187 ok_pp = self.allowed_pipelines
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
188 nok_pp = self.forbidden_pipelines
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
189 ok_src = self.allowed_sources
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
190 for source in self.app.sources:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
191 if ok_src is not None and source.name not in ok_src:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
192 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
193
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
194 pname = get_pipeline_name_for_source(source)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
195 if ok_pp is not None and pname not in ok_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
196 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
197 if nok_pp is not None and pname in nok_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
198 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
199
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
200 ppinfo = ppmngr.createPipeline(source)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
201 logger.debug(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
202 "Created pipeline '%s' for source: %s" %
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
203 (ppinfo.pipeline.PIPELINE_NAME, source.name))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
204 has_any_pp = True
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
205 if not has_any_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
206 raise Exception("The website has no content sources, or the bake "
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
207 "command was invoked with all pipelines filtered "
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
208 "out. There's nothing to do.")
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
209 return ppmngr
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
210
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
211 def _populateTemplateCaches(self):
1050
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
212 engine_name = self.app.config.get('site/default_template_engine')
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
213 for engine in self.app.plugin_loader.getTemplateEngines():
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
214 if engine_name in engine.ENGINE_NAMES:
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
215 engine.populateCache()
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
216 break
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
217
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
218 def _bakeRealms(self, pool, ppmngr, record_histories):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
219 # Bake the realms -- user first, theme second, so that a user item
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
220 # can override a theme item.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
221 # Do this for as many times as we have pipeline passes left to do.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
222 realm_list = [REALM_USER, REALM_THEME]
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
223 pp_by_pass_and_realm = _get_pipeline_infos_by_pass_and_realm(
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
224 ppmngr.getPipelineInfos())
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
225
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
226 for pp_pass_num in sorted(pp_by_pass_and_realm.keys()):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
227 logger.debug("Pipelines pass %d" % pp_pass_num)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
228 pp_by_realm = pp_by_pass_and_realm[pp_pass_num]
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
229 for realm in realm_list:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
230 pplist = pp_by_realm.get(realm)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
231 if pplist is not None:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
232 self._bakeRealm(pool, ppmngr, record_histories,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
233 pp_pass_num, realm, pplist)
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
234
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
235 def _bakeRealm(self, pool, ppmngr, record_histories,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
236 pp_pass_num, realm, pplist):
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
237 # Start with the first step, where we iterate on the content sources'
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
238 # items and run jobs on those.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
239 pool.userdata.cur_pass = pp_pass_num
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
240 pool.userdata.cur_step = 0
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
241 next_step_jobs = {}
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
242 pool.userdata.next_step_jobs = next_step_jobs
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
243
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
244 start_time = time.perf_counter()
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
245 job_count = 0
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
246 stats = self.app.env.stats
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
247 realm_name = REALM_NAMES[realm].lower()
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
248 participating_source_names = []
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
249
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
250 for ppinfo in pplist:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
251 src = ppinfo.source
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
252 pp = ppinfo.pipeline
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
253 jcctx = PipelineJobCreateContext(pp_pass_num, pp.record_name,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
254 record_histories)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
255
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
256 next_step_jobs[src.name] = []
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
257 jobs = pp.createJobs(jcctx)
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
258 if jobs is not None:
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
259 new_job_count = len(jobs)
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
260 job_count += new_job_count
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
261 pool.queueJobs(jobs)
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
262 participating_source_names.append(src.name)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
263 else:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
264 new_job_count = 0
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
265
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
266 logger.debug(
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
267 "Queued %d jobs for source '%s' using pipeline '%s' "
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
268 "(%s, step 0)." %
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
269 (new_job_count, src.name, pp.PIPELINE_NAME, realm_name))
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
270
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
271 if job_count == 0:
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
272 logger.debug("No jobs queued! Bailing out of this bake pass.")
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
273 return
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
274
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
275 pool.wait()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
276
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
277 logger.info(format_timed(
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
278 start_time, "%d jobs completed (%s)." %
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
279 (job_count, ', '.join(participating_source_names))))
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
280
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
281 # Now let's see if any job created a follow-up job. Let's keep
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
282 # processing those jobs as long as they create new ones.
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
283 pool.userdata.cur_step = 1
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
284 while True:
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
285 # Make a copy of out next step jobs and reset the list, so
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
286 # the first jobs to be processed don't mess it up as we're
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
287 # still iterating on it.
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
288 next_step_jobs = pool.userdata.next_step_jobs
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
289 pool.userdata.next_step_jobs = {}
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
290
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
291 start_time = time.perf_counter()
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
292 job_count = 0
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
293 participating_source_names = []
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
294
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
295 for sn, jobs in next_step_jobs.items():
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
296 if jobs:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
297 logger.debug(
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
298 "Queuing jobs for source '%s' (%s, step %d)." %
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
299 (sn, realm_name, pool.userdata.cur_step))
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
300
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
301 pp = ppmngr.getPipeline(sn)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
302 valctx = PipelineJobValidateContext(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
303 pp_pass_num, pool.userdata.cur_step,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
304 pp.record_name, record_histories)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
305 pp.validateNextStepJobs(jobs, valctx)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
306
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
307 job_count += len(jobs)
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
308 pool.userdata.next_step_jobs[sn] = []
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
309 pool.queueJobs(jobs)
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
310 participating_source_names.append(sn)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
311
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
312 if job_count == 0:
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
313 break
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
314
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
315 pool.wait()
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
316
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
317 logger.info(format_timed(
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
318 start_time,
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
319 "%d jobs completed (%s)." %
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
320 (job_count, ', '.join(participating_source_names))))
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
321
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
322 pool.userdata.cur_step += 1
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
323
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
324 def _logErrors(self, item_spec, errors):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
325 logger.error("Errors found in %s:" % item_spec)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
326 for e in errors:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
327 logger.error(" " + e)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
328
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
329 def _logWorkerException(self, item_spec, exc_data):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
330 logger.error("Errors found in %s:" % item_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
331 logger.error(exc_data['value'])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
332 if self.app.debug:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
333 logger.error(exc_data['traceback'])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
334
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
335 def _createWorkerPool(self, previous_records_path, pool_userdata):
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
336 from piecrust.workerpool import WorkerPool
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
337 from piecrust.baking.worker import BakeWorkerContext, BakeWorker
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
338
456
5e902e228053 bake: Correctly use the `num_worers` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
339 worker_count = self.app.config.get('baker/workers')
462
04abc97dd3b6 bake: Add CLI argument to specify job batch size.
Ludovic Chabant <ludovic@chabant.com>
parents: 456
diff changeset
340 batch_size = self.app.config.get('baker/batch_size')
456
5e902e228053 bake: Correctly use the `num_worers` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
341
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
342 ctx = BakeWorkerContext(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
343 self.appfactory,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
344 self.out_dir,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
345 force=self.force,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
346 previous_records_path=previous_records_path,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
347 allowed_pipelines=self.allowed_pipelines,
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
348 forbidden_pipelines=self.forbidden_pipelines)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
349 pool = WorkerPool(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
350 worker_count=worker_count,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
351 batch_size=batch_size,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
352 worker_class=BakeWorker,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
353 initargs=(ctx,),
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
354 callback=self._handleWorkerResult,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
355 error_callback=self._handleWorkerError,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
356 userdata=pool_userdata)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
357 return pool
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
358
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
359 def _handleWorkerResult(self, job, res, userdata):
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
360 cur_pass = userdata.cur_pass
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
361 cur_step = userdata.cur_step
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
362 source_name, item_spec = job['job_spec']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
363
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
364 # See if there's a next step to take.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
365 npj = res.get('next_step_job')
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
366 if npj is not None:
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
367 npj['pass_num'] = cur_pass
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
368 npj['step_num'] = cur_step + 1
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
369 userdata.next_step_jobs[source_name].append(npj)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
370
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
371 # Make the pipeline do custom handling to update the record entry.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
372 ppinfo = userdata.ppmngr.getPipelineInfo(source_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
373 pipeline = ppinfo.pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
374 record = ppinfo.current_record
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
375 ppmrctx = PipelineJobResultHandleContext(record, job, cur_pass,
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
376 cur_step)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
377 pipeline.handleJobResult(res, ppmrctx)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
378
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
379 # Set the overall success flags if there was an error.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
380 record_entry = ppmrctx.record_entry
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
381 if not record_entry.success:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
382 record.success = False
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
383 userdata.records.success = False
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
384 self._logErrors(job['item_spec'], record_entry.errors)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
385
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
386 def _handleWorkerError(self, job, exc_data, userdata):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
387 # Set the overall success flag.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
388 source_name, item_spec = job['job_spec']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
389 ppinfo = userdata.ppmngr.getPipelineInfo(source_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
390 pipeline = ppinfo.pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
391 record = ppinfo.current_record
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
392 record.success = False
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
393 userdata.records.success = False
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
394
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
395 # Add those errors to the record, if possible.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
396 record_entry_spec = job.get('record_entry_spec', item_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
397 e = record.getEntry(record_entry_spec)
1044
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
398 if not e:
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
399 e = pipeline.createRecordEntry(item_spec)
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
400 record.addEntry(e)
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
401 e.errors.append(exc_data['value'])
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
402 self._logWorkerException(item_spec, exc_data)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
403
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
404 # Log debug stuff.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
405 if self.app.debug:
1040
506a5cb1e942 bake: Fix exception verbose reporting for bake worker errors.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
406 logger.error(exc_data['traceback'])
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
407
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
408
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
409 class _PoolUserData:
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
410 def __init__(self, baker, ppmngr):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
411 self.baker = baker
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
412 self.ppmngr = ppmngr
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
413 self.records = ppmngr.record_histories.current
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
414 self.cur_pass = 0
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
415 self.cur_step = 0
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
416 self.next_step_jobs = {}
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
417
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
418
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
419 def _get_pipeline_infos_by_pass_and_realm(pp_infos):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
420 pp_by_pass_and_realm = {}
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
421 for pp_info in pp_infos:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
422 pp_pass_num = pp_info.pipeline.PASS_NUM
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
423 if isinstance(pp_pass_num, list):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
424 for ppn in pp_pass_num:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
425 _add_pipeline_info_to_pass_and_realm_dict(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
426 ppn, pp_info, pp_by_pass_and_realm)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
427 else:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
428 _add_pipeline_info_to_pass_and_realm_dict(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
429 pp_pass_num, pp_info, pp_by_pass_and_realm)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
430 return pp_by_pass_and_realm
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
431
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
432
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
433 def _add_pipeline_info_to_pass_and_realm_dict(pp_pass_num, pp_info,
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
434 pp_by_pass_and_realm):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
435 pp_by_realm = pp_by_pass_and_realm.setdefault(pp_pass_num, {})
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
436 pplist = pp_by_realm.setdefault(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
437 pp_info.pipeline.source.config['realm'], [])
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
438 pplist.append(pp_info)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
439
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
440
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
441 def _merge_execution_stats(base_stats, *other_stats):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
442 total_stats = ExecutionStats()
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
443 total_stats.mergeStats(base_stats)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
444 for ps in other_stats:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
445 if ps is not None:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
446 total_stats.mergeStats(ps)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
447 return total_stats
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
448
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
449
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
450 def _save_bake_records(records, records_path, *, rotate_previous):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
451 if rotate_previous:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
452 records_dir, records_fn = os.path.split(records_path)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
453 records_id, _ = os.path.splitext(records_fn)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
454 for i in range(8, -1, -1):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
455 suffix = '' if i == 0 else '.%d' % i
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
456 records_path_i = os.path.join(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
457 records_dir,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
458 '%s%s.records' % (records_id, suffix))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
459 if os.path.exists(records_path_i):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
460 records_path_next = os.path.join(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
461 records_dir,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
462 '%s.%s.records' % (records_id, i + 1))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
463 if os.path.exists(records_path_next):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
464 os.remove(records_path_next)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
465 os.rename(records_path_i, records_path_next)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
466
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
467 with format_timed_scope(logger, "saved bake records.",
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
468 level=logging.DEBUG, colored=False):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
469 records.save(records_path)