Mercurial > piecrust2

annotate piecrust/baking/baker.py @ 1042:895f49c9833d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
sources: Adds support for whitelist filtering to file-system sources.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jan 2018 17:23:34 -0800
parents 506a5cb1e942
children 7b64eb5bbc81
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import time
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import hashlib
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
5 from piecrust.chefutil import (
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
6 format_timed_scope, format_timed)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
7 from piecrust.environment import ExecutionStats
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
8 from piecrust.pipelines.base import (
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
9 PipelineJobCreateContext, PipelineJobResultHandleContext,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
10 PipelineJobValidateContext, PipelineManager,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
11 get_pipeline_name_for_source)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
12 from piecrust.pipelines.records import (
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
13 MultiRecordHistory, MultiRecord, RecordEntry,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
14 load_records)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
15 from piecrust.sources.base import REALM_USER, REALM_THEME, REALM_NAMES
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
21 def get_bake_records_path(app, out_dir, *, suffix=''):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
22 records_cache = app.cache.getCache('baker')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
23 records_id = hashlib.md5(out_dir.encode('utf8')).hexdigest()
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
24 records_name = '%s%s.records' % (records_id, suffix)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
25 return records_cache.getCachePath(records_name)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
26
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
27
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 class Baker(object):
918
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
29 def __init__(self, appfactory, app, out_dir, *,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
30 force=False,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
31 allowed_pipelines=None,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
32 forbidden_pipelines=None,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
33 allowed_sources=None,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
34 rotate_bake_records=True):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
35 self.appfactory = appfactory
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self.app = app
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
37 self.out_dir = out_dir
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 self.force = force
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
39 self.allowed_pipelines = allowed_pipelines
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
40 self.forbidden_pipelines = forbidden_pipelines
918
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
41 self.allowed_sources = allowed_sources
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
42 self.rotate_bake_records = rotate_bake_records
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 def bake(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
45 start_time = time.perf_counter()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
46
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
47 # Setup baker.
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 logger.debug(" Bake Output: %s" % self.out_dir)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 logger.debug(" Root URL: %s" % self.app.config.get('site/root'))
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 # Get into bake mode.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 self.app.config.set('baker/is_baking', True)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
53 self.app.config.set('site/asset_url_format', '%page_uri%/%filename%')
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
55 stats = self.app.env.stats
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
56 stats.registerTimer('LoadSourceContents', raise_if_registered=False)
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
57 stats.registerTimer('CacheTemplates', raise_if_registered=False)
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
58
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 # Make sure the output directory exists.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 if not os.path.isdir(self.out_dir):
5
474c9882decf Upgrade to Python 3.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
61 os.makedirs(self.out_dir, 0o755)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
63 # Load/create the bake records.
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
64 records_path = get_bake_records_path(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
65 self.app, self.out_dir)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
66 if not self.force and os.path.isfile(records_path):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
67 with format_timed_scope(logger, "loaded previous bake records",
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
68 level=logging.DEBUG, colored=False):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
69 previous_records = load_records(records_path)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
70 else:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
71 previous_records = MultiRecord()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
72 current_records = MultiRecord()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
74 # Figure out if we need to clean the cache because important things
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
75 # have changed.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
76 is_cache_valid = self._handleCacheValidity(previous_records,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
77 current_records)
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
78 if not is_cache_valid:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
79 previous_records = MultiRecord()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
80
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
81 # Create the bake records history which tracks what's up-to-date
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
82 # or not since last time we baked to the given output folder.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
83 record_histories = MultiRecordHistory(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
84 previous_records, current_records)
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
85
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
86 # Pre-create all caches.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
87 for cache_name in ['app', 'baker', 'pages', 'renders']:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
88 self.app.cache.getCache(cache_name)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
89
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
90 # Create the pipelines.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
91 ppmngr = self._createPipelineManager(record_histories)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
93 # Done with all the setup, let's start the actual work.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
94 logger.info(format_timed(start_time, "setup baker"))
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
95
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
96 # Load all sources, pre-cache templates.
1014
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1011
diff changeset
97 load_start_time = time.perf_counter()
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
98 self._populateTemplateCaches()
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
99 logger.info(format_timed(load_start_time, "cache templates"))
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
100
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
101 # Create the worker processes.
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
102 pool_userdata = _PoolUserData(self, ppmngr)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
103 pool = self._createWorkerPool(records_path, pool_userdata)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
104
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
105 # Bake the realms.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
106 self._bakeRealms(pool, ppmngr, record_histories)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
107
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
108 # Handle deletions, collapse records, etc.
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
109 ppmngr.postJobRun()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
110 ppmngr.deleteStaleOutputs()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
111 ppmngr.collapseRecords()
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
112
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
113 # All done with the workers. Close the pool and get reports.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
114 pool_stats = pool.close()
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
115 current_records.stats = _merge_execution_stats(stats, *pool_stats)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
117 # Shutdown the pipelines.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
118 ppmngr.shutdownPipelines()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
120 # Backup previous records, save the current ones.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
121 current_records.bake_time = time.time()
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
122 current_records.out_dir = self.out_dir
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
123 _save_bake_records(current_records, records_path,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
124 rotate_previous=self.rotate_bake_records)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 # All done.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 self.app.config.set('baker/is_baking', False)
151
fd146f54bdaa Forgot this wasn't C++.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
128 logger.debug(format_timed(start_time, 'done baking'))
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
130 return current_records
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
131
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
132 def _handleCacheValidity(self, previous_records, current_records):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
133 start_time = time.perf_counter()
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
134
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
135 reason = None
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
136 if self.force:
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
137 reason = "ordered to"
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
138 elif not self.app.config.get('__cache_valid'):
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
139 # The configuration file was changed, or we're running a new
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
140 # version of the app.
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
141 reason = "not valid anymore"
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
142 elif previous_records.invalidated:
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
143 # We have no valid previous bake records.
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
144 reason = "need bake records regeneration"
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
145 else:
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
146 # Check if any template has changed since the last bake. Since
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
147 # there could be some advanced conditional logic going on, we'd
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
148 # better just force a bake from scratch if that's the case.
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
149 max_time = 0
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
150 for d in self.app.templates_dirs:
42
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
151 for dpath, _, filenames in os.walk(d):
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
152 for fn in filenames:
42
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
153 full_fn = os.path.join(dpath, fn)
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
154 max_time = max(max_time, os.path.getmtime(full_fn))
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
155 if max_time >= previous_records.bake_time:
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
156 reason = "templates modified"
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
157
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
158 if reason is not None:
85
3471ffa059b2 Add a `BakeScheduler` to handle build dependencies. Add unit-tests.
Ludovic Chabant <ludovic@chabant.com>
parents: 50
diff changeset
159 # We have to bake everything from scratch.
707
5f552aedd918 bake: Don't clean the `baker` cache on a force bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 691
diff changeset
160 self.app.cache.clearCaches(except_names=['app', 'baker'])
45
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 42
diff changeset
161 self.force = True
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
162 current_records.incremental_count = 0
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
163 previous_records = MultiRecord()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
164 logger.debug(format_timed(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
165 start_time, "cleaned cache (reason: %s)" % reason,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
166 colored=False))
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
167 return False
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
168 else:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
169 current_records.incremental_count += 1
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 151
diff changeset
170 logger.debug(format_timed(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
171 start_time, "cache is assumed valid", colored=False))
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
172 return True
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
173
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
174 def _createPipelineManager(self, record_histories):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
175 # Gather all sources by realm -- we're going to bake each realm
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
176 # separately so we can handle "overriding" (i.e. one realm overrides
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
177 # another realm's pages, like the user realm overriding the theme
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
178 # realm).
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
179 #
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
180 # Also, create and initialize each pipeline for each source.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
181 has_any_pp = False
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
182 ppmngr = PipelineManager(
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
183 self.app, self.out_dir,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
184 record_histories=record_histories)
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
185 ok_pp = self.allowed_pipelines
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
186 nok_pp = self.forbidden_pipelines
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
187 ok_src = self.allowed_sources
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
188 for source in self.app.sources:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
189 if ok_src is not None and source.name not in ok_src:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
190 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
191
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
192 pname = get_pipeline_name_for_source(source)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
193 if ok_pp is not None and pname not in ok_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
194 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
195 if nok_pp is not None and pname in nok_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
196 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
197
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
198 ppinfo = ppmngr.createPipeline(source)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
199 logger.debug(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
200 "Created pipeline '%s' for source: %s" %
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
201 (ppinfo.pipeline.PIPELINE_NAME, source.name))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
202 has_any_pp = True
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
203 if not has_any_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
204 raise Exception("The website has no content sources, or the bake "
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
205 "command was invoked with all pipelines filtered "
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
206 "out. There's nothing to do.")
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
207 return ppmngr
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
208
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
209 def _populateTemplateCaches(self):
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
210 for eng in self.app.plugin_loader.getTemplateEngines():
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
211 eng.populateCache()
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
212
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
213 def _bakeRealms(self, pool, ppmngr, record_histories):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
214 # Bake the realms -- user first, theme second, so that a user item
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
215 # can override a theme item.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
216 # Do this for as many times as we have pipeline passes left to do.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
217 realm_list = [REALM_USER, REALM_THEME]
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
218 pp_by_pass_and_realm = _get_pipeline_infos_by_pass_and_realm(
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
219 ppmngr.getPipelineInfos())
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
220
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
221 for pp_pass_num in sorted(pp_by_pass_and_realm.keys()):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
222 logger.debug("Pipelines pass %d" % pp_pass_num)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
223 pp_by_realm = pp_by_pass_and_realm[pp_pass_num]
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
224 for realm in realm_list:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
225 pplist = pp_by_realm.get(realm)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
226 if pplist is not None:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
227 self._bakeRealm(pool, ppmngr, record_histories,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
228 pp_pass_num, realm, pplist)
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
229
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
230 def _bakeRealm(self, pool, ppmngr, record_histories,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
231 pp_pass_num, realm, pplist):
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
232 # Start with the first step, where we iterate on the content sources'
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
233 # items and run jobs on those.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
234 pool.userdata.cur_pass = pp_pass_num
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
235 pool.userdata.cur_step = 0
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
236 next_step_jobs = {}
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
237 pool.userdata.next_step_jobs = next_step_jobs
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
238
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
239 start_time = time.perf_counter()
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
240 job_count = 0
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
241 stats = self.app.env.stats
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
242 realm_name = REALM_NAMES[realm].lower()
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
243 participating_source_names = []
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
244
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
245 for ppinfo in pplist:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
246 src = ppinfo.source
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
247 pp = ppinfo.pipeline
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
248 jcctx = PipelineJobCreateContext(pp_pass_num, pp.record_name,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
249 record_histories)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
250
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
251 next_step_jobs[src.name] = []
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
252 jobs = pp.createJobs(jcctx)
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
253 if jobs is not None:
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
254 new_job_count = len(jobs)
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
255 job_count += new_job_count
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
256 pool.queueJobs(jobs)
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
257 participating_source_names.append(src.name)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
258 else:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
259 new_job_count = 0
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
260
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
261 logger.debug(
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
262 "Queued %d jobs for source '%s' using pipeline '%s' "
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
263 "(%s, step 0)." %
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
264 (new_job_count, src.name, pp.PIPELINE_NAME, realm_name))
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
265
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
266 if job_count == 0:
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
267 logger.debug("No jobs queued! Bailing out of this bake pass.")
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
268 return
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
269
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
270 pool.wait()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
271
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
272 logger.info(format_timed(
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
273 start_time, "%d jobs completed (%s)." %
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
274 (job_count, ', '.join(participating_source_names))))
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
275
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
276 # Now let's see if any job created a follow-up job. Let's keep
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
277 # processing those jobs as long as they create new ones.
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
278 pool.userdata.cur_step = 1
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
279 while True:
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
280 # Make a copy of out next step jobs and reset the list, so
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
281 # the first jobs to be processed don't mess it up as we're
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
282 # still iterating on it.
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
283 next_step_jobs = pool.userdata.next_step_jobs
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
284 pool.userdata.next_step_jobs = {}
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
285
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
286 start_time = time.perf_counter()
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
287 job_count = 0
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
288 participating_source_names = []
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
289
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
290 for sn, jobs in next_step_jobs.items():
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
291 if jobs:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
292 logger.debug(
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
293 "Queuing jobs for source '%s' (%s, step %d)." %
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
294 (sn, realm_name, pool.userdata.cur_step))
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
295
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
296 pp = ppmngr.getPipeline(sn)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
297 valctx = PipelineJobValidateContext(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
298 pp_pass_num, pool.userdata.cur_step,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
299 pp.record_name, record_histories)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
300 pp.validateNextStepJobs(jobs, valctx)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
301
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
302 job_count += len(jobs)
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
303 pool.userdata.next_step_jobs[sn] = []
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
304 pool.queueJobs(jobs)
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
305 participating_source_names.append(sn)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
306
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
307 if job_count == 0:
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
308 break
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
309
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
310 pool.wait()
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
311
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
312 logger.info(format_timed(
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
313 start_time,
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
314 "%d jobs completed (%s)." %
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
315 (job_count, ', '.join(participating_source_names))))
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
316
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
317 pool.userdata.cur_step += 1
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
318
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
319 def _logErrors(self, item_spec, errors):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
320 logger.error("Errors found in %s:" % item_spec)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
321 for e in errors:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
322 logger.error(" " + e)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
323
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
324 def _logWorkerException(self, item_spec, exc_data):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
325 logger.error("Errors found in %s:" % item_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
326 logger.error(exc_data['value'])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
327 if self.app.debug:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
328 logger.error(exc_data['traceback'])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
329
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
330 def _createWorkerPool(self, previous_records_path, pool_userdata):
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
331 from piecrust.workerpool import WorkerPool
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
332 from piecrust.baking.worker import BakeWorkerContext, BakeWorker
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
333
456
5e902e228053 bake: Correctly use the `num_worers` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
334 worker_count = self.app.config.get('baker/workers')
462
04abc97dd3b6 bake: Add CLI argument to specify job batch size.
Ludovic Chabant <ludovic@chabant.com>
parents: 456
diff changeset
335 batch_size = self.app.config.get('baker/batch_size')
456
5e902e228053 bake: Correctly use the `num_worers` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
336
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
337 ctx = BakeWorkerContext(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
338 self.appfactory,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
339 self.out_dir,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
340 force=self.force,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
341 previous_records_path=previous_records_path,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
342 allowed_pipelines=self.allowed_pipelines,
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
343 forbidden_pipelines=self.forbidden_pipelines)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
344 pool = WorkerPool(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
345 worker_count=worker_count,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
346 batch_size=batch_size,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
347 worker_class=BakeWorker,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
348 initargs=(ctx,),
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
349 callback=self._handleWorkerResult,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
350 error_callback=self._handleWorkerError,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
351 userdata=pool_userdata)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
352 return pool
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
353
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
354 def _handleWorkerResult(self, job, res, userdata):
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
355 cur_pass = userdata.cur_pass
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
356 cur_step = userdata.cur_step
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
357 source_name, item_spec = job['job_spec']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
358
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
359 # See if there's a next step to take.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
360 npj = res.get('next_step_job')
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
361 if npj is not None:
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
362 npj['pass_num'] = cur_pass
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
363 npj['step_num'] = cur_step + 1
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
364 userdata.next_step_jobs[source_name].append(npj)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
365
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
366 # Make the pipeline do custom handling to update the record entry.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
367 ppinfo = userdata.ppmngr.getPipelineInfo(source_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
368 pipeline = ppinfo.pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
369 record = ppinfo.current_record
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
370 ppmrctx = PipelineJobResultHandleContext(record, job, cur_pass,
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
371 cur_step)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
372 pipeline.handleJobResult(res, ppmrctx)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
373
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
374 # Set the overall success flags if there was an error.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
375 record_entry = ppmrctx.record_entry
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
376 if not record_entry.success:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
377 record.success = False
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
378 userdata.records.success = False
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
379 self._logErrors(job['item_spec'], record_entry.errors)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
380
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
381 def _handleWorkerError(self, job, exc_data, userdata):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
382 # Set the overall success flag.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
383 source_name, item_spec = job['job_spec']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
384 ppinfo = userdata.ppmngr.getPipelineInfo(source_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
385 pipeline = ppinfo.pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
386 record = ppinfo.current_record
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
387 record.success = False
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
388 userdata.records.success = False
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
389
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
390 # Add those errors to the record, if possible.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
391 record_entry_spec = job.get('record_entry_spec', item_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
392 e = record.getEntry(record_entry_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
393 if e:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
394 e.errors.append(exc_data['value'])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
395 self._logWorkerException(item_spec, exc_data)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
396
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
397 # Log debug stuff.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
398 if self.app.debug:
1040
506a5cb1e942 bake: Fix exception verbose reporting for bake worker errors.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
399 logger.error(exc_data['traceback'])
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
400
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
401
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
402 class _PoolUserData:
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
403 def __init__(self, baker, ppmngr):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
404 self.baker = baker
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
405 self.ppmngr = ppmngr
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
406 self.records = ppmngr.record_histories.current
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
407 self.cur_pass = 0
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
408 self.cur_step = 0
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
409 self.next_step_jobs = {}
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
410
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
411
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
412 def _get_pipeline_infos_by_pass_and_realm(pp_infos):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
413 pp_by_pass_and_realm = {}
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
414 for pp_info in pp_infos:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
415 pp_pass_num = pp_info.pipeline.PASS_NUM
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
416 if isinstance(pp_pass_num, list):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
417 for ppn in pp_pass_num:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
418 _add_pipeline_info_to_pass_and_realm_dict(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
419 ppn, pp_info, pp_by_pass_and_realm)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
420 else:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
421 _add_pipeline_info_to_pass_and_realm_dict(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
422 pp_pass_num, pp_info, pp_by_pass_and_realm)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
423 return pp_by_pass_and_realm
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
424
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
425
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
426 def _add_pipeline_info_to_pass_and_realm_dict(pp_pass_num, pp_info,
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
427 pp_by_pass_and_realm):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
428 pp_by_realm = pp_by_pass_and_realm.setdefault(pp_pass_num, {})
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
429 pplist = pp_by_realm.setdefault(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
430 pp_info.pipeline.source.config['realm'], [])
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
431 pplist.append(pp_info)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
432
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
433
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
434 def _merge_execution_stats(base_stats, *other_stats):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
435 total_stats = ExecutionStats()
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
436 total_stats.mergeStats(base_stats)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
437 for ps in other_stats:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
438 if ps is not None:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
439 total_stats.mergeStats(ps)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
440 return total_stats
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
441
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
442
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
443 def _save_bake_records(records, records_path, *, rotate_previous):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
444 if rotate_previous:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
445 records_dir, records_fn = os.path.split(records_path)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
446 records_id, _ = os.path.splitext(records_fn)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
447 for i in range(8, -1, -1):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
448 suffix = '' if i == 0 else '.%d' % i
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
449 records_path_i = os.path.join(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
450 records_dir,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
451 '%s%s.records' % (records_id, suffix))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
452 if os.path.exists(records_path_i):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
453 records_path_next = os.path.join(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
454 records_dir,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
455 '%s.%s.records' % (records_id, i + 1))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
456 if os.path.exists(records_path_next):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
457 os.remove(records_path_next)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
458 os.rename(records_path_i, records_path_next)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
459
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
460 with format_timed_scope(logger, "saved bake records.",
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
461 level=logging.DEBUG, colored=False):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
462 records.save(records_path)