Mercurial > piecrust2

annotate piecrust/baking/baker.py @ 871:504ddb370df8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactor: Fixing some issues with baking assets.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 13 Jun 2017 22:30:27 -0700
parents 8d25f76fce98
children d1095774bfcf
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import time
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import hashlib
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
5 from piecrust.chefutil import (
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
6 format_timed_scope, format_timed)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
7 from piecrust.environment import ExecutionStats
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
8 from piecrust.pipelines.base import (
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
9 PipelineJobCreateContext, PipelineMergeRecordContext, PipelineManager,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
10 get_pipeline_name_for_source)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
11 from piecrust.pipelines.records import (
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
12 MultiRecordHistory, MultiRecord, RecordEntry,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
13 load_records)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
14 from piecrust.sources.base import REALM_USER, REALM_THEME, REALM_NAMES
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
20 def get_bake_records_path(app, out_dir, *, suffix=''):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
21 records_cache = app.cache.getCache('baker')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
22 records_id = hashlib.md5(out_dir.encode('utf8')).hexdigest()
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
23 records_name = '%s%s.records' % (records_id, suffix)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
24 return records_cache.getCachePath(records_name)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
25
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
26
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 class Baker(object):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
28 def __init__(self, appfactory, app, out_dir,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
29 force=False, allowed_pipelines=None,
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
30 forbidden_pipelines=None):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
31 self.appfactory = appfactory
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 self.app = app
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
33 self.out_dir = out_dir
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 self.force = force
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
35 self.allowed_pipelines = allowed_pipelines
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
36 self.forbidden_pipelines = forbidden_pipelines
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 def bake(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
39 start_time = time.perf_counter()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 logger.debug(" Bake Output: %s" % self.out_dir)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 logger.debug(" Root URL: %s" % self.app.config.get('site/root'))
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 # Get into bake mode.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 self.app.config.set('baker/is_baking', True)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
45 self.app.config.set('site/asset_url_format', '%page_uri%/%filename%')
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 # Make sure the output directory exists.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 if not os.path.isdir(self.out_dir):
5
474c9882decf Upgrade to Python 3.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
49 os.makedirs(self.out_dir, 0o755)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
51 # Load/create the bake records.
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
52 records_path = get_bake_records_path(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
53 self.app, self.out_dir)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
54 if not self.force and os.path.isfile(records_path):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
55 with format_timed_scope(logger, "loaded previous bake records",
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
56 level=logging.DEBUG, colored=False):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
57 previous_records = load_records(records_path)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
58 else:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
59 previous_records = MultiRecord()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
60 current_records = MultiRecord()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
62 # Figure out if we need to clean the cache because important things
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
63 # have changed.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
64 is_cache_valid = self._handleCacheValidity(previous_records,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
65 current_records)
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
66 if not is_cache_valid:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
67 previous_records = MultiRecord()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
68
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
69 # Create the bake records history which tracks what's up-to-date
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
70 # or not since last time we baked to the given output folder.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
71 record_histories = MultiRecordHistory(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
72 previous_records, current_records)
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
73
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
74 # Pre-create all caches.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
75 for cache_name in ['app', 'baker', 'pages', 'renders']:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
76 self.app.cache.getCache(cache_name)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
77
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 # Gather all sources by realm -- we're going to bake each realm
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
79 # separately so we can handle "overriding" (i.e. one realm overrides
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
80 # another realm's pages, like the user realm overriding the theme
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
81 # realm).
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
82 #
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
83 # Also, create and initialize each pipeline for each source.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
84 has_any_pp = False
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
85 ppmngr = PipelineManager(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
86 self.app, self.out_dir, record_histories)
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
87 ok_pp = self.allowed_pipelines
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
88 nok_pp = self.forbidden_pipelines
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 for source in self.app.sources:
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
90 pname = get_pipeline_name_for_source(source)
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
91 if ok_pp is not None and pname not in ok_pp:
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
92 continue
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
93 if nok_pp is not None and pname in nok_pp:
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
94 continue
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
95
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
96 ppinfo = ppmngr.createPipeline(source)
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
97 logger.debug(
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
98 "Created pipeline '%s' for source: %s" %
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
99 (ppinfo.pipeline.PIPELINE_NAME, source.name))
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
100 has_any_pp = True
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
101 if not has_any_pp:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
102 raise Exception("The website has no content sources, or the bake "
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
103 "command was invoked with all pipelines filtered "
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
104 "out. There's nothing to do.")
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
106 # Create the worker processes.
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
107 pool_userdata = _PoolUserData(self, ppmngr)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
108 pool = self._createWorkerPool(records_path, pool_userdata)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
109 realm_list = [REALM_USER, REALM_THEME]
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
110
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
111 # Bake the realms -- user first, theme second, so that a user item
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
112 # can override a theme item.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
113 # Do this for as many times as we have pipeline passes left to do.
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
114 pp_by_pass_and_realm = {}
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
115 for ppinfo in ppmngr.getPipelines():
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
116 pp_by_realm = pp_by_pass_and_realm.setdefault(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
117 ppinfo.pipeline.PASS_NUM, {})
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
118 pplist = pp_by_realm.setdefault(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
119 ppinfo.pipeline.source.config['realm'], [])
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
120 pplist.append(ppinfo)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
121
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
122 for pp_pass_num in sorted(pp_by_pass_and_realm.keys()):
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
123 logger.debug("Pipelines pass %d" % pp_pass_num)
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
124 pp_by_realm = pp_by_pass_and_realm[pp_pass_num]
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
125 for realm in realm_list:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
126 pplist = pp_by_realm.get(realm)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
127 if pplist is not None:
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
128 self._bakeRealm(
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
129 pool, record_histories, pp_pass_num, realm, pplist)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
130
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
131 # Handle deletions, collapse records, etc.
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
132 ppmngr.postJobRun()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
133 ppmngr.deleteStaleOutputs()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
134 ppmngr.collapseRecords()
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
135
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
136 # All done with the workers. Close the pool and get reports.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
137 pool_stats = pool.close()
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
138 total_stats = ExecutionStats()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
139 for ps in pool_stats:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
140 if ps is not None:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
141 total_stats.mergeStats(ps)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
142 current_records.stats = total_stats
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
144 # Shutdown the pipelines.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
145 ppmngr.shutdownPipelines()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
146
333
91b07f9efdc1 bake: Use a rotating bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
147 # Backup previous records.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
148 records_dir, records_fn = os.path.split(records_path)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
149 records_id, _ = os.path.splitext(records_fn)
333
91b07f9efdc1 bake: Use a rotating bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
150 for i in range(8, -1, -1):
91b07f9efdc1 bake: Use a rotating bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
151 suffix = '' if i == 0 else '.%d' % i
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
152 records_path_i = os.path.join(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
153 records_dir,
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
154 '%s%s.records' % (records_id, suffix))
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
155 if os.path.exists(records_path_i):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
156 records_path_next = os.path.join(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
157 records_dir,
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
158 '%s.%s.records' % (records_id, i + 1))
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
159 if os.path.exists(records_path_next):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
160 os.remove(records_path_next)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
161 os.rename(records_path_i, records_path_next)
333
91b07f9efdc1 bake: Use a rotating bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
162
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
163 # Save the bake records.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
164 with format_timed_scope(logger, "saved bake records.",
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
165 level=logging.DEBUG, colored=False):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
166 current_records.bake_time = time.time()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
167 current_records.out_dir = self.out_dir
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
168 current_records.save(records_path)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170 # All done.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171 self.app.config.set('baker/is_baking', False)
151
fd146f54bdaa Forgot this wasn't C++.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
172 logger.debug(format_timed(start_time, 'done baking'))
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
174 return current_records
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
175
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
176 def _handleCacheValidity(self, previous_records, current_records):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
177 start_time = time.perf_counter()
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
178
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
179 reason = None
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
180 if self.force:
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
181 reason = "ordered to"
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
182 elif not self.app.config.get('__cache_valid'):
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
183 # The configuration file was changed, or we're running a new
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
184 # version of the app.
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
185 reason = "not valid anymore"
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
186 elif previous_records.invalidated:
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
187 # We have no valid previous bake records.
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
188 reason = "need bake records regeneration"
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
189 else:
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
190 # Check if any template has changed since the last bake. Since
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
191 # there could be some advanced conditional logic going on, we'd
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
192 # better just force a bake from scratch if that's the case.
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
193 max_time = 0
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
194 for d in self.app.templates_dirs:
42
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
195 for dpath, _, filenames in os.walk(d):
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
196 for fn in filenames:
42
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
197 full_fn = os.path.join(dpath, fn)
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
198 max_time = max(max_time, os.path.getmtime(full_fn))
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
199 if max_time >= previous_records.bake_time:
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
200 reason = "templates modified"
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
201
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
202 if reason is not None:
85
3471ffa059b2 Add a `BakeScheduler` to handle build dependencies. Add unit-tests.
Ludovic Chabant <ludovic@chabant.com>
parents: 50
diff changeset
203 # We have to bake everything from scratch.
707
5f552aedd918 bake: Don't clean the `baker` cache on a force bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 691
diff changeset
204 self.app.cache.clearCaches(except_names=['app', 'baker'])
45
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 42
diff changeset
205 self.force = True
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
206 current_records.incremental_count = 0
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
207 previous_records = MultiRecord()
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 151
diff changeset
208 logger.info(format_timed(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
209 start_time, "cleaned cache (reason: %s)" % reason))
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
210 return False
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
211 else:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
212 current_records.incremental_count += 1
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 151
diff changeset
213 logger.debug(format_timed(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
214 start_time, "cache is assumed valid", colored=False))
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
215 return True
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
216
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
217 def _bakeRealm(self, pool, record_histories, pp_pass_num, realm, pplist):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
218 # Start with the first pass, where we iterate on the content sources'
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
219 # items and run jobs on those.
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
220 pool.userdata.cur_pass = 0
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
221 next_pass_jobs = {}
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
222 pool.userdata.next_pass_jobs = next_pass_jobs
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
223
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
224 start_time = time.perf_counter()
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
225 job_count = 0
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
226 realm_name = REALM_NAMES[realm].lower()
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
227
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
228 for ppinfo in pplist:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
229 src = ppinfo.source
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
230 pp = ppinfo.pipeline
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
231
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
232 logger.debug(
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
233 "Queuing jobs for source '%s' using pipeline '%s' "
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
234 "(%s, pass 0)." %
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
235 (src.name, pp.PIPELINE_NAME, realm_name))
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
236
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
237 next_pass_jobs[src.name] = []
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
238 jcctx = PipelineJobCreateContext(pp_pass_num, record_histories)
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
239 jobs = pp.createJobs(jcctx)
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
240 if jobs is not None:
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
241 job_count += len(jobs)
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
242 pool.queueJobs(jobs)
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
243
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
244 if job_count == 0:
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
245 logger.debug("No jobs queued! Bailing out of this bake pass.")
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
246 return
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
247
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
248 pool.wait()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
249
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
250 logger.info(format_timed(
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
251 start_time, "%d pipeline jobs completed (%s, pass 0)." %
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
252 (job_count, realm_name)))
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
253
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
254 # Now let's see if any job created a follow-up job. Let's keep
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
255 # processing those jobs as long as they create new ones.
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
256 pool.userdata.cur_pass = 1
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
257 while True:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
258 # Make a copy of out next pass jobs and reset the list, so
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
259 # the first jobs to be processed don't mess it up as we're
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
260 # still iterating on it.
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
261 next_pass_jobs = pool.userdata.next_pass_jobs
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
262 pool.userdata.next_pass_jobs = {}
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
263
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
264 start_time = time.perf_counter()
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
265 job_count = 0
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
266
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
267 for sn, jobs in next_pass_jobs.items():
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
268 if jobs:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
269 logger.debug(
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
270 "Queuing jobs for source '%s' (%s, pass %d)." %
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
271 (sn, realm_name, pool.userdata.cur_pass))
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
272 job_count += len(jobs)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
273 pool.userdata.next_pass_jobs[sn] = []
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
274 pool.queueJobs(jobs)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
275
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
276 if job_count == 0:
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
277 break
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
278
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
279 pool.wait()
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
280
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
281 logger.info(format_timed(
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
282 start_time,
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
283 "%d pipeline jobs completed (%s, pass %d)." %
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
284 (job_count, realm_name, pool.userdata.cur_pass)))
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
285
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
286 pool.userdata.cur_pass += 1
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
287
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
288 def _logErrors(self, item_spec, errors):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
289 logger.error("Errors found in %s:" % item_spec)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
290 for e in errors:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
291 logger.error(" " + e)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
292
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
293 def _createWorkerPool(self, previous_records_path, pool_userdata):
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
294 from piecrust.workerpool import WorkerPool
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
295 from piecrust.baking.worker import BakeWorkerContext, BakeWorker
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
296
456
5e902e228053 bake: Correctly use the `num_worers` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
297 worker_count = self.app.config.get('baker/workers')
462
04abc97dd3b6 bake: Add CLI argument to specify job batch size.
Ludovic Chabant <ludovic@chabant.com>
parents: 456
diff changeset
298 batch_size = self.app.config.get('baker/batch_size')
456
5e902e228053 bake: Correctly use the `num_worers` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
299
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
300 ctx = BakeWorkerContext(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
301 self.appfactory,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
302 self.out_dir,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
303 force=self.force,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
304 previous_records_path=previous_records_path,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
305 allowed_pipelines=self.allowed_pipelines,
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
306 forbidden_pipelines=self.forbidden_pipelines)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
307 pool = WorkerPool(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
308 worker_count=worker_count,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
309 batch_size=batch_size,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
310 worker_class=BakeWorker,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
311 initargs=(ctx,),
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
312 callback=self._handleWorkerResult,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
313 error_callback=self._handleWorkerError,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
314 userdata=pool_userdata)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
315 return pool
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
316
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
317 def _handleWorkerResult(self, job, res, userdata):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
318 cur_pass = userdata.cur_pass
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
319 record = userdata.records.getRecord(job.record_name)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
320
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
321 if cur_pass == 0:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
322 record.addEntry(res.record_entry)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
323 else:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
324 ppinfo = userdata.ppmngr.getPipeline(job.source_name)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
325 ppmrctx = PipelineMergeRecordContext(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
326 record, job, cur_pass)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
327 ppinfo.pipeline.mergeRecordEntry(res.record_entry, ppmrctx)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
328
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
329 npj = res.next_pass_job
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
330 if npj is not None:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
331 npj.data['pass'] = cur_pass + 1
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
332 userdata.next_pass_jobs[job.source_name].append(npj)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
333
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
334 if not res.record_entry.success:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
335 record.success = False
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
336 userdata.records.success = False
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
337 self._logErrors(job.content_item.spec, res.record_entry.errors)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
338
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
339 def _handleWorkerError(self, job, exc_data, userdata):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
340 cur_pass = userdata.cur_pass
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
341 record = userdata.records.getRecord(job.record_name)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
342
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
343 if cur_pass == 0:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
344 ppinfo = userdata.ppmngr.getPipeline(job.source_name)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
345 entry_class = ppinfo.pipeline.RECORD_ENTRY_CLASS or RecordEntry
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
346 e = entry_class()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
347 e.item_spec = job.content_item.spec
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
348 e.errors.append(str(exc_data))
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
349 record.addEntry(e)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
350 else:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
351 e = record.getEntry(job.content_item.spec)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
352 e.errors.append(str(exc_data))
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
353
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
354 record.success = False
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
355 userdata.records.success = False
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
356
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
357 self._logErrors(job.content_item.spec, e.errors)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
358 if self.app.debug:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
359 logger.error(exc_data.traceback)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
360
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
361
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
362 class _PoolUserData:
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
363 def __init__(self, baker, ppmngr):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
364 self.baker = baker
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
365 self.ppmngr = ppmngr
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
366 self.records = ppmngr.record_histories.current
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
367 self.cur_pass = 0
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
368 self.next_pass_jobs = {}