Mercurial > piecrust2

annotate piecrust/baking/baker.py @ 1175:d0f86d9a9d40

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
wsgi: Better logging for the admin app.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 21 May 2020 21:55:41 -0700
parents 5f97b5b59dfe
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import time
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import hashlib
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
5 from piecrust.chefutil import (
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
6 format_timed_scope, format_timed)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
7 from piecrust.environment import ExecutionStats
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
8 from piecrust.pipelines.base import (
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
9 PipelineJobCreateContext, PipelineJobResultHandleContext, PipelineManager,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
10 get_pipeline_name_for_source)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
11 from piecrust.pipelines.records import (
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
12 MultiRecordHistory, MultiRecord,
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
13 load_records)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
14 from piecrust.sources.base import REALM_USER, REALM_THEME, REALM_NAMES
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
20 def get_bake_records_path(app, out_dir, *, suffix=''):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
21 records_cache = app.cache.getCache('baker')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
22 records_id = hashlib.md5(out_dir.encode('utf8')).hexdigest()
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
23 records_name = '%s%s.records' % (records_id, suffix)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
24 return records_cache.getCachePath(records_name)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
25
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
26
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 class Baker(object):
918
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
28 def __init__(self, appfactory, app, out_dir, *,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
29 force=False,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
30 allowed_pipelines=None,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
31 forbidden_pipelines=None,
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
32 allowed_sources=None,
1051
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1050
diff changeset
33 rotate_bake_records=True,
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1050
diff changeset
34 keep_unused_records=False):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
35 self.appfactory = appfactory
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self.app = app
127
bc63dc20baa0 Fix how we pass the out directory to the baking modules.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
37 self.out_dir = out_dir
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 self.force = force
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
39 self.allowed_pipelines = allowed_pipelines
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
40 self.forbidden_pipelines = forbidden_pipelines
918
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
41 self.allowed_sources = allowed_sources
7f1da7e7b154 internal: The processing loop for the server is now using the baker.
Ludovic Chabant <ludovic@chabant.com>
parents: 899
diff changeset
42 self.rotate_bake_records = rotate_bake_records
1051
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1050
diff changeset
43 self.keep_unused_records = keep_unused_records
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 def bake(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
46 start_time = time.perf_counter()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
47
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
48 # Setup baker.
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 logger.debug(" Bake Output: %s" % self.out_dir)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 logger.debug(" Root URL: %s" % self.app.config.get('site/root'))
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 # Get into bake mode.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 self.app.config.set('baker/is_baking', True)
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
54 self.app.config.set('site/asset_url_format', '%page_uri%/%filename%')
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
56 stats = self.app.env.stats
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
57 stats.registerTimer('LoadSourceContents', raise_if_registered=False)
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
58 stats.registerTimer('CacheTemplates', raise_if_registered=False)
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
59
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 # Make sure the output directory exists.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 if not os.path.isdir(self.out_dir):
5
474c9882decf Upgrade to Python 3.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
62 os.makedirs(self.out_dir, 0o755)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
64 # Load/create the bake records.
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
65 records_path = get_bake_records_path(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
66 self.app, self.out_dir)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
67 if not self.force and os.path.isfile(records_path):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
68 with format_timed_scope(logger, "loaded previous bake records",
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
69 level=logging.DEBUG, colored=False):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
70 previous_records = load_records(records_path)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
71 else:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
72 previous_records = MultiRecord()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
73 current_records = MultiRecord()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
75 # Figure out if we need to clean the cache because important things
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
76 # have changed.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
77 is_cache_valid = self._handleCacheValidity(previous_records,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
78 current_records)
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
79 if not is_cache_valid:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
80 previous_records = MultiRecord()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
81
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
82 # Create the bake records history which tracks what's up-to-date
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
83 # or not since last time we baked to the given output folder.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
84 record_histories = MultiRecordHistory(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
85 previous_records, current_records)
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
86
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
87 # Pre-create all caches.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
88 for cache_name in ['app', 'baker', 'pages', 'renders']:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
89 self.app.cache.getCache(cache_name)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
90
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
91 # Create the pipelines.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
92 ppmngr = self._createPipelineManager(record_histories)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
94 # Done with all the setup, let's start the actual work.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
95 logger.info(format_timed(start_time, "setup baker"))
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
96
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
97 # Load all sources, pre-cache templates.
1014
071f30aa04bb bake: Do template caching in a background job if possible.
Ludovic Chabant <ludovic@chabant.com>
parents: 1011
diff changeset
98 load_start_time = time.perf_counter()
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
99 self._populateTemplateCaches()
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
100 logger.info(format_timed(load_start_time, "cache templates"))
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
101
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
102 # Create the worker processes.
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
103 pool_userdata = _PoolUserData(self, ppmngr)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
104 pool = self._createWorkerPool(records_path, pool_userdata)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
105
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
106 # Bake the realms.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
107 self._bakeRealms(pool, ppmngr, record_histories)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
108
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
109 # Handle deletions, collapse records, etc.
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
110 ppmngr.postJobRun()
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
111 ppmngr.deleteStaleOutputs()
1051
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1050
diff changeset
112 ppmngr.collapseRecords(self.keep_unused_records)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
113
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
114 # All done with the workers. Close the pool and get reports.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
115 pool_stats = pool.close()
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
116 current_records.stats = _merge_execution_stats(stats, *pool_stats)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
118 # Shutdown the pipelines.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
119 ppmngr.shutdownPipelines()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
121 # Backup previous records, save the current ones.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
122 current_records.bake_time = time.time()
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
123 current_records.out_dir = self.out_dir
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
124 _save_bake_records(current_records, records_path,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
125 rotate_previous=self.rotate_bake_records)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 # All done.
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 self.app.config.set('baker/is_baking', False)
151
fd146f54bdaa Forgot this wasn't C++.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
129 logger.debug(format_timed(start_time, 'done baking'))
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
131 return current_records
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
132
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
133 def _handleCacheValidity(self, previous_records, current_records):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
134 start_time = time.perf_counter()
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
135
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
136 reason = None
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
137 if self.force:
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
138 reason = "ordered to"
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
139 elif not self.app.config.get('__cache_valid'):
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
140 # The configuration file was changed, or we're running a new
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
141 # version of the app.
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
142 reason = "not valid anymore"
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
143 elif previous_records.invalidated:
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
144 # We have no valid previous bake records.
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
145 reason = "need bake records regeneration"
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
146 else:
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
147 # Check if any template has changed since the last bake. Since
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
148 # there could be some advanced conditional logic going on, we'd
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
149 # better just force a bake from scratch if that's the case.
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
150 max_time = 0
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
151 for d in self.app.templates_dirs:
42
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
152 for dpath, _, filenames in os.walk(d):
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
153 for fn in filenames:
42
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
154 full_fn = os.path.join(dpath, fn)
9e058e221108 Fix a crash when checking for timestamps on template files.
Ludovic Chabant <ludovic@chabant.com>
parents: 39
diff changeset
155 max_time = max(max_time, os.path.getmtime(full_fn))
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
156 if max_time >= previous_records.bake_time:
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
157 reason = "templates modified"
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
158
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
159 if reason is not None:
85
3471ffa059b2 Add a `BakeScheduler` to handle build dependencies. Add unit-tests.
Ludovic Chabant <ludovic@chabant.com>
parents: 50
diff changeset
160 # We have to bake everything from scratch.
707
5f552aedd918 bake: Don't clean the `baker` cache on a force bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 691
diff changeset
161 self.app.cache.clearCaches(except_names=['app', 'baker'])
45
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 42
diff changeset
162 self.force = True
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
163 current_records.incremental_count = 0
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
164 previous_records = MultiRecord()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
165 logger.debug(format_timed(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
166 start_time, "cleaned cache (reason: %s)" % reason,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
167 colored=False))
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
168 return False
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
169 else:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
170 current_records.incremental_count += 1
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 151
diff changeset
171 logger.debug(format_timed(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
172 start_time, "cache is assumed valid", colored=False))
453
8351a77e13f5 bake: Don't pass the previous record entries to the workers.
Ludovic Chabant <ludovic@chabant.com>
parents: 451
diff changeset
173 return True
39
2f717f961996 Better error reporting and cache validation.
Ludovic Chabant <ludovic@chabant.com>
parents: 36
diff changeset
174
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
175 def _createPipelineManager(self, record_histories):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
176 # Gather all sources by realm -- we're going to bake each realm
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
177 # separately so we can handle "overriding" (i.e. one realm overrides
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
178 # another realm's pages, like the user realm overriding the theme
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
179 # realm).
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
180 #
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
181 # Also, create and initialize each pipeline for each source.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
182 has_any_pp = False
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
183 ppmngr = PipelineManager(
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
184 self.app, self.out_dir,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
185 record_histories=record_histories)
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
186 ok_pp = self.allowed_pipelines
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
187 nok_pp = self.forbidden_pipelines
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
188 ok_src = self.allowed_sources
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
189 for source in self.app.sources:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
190 if ok_src is not None and source.name not in ok_src:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
191 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
192
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
193 pname = get_pipeline_name_for_source(source)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
194 if ok_pp is not None and pname not in ok_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
195 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
196 if nok_pp is not None and pname in nok_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
197 continue
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
198
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
199 ppinfo = ppmngr.createPipeline(source)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
200 logger.debug(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
201 "Created pipeline '%s' for source: %s" %
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
202 (ppinfo.pipeline.PIPELINE_NAME, source.name))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
203 has_any_pp = True
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
204 if not has_any_pp:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
205 raise Exception("The website has no content sources, or the bake "
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
206 "command was invoked with all pipelines filtered "
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
207 "out. There's nothing to do.")
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
208 return ppmngr
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
209
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
210 def _populateTemplateCaches(self):
1050
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
211 engine_name = self.app.config.get('site/default_template_engine')
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
212 for engine in self.app.plugin_loader.getTemplateEngines():
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
213 if engine_name in engine.ENGINE_NAMES:
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
214 engine.populateCache()
e0796761cdcc bake: Only cache templates for the default template engine.
Ludovic Chabant <ludovic@chabant.com>
parents: 1044
diff changeset
215 break
1011
c4cf3cfe2726 bake: Better performance stats, and add callback to preload templates.
Ludovic Chabant <ludovic@chabant.com>
parents: 1007
diff changeset
216
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
217 def _bakeRealms(self, pool, ppmngr, record_histories):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
218 # Bake the realms -- user first, theme second, so that a user item
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
219 # can override a theme item.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
220 # Do this for as many times as we have pipeline passes left to do.
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
221 realm_list = [REALM_USER, REALM_THEME]
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
222 pp_by_pass_and_realm = _get_pipeline_infos_by_pass_and_realm(
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
223 ppmngr.getPipelineInfos())
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
224
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
225 for pp_pass_num in sorted(pp_by_pass_and_realm.keys()):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
226 logger.debug("Pipelines pass %d" % pp_pass_num)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
227 pp_by_realm = pp_by_pass_and_realm[pp_pass_num]
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
228 for realm in realm_list:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
229 pplist = pp_by_realm.get(realm)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
230 if pplist is not None:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
231 self._bakeRealm(pool, ppmngr, record_histories,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
232 pp_pass_num, realm, pplist)
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
233
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
234 def _bakeRealm(self, pool, ppmngr, record_histories,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
235 pp_pass_num, realm, pplist):
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
236 start_time = time.perf_counter()
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
237
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
238 job_count = 0
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
239 job_descs = {}
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
240 realm_name = REALM_NAMES[realm].lower()
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
241 pool.userdata.cur_pass = pp_pass_num
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
242
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
243 for ppinfo in pplist:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
244 src = ppinfo.source
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
245 pp = ppinfo.pipeline
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
246 jcctx = PipelineJobCreateContext(pp_pass_num, pp.record_name,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
247 record_histories)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
248
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
249 jobs, job_desc = pp.createJobs(jcctx)
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
250 if jobs is not None:
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
251 new_job_count = len(jobs)
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
252 job_count += new_job_count
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
253 pool.queueJobs(jobs)
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
254 if job_desc:
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
255 job_descs.setdefault(job_desc, []).append(src.name)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
256 else:
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
257 new_job_count = 0
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
258
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
259 logger.debug(
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
260 "Queued %d jobs for source '%s' using pipeline '%s' "
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
261 "(%s)." %
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 972
diff changeset
262 (new_job_count, src.name, pp.PIPELINE_NAME, realm_name))
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
263
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
264 if job_count == 0:
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
265 logger.debug("No jobs queued! Bailing out of this bake pass.")
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
266 return
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
267
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
268 pool.wait()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
269
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 868
diff changeset
270 logger.info(format_timed(
1007
09dc0240f08a bake: Simplify output.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
271 start_time, "%d jobs completed (%s)." %
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
272 (job_count, ', '.join(
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
273 ['%s %s' % (d, ', '.join(sn))
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
274 for d, sn in job_descs.items()]))))
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
275
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
276 def _logErrors(self, item_spec, errors):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
277 logger.error("Errors found in %s:" % item_spec)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
278 for e in errors:
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
279 logger.error(" " + e)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
280
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
281 def _logWorkerException(self, item_spec, exc_data):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
282 logger.error("Errors found in %s:" % item_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
283 logger.error(exc_data['value'])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
284 if self.app.debug:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
285 logger.error(exc_data['traceback'])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
286
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
287 def _createWorkerPool(self, previous_records_path, pool_userdata):
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
288 from piecrust.workerpool import WorkerPool
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
289 from piecrust.baking.worker import BakeWorkerContext, BakeWorker
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
290
456
5e902e228053 bake: Correctly use the `num_worers` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
291 worker_count = self.app.config.get('baker/workers')
462
04abc97dd3b6 bake: Add CLI argument to specify job batch size.
Ludovic Chabant <ludovic@chabant.com>
parents: 456
diff changeset
292 batch_size = self.app.config.get('baker/batch_size')
456
5e902e228053 bake: Correctly use the `num_worers` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 453
diff changeset
293
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
294 ctx = BakeWorkerContext(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
295 self.appfactory,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
296 self.out_dir,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
297 force=self.force,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
298 previous_records_path=previous_records_path,
868
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
299 allowed_pipelines=self.allowed_pipelines,
8d25f76fce98 bake: Add ability to specify pipelines to exclude during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
300 forbidden_pipelines=self.forbidden_pipelines)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
301 pool = WorkerPool(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
302 worker_count=worker_count,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
303 batch_size=batch_size,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
304 worker_class=BakeWorker,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
305 initargs=(ctx,),
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
306 callback=self._handleWorkerResult,
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
307 error_callback=self._handleWorkerError,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
308 userdata=pool_userdata)
447
aefe70229fdd bake: Commonize worker pool code between html and asset baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
309 return pool
217
1f4c3dae1fe8 bake: Better error handling for site baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
310
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
311 def _handleWorkerResult(self, job, res, userdata):
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
312 cur_pass = userdata.cur_pass
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
313 source_name, item_spec = job['job_spec']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
314
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
315 # Make the pipeline do custom handling to update the record entry.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
316 ppinfo = userdata.ppmngr.getPipelineInfo(source_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
317 pipeline = ppinfo.pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
318 record = ppinfo.current_record
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
319 ppmrctx = PipelineJobResultHandleContext(record, job, cur_pass)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
320 pipeline.handleJobResult(res, ppmrctx)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
321
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
322 # Set the overall success flags if there was an error.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
323 record_entry = ppmrctx.record_entry
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
324 if not record_entry.success:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
325 record.success = False
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
326 userdata.records.success = False
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
327 self._logErrors(job['item_spec'], record_entry.errors)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
328
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
329 def _handleWorkerError(self, job, exc_data, userdata):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
330 # Set the overall success flag.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
331 source_name, item_spec = job['job_spec']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
332 ppinfo = userdata.ppmngr.getPipelineInfo(source_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
333 pipeline = ppinfo.pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
334 record = ppinfo.current_record
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
335 record.success = False
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
336 userdata.records.success = False
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
337
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
338 # Add those errors to the record, if possible.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
339 record_entry_spec = job.get('record_entry_spec', item_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
340 e = record.getEntry(record_entry_spec)
1044
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
341 if not e:
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
342 e = pipeline.createRecordEntry(item_spec)
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
343 record.addEntry(e)
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
344 e.errors.append(exc_data['value'])
7b64eb5bbc81 bake: Don't ignore errors reported on things that are not yet in the records.
Ludovic Chabant <ludovic@chabant.com>
parents: 1040
diff changeset
345 self._logWorkerException(item_spec, exc_data)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
346
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
347 # Log debug stuff.
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
348 if self.app.debug:
1040
506a5cb1e942 bake: Fix exception verbose reporting for bake worker errors.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
349 logger.error(exc_data['traceback'])
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 753
diff changeset
350
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
351
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
352 class _PoolUserData:
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
353 def __init__(self, baker, ppmngr):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
354 self.baker = baker
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
355 self.ppmngr = ppmngr
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
356 self.records = ppmngr.record_histories.current
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 1014
diff changeset
357 self.cur_pass = 0
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
358
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
359
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
360 def _get_pipeline_infos_by_pass_and_realm(pp_infos):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
361 pp_by_pass_and_realm = {}
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
362 for pp_info in pp_infos:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
363 pp_pass_num = pp_info.pipeline.PASS_NUM
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
364 if isinstance(pp_pass_num, list):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
365 for ppn in pp_pass_num:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
366 _add_pipeline_info_to_pass_and_realm_dict(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
367 ppn, pp_info, pp_by_pass_and_realm)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
368 else:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
369 _add_pipeline_info_to_pass_and_realm_dict(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
370 pp_pass_num, pp_info, pp_by_pass_and_realm)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
371 return pp_by_pass_and_realm
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
372
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
373
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
374 def _add_pipeline_info_to_pass_and_realm_dict(pp_pass_num, pp_info,
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
375 pp_by_pass_and_realm):
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
376 pp_by_realm = pp_by_pass_and_realm.setdefault(pp_pass_num, {})
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
377 pplist = pp_by_realm.setdefault(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
378 pp_info.pipeline.source.config['realm'], [])
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
379 pplist.append(pp_info)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
380
972
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
381
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
382 def _merge_execution_stats(base_stats, *other_stats):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
383 total_stats = ExecutionStats()
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
384 total_stats.mergeStats(base_stats)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
385 for ps in other_stats:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
386 if ps is not None:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
387 total_stats.mergeStats(ps)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
388 return total_stats
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
389
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
390
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
391 def _save_bake_records(records, records_path, *, rotate_previous):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
392 if rotate_previous:
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
393 records_dir, records_fn = os.path.split(records_path)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
394 records_id, _ = os.path.splitext(records_fn)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
395 for i in range(8, -1, -1):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
396 suffix = '' if i == 0 else '.%d' % i
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
397 records_path_i = os.path.join(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
398 records_dir,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
399 '%s%s.records' % (records_id, suffix))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
400 if os.path.exists(records_path_i):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
401 records_path_next = os.path.join(
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
402 records_dir,
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
403 '%s.%s.records' % (records_id, i + 1))
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
404 if os.path.exists(records_path_next):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
405 os.remove(records_path_next)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
406 os.rename(records_path_i, records_path_next)
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
407
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
408 with format_timed_scope(logger, "saved bake records.",
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
409 level=logging.DEBUG, colored=False):
bbf5a96b56db internal: Clean up baker code.
Ludovic Chabant <ludovic@chabant.com>
parents: 918
diff changeset
410 records.save(records_path)