annotate piecrust/baking/single.py @ 415:0e9a94b7fdfa

bake: Improve bake record information. * Store things in the bake record that require less interaction between the master process and the workers. For instance, don't store the paginator object in the render pass info -- instead, just store whether pagination was used, and whether it had more items. * Simplify information passing between workers and bake passes by saving the rendering info to the JSON cache. This means the "render first sub" job doesn't have to return anything except errors now. * Add more performance counter info.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:23:16 -0700
parents e7b865f8f335
children 21e26ed867b6
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import shutil
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import codecs
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import urllib.parse
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
6 from piecrust import ASSET_DIR_SUFFIX
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
7 from piecrust.baking.records import SubPageBakeInfo
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
8 from piecrust.rendering import (
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
9 QualifiedPage, PageRenderingContext, render_page,
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
10 PASS_FORMATTING)
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
11 from piecrust.uriutil import split_uri
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 logger = logging.getLogger(__name__)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
264
74bea91c9630 bake: Don't store internal config values in the bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 262
diff changeset
17 def copy_public_page_config(config):
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
18 res = config.getDeepcopy()
264
74bea91c9630 bake: Don't store internal config values in the bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 262
diff changeset
19 for k in list(res.keys()):
74bea91c9630 bake: Don't store internal config values in the bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 262
diff changeset
20 if k.startswith('__'):
74bea91c9630 bake: Don't store internal config values in the bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 262
diff changeset
21 del res[k]
74bea91c9630 bake: Don't store internal config values in the bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 262
diff changeset
22 return res
74bea91c9630 bake: Don't store internal config values in the bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 262
diff changeset
23
74bea91c9630 bake: Don't store internal config values in the bake record.
Ludovic Chabant <ludovic@chabant.com>
parents: 262
diff changeset
24
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 class BakingError(Exception):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 pass
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 class PageBaker(object):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
30 def __init__(self, app, out_dir, force=False, copy_assets=True):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 self.app = app
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 self.out_dir = out_dir
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 self.force = force
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 self.copy_assets = copy_assets
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 self.site_root = app.config.get('site/root')
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self.pretty_urls = app.config.get('site/pretty_urls')
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 def getOutputPath(self, uri):
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
39 uri_root, uri_path = split_uri(self.app, uri)
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
40
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 bake_path = [self.out_dir]
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
42 decoded_uri = urllib.parse.unquote(uri_path)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 if self.pretty_urls:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 bake_path.append(decoded_uri)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 bake_path.append('index.html')
262
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
46 elif decoded_uri == '':
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
47 bake_path.append('index.html')
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 else:
262
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
49 bake_path.append(decoded_uri)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 return os.path.normpath(os.path.join(*bake_path))
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
53 def bake(self, factory, route, route_metadata, prev_entry,
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
54 dirty_source_names, tax_info=None):
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
55 # Get the page.
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
56 page = factory.buildPage()
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
57
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
58 # Start baking the sub-pages.
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 cur_sub = 1
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 has_more_subs = True
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
61 sub_entries = []
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 while has_more_subs:
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
64 # Get the URL and path for this sub-page.
256
da5e6e00fb41 bake/serve: Make previewed and baked URLs consistent.
Ludovic Chabant <ludovic@chabant.com>
parents: 235
diff changeset
65 sub_uri = route.getUri(route_metadata, sub_num=cur_sub,
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
66 provider=page)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
67 logger.debug("Baking '%s' [%d]..." % (sub_uri, cur_sub))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 out_path = self.getOutputPath(sub_uri)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
70 # Create the sub-entry for the bake record.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
71 sub_entry = SubPageBakeInfo(sub_uri, out_path)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
72 sub_entries.append(sub_entry)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
73
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
74 # Find a corresponding sub-entry in the previous bake record.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
75 prev_sub_entry = None
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
76 if prev_entry:
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
77 try:
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
78 prev_sub_entry = prev_entry.getSub(cur_sub)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
79 except IndexError:
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
80 pass
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
81
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
82 # Figure out if we need to invalidate or force anything.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
83 force_this_sub, invalidate_formatting = _compute_force_flags(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
84 prev_sub_entry, sub_entry, dirty_source_names)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
85 force_this_sub = force_this_sub or self.force
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
86
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 # Check for up-to-date outputs.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 do_bake = True
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
89 if not force_this_sub:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 try:
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 329
diff changeset
91 in_path_time = page.path_mtime
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92 out_path_time = os.path.getmtime(out_path)
286
a2d283d1033d tests: Fixes for running on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 280
diff changeset
93 if out_path_time >= in_path_time:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 do_bake = False
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 except OSError:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 # File doesn't exist, we'll need to bake.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 pass
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 # If this page didn't bake because it's already up-to-date.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 # Keep trying for as many subs as we know this page has.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 if not do_bake:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
102 sub_entry.render_info = prev_sub_entry.copyRenderInfo()
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
103 sub_entry.flags = SubPageBakeInfo.FLAG_NONE
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
104
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
105 if prev_entry.num_subs >= cur_sub + 1:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106 cur_sub += 1
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 has_more_subs = True
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 logger.debug(" %s is up to date, skipping to next "
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 329
diff changeset
109 "sub-page." % out_path)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 continue
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112 logger.debug(" %s is up to date, skipping bake." % out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 break
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 # All good, proceed.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116 try:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 if invalidate_formatting:
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 329
diff changeset
118 cache_key = sub_uri
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119 self.app.env.rendered_segments_repository.invalidate(
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120 cache_key)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
121 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
122 SubPageBakeInfo.FLAG_FORMATTING_INVALIDATED
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 logger.debug(" p%d -> %s" % (cur_sub, out_path))
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
125 qp = QualifiedPage(page, route, route_metadata)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
126 rp = self._bakeSingle(qp, cur_sub, out_path, tax_info)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 except Exception as ex:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 page_rel_path = os.path.relpath(page.path, self.app.root_dir)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129 raise BakingError("%s: error baking '%s'." %
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
130 (page_rel_path, sub_uri)) from ex
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
132 # Record what we did.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
133 sub_entry.flags |= SubPageBakeInfo.FLAG_BAKED
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
134 sub_entry.render_info = rp.copyRenderInfo()
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
135
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 # Copy page assets.
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
137 if (cur_sub == 1 and self.copy_assets and
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
138 sub_entry.anyPass(lambda p: p.used_assets)):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
139 if self.pretty_urls:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140 out_assets_dir = os.path.dirname(out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141 else:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142 out_assets_dir, out_name = os.path.split(out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143 if sub_uri != self.site_root:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
144 out_name_noext, _ = os.path.splitext(out_name)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
145 out_assets_dir += out_name_noext
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
146
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
147 logger.debug("Copying page assets to: %s" % out_assets_dir)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
148 _ensure_dir_exists(out_assets_dir)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
149
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
150 page_dirname = os.path.dirname(page.path)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
151 page_pathname, _ = os.path.splitext(page.path)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
152 in_assets_dir = page_pathname + ASSET_DIR_SUFFIX
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
153 for fn in os.listdir(in_assets_dir):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
154 full_fn = os.path.join(page_dirname, fn)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
155 if os.path.isfile(full_fn):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
156 dest_ap = os.path.join(out_assets_dir, fn)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
157 logger.debug(" %s -> %s" % (full_fn, dest_ap))
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
158 shutil.copy(full_fn, dest_ap)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
160 # Figure out if we have more work.
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161 has_more_subs = False
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
162 if sub_entry.anyPass(lambda p: p.pagination_has_more):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
163 cur_sub += 1
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
164 has_more_subs = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
165
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
166 return sub_entries
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
167
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
168 def _bakeSingle(self, qualified_page, num, out_path, tax_info=None):
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
169 ctx = PageRenderingContext(qualified_page, page_num=num)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
170 if tax_info:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
171 tax = self.app.getTaxonomy(tax_info.taxonomy_name)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
172 ctx.setTaxonomyFilter(tax, tax_info.term)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174 rp = render_page(ctx)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
175
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176 out_dir = os.path.dirname(out_path)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
177 _ensure_dir_exists(out_dir)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179 with codecs.open(out_path, 'w', 'utf8') as fp:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
180 fp.write(rp.content)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
181
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
182 return rp
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
183
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
184
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
185 def _compute_force_flags(prev_sub_entry, sub_entry, dirty_source_names):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
186 # Figure out what to do with this page.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
187 force_this_sub = False
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
188 invalidate_formatting = False
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
189 sub_uri = sub_entry.out_uri
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
190 if (prev_sub_entry and
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
191 (prev_sub_entry.was_baked_successfully or
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
192 prev_sub_entry.was_clean)):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
193 # If the current page is known to use pages from other sources,
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
194 # see if any of those got baked, or are going to be baked for
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
195 # some reason. If so, we need to bake this one too.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
196 # (this happens for instance with the main page of a blog).
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
197 dirty_for_this, invalidated_render_passes = (
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
198 _get_dirty_source_names_and_render_passes(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
199 prev_sub_entry, dirty_source_names))
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
200 if len(invalidated_render_passes) > 0:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
201 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
202 "'%s' is known to use sources %s, which have "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
203 "items that got (re)baked. Will force bake this "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
204 "page. " % (sub_uri, dirty_for_this))
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
205 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
206 SubPageBakeInfo.FLAG_FORCED_BY_SOURCE
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
207 force_this_sub = True
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
208
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
209 if PASS_FORMATTING in invalidated_render_passes:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
210 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
211 "Will invalidate cached formatting for '%s' "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
212 "since sources were using during that pass."
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
213 % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
214 invalidate_formatting = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
215 elif (prev_sub_entry and
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
216 prev_sub_entry.errors):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
217 # Previous bake failed. We'll have to bake it again.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
218 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
219 "Previous record entry indicates baking failed for "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
220 "'%s'. Will bake it again." % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
221 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
222 SubPageBakeInfo.FLAG_FORCED_BY_PREVIOUS_ERRORS
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
223 force_this_sub = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
224 elif not prev_sub_entry:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
225 # No previous record. We'll have to bake it.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
226 logger.debug("No previous record entry found for '%s'. Will "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
227 "force bake it." % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
228 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
229 SubPageBakeInfo.FLAG_FORCED_BY_NO_PREVIOUS
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
230 force_this_sub = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
231
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
232 return force_this_sub, invalidate_formatting
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
233
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
234
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
235 def _get_dirty_source_names_and_render_passes(sub_entry, dirty_source_names):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
236 dirty_for_this = set()
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
237 invalidated_render_passes = set()
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
238 assert sub_entry.render_info is not None
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
239 for p, pinfo in sub_entry.render_info.items():
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
240 for src_name in pinfo.used_source_names:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
241 is_dirty = (src_name in dirty_source_names)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
242 if is_dirty:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
243 invalidated_render_passes.add(p)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
244 dirty_for_this.add(src_name)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
245 break
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
246 return dirty_for_this, invalidated_render_passes
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
247
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
248
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
249 def _ensure_dir_exists(path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
250 try:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
251 os.makedirs(path, mode=0o755, exist_ok=True)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
252 except OSError:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
253 # In a multiprocess environment, several process may very
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
254 # occasionally try to create the same directory at the same time.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
255 # Let's ignore any error and if something's really wrong (like file
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
256 # acces permissions or whatever), then it will more legitimately fail
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
257 # just after this when we try to write files.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
258 pass
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
259