Mercurial > piecrust2

annotate piecrust/baking/single.py @ 661:2f780b191541

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
internal: Fix a bug with registering taxonomy terms that are not strings. Some objects, like the blog data provider's taxnonomy entries, can render as strings, but are objects themselves. When registering them as "used terms", we need to use their string representation.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 01 Mar 2016 22:26:09 -0800
parents 59268b4d8c71
children 33ab9badfd7a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import shutil
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import codecs
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import urllib.parse
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
6 from piecrust import ASSET_DIR_SUFFIX
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
7 from piecrust.baking.records import SubPageBakeInfo
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
8 from piecrust.rendering import (
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
9 QualifiedPage, PageRenderingContext, render_page,
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
10 PASS_FORMATTING)
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
11 from piecrust.uriutil import split_uri
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 logger = logging.getLogger(__name__)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 class BakingError(Exception):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 pass
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 class PageBaker(object):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
22 def __init__(self, app, out_dir, force=False, copy_assets=True):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 self.app = app
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 self.out_dir = out_dir
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 self.force = force
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 self.copy_assets = copy_assets
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 self.site_root = app.config.get('site/root')
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 self.pretty_urls = app.config.get('site/pretty_urls')
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 def getOutputPath(self, uri):
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
31 uri_root, uri_path = split_uri(self.app, uri)
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
32
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 bake_path = [self.out_dir]
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
34 decoded_uri = urllib.parse.unquote(uri_path)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 if self.pretty_urls:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 bake_path.append(decoded_uri)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 bake_path.append('index.html')
262
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
38 elif decoded_uri == '':
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
39 bake_path.append('index.html')
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 else:
262
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
41 bake_path.append(decoded_uri)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 return os.path.normpath(os.path.join(*bake_path))
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
45 def bake(self, qualified_page, prev_entry, dirty_source_names,
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
46 tax_info=None):
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
47 # Start baking the sub-pages.
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 cur_sub = 1
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 has_more_subs = True
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
50 sub_entries = []
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 while has_more_subs:
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
53 # Get the URL and path for this sub-page.
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
54 sub_uri = qualified_page.getUri(cur_sub)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
55 logger.debug("Baking '%s' [%d]..." % (sub_uri, cur_sub))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 out_path = self.getOutputPath(sub_uri)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
58 # Create the sub-entry for the bake record.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
59 sub_entry = SubPageBakeInfo(sub_uri, out_path)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
60 sub_entries.append(sub_entry)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
61
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
62 # Find a corresponding sub-entry in the previous bake record.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
63 prev_sub_entry = None
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
64 if prev_entry:
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
65 try:
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
66 prev_sub_entry = prev_entry.getSub(cur_sub)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
67 except IndexError:
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
68 pass
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
69
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
70 # Figure out if we need to invalidate or force anything.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
71 force_this_sub, invalidate_formatting = _compute_force_flags(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
72 prev_sub_entry, sub_entry, dirty_source_names)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
73 force_this_sub = force_this_sub or self.force
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
74
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 # Check for up-to-date outputs.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 do_bake = True
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
77 if not force_this_sub:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 try:
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
79 in_path_time = qualified_page.path_mtime
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 out_path_time = os.path.getmtime(out_path)
286
a2d283d1033d tests: Fixes for running on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 280
diff changeset
81 if out_path_time >= in_path_time:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 do_bake = False
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 except OSError:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 # File doesn't exist, we'll need to bake.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 pass
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 # If this page didn't bake because it's already up-to-date.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 # Keep trying for as many subs as we know this page has.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 if not do_bake:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
90 sub_entry.render_info = prev_sub_entry.copyRenderInfo()
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
91 sub_entry.flags = SubPageBakeInfo.FLAG_NONE
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
92
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
93 if prev_entry.num_subs >= cur_sub + 1:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 cur_sub += 1
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 has_more_subs = True
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 logger.debug(" %s is up to date, skipping to next "
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 329
diff changeset
97 "sub-page." % out_path)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 continue
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 logger.debug(" %s is up to date, skipping bake." % out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 break
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 # All good, proceed.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104 try:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105 if invalidate_formatting:
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 329
diff changeset
106 cache_key = sub_uri
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 self.app.env.rendered_segments_repository.invalidate(
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 cache_key)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
109 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
110 SubPageBakeInfo.FLAG_FORMATTING_INVALIDATED
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112 logger.debug(" p%d -> %s" % (cur_sub, out_path))
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
113 rp = self._bakeSingle(qualified_page, cur_sub, out_path,
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
114 tax_info)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 except Exception as ex:
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
116 page_rel_path = os.path.relpath(qualified_page.path,
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
117 self.app.root_dir)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 raise BakingError("%s: error baking '%s'." %
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
119 (page_rel_path, sub_uri)) from ex
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
121 # Record what we did.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
122 sub_entry.flags |= SubPageBakeInfo.FLAG_BAKED
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
123 sub_entry.render_info = rp.copyRenderInfo()
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
124
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125 # Copy page assets.
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
126 if (cur_sub == 1 and self.copy_assets and
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
127 sub_entry.anyPass(lambda p: p.used_assets)):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 if self.pretty_urls:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129 out_assets_dir = os.path.dirname(out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130 else:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 out_assets_dir, out_name = os.path.split(out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
132 if sub_uri != self.site_root:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 out_name_noext, _ = os.path.splitext(out_name)
473
eb3ace870708 bake: Fix a bug with copying assets when `pretty_urls` are disabled.
Ludovic Chabant <ludovic@chabant.com>
parents: 472
diff changeset
134 out_assets_dir = os.path.join(out_assets_dir,
eb3ace870708 bake: Fix a bug with copying assets when `pretty_urls` are disabled.
Ludovic Chabant <ludovic@chabant.com>
parents: 472
diff changeset
135 out_name_noext)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
137 logger.debug("Copying page assets to: %s" % out_assets_dir)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
138 _ensure_dir_exists(out_assets_dir)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
139
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
140 page_pathname, _ = os.path.splitext(qualified_page.path)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
141 in_assets_dir = page_pathname + ASSET_DIR_SUFFIX
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
142 for fn in os.listdir(in_assets_dir):
472
dc78ade3f320 bug: Fix copying of page assets during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
143 full_fn = os.path.join(in_assets_dir, fn)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
144 if os.path.isfile(full_fn):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
145 dest_ap = os.path.join(out_assets_dir, fn)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
146 logger.debug(" %s -> %s" % (full_fn, dest_ap))
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
147 shutil.copy(full_fn, dest_ap)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
148
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
149 # Figure out if we have more work.
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
150 has_more_subs = False
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
151 if sub_entry.anyPass(lambda p: p.pagination_has_more):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
152 cur_sub += 1
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
153 has_more_subs = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
154
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
155 return sub_entries
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
156
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
157 def _bakeSingle(self, qualified_page, num, out_path, tax_info=None):
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
158 ctx = PageRenderingContext(qualified_page, page_num=num)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
159 if tax_info:
515
16e705c58cae internal: Improve handling of taxonomy term slugification.
Ludovic Chabant <ludovic@chabant.com>
parents: 473
diff changeset
160 ctx.setTaxonomyFilter(tax_info.term)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161
586
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
162 with self.app.env.timerScope("PageRender"):
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
163 rp = render_page(ctx)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164
586
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
165 with self.app.env.timerScope("PageSerialize"):
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
166 out_dir = os.path.dirname(out_path)
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
167 _ensure_dir_exists(out_dir)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
168
586
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
169 with open(out_path, 'w', encoding='utf8') as fp:
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
170 fp.write(rp.content)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
172 return rp
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
174
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
175 def _compute_force_flags(prev_sub_entry, sub_entry, dirty_source_names):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
176 # Figure out what to do with this page.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
177 force_this_sub = False
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
178 invalidate_formatting = False
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
179 sub_uri = sub_entry.out_uri
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
180 if (prev_sub_entry and
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
181 (prev_sub_entry.was_baked_successfully or
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
182 prev_sub_entry.was_clean)):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
183 # If the current page is known to use pages from other sources,
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
184 # see if any of those got baked, or are going to be baked for
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
185 # some reason. If so, we need to bake this one too.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
186 # (this happens for instance with the main page of a blog).
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
187 dirty_for_this, invalidated_render_passes = (
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
188 _get_dirty_source_names_and_render_passes(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
189 prev_sub_entry, dirty_source_names))
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
190 if len(invalidated_render_passes) > 0:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
191 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
192 "'%s' is known to use sources %s, which have "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
193 "items that got (re)baked. Will force bake this "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
194 "page. " % (sub_uri, dirty_for_this))
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
195 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
196 SubPageBakeInfo.FLAG_FORCED_BY_SOURCE
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
197 force_this_sub = True
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
198
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
199 if PASS_FORMATTING in invalidated_render_passes:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
200 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
201 "Will invalidate cached formatting for '%s' "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
202 "since sources were using during that pass."
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
203 % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
204 invalidate_formatting = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
205 elif (prev_sub_entry and
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
206 prev_sub_entry.errors):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
207 # Previous bake failed. We'll have to bake it again.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
208 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
209 "Previous record entry indicates baking failed for "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
210 "'%s'. Will bake it again." % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
211 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
212 SubPageBakeInfo.FLAG_FORCED_BY_PREVIOUS_ERRORS
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
213 force_this_sub = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
214 elif not prev_sub_entry:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
215 # No previous record. We'll have to bake it.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
216 logger.debug("No previous record entry found for '%s'. Will "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
217 "force bake it." % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
218 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
219 SubPageBakeInfo.FLAG_FORCED_BY_NO_PREVIOUS
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
220 force_this_sub = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
221
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
222 return force_this_sub, invalidate_formatting
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
223
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
224
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
225 def _get_dirty_source_names_and_render_passes(sub_entry, dirty_source_names):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
226 dirty_for_this = set()
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
227 invalidated_render_passes = set()
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
228 assert sub_entry.render_info is not None
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
229 for p, pinfo in sub_entry.render_info.items():
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
230 for src_name in pinfo.used_source_names:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
231 is_dirty = (src_name in dirty_source_names)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
232 if is_dirty:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
233 invalidated_render_passes.add(p)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
234 dirty_for_this.add(src_name)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
235 break
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
236 return dirty_for_this, invalidated_render_passes
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
237
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
238
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
239 def _ensure_dir_exists(path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
240 try:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
241 os.makedirs(path, mode=0o755, exist_ok=True)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
242 except OSError:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
243 # In a multiprocess environment, several process may very
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
244 # occasionally try to create the same directory at the same time.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
245 # Let's ignore any error and if something's really wrong (like file
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
246 # acces permissions or whatever), then it will more legitimately fail
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
247 # just after this when we try to write files.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
248 pass
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
249