Mercurial > piecrust2

annotate piecrust/baking/single.py @ 702:c62d83e17abf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Some more optimizations. Write the output files in a background thread to get away from the GIL.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 16 Apr 2016 22:50:07 -0700
parents 3e4049022869
children ab5c6a8ae90a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
2 import queue
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import shutil
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
5 import threading
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 import urllib.parse
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
7 from piecrust import ASSET_DIR_SUFFIX
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
8 from piecrust.baking.records import SubPageBakeInfo
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
9 from piecrust.rendering import (
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
10 QualifiedPage, PageRenderingContext, render_page,
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
11 PASS_FORMATTING)
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
12 from piecrust.uriutil import split_uri
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 logger = logging.getLogger(__name__)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 class BakingError(Exception):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 pass
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
22 def _text_writer(q):
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
23 while True:
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
24 item = q.get()
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
25 if item is not None:
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
26 out_path, txt = item
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
27 out_dir = os.path.dirname(out_path)
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
28 _ensure_dir_exists(out_dir)
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
29
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
30 with open(out_path, 'w', encoding='utf8') as fp:
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
31 fp.write(txt)
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
32
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
33 q.task_done()
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
34 else:
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
35 # Sentinel object, terminate the thread.
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
36 q.task_done()
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
37 break
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
38
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
39
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 class PageBaker(object):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
41 def __init__(self, app, out_dir, force=False, copy_assets=True):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 self.app = app
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 self.out_dir = out_dir
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 self.force = force
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 self.copy_assets = copy_assets
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 self.site_root = app.config.get('site/root')
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 self.pretty_urls = app.config.get('site/pretty_urls')
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
48 self._writer_queue = queue.Queue()
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
49 self._writer = threading.Thread(
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
50 name='PageSerializer',
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
51 target=_text_writer,
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
52 args=(self._writer_queue,))
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
53 self._writer.start()
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
54
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
55 def shutdown(self):
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
56 self._writer_queue.put_nowait(None)
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
57 self._writer.join()
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 def getOutputPath(self, uri):
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
60 uri_root, uri_path = split_uri(self.app, uri)
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
61
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 bake_path = [self.out_dir]
329
422052d2e978 internal: Try handling URLs in a consistent way.
Ludovic Chabant <ludovic@chabant.com>
parents: 324
diff changeset
63 decoded_uri = urllib.parse.unquote(uri_path)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 if self.pretty_urls:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 bake_path.append(decoded_uri)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 bake_path.append('index.html')
262
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
67 elif decoded_uri == '':
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
68 bake_path.append('index.html')
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 else:
262
61145dcd56e0 routing: Better generate URLs according to the site configuration.
Ludovic Chabant <ludovic@chabant.com>
parents: 256
diff changeset
70 bake_path.append(decoded_uri)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 return os.path.normpath(os.path.join(*bake_path))
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
74 def bake(self, qualified_page, prev_entry, dirty_source_names,
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
75 tax_info=None):
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
76 # Start baking the sub-pages.
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 cur_sub = 1
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 has_more_subs = True
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
79 sub_entries = []
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 while has_more_subs:
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
82 # Get the URL and path for this sub-page.
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
83 sub_uri = qualified_page.getUri(cur_sub)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
84 logger.debug("Baking '%s' [%d]..." % (sub_uri, cur_sub))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 out_path = self.getOutputPath(sub_uri)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
87 # Create the sub-entry for the bake record.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
88 sub_entry = SubPageBakeInfo(sub_uri, out_path)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
89 sub_entries.append(sub_entry)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
90
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
91 # Find a corresponding sub-entry in the previous bake record.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
92 prev_sub_entry = None
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
93 if prev_entry:
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
94 try:
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
95 prev_sub_entry = prev_entry.getSub(cur_sub)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
96 except IndexError:
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
97 pass
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
98
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
99 # Figure out if we need to invalidate or force anything.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
100 force_this_sub, invalidate_formatting = _compute_force_flags(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
101 prev_sub_entry, sub_entry, dirty_source_names)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
102 force_this_sub = force_this_sub or self.force
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
103
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104 # Check for up-to-date outputs.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105 do_bake = True
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
106 if not force_this_sub:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 try:
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
108 in_path_time = qualified_page.path_mtime
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109 out_path_time = os.path.getmtime(out_path)
286
a2d283d1033d tests: Fixes for running on Windows.
Ludovic Chabant <ludovic@chabant.com>
parents: 280
diff changeset
110 if out_path_time >= in_path_time:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 do_bake = False
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112 except OSError:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 # File doesn't exist, we'll need to bake.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 pass
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116 # If this page didn't bake because it's already up-to-date.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 # Keep trying for as many subs as we know this page has.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 if not do_bake:
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
119 sub_entry.render_info = prev_sub_entry.copyRenderInfo()
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
120 sub_entry.flags = SubPageBakeInfo.FLAG_NONE
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
121
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
122 if prev_entry.num_subs >= cur_sub + 1:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 cur_sub += 1
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 has_more_subs = True
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125 logger.debug(" %s is up to date, skipping to next "
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 329
diff changeset
126 "sub-page." % out_path)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 continue
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129 logger.debug(" %s is up to date, skipping bake." % out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130 break
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
132 # All good, proceed.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 try:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134 if invalidate_formatting:
334
b034f6f15e22 bake: Several bug taxonomy-related fixes for incorrect incremental bakes.
Ludovic Chabant <ludovic@chabant.com>
parents: 329
diff changeset
135 cache_key = sub_uri
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 self.app.env.rendered_segments_repository.invalidate(
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
137 cache_key)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
138 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
139 SubPageBakeInfo.FLAG_FORMATTING_INVALIDATED
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141 logger.debug(" p%d -> %s" % (cur_sub, out_path))
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
142 rp = self._bakeSingle(qualified_page, cur_sub, out_path,
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
143 tax_info)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
144 except Exception as ex:
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
145 page_rel_path = os.path.relpath(qualified_page.path,
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
146 self.app.root_dir)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
147 raise BakingError("%s: error baking '%s'." %
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
148 (page_rel_path, sub_uri)) from ex
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
149
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
150 # Record what we did.
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
151 sub_entry.flags |= SubPageBakeInfo.FLAG_BAKED
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
152 sub_entry.render_info = rp.copyRenderInfo()
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
153
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
154 # Copy page assets.
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
155 if (cur_sub == 1 and self.copy_assets and
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
156 sub_entry.anyPass(lambda p: p.used_assets)):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
157 if self.pretty_urls:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
158 out_assets_dir = os.path.dirname(out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159 else:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160 out_assets_dir, out_name = os.path.split(out_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161 if sub_uri != self.site_root:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
162 out_name_noext, _ = os.path.splitext(out_name)
473
eb3ace870708 bake: Fix a bug with copying assets when `pretty_urls` are disabled.
Ludovic Chabant <ludovic@chabant.com>
parents: 472
diff changeset
163 out_assets_dir = os.path.join(out_assets_dir,
eb3ace870708 bake: Fix a bug with copying assets when `pretty_urls` are disabled.
Ludovic Chabant <ludovic@chabant.com>
parents: 472
diff changeset
164 out_name_noext)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
165
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166 logger.debug("Copying page assets to: %s" % out_assets_dir)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
167 _ensure_dir_exists(out_assets_dir)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
168
430
21e26ed867b6 internal: Create full route metadata in one place.
Ludovic Chabant <ludovic@chabant.com>
parents: 415
diff changeset
169 page_pathname, _ = os.path.splitext(qualified_page.path)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
170 in_assets_dir = page_pathname + ASSET_DIR_SUFFIX
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
171 for fn in os.listdir(in_assets_dir):
472
dc78ade3f320 bug: Fix copying of page assets during the bake.
Ludovic Chabant <ludovic@chabant.com>
parents: 430
diff changeset
172 full_fn = os.path.join(in_assets_dir, fn)
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
173 if os.path.isfile(full_fn):
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
174 dest_ap = os.path.join(out_assets_dir, fn)
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
175 logger.debug(" %s -> %s" % (full_fn, dest_ap))
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
176 shutil.copy(full_fn, dest_ap)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
177
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
178 # Figure out if we have more work.
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179 has_more_subs = False
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
180 if sub_entry.anyPass(lambda p: p.pagination_has_more):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
181 cur_sub += 1
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
182 has_more_subs = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
183
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
184 return sub_entries
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
185
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
186 def _bakeSingle(self, qualified_page, num, out_path, tax_info=None):
369
4b1019bb2533 serve: Giant refactor to change how we handle data when serving pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 350
diff changeset
187 ctx = PageRenderingContext(qualified_page, page_num=num)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
188 if tax_info:
515
16e705c58cae internal: Improve handling of taxonomy term slugification.
Ludovic Chabant <ludovic@chabant.com>
parents: 473
diff changeset
189 ctx.setTaxonomyFilter(tax_info.term)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
190
586
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
191 with self.app.env.timerScope("PageRender"):
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
192 rp = render_page(ctx)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
193
586
59268b4d8c71 bake: Add new performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 515
diff changeset
194 with self.app.env.timerScope("PageSerialize"):
702
c62d83e17abf bake: Some more optimizations.
Ludovic Chabant <ludovic@chabant.com>
parents: 699
diff changeset
195 self._writer_queue.put_nowait((out_path, rp.content))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
197 return rp
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
198
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
199
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
200 def _compute_force_flags(prev_sub_entry, sub_entry, dirty_source_names):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
201 # Figure out what to do with this page.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
202 force_this_sub = False
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
203 invalidate_formatting = False
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
204 sub_uri = sub_entry.out_uri
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
205 if (prev_sub_entry and
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
206 (prev_sub_entry.was_baked_successfully or
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
207 prev_sub_entry.was_clean)):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
208 # If the current page is known to use pages from other sources,
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
209 # see if any of those got baked, or are going to be baked for
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
210 # some reason. If so, we need to bake this one too.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
211 # (this happens for instance with the main page of a blog).
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
212 dirty_for_this, invalidated_render_passes = (
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
213 _get_dirty_source_names_and_render_passes(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
214 prev_sub_entry, dirty_source_names))
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
215 if len(invalidated_render_passes) > 0:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
216 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
217 "'%s' is known to use sources %s, which have "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
218 "items that got (re)baked. Will force bake this "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
219 "page. " % (sub_uri, dirty_for_this))
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
220 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
221 SubPageBakeInfo.FLAG_FORCED_BY_SOURCE
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
222 force_this_sub = True
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 334
diff changeset
223
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
224 if PASS_FORMATTING in invalidated_render_passes:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
225 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
226 "Will invalidate cached formatting for '%s' "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
227 "since sources were using during that pass."
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
228 % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
229 invalidate_formatting = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
230 elif (prev_sub_entry and
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
231 prev_sub_entry.errors):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
232 # Previous bake failed. We'll have to bake it again.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
233 logger.debug(
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
234 "Previous record entry indicates baking failed for "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
235 "'%s'. Will bake it again." % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
236 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
237 SubPageBakeInfo.FLAG_FORCED_BY_PREVIOUS_ERRORS
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
238 force_this_sub = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
239 elif not prev_sub_entry:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
240 # No previous record. We'll have to bake it.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
241 logger.debug("No previous record entry found for '%s'. Will "
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
242 "force bake it." % sub_uri)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
243 sub_entry.flags |= \
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
244 SubPageBakeInfo.FLAG_FORCED_BY_NO_PREVIOUS
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
245 force_this_sub = True
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
246
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
247 return force_this_sub, invalidate_formatting
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
248
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
249
415
0e9a94b7fdfa bake: Improve bake record information.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
250 def _get_dirty_source_names_and_render_passes(sub_entry, dirty_source_names):
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
251 dirty_for_this = set()
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
252 invalidated_render_passes = set()
698
33ab9badfd7a render: Change how we store render passes info.
Ludovic Chabant <ludovic@chabant.com>
parents: 586
diff changeset
253 for p, pinfo in enumerate(sub_entry.render_info):
33ab9badfd7a render: Change how we store render passes info.
Ludovic Chabant <ludovic@chabant.com>
parents: 586
diff changeset
254 if pinfo:
33ab9badfd7a render: Change how we store render passes info.
Ludovic Chabant <ludovic@chabant.com>
parents: 586
diff changeset
255 for src_name in pinfo.used_source_names:
33ab9badfd7a render: Change how we store render passes info.
Ludovic Chabant <ludovic@chabant.com>
parents: 586
diff changeset
256 is_dirty = (src_name in dirty_source_names)
33ab9badfd7a render: Change how we store render passes info.
Ludovic Chabant <ludovic@chabant.com>
parents: 586
diff changeset
257 if is_dirty:
33ab9badfd7a render: Change how we store render passes info.
Ludovic Chabant <ludovic@chabant.com>
parents: 586
diff changeset
258 invalidated_render_passes.add(p)
33ab9badfd7a render: Change how we store render passes info.
Ludovic Chabant <ludovic@chabant.com>
parents: 586
diff changeset
259 dirty_for_this.add(src_name)
33ab9badfd7a render: Change how we store render passes info.
Ludovic Chabant <ludovic@chabant.com>
parents: 586
diff changeset
260 break
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
261 return dirty_for_this, invalidated_render_passes
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
262
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
263
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
264 def _ensure_dir_exists(path):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
265 try:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
266 os.makedirs(path, mode=0o755, exist_ok=True)
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
267 except OSError:
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
268 # In a multiprocess environment, several process may very
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
269 # occasionally try to create the same directory at the same time.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
270 # Let's ignore any error and if something's really wrong (like file
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
271 # acces permissions or whatever), then it will more legitimately fail
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
272 # just after this when we try to write files.
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
273 pass
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 369
diff changeset
274