Mercurial > piecrust2

annotate piecrust/pipelines/page.py @ 1146:3516759ea1b2 3.2.1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cm: Regenerate the CHANGELOG.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 05 Jun 2018 22:20:45 -0700
parents 5f97b5b59dfe
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
1 import copy
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
2 import logging
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
3 from piecrust.pipelines.base import (
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
4 ContentPipeline, create_job, content_item_from_job)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
5 from piecrust.pipelines._pagebaker import PageBaker, get_output_path
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
6 from piecrust.pipelines._pagerecords import (
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
7 PagePipelineRecordEntry, SubPageFlags)
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
8 from piecrust.rendering import RenderingContext, render_page_segments
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
9 from piecrust.sources.base import AbortedSourceUseError
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
10
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
11
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
12 logger = logging.getLogger(__name__)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 class PagePipeline(ContentPipeline):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 PIPELINE_NAME = 'page'
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
17 RECORD_ENTRY_CLASS = PagePipelineRecordEntry
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
18 PASS_NUM = [0, 1, 2]
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
19
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
20 def __init__(self, source, ppctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
21 super().__init__(source, ppctx)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
22 self._pagebaker = None
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
23 self._stats = source.app.env.stats
939
abc52a6262a1 bake: Support the `draft` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 903
diff changeset
24 self._draft_setting = self.app.config['baker/no_bake_setting']
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
26 def initialize(self):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
27 stats = self._stats
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
28 stats.registerCounter('SourceUseAbortions', raise_if_registered=False)
903
812ca80863d4 bake: Keep track of which pages were aborted for using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 881
diff changeset
29 stats.registerManifest('SourceUseAbortions', raise_if_registered=False)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
31 self._pagebaker = PageBaker(self.app,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
32 self.ctx.out_dir,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
33 force=self.ctx.force)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
34 self._pagebaker.startWriterQueue()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
36 def createJobs(self, ctx):
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
37 pass_num = ctx.pass_num
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
38 if pass_num == 0:
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
39 ctx.current_record.user_data['dirty_source_names'] = set()
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
40 return self._createLoadJobs(ctx), "load"
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
41 if pass_num == 1:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
42 return self._createSegmentJobs(ctx), "render"
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
43 if pass_num == 2:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
44 return self._createLayoutJobs(ctx), "layout"
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
45 raise Exception("Unexpected pipeline pass: %d" % pass_num)
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
46
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
47 def _createLoadJobs(self, ctx):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
48 # Here we load all the pages in the source, making sure they all
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
49 # have a valid cache for their configuration and contents.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
50 jobs = []
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
51 for item in self.source.getAllContents():
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
52 jobs.append(create_job(self, item.spec))
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
53 if len(jobs) > 0:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
54 return jobs
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
55 return None
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
56
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
57 def _createSegmentJobs(self, ctx):
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
58 jobs = []
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
59
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
60 app = self.app
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
61 pass_num = ctx.pass_num
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
62 out_dir = self.ctx.out_dir
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
63 uri_getter = self.source.route.getUri
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
64 pretty_urls = app.config.get('site/pretty_urls')
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
65
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
66 history = ctx.record_histories.getHistory(ctx.record_name).copy()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
67 history.build()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
68
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
69 cur_rec_used_paths = {}
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
70 history.current.user_data['used_paths'] = cur_rec_used_paths
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
71 all_records = ctx.record_histories.current.records
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
72
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
73 for prev, cur in history.diffs:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
74 # Ignore pages that disappeared since last bake.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
75 if cur is None:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
76 continue
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
77
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
78 # Skip draft pages.
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
79 if cur.hasFlag(PagePipelineRecordEntry.FLAG_IS_DRAFT):
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
80 continue
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
81
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
82 # Skip pages that haven't changed since last bake.
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
83 if (prev and not cur.hasFlag(
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
84 PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED)):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
85 continue
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
86
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
87 # For pages that are known to use other sources in their own
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
88 # content segments (we don't care about the layout yet), we
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
89 # postpone them to the next pipeline pass immediately, because they
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
90 # might need populated render caches for those sources' pages.
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
91 if prev:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
92 usn1, _ = prev.getAllUsedSourceNames()
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
93 if usn1:
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
94 logger.debug("Postponing: %s" % cur.item_spec)
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
95 cur.flags |= \
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
96 PagePipelineRecordEntry.FLAG_ABORTED_FOR_SOURCE_USE
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
97 continue
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
98
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
99 # Check if this item has been overriden by a previous pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
100 # run... for instance, we could be the pipeline for a "theme pages"
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
101 # source, and some of our pages have been overriden by a user
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
102 # page that writes out to the same URL.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
103 uri = uri_getter(cur.route_params)
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
104 out_path = get_output_path(app, out_dir, uri, pretty_urls)
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
105 override = _find_used_path_spec(all_records, out_path)
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
106 if override is not None:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
107 override_source_name, override_entry_spec = override
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
108 override_source = app.getSource(override_source_name)
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
109 if override_source.config['realm'] == \
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
110 self.source.config['realm']:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
111 logger.error(
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
112 "Page '%s' would get baked to '%s' "
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
113 "but is overriden by '%s'." %
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
114 (cur.item_spec, out_path, override_entry_spec))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
115 else:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
116 logger.debug(
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
117 "Page '%s' would get baked to '%s' "
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
118 "but is overriden by '%s'." %
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
119 (cur.item_spec, out_path, override_entry_spec))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
120
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
121 cur.flags |= PagePipelineRecordEntry.FLAG_OVERRIDEN
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
122 continue
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
123
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
124 # Nope, all good, let's create a job for this item.
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
125 cur.flags |= PagePipelineRecordEntry.FLAG_SEGMENTS_RENDERED
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
126 cur_rec_used_paths[out_path] = cur.item_spec
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
127
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
128 jobs.append(create_job(self, cur.item_spec,
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
129 pass_num=pass_num))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
130
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
131 if len(jobs) > 0:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
132 return jobs
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
133 return None
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
134
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
135 def _createLayoutJobs(self, ctx):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
136 # Get the list of all sources that had anything baked.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
137 dirty_source_names = set()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
138 all_records = ctx.record_histories.current.records
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
139 for rec in all_records:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
140 rec_dsn = rec.user_data.get('dirty_source_names')
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
141 if rec_dsn:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
142 dirty_source_names |= rec_dsn
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
143
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
144 jobs = []
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
145 pass_num = ctx.pass_num
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
146 history = ctx.record_histories.getHistory(ctx.record_name).copy()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
147 history.build()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
148 for prev, cur in history.diffs:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
149 if not cur or cur.hasFlag(PagePipelineRecordEntry.FLAG_OVERRIDEN):
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
150 continue
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
151
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
152 do_bake = False
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
153 force_segments = False
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
154 force_layout = False
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
155
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
156 # Make sure we bake the layout for pages that got their segments
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
157 # re-rendered.
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
158 if cur.hasFlag(PagePipelineRecordEntry.FLAG_SEGMENTS_RENDERED):
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
159 do_bake = True
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
160
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
161 # Now look at the stuff we baked for our own source on the second
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
162 # pass. For anything that wasn't baked (i.e. it was considered 'up
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
163 # to date') we look at the records from last time, and if they say
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
164 # that some page was using a source that is "dirty", then we force
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
165 # bake it.
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
166 #
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
167 # The common example for this is a blog index page which hasn't
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
168 # been touched, but needs to be re-baked because someone added or
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
169 # edited a post.
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
170 if prev:
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
171 usn1, usn2 = prev.getAllUsedSourceNames()
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
172 force_segments = any(map(lambda u: u in dirty_source_names,
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
173 usn1))
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
174 force_layout = any(map(lambda u: u in dirty_source_names,
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
175 usn2))
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
176
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
177 if force_segments or force_layout:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
178 # Yep, we need to force-rebake some aspect of this page.
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
179 do_bake = True
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
180
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
181 elif not do_bake:
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
182 # This page uses other sources, but no source was dirty
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
183 # this time around (it was a null build, maybe). We
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
184 # don't have any work to do, but we need to carry over
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
185 # any information we have, otherwise the post bake step
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
186 # will think we need to delete last bake's outputs.
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
187 cur.subs = copy.deepcopy(prev.subs)
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
188 for cur_sub in cur.subs:
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
189 cur_sub['flags'] = \
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
190 SubPageFlags.FLAG_COLLAPSED_FROM_LAST_RUN
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
191
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
192 if do_bake:
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
193 jobs.append(create_job(self, cur.item_spec,
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
194 pass_num=pass_num,
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
195 force_segments=force_segments,
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
196 force_layout=force_layout))
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
197
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
198 if len(jobs) > 0:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
199 return jobs
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
200 return None
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
201
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
202 def handleJobResult(self, result, ctx):
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
203 pass_num = ctx.pass_num
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
204
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
205 if pass_num == 0:
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
206 # Just went through a "load page" job. Let's create a record
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
207 # entry with the information we got from the worker.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
208 new_entry = self.createRecordEntry(result['item_spec'])
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
209 new_entry.flags = result['flags']
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
210 new_entry.config = result['config']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
211 new_entry.route_params = result['route_params']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
212 new_entry.timestamp = result['timestamp']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
213 ctx.record.addEntry(new_entry)
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
214
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
215 # If this page was modified, flag its entire source as "dirty",
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
216 # so any pages using that source can be re-baked.
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
217 if new_entry.flags & PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED:
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
218 ctx.record.user_data['dirty_source_names'].add(
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
219 self.source.name)
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
220
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
221 # If this page is new
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
222
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
223 elif pass_num == 1:
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
224 # Just went through the "render segments" job.
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
225 existing = ctx.record_entry
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
226 existing.flags |= result.get('flags',
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
227 PagePipelineRecordEntry.FLAG_NONE)
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
228
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
229 else:
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
230 # Update the entry with the new information.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
231 existing = ctx.record_entry
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
232 existing.flags |= result.get('flags',
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
233 PagePipelineRecordEntry.FLAG_NONE)
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
234 existing.errors += result.get('errors', [])
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
235 existing.subs += result.get('subs', [])
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
236
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
237 def run(self, job, ctx, result):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
238 pass_num = job.get('pass_num', 0)
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
239
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
240 if pass_num == 0:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
241 return self._loadPage(job, ctx, result)
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
242
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
243 elif pass_num == 1:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
244 return self._renderSegments(job, ctx, result)
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
245
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
246 elif pass_num >= 2:
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
247 return self._renderLayout(job, ctx, result)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
248
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
249 def getDeletions(self, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
250 for prev, cur in ctx.record_history.diffs:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
251 if prev and not cur:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
252 for sub in prev.subs:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
253 yield (sub['out_path'], 'previous source file was removed')
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
254 elif prev and cur:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
255 prev_out_paths = [o['out_path'] for o in prev.subs]
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
256 cur_out_paths = [o['out_path'] for o in cur.subs]
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
257 diff = set(prev_out_paths) - set(cur_out_paths)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
258 for p in diff:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
259 yield (p, 'source file changed outputs')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
260
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
261 def collapseRecords(self, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
262 pass
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
263
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
264 def shutdown(self):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
265 self._pagebaker.stopWriterQueue()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
266
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
267 def _loadPage(self, job, ctx, result):
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
268 content_item = content_item_from_job(self, job)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
269 page = self.app.getPage(self.source, content_item)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
270
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
271 result['flags'] = PagePipelineRecordEntry.FLAG_NONE
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
272 result['config'] = page.config.getAll()
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
273 result['route_params'] = content_item.metadata['route_params']
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
274 result['timestamp'] = page.datetime.timestamp()
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
275
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
276 if page.was_modified:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
277 result['flags'] |= PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
278 if page.config.get(self._draft_setting):
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
279 result['flags'] |= PagePipelineRecordEntry.FLAG_IS_DRAFT
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
280
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
281 def _renderSegments(self, job, ctx, result):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
282 # Here our job is to render the page's segments so that they're
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
283 # cached in memory and on disk... unless we detect that the page
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
284 # is using some other sources, in which case we abort and we'll try
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
285 # again on the second pass.
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
286 content_item = content_item_from_job(self, job)
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
287 logger.debug("Render segments for: %s" % content_item.spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
288 page = self.app.getPage(self.source, content_item)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
289 if page.config.get(self._draft_setting):
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
290 raise Exception("Shouldn't have a draft page in a render job!")
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
291
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
292 env = self.app.env
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
293 env.abort_source_use = True
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
294 try:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
295 rdr_ctx = RenderingContext(page)
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
296 render_page_segments(rdr_ctx)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
297 except AbortedSourceUseError:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
298 logger.debug("Page was aborted for using source: %s" %
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
299 content_item.spec)
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
300 result['flags'] = \
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
301 PagePipelineRecordEntry.FLAG_ABORTED_FOR_SOURCE_USE
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
302 env.stats.stepCounter("SourceUseAbortions")
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
303 env.stats.addManifestEntry("SourceUseAbortions", content_item.spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
304 finally:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
305 env.abort_source_use = False
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
306
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
307 def _renderLayout(self, job, ctx, result):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
308 content_item = content_item_from_job(self, job)
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
309 logger.debug("Render layout for: %s" % content_item.spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
310 page = self.app.getPage(self.source, content_item)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
311 prev_entry = ctx.previous_entry
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
312 rdr_subs = self._pagebaker.bake(
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
313 page, prev_entry,
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
314 force_segments=job.get('force_segments'),
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
315 force_layout=job.get('force_layout'))
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
316 result['subs'] = rdr_subs
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
317
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
318
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
319 def _find_used_path_spec(records, path):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
320 for rec in records:
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
321 up = rec.user_data.get('used_paths')
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
322 if up is not None:
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
323 entry_spec = up.get(path)
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
324 if entry_spec is not None:
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
325 src_name = rec.name.split('@')[0]
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
326 return (src_name, entry_spec)
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1132
diff changeset
327 return None