Mercurial > piecrust2

annotate piecrust/pipelines/page.py @ 1132:3bcb2d446397

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix: Correctly invalidate pages that use dirtied sources.
author Ludovic Chabant <ludovic@chabant.com>
date Mon, 16 Apr 2018 22:22:54 -0700
parents d85de09f40c7
children 5f97b5b59dfe
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
1 import copy
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
2 import time
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
3 import logging
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
4 from piecrust.pipelines.base import (
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
5 ContentPipeline, create_job, content_item_from_job)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
6 from piecrust.pipelines._pagebaker import PageBaker, get_output_path
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
7 from piecrust.pipelines._pagerecords import (
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
8 PagePipelineRecordEntry,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
9 add_page_job_result, merge_job_result_into_record_entry)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
10 from piecrust.sources.base import AbortedSourceUseError
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
11
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
12
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
13 logger = logging.getLogger(__name__)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 class PagePipeline(ContentPipeline):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 PIPELINE_NAME = 'page'
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
18 RECORD_ENTRY_CLASS = PagePipelineRecordEntry
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
19 PASS_NUM = [0, 1, 2]
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
20
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
21 def __init__(self, source, ppctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
22 super().__init__(source, ppctx)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
23 self._pagebaker = None
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
24 self._stats = source.app.env.stats
939
abc52a6262a1 bake: Support the `draft` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 903
diff changeset
25 self._draft_setting = self.app.config['baker/no_bake_setting']
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
27 def initialize(self):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
28 stats = self._stats
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
29 stats.registerCounter('SourceUseAbortions', raise_if_registered=False)
903
812ca80863d4 bake: Keep track of which pages were aborted for using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 881
diff changeset
30 stats.registerManifest('SourceUseAbortions', raise_if_registered=False)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
32 self._pagebaker = PageBaker(self.app,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
33 self.ctx.out_dir,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
34 force=self.ctx.force)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
35 self._pagebaker.startWriterQueue()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
37 def createJobs(self, ctx):
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
38 pass_num = ctx.pass_num
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
39 if pass_num == 0:
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
40 ctx.current_record.user_data['dirty_source_names'] = set()
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
41 return self._createLoadJobs(ctx)
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
42 if pass_num == 1:
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
43 return self._createSecondPassJobs(ctx)
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
44 if pass_num == 2:
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
45 return self._createThirdPassJobs(ctx)
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
46 raise Exception("Unexpected pipeline pass: %d" % pass_num)
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
47
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
48 def _createLoadJobs(self, ctx):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
49 # Here we load all the pages in the source, making sure they all
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
50 # have a valid cache for their configuration and contents.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
51 jobs = []
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
52 for item in self.source.getAllContents():
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
53 jobs.append(create_job(self, item.spec))
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
54 if len(jobs) > 0:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
55 return jobs
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
56 return None
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
57
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
58 def _createSecondPassJobs(self, ctx):
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
59 jobs = []
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
60
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
61 app = self.app
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
62 out_dir = self.ctx.out_dir
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
63 uri_getter = self.source.route.getUri
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
64 pretty_urls = app.config.get('site/pretty_urls')
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
65
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
66 used_paths = _get_used_paths_from_records(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
67 ctx.record_histories.current.records)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
68 history = ctx.record_histories.getHistory(ctx.record_name).copy()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
69 history.build()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
70
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
71 pass_num = ctx.pass_num
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
72
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
73 for prev, cur in history.diffs:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
74 # Ignore pages that disappeared since last bake.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
75 if cur is None:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
76 continue
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
77
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
78 # Skip draft pages.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
79 if cur.flags & PagePipelineRecordEntry.FLAG_IS_DRAFT:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
80 continue
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
81
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
82 # For pages that are known to use other sources, we make a dummy
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
83 # job that will effectively get directly passed on to the next
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
84 # step.
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
85 if prev:
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
86 usn1, usn2 = prev.getAllUsedSourceNames()
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
87 if usn1 or usn2:
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
88 jobs.append(create_job(self, cur.item_spec,
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
89 pass_num=pass_num,
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
90 uses_sources=True))
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
91 continue
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
92
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
93 # Check if this item has been overriden by a previous pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
94 # run... for instance, we could be the pipeline for a "theme pages"
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
95 # source, and some of our pages have been overriden by a user
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
96 # page that writes out to the same URL.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
97 uri = uri_getter(cur.route_params)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
98 path = get_output_path(app, out_dir, uri, pretty_urls)
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
99 override = used_paths.get(path)
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
100 if override is not None:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
101 override_source_name, override_entry = override
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
102 override_source = app.getSource(override_source_name)
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
103 if override_source.config['realm'] == \
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
104 self.source.config['realm']:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
105 logger.error(
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
106 "Page '%s' would get baked to '%s' "
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
107 "but is overriden by '%s'." %
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
108 (cur.item_spec, path, override_entry.item_spec))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
109 else:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
110 logger.debug(
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
111 "Page '%s' would get baked to '%s' "
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
112 "but is overriden by '%s'." %
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
113 (cur.item_spec, path, override_entry.item_spec))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
114
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
115 cur.flags |= PagePipelineRecordEntry.FLAG_OVERRIDEN
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
116 continue
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
117
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
118 # Nope, all good, let's create a job for this item.
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
119 jobs.append(create_job(self, cur.item_spec,
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
120 pass_num=pass_num))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
121
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
122 if len(jobs) > 0:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
123 return jobs
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
124 return None
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
125
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
126 def _createThirdPassJobs(self, ctx):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
127 # Get the list of all sources that had anything baked.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
128 dirty_source_names = set()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
129 all_records = ctx.record_histories.current.records
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
130 for rec in all_records:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
131 rec_dsn = rec.user_data.get('dirty_source_names')
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
132 if rec_dsn:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
133 dirty_source_names |= rec_dsn
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
134
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
135 # Now look at the stuff we bake for our own source on the first pass.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
136 # For anything that wasn't baked (i.e. it was considered 'up to date')
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
137 # we look at the records from last time, and if they say that some
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
138 # page was using a source that is "dirty", then we force bake it.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
139 #
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
140 # The common example for this is a blog index page which hasn't been
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
141 # touched, but needs to be re-baked because someone added or edited
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
142 # a post.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
143 jobs = []
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
144 pass_num = ctx.pass_num
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
145 history = ctx.record_histories.getHistory(ctx.record_name).copy()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
146 history.build()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
147 for prev, cur in history.diffs:
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
148 if not cur:
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
149 continue
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
150 if cur.was_any_sub_baked:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
151 continue
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
152 if prev:
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
153 usn1, usn2 = prev.getAllUsedSourceNames()
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
154 force_segments = any(map(lambda u: u in dirty_source_names,
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
155 usn1))
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
156 force_layout = any(map(lambda u: u in dirty_source_names,
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
157 usn2))
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
158 if force_segments or force_layout:
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
159 jobs.append(create_job(self, prev.item_spec,
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
160 pass_num=pass_num,
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
161 force_segments=force_segments,
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
162 force_layout=force_layout))
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
163 else:
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
164 # This page uses other sources, but no source was dirty
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
165 # this time around (it was a null build, maybe). We
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
166 # don't have any work to do, but we need to carry over
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
167 # any information we have, otherwise the post bake step
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
168 # will think we need to delete last bake's outputs.
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
169 cur.subs = copy.deepcopy(prev.subs)
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
170
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
171 if len(jobs) > 0:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
172 return jobs
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
173 return None
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
174
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
175 def handleJobResult(self, result, ctx):
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
176 pass_num = ctx.pass_num
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
177 step_num = ctx.step_num
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
178
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
179 if pass_num == 0:
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
180 # Just went through a "load page" job. Let's create a record
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
181 # entry with the information we got from the worker.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
182 new_entry = self.createRecordEntry(result['item_spec'])
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
183 new_entry.flags = result['flags']
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
184 new_entry.config = result['config']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
185 new_entry.route_params = result['route_params']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
186 new_entry.timestamp = result['timestamp']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
187 ctx.record.addEntry(new_entry)
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
188
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
189 # If this page was modified, flag its entire source as "dirty",
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
190 # so any pages using that source can be re-baked.
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
191 if (new_entry.flags & PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED):
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
192 ctx.record.user_data['dirty_source_names'].add(
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
193 self.source.name)
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
194 else:
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
195 # Update the entry with the new information.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
196 existing = ctx.record_entry
1026
d85de09f40c7 bake: Fix bug with postponed pages due to them using other sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1020
diff changeset
197 if not result.get('postponed', False):
d85de09f40c7 bake: Fix bug with postponed pages due to them using other sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1020
diff changeset
198 merge_job_result_into_record_entry(existing, result)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
199
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
200 def run(self, job, ctx, result):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
201 pass_num = job.get('pass_num', 0)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
202 step_num = job.get('step_num', 0)
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
203
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
204 if pass_num == 0:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
205 if step_num == 0:
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
206 return self._loadPage(job, ctx, result)
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
207
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
208 elif pass_num == 1:
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
209 if step_num == 0:
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
210 return self._renderOrPostpone(job, ctx, result)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
211 elif step_num == 1:
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
212 return self._renderAlways(job, ctx, result)
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
213
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
214 elif pass_num == 2:
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
215 if step_num == 0:
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
216 return self._renderAlways(job, ctx, result)
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
217
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
218 raise Exception("Unexpected pipeline pass/step: %d/%d" %
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
219 (pass_num, step_num))
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
220
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
221 def getDeletions(self, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
222 for prev, cur in ctx.record_history.diffs:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
223 if prev and not cur:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
224 for sub in prev.subs:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
225 yield (sub['out_path'], 'previous source file was removed')
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
226 elif prev and cur:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
227 prev_out_paths = [o['out_path'] for o in prev.subs]
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
228 cur_out_paths = [o['out_path'] for o in cur.subs]
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
229 diff = set(prev_out_paths) - set(cur_out_paths)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
230 for p in diff:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
231 yield (p, 'source file changed outputs')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
232
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
233 def collapseRecords(self, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
234 pass
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
235
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
236 def shutdown(self):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
237 self._pagebaker.stopWriterQueue()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
238
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
239 def _loadPage(self, job, ctx, result):
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
240 content_item = content_item_from_job(self, job)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
241 page = self.app.getPage(self.source, content_item)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
242
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
243 result['flags'] = PagePipelineRecordEntry.FLAG_NONE
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
244 result['config'] = page.config.getAll()
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
245 result['route_params'] = content_item.metadata['route_params']
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
246 result['timestamp'] = page.datetime.timestamp()
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
247
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
248 if page.was_modified:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
249 result['flags'] |= PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
250 if page.config.get(self._draft_setting):
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
251 result['flags'] |= PagePipelineRecordEntry.FLAG_IS_DRAFT
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
252
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
253 def _renderOrPostpone(self, job, ctx, result):
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
254 # See if we should immediately kick this job off to the next step.
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
255 if job.get('uses_sources', False):
1026
d85de09f40c7 bake: Fix bug with postponed pages due to them using other sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1020
diff changeset
256 result['postponed'] = True
1020
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
257 result['next_step_job'] = create_job(self, job['job_spec'][1])
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
258 return
298b07a899b5 bake: Fix overriding issues between theme and user pages for index pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 1018
diff changeset
259
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
260 # Here our job is to render the page's segments so that they're
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
261 # cached in memory and on disk... unless we detect that the page
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
262 # is using some other sources, in which case we abort and we'll try
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
263 # again on the second pass.
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
264 content_item = content_item_from_job(self, job)
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
265 logger.debug("Conditional render for: %s" % content_item.spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
266 page = self.app.getPage(self.source, content_item)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
267 if page.config.get(self._draft_setting):
1018
3c6e6e7b9639 bake: Fix totally broken page baking caused by previous commit.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
268 raise Exception("Shouldn't have a draft page in a render job!")
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
269
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
270 prev_entry = ctx.previous_entry
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
271
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
272 env = self.app.env
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
273 env.abort_source_use = True
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
274 add_page_job_result(result)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
275 try:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
276 rdr_subs = self._pagebaker.bake(page, prev_entry)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
277 result['subs'] = rdr_subs
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
278 except AbortedSourceUseError:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
279 logger.debug("Page was aborted for using source: %s" %
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
280 content_item.spec)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
281 result['flags'] |= \
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
282 PagePipelineRecordEntry.FLAG_ABORTED_FOR_SOURCE_USE
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
283 env.stats.stepCounter("SourceUseAbortions")
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
284 env.stats.addManifestEntry("SourceUseAbortions", content_item.spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
285 result['next_step_job'] = create_job(self, content_item.spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
286 finally:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
287 env.abort_source_use = False
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
288
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
289 def _renderAlways(self, job, ctx, result):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
290 content_item = content_item_from_job(self, job)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
291 logger.debug("Full render for: %s" % content_item.spec)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
292 page = self.app.getPage(self.source, content_item)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
293 prev_entry = ctx.previous_entry
1132
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
294 rdr_subs = self._pagebaker.bake(
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
295 page, prev_entry,
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
296 force_segments=job.get('force_segments'),
3bcb2d446397 fix: Correctly invalidate pages that use dirtied sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 1026
diff changeset
297 force_layout=job.get('force_layout'))
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
298
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
299 add_page_job_result(result)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
300 result['subs'] = rdr_subs
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
301
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
302 def _get_used_paths_from_records(records):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
303 used_paths = {}
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
304 for rec in records:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
305 src_name = rec.name.split('@')[0]
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
306 for e in rec.getEntries():
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
307 paths = e.getAllOutputPaths()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
308 if paths is not None:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
309 for p in paths:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
310 used_paths[p] = (src_name, e)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
311 return used_paths