Mercurial > piecrust2

annotate piecrust/pipelines/page.py @ 1015:fa489c5e829e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Load pages in parallel again.
author Ludovic Chabant <ludovic@chabant.com>
date Wed, 29 Nov 2017 20:37:57 -0800
parents 1857dbd4580f
children 3c6e6e7b9639
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
1 import copy
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
2 import time
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
3 import logging
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
4 from piecrust.pipelines.base import (
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
5 ContentPipeline, create_job, content_item_from_job)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
6 from piecrust.pipelines._pagebaker import PageBaker, get_output_path
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
7 from piecrust.pipelines._pagerecords import (
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
8 PagePipelineRecordEntry,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
9 add_page_job_result, merge_job_result_into_record_entry)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
10 from piecrust.sources.base import AbortedSourceUseError
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
11
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
12
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
13 logger = logging.getLogger(__name__)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 class PagePipeline(ContentPipeline):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 PIPELINE_NAME = 'page'
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
18 RECORD_ENTRY_CLASS = PagePipelineRecordEntry
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
19 PASS_NUM = [0, 1, 2]
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
20
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
21 def __init__(self, source, ppctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
22 super().__init__(source, ppctx)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
23 self._pagebaker = None
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
24 self._stats = source.app.env.stats
939
abc52a6262a1 bake: Support the `draft` setting.
Ludovic Chabant <ludovic@chabant.com>
parents: 903
diff changeset
25 self._draft_setting = self.app.config['baker/no_bake_setting']
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
27 def initialize(self):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
28 stats = self._stats
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
29 stats.registerCounter('SourceUseAbortions', raise_if_registered=False)
903
812ca80863d4 bake: Keep track of which pages were aborted for using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 881
diff changeset
30 stats.registerManifest('SourceUseAbortions', raise_if_registered=False)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
32 self._pagebaker = PageBaker(self.app,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
33 self.ctx.out_dir,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
34 force=self.ctx.force)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
35 self._pagebaker.startWriterQueue()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
37 def createJobs(self, ctx):
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
38 if ctx.pass_num == 0:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
39 return self._createLoadJobs(ctx)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
40 if ctx.pass_num == 1:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
41 return self._createSecondPassJobs(ctx)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
42 return self._createThirdPassJobs(ctx)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
43
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
44 def _createLoadJobs(self, ctx):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
45 # Here we load all the pages in the source, making sure they all
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
46 # have a valid cache for their configuration and contents.
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
47 jobs = []
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
48 for item in self.source.getAllContents():
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
49 jobs.append(create_job(self, item.spec))
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
50 if len(jobs) > 0:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
51 return jobs
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
52 return None
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
53
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
54 def _createSecondPassJobs(self, ctx):
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
55 jobs = []
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
56
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
57 app = self.app
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
58 out_dir = self.ctx.out_dir
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
59 uri_getter = self.source.route.getUri
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
60 pretty_urls = app.config.get('site/pretty_urls')
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
61
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
62 used_paths = _get_used_paths_from_records(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
63 ctx.record_histories.current.records)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
64 history = ctx.record_histories.getHistory(ctx.record_name).copy()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
65 history.build()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
66
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
67 record = ctx.current_record
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
68 record.user_data['dirty_source_names'] = set()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
69
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
70 for prev, cur in history.diffs:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
71 # Ignore pages that disappeared since last bake.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
72 if cur is None:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
73 continue
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
74
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
75 # Skip draft pages.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
76 if cur.flags & PagePipelineRecordEntry.FLAG_IS_DRAFT:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
77 continue
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
78
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
79 # Skip pages that are known to use other sources... we'll
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
80 # schedule them in the second pass.
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
81 if prev:
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
82 usn1, usn2 = prev.getAllUsedSourceNames()
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
83 if usn1 or usn2:
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
84 continue
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
85
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
86 # Check if this item has been overriden by a previous pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
87 # run... for instance, we could be the pipeline for a "theme pages"
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
88 # source, and some of our pages have been overriden by a user
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
89 # page that writes out to the same URL.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
90 uri = uri_getter(cur.route_params)
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
91 path = get_output_path(app, out_dir, uri, pretty_urls)
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
92 override = used_paths.get(path)
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
93 if override is not None:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
94 override_source_name, override_entry = override
979
45ad976712ec tests: Big push to get the tests to pass again.
Ludovic Chabant <ludovic@chabant.com>
parents: 939
diff changeset
95 override_source = app.getSource(override_source_name)
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
96 if override_source.config['realm'] == \
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
97 self.source.config['realm']:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
98 logger.error(
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
99 "Page '%s' would get baked to '%s' "
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
100 "but is overriden by '%s'." %
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
101 (enrty.item_spec, path, override_entry.item_spec))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
102 else:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
103 logger.debug(
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
104 "Page '%s' would get baked to '%s' "
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
105 "but is overriden by '%s'." %
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
106 (cur.item_spec, path, override_entry.item_spec))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
107
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
108 cur.flags |= PagePipelineRecordEntry.FLAG_OVERRIDEN
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
109 continue
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
110
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
111 # Nope, all good, let's create a job for this item.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
112 jobs.append(create_job(self, cur.item_spec))
881
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
113
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
114 if len(jobs) > 0:
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
115 return jobs
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
116 return None
b4e10471e970 bake: Don't create bake jobs for overriden pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
117
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
118 def _createThirdPassJobs(self, ctx):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
119 # Get the list of all sources that had anything baked.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
120 dirty_source_names = set()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
121 all_records = ctx.record_histories.current.records
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
122 for rec in all_records:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
123 rec_dsn = rec.user_data.get('dirty_source_names')
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
124 if rec_dsn:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
125 dirty_source_names |= rec_dsn
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
126
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
127 # Now look at the stuff we bake for our own source on the first pass.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
128 # For anything that wasn't baked (i.e. it was considered 'up to date')
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
129 # we look at the records from last time, and if they say that some
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
130 # page was using a source that is "dirty", then we force bake it.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
131 #
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
132 # The common example for this is a blog index page which hasn't been
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
133 # touched, but needs to be re-baked because someone added or edited
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
134 # a post.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
135 jobs = []
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
136 pass_num = ctx.pass_num
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
137 history = ctx.record_histories.getHistory(ctx.record_name).copy()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
138 history.build()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
139 for prev, cur in history.diffs:
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
140 if not cur:
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
141 continue
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
142 if cur.was_any_sub_baked:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
143 continue
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
144 if prev:
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
145 if any(map(
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
146 lambda usn: usn in dirty_source_names,
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
147 prev.getAllUsedSourceNames()[0])):
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
148 jobs.append(create_job(self, prev.item_spec,
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
149 pass_num=pass_num,
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
150 force_bake=True))
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
151 else:
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
152 # This page uses other sources, but no source was dirty
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
153 # this time around (it was a null build, maybe). We
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
154 # don't have any work to do, but we need to carry over
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
155 # any information we have, otherwise the post bake step
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
156 # will think we need to delete last bake's outputs.
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
157 cur.subs = copy.deepcopy(prev.subs)
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
158
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
159 if len(jobs) > 0:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
160 return jobs
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
161 return None
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
162
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
163 def handleJobResult(self, result, ctx):
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
164 step_num = ctx.step_num
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
165
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
166 if step_num == 0:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
167 print(result)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
168 new_entry = self.createRecordEntry(result['item_spec'])
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
169 new_entry.config = result['config']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
170 new_entry.route_params = result['route_params']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
171 new_entry.timestamp = result['timestamp']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
172 ctx.record.addEntry(new_entry)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
173 else:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
174 existing = ctx.record_entry
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
175 merge_job_result_into_record_entry(existing, result)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
176
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
177 if existing.was_any_sub_baked:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
178 ctx.record.user_data['dirty_source_names'].add(self.source.name)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
180 def run(self, job, ctx, result):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
181 pass_num = job.get('pass_num', 0)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
182 step_num = job.get('step_num', 0)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
183 if pass_num == 0:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
184 if step_num == 0:
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
185 self._loadPage(job, ctx, result)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
186 elif step_num == 1:
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
187 self._renderOrPostpone(job, ctx, result)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
188 elif step_num == 2:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
189 self._renderAlways(job, ctx, result)
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
190 else:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
191 raise Exception("Unexpected pipeline step: %d" % step_num)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
192 elif pass_num == 1:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
193 self._renderAlways(job, ctx, result)
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
194 else:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
195 raise Exception("Unexpected pipeline pass: %d" % pass_num)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
197 def getDeletions(self, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
198 for prev, cur in ctx.record_history.diffs:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
199 if prev and not cur:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
200 for sub in prev.subs:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
201 yield (sub['out_path'], 'previous source file was removed')
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
202 elif prev and cur:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
203 prev_out_paths = [o['out_path'] for o in prev.subs]
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
204 cur_out_paths = [o['out_path'] for o in cur.subs]
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
205 diff = set(prev_out_paths) - set(cur_out_paths)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
206 for p in diff:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
207 yield (p, 'source file changed outputs')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
208
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
209 def collapseRecords(self, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
210 pass
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
211
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
212 def shutdown(self):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
213 self._pagebaker.stopWriterQueue()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
214
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
215 def _loadPage(self, job, ctx, result):
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
216 content_item = content_item_from_job(self, job)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
217 page = self.app.getPage(self.source, content_item)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
218
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
219 trigger_next_job = True
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
220 result['flags'] = PagePipelineRecordEntry.FLAG_NONE
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
221 result['config'] = page.config.getAll()
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
222 result['route_params'] = item.metadata['route_params']
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
223 result['timestamp'] = page.datetime.timestamp()
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
224
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
225 if page.was_modified:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
226 result['flags'] |= PagePipelineRecordEntry.FLAG_SOURCE_MODIFIED
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
227 if page.config.get(self._draft_setting):
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
228 result['flags'] |= PagePipelineRecordEntry.FLAG_IS_DRAFT
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
229 trigger_next_job = False
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
230
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
231 if trigger_next_job:
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
232 result['next_step_job'] = create_job(self, content_item.spec)
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
233
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
234 def _renderOrPostpone(self, job, ctx, result):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
235 # Here our job is to render the page's segments so that they're
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
236 # cached in memory and on disk... unless we detect that the page
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
237 # is using some other sources, in which case we abort and we'll try
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
238 # again on the second pass.
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
239 content_item = content_item_from_job(self, job)
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
240 logger.debug("Conditional render for: %s" % content_item.spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
241 page = self.app.getPage(self.source, content_item)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
242 if page.config.get(self._draft_setting):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
243 return
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
244
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
245 prev_entry = ctx.previous_entry
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
246
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
247 env = self.app.env
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
248 env.abort_source_use = True
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
249 add_page_job_result(result)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
250 try:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
251 rdr_subs = self._pagebaker.bake(page, prev_entry)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
252 result['subs'] = rdr_subs
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
253 except AbortedSourceUseError:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
254 logger.debug("Page was aborted for using source: %s" %
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
255 content_item.spec)
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
256 result['flags'] |= \
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
257 PagePipelineRecordEntry.FLAG_ABORTED_FOR_SOURCE_USE
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
258 env.stats.stepCounter("SourceUseAbortions")
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
259 env.stats.addManifestEntry("SourceUseAbortions", content_item.spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
260 result['next_step_job'] = create_job(self, content_item.spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
261 finally:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
262 env.abort_source_use = False
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
263
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
264 def _renderAlways(self, job, ctx, result):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
265 content_item = content_item_from_job(self, job)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
266 logger.debug("Full render for: %s" % content_item.spec)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
267 page = self.app.getPage(self.source, content_item)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
268 prev_entry = ctx.previous_entry
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
269 rdr_subs = self._pagebaker.bake(page, prev_entry,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
270 force=job.get('force_bake'))
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
271
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
272 add_page_job_result(result)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
273 result['subs'] = rdr_subs
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
274
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
275 def _get_used_paths_from_records(records):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
276 used_paths = {}
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
277 for rec in records:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
278 src_name = rec.name.split('@')[0]
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
279 for e in rec.getEntries():
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
280 paths = e.getAllOutputPaths()
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
281 if paths is not None:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
282 for p in paths:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
283 used_paths[p] = (src_name, e)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 979
diff changeset
284 return used_paths