Mercurial > piecrust2

annotate piecrust/pipelines/page.py @ 877:d6d35b2efd04

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Rename "pass" to "step" and make the page pipeline use different steps. That pipeline is now first loading all pages, and then rendering full pages unless they trigger a sub-render.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 15 Jun 2017 22:16:23 -0700
parents 504ddb370df8
children b4e10471e970
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
1 import logging
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 from piecrust.pipelines.base import ContentPipeline
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
3 from piecrust.pipelines._pagebaker import PageBaker
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
4 from piecrust.pipelines._pagerecords import PagePipelineRecordEntry
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
5 from piecrust.sources.base import AbortedSourceUseError
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
6
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
7
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
8 logger = logging.getLogger(__name__)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 class PagePipeline(ContentPipeline):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 PIPELINE_NAME = 'page'
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
13 RECORD_ENTRY_CLASS = PagePipelineRecordEntry
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
14
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
15 def __init__(self, source, ppctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
16 super().__init__(source, ppctx)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
17 self._pagebaker = None
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
18 self._stats = source.app.env.stats
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
20 def initialize(self):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
21 stats = self.app.env.stats
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
22 stats.registerCounter('SourceUseAbortions', raise_if_registered=False)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
24 self._pagebaker = PageBaker(self.app,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
25 self.ctx.out_dir,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
26 force=self.ctx.force)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
27 self._pagebaker.startWriterQueue()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
29 def mergeRecordEntry(self, record_entry, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
30 existing = ctx.record.getEntry(record_entry.item_spec)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
31 existing.errors += record_entry.errors
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
32 existing.flags |= record_entry.flags
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
33 existing.subs = record_entry.subs
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
35 def run(self, job, ctx, result):
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
36 step_num = job.step_num
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
37 if step_num == 0:
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
38 self._loadPage(job.content_item, ctx, result)
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
39 elif step_num == 1:
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
40 self._renderOrPostpone(job.content_item, ctx, result)
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
41 elif step_num == 2:
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
42 self._renderAlways(job.content_item, ctx, result)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
44 def getDeletions(self, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
45 for prev, cur in ctx.record_history.diffs:
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 if prev and not cur:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 for sub in prev.subs:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 yield (sub.out_path, 'previous source file was removed')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 elif prev and cur:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 prev_out_paths = [o.out_path for o in prev.subs]
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 cur_out_paths = [o.out_path for o in cur.subs]
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 diff = set(prev_out_paths) - set(cur_out_paths)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 for p in diff:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 yield (p, 'source file changed outputs')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
56 def collapseRecords(self, ctx):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
57 pass
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
59 def shutdown(self):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
60 self._pagebaker.stopWriterQueue()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
62 def _loadPage(self, content_item, ctx, result):
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
63 logger.debug("Loading page: %s" % content_item.spec)
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
64 page = self.app.getPage(self.source, content_item)
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
65 record_entry = result.record_entry
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
66 record_entry.config = page.config.getAll()
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
67 record_entry.timestamp = page.datetime.timestamp()
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
68 result.next_step_job = self.createJob(content_item)
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
69
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
70 def _renderOrPostpone(self, content_item, ctx, result):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
71 # Here our job is to render the page's segments so that they're
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
72 # cached in memory and on disk... unless we detect that the page
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
73 # is using some other sources, in which case we abort and we'll try
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
74 # again on the second pass.
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
75 logger.debug("Conditional render for: %s" % content_item.spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
76 page = self.app.getPage(self.source, content_item)
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
77 prev_entry = ctx.previous_entry
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
78 cur_entry = result.record_entry
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
79 self.app.env.abort_source_use = True
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
80 try:
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
81 self._pagebaker.bake(page, prev_entry, cur_entry)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
82 except AbortedSourceUseError:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
83 logger.debug("Page was aborted for using source: %s" %
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
84 content_item.spec)
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
85 self.app.env.stats.stepCounter("SourceUseAbortions")
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
86 result.next_step_job = self.createJob(content_item)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
87 finally:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
88 self.app.env.abort_source_use = False
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
90 def _renderAlways(self, content_item, ctx, result):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
91 logger.debug("Full render for: %s" % content_item.spec)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
92 page = self.app.getPage(self.source, content_item)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
93 prev_entry = ctx.previous_entry
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
94 cur_entry = result.record_entry
871
504ddb370df8 refactor: Fixing some issues with baking assets.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
95 self._pagebaker.bake(page, prev_entry, cur_entry)