Mercurial > piecrust2

annotate piecrust/pipelines/base.py @ 1157:40a40305c4e1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
config: Support environment variables in theme directories.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 11 Jun 2019 15:14:20 -0700
parents 5f97b5b59dfe
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import logging
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
3 from werkzeug.utils import cached_property
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
4 from piecrust.configuration import ConfigurationError
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 logger = logging.getLogger(__name__)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 class PipelineContext:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 """ The context for running a content pipeline.
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 """
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
13 def __init__(self, out_dir, *,
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 worker_id=-1, force=None):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 self.out_dir = out_dir
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 self.worker_id = worker_id
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 self.force = force
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 @property
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 def is_worker(self):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 """ Returns `True` if the content pipeline is running inside
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 a worker process, and this is the first one.
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 """
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 return self.worker_id >= 0
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 @property
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 def is_main_process(self):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 """ Returns `True` is the content pipeline is running inside
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 the main process (and not a worker process). This is the case
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 if there are no worker processes at all.
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 """
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 return self.worker_id < 0
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
34
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
35 class _PipelineMasterProcessJobContextBase:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
36 def __init__(self, record_name, record_histories):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
37 self.record_name = record_name
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
38 self.record_histories = record_histories
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
39
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
40 @property
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
41 def previous_record(self):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
42 return self.record_histories.getPreviousRecord(self.record_name)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
43
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
44 @property
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
45 def current_record(self):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
46 return self.record_histories.getCurrentRecord(self.record_name)
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
47
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
49 class PipelineJobCreateContext(_PipelineMasterProcessJobContextBase):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
50 """ Context for creating pipeline baking jobs.
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
51
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
52 This is run on the master process, so it can access both the
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
53 previous and current records.
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
54 """
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
55 def __init__(self, pass_num, record_name, record_histories):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
56 super().__init__(record_name, record_histories)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
57 self.pass_num = pass_num
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
58
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
59
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
60 class PipelineJobRunContext:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
61 """ Context for running pipeline baking jobs.
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
62
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
63 This is run on the worker processes, so it can only access the
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
64 previous records.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
65 """
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
66 def __init__(self, job, record_name, previous_records):
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
67 self.job = job
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
68 self.record_name = record_name
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
69 self.previous_records = previous_records
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
70
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
71 @cached_property
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
72 def record_entry_spec(self):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
73 return self.job.get('record_entry_spec',
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
74 self.job['job_spec'][1])
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
75
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
76 @cached_property
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
77 def previous_record(self):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
78 return self.previous_records.getRecord(self.record_name)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
79
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
80 @cached_property
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
81 def previous_entry(self):
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 855
diff changeset
82 return self.previous_record.getEntry(self.record_entry_spec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
83
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
84
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
85 class PipelineJobResultHandleContext:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
86 """ The context for handling the result from a job that ran in a
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
87 worker process.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
88
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
89 This is run on the master process, so it can access the current
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
90 record.
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
91 """
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
92 def __init__(self, record, job, pass_num):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
93 self.record = record
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
94 self.job = job
1015
fa489c5e829e bake: Load pages in parallel again.
Ludovic Chabant <ludovic@chabant.com>
parents: 989
diff changeset
95 self.pass_num = pass_num
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
96
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
97 @cached_property
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
98 def record_entry(self):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
99 record_entry_spec = self.job.get('record_entry_spec',
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
100 self.job['job_spec'][1])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
101 return self.record.getEntry(record_entry_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
102
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
103
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
104 class PipelinePostJobRunContext:
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
105 def __init__(self, record_history):
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
106 self.record_history = record_history
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
107
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
108
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
109 class PipelineDeletionContext:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
110 def __init__(self, record_history):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
111 self.record_history = record_history
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
112
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
113
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
114 class PipelineCollapseRecordContext:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
115 def __init__(self, record_history):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
116 self.record_history = record_history
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119 class ContentPipeline:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120 """ A pipeline that processes content from a `ContentSource`.
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 """
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 PIPELINE_NAME = None
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
123 RECORD_ENTRY_CLASS = None
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
124 PASS_NUM = 0
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
126 def __init__(self, source, ctx):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 self.source = source
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
128 self.ctx = ctx
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
129 self.record_name = '%s@%s' % (source.name, self.PIPELINE_NAME)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 app = source.app
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
132 tmp_dir = app.cache_dir
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 if not tmp_dir:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134 import tempfile
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135 tmp_dir = os.path.join(tempfile.gettempdir(), 'piecrust')
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 self.tmp_dir = os.path.join(tmp_dir, self.PIPELINE_NAME)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
137
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
138 @property
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
139 def app(self):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140 return self.source.app
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
142 def initialize(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143 pass
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
144
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
145 def createJobs(self, ctx):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
146 return [
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
147 create_job(self, item.spec)
1136
5f97b5b59dfe bake: Optimize cache handling for the baking process.
Ludovic Chabant <ludovic@chabant.com>
parents: 1051
diff changeset
148 for item in self.source.getAllContents()], None
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
149
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
150 def createRecordEntry(self, item_spec):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
151 entry_class = self.RECORD_ENTRY_CLASS
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
152 record_entry = entry_class()
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
153 record_entry.item_spec = item_spec
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
154 return record_entry
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
155
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
156 def handleJobResult(self, result, ctx):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
157 raise NotImplementedError()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
158
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
159 def run(self, job, ctx, result):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160 raise NotImplementedError()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
161
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
162 def postJobRun(self, ctx):
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
163 pass
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
164
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
165 def getDeletions(self, ctx):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166 pass
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
167
853
f070a4fc033c core: Continue PieCrust3 refactor, simplify pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
168 def collapseRecords(self, ctx):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169 pass
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
171 def shutdown(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
172 pass
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
173
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
174
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
175 def create_job(pipeline, item_spec, **kwargs):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
176 job = {
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
177 'job_spec': (pipeline.source.name, item_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
178 }
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
179 job.update(kwargs)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
180 return job
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
181
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
182
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
183 def content_item_from_job(pipeline, job):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
184 return pipeline.source.findContentFromSpec(job['job_spec'][1])
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
185
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
186
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
187 def get_record_name_for_source(source):
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
188 ppname = get_pipeline_name_for_source(source)
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
189 return '%s@%s' % (source.name, ppname)
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
190
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
191
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
192 def get_pipeline_name_for_source(source):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
193 pname = source.config['pipeline']
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
194 if not pname:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
195 pname = source.DEFAULT_PIPELINE_NAME
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
196 if not pname:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
197 raise ConfigurationError(
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
198 "Source '%s' doesn't specify any pipeline." % source.name)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
199 return pname
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
200
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
201
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
202 class PipelineManager:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
203 def __init__(self, app, out_dir, *,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
204 record_histories=None, worker_id=-1, force=False):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
205 self.app = app
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
206 self.record_histories = record_histories
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
207 self.out_dir = out_dir
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
208 self.worker_id = worker_id
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
209 self.force = force
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
210
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
211 self._pipeline_classes = {}
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
212 for pclass in app.plugin_loader.getPipelines():
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
213 self._pipeline_classes[pclass.PIPELINE_NAME] = pclass
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
214
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
215 self._pipelines = {}
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
216
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
217 def getPipeline(self, source_name):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
218 return self.getPipelineInfo(source_name).pipeline
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
219
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
220 def getPipelineInfo(self, source_name):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
221 return self._pipelines[source_name]
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
222
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
223 def getPipelineInfos(self):
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
224 return self._pipelines.values()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
225
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
226 def createPipeline(self, source):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
227 if source.name in self._pipelines:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
228 raise ValueError("Pipeline for source '%s' was already created." %
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
229 source.name)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
230
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
231 pname = get_pipeline_name_for_source(source)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
232 ppctx = PipelineContext(self.out_dir,
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
233 worker_id=self.worker_id, force=self.force)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
234 pp = self._pipeline_classes[pname](source, ppctx)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
235 pp.initialize()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
236
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
237 record_history = None
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
238 if self.record_histories:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
239 record_history = self.record_histories.getHistory(pp.record_name)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
240
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
241 info = _PipelineInfo(pp, record_history)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
242 self._pipelines[source.name] = info
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
243 return info
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
244
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
245 def postJobRun(self):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
246 for ppinfo in self.getPipelineInfos():
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
247 ppinfo.record_history.build()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
248
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
249 for ppinfo in self.getPipelineInfos():
855
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
250 ctx = PipelinePostJobRunContext(ppinfo.record_history)
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
251 ppinfo.pipeline.postJobRun(ctx)
448710d84121 refactor: Get the taxonomy support back to a functional state.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
252
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
253 def deleteStaleOutputs(self):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
254 for ppinfo in self.getPipelineInfos():
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
255 ctx = PipelineDeletionContext(ppinfo.record_history)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
256 to_delete = ppinfo.pipeline.getDeletions(ctx)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
257 current_record = ppinfo.record_history.current
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
258 if to_delete is not None:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
259 for path, reason in to_delete:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
260 logger.debug("Removing '%s': %s" % (path, reason))
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
261 current_record.deleted_out_paths.append(path)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
262 try:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
263 os.remove(path)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
264 except FileNotFoundError:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
265 pass
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
266 logger.info('[delete] %s' % path)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
267
1051
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
268 def collapseRecords(self, keep_unused_records=False):
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
269 seen_records = []
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
270 for ppinfo in self.getPipelineInfos():
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
271 ctx = PipelineCollapseRecordContext(ppinfo.record_history)
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
272 ppinfo.pipeline.collapseRecords(ctx)
1051
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
273 seen_records.append(ppinfo.pipeline.record_name)
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
274
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
275 if keep_unused_records:
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
276 cur_recs = self.record_histories.current
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
277 prev_recs = self.record_histories.previous
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
278 for prev_rec in prev_recs.records:
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
279 if prev_rec.name in seen_records:
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
280 continue
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
281
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
282 logger.debug("Keeping record: %s" % prev_rec.name)
971b4d67e82a serve: Fix problems with assets disappearing between servings.
Ludovic Chabant <ludovic@chabant.com>
parents: 1015
diff changeset
283 cur_recs.records.append(prev_rec)
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
284
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
285 def shutdownPipelines(self):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
286 for ppinfo in self.getPipelineInfos():
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
287 ppinfo.pipeline.shutdown()
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
288
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
289 self._pipelines = {}
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
290
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
291
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
292 class _PipelineInfo:
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
293 def __init__(self, pipeline, record_history):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
294 self.pipeline = pipeline
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
295 self.record_history = record_history
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
296 self.userdata = None
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
297
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
298 @property
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
299 def source(self):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
300 return self.pipeline.source
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
301
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
302 @property
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
303 def current_record(self):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
304 if self.record_history is not None:
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
305 return self.record_history.current
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
306 raise Exception("The current record is not available.")
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
307
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
308 @property
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
309 def pipeline_name(self):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
310 return self.pipeline.PIPELINE_NAME
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
311