Mercurial > piecrust2

annotate piecrust/processing/records.py @ 371:c2ca72fb7f0b 2.0.0a8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
caching: Use separate caches for config variants and other contexts. * The `_cache` directory is now organized in multiple "sub-caches" for different contexts. * A new context is created when config variants or overrides are applied. * `serve` context uses a different context that the other commends, to prevent the `bake` command's output from messing up the preview server (e.g. with how asset URLs are generated differently between the two). * Fix a few places where the cache directory was referenced directly.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 03 May 2015 23:59:46 -0700
parents f82262f59600
children c4b3a7fd2f87
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 from piecrust.records import Record, TransitionalRecord
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 class ProcessorPipelineRecord(Record):
216
c5ada46b281a internal: Bump the processing record version.
Ludovic Chabant <ludovic@chabant.com>
parents: 215
diff changeset
6 RECORD_VERSION = 4
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 def __init__(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 super(ProcessorPipelineRecord, self).__init__()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 self.out_dir = None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 self.process_time = None
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 191
diff changeset
12 self.success = False
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 def hasOverrideEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 return self.findEntry(rel_path) is not None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 def findEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 rel_path = rel_path.lower()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 for entry in self.entries:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 for out_path in entry.rel_outputs:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 if out_path.lower() == rel_path:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 return entry
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 return None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
133
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
25 def replaceEntry(self, new_entry):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
26 for e in self.entries:
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
27 if (e.base_dir == new_entry.base_dir and
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
28 e.rel_input == new_entry.rel_input):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
29 e.flags = new_entry.flags
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
30 e.rel_outputs = list(new_entry.rel_outputs)
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
31 e.errors = list(new_entry.errors)
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
32 break
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
33
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 FLAG_NONE = 0
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
36 FLAG_PREPARED = 2**0
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
37 FLAG_PROCESSED = 2**1
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
38 FLAG_OVERRIDEN = 2**2
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
39 FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**3
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 class ProcessorPipelineRecordEntry(object):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 def __init__(self, base_dir, rel_input):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 self.base_dir = base_dir
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 self.rel_input = rel_input
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 self.flags = FLAG_NONE
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 self.rel_outputs = []
191
308d5180bf81 processing: Add more information to the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 133
diff changeset
49 self.proc_tree = None
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 self.errors = []
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 @property
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 def path(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 return os.path.join(self.base_dir, self.rel_input)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 @property
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
57 def was_prepared(self):
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
58 return bool(self.flags & FLAG_PREPARED)
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
59
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
60 @property
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 def was_processed(self):
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
62 return (self.was_prepared and
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
63 (bool(self.flags & FLAG_PROCESSED) or len(self.errors) > 0))
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 @property
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 def was_processed_successfully(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 return self.was_processed and not self.errors
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 class TransitionalProcessorPipelineRecord(TransitionalRecord):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 def __init__(self, previous_path=None):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 super(TransitionalProcessorPipelineRecord, self).__init__(
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 ProcessorPipelineRecord, previous_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 def getTransitionKey(self, entry):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 return entry.rel_input
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 def getPreviousEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 pair = self.transitions.get(rel_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 if pair is not None:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 return pair[0]
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 return None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 def collapseRecords(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 for prev, cur in self.transitions.values():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 if prev and cur and not cur.was_processed:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 # This asset wasn't processed, so the information from
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 # last time is still valid.
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 cur.flags = prev.flags
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 cur.rel_outputs = list(prev.rel_outputs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 cur.errors = list(prev.errors)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 def getDeletions(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 for prev, cur in self.transitions.values():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 if prev and not cur:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 for p in prev.rel_outputs:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 abs_p = os.path.join(self.previous.out_dir, p)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 yield (abs_p, 'previous asset was removed')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 elif prev and cur and cur.was_processed_successfully:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 diff = set(prev.rel_outputs) - set(cur.rel_outputs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 for p in diff:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 abs_p = os.path.join(self.previous.out_dir, p)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 yield (abs_p, 'asset changed outputs')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104