Mercurial > piecrust2

annotate piecrust/processing/records.py @ 215:a47580a0955b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Better error handling for the processing pipeline. Pipeline jobs now keep track of whether they've seen any errors. This is aggregated into an overall "success" flag for the processing record. Also, jobs keep going as long as there's no critical (i.e. internal) failure happening. Errors raised by processors are also better tracked: the actual processor that failed, along with the input file, are tracks in the processing record. The `bake` command returns a failure exit code if processing saw any error.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 31 Jan 2015 17:08:02 -0800
parents 308d5180bf81
children c5ada46b281a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 from piecrust.records import Record, TransitionalRecord
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 class ProcessorPipelineRecord(Record):
191
308d5180bf81 processing: Add more information to the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 133
diff changeset
6 RECORD_VERSION = 3
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 def __init__(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 super(ProcessorPipelineRecord, self).__init__()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 self.out_dir = None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 self.process_time = None
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 191
diff changeset
12 self.success = False
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 def hasOverrideEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 return self.findEntry(rel_path) is not None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 def findEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 rel_path = rel_path.lower()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 for entry in self.entries:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 for out_path in entry.rel_outputs:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 if out_path.lower() == rel_path:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 return entry
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 return None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
133
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
25 def replaceEntry(self, new_entry):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
26 for e in self.entries:
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
27 if (e.base_dir == new_entry.base_dir and
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
28 e.rel_input == new_entry.rel_input):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
29 e.flags = new_entry.flags
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
30 e.rel_outputs = list(new_entry.rel_outputs)
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
31 e.errors = list(new_entry.errors)
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
32 break
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
33
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 FLAG_NONE = 0
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 FLAG_PROCESSED = 2**0
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 FLAG_OVERRIDEN = 2**1
191
308d5180bf81 processing: Add more information to the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 133
diff changeset
38 FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**2
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 class ProcessorPipelineRecordEntry(object):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 def __init__(self, base_dir, rel_input):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 self.base_dir = base_dir
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 self.rel_input = rel_input
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 self.flags = FLAG_NONE
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 self.rel_outputs = []
191
308d5180bf81 processing: Add more information to the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 133
diff changeset
48 self.proc_tree = None
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 self.errors = []
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 @property
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 def path(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 return os.path.join(self.base_dir, self.rel_input)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 @property
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 def was_processed(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 return bool(self.flags & FLAG_PROCESSED)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 @property
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 def was_processed_successfully(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 return self.was_processed and not self.errors
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 class TransitionalProcessorPipelineRecord(TransitionalRecord):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 def __init__(self, previous_path=None):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 super(TransitionalProcessorPipelineRecord, self).__init__(
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 ProcessorPipelineRecord, previous_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 def getTransitionKey(self, entry):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 return entry.rel_input
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 def getPreviousEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 pair = self.transitions.get(rel_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 if pair is not None:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 return pair[0]
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 return None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 def collapseRecords(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 for prev, cur in self.transitions.values():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 if prev and cur and not cur.was_processed:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 # This asset wasn't processed, so the information from
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 # last time is still valid.
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 cur.flags = prev.flags
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 cur.rel_outputs = list(prev.rel_outputs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 cur.errors = list(prev.errors)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 def getDeletions(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 for prev, cur in self.transitions.values():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 if prev and not cur:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 for p in prev.rel_outputs:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 abs_p = os.path.join(self.previous.out_dir, p)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92 yield (abs_p, 'previous asset was removed')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 elif prev and cur and cur.was_processed_successfully:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 diff = set(prev.rel_outputs) - set(cur.rel_outputs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 for p in diff:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 abs_p = os.path.join(self.previous.out_dir, p)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 yield (abs_p, 'asset changed outputs')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98