Mercurial > piecrust2

annotate piecrust/processing/records.py @ 999:46025a1b5434

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugins: Support multiple customizable plugins directories. Also add support for specifying the theme directory in some customizable paths.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 21 Nov 2017 09:54:56 -0800
parents 9a92e2804562
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
2 import hashlib
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 from piecrust.records import Record, TransitionalRecord
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 class ProcessorPipelineRecord(Record):
753
9a92e2804562 bake: Add the list of deleted files to the bake/processing records.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
7 RECORD_VERSION = 7
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 def __init__(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 super(ProcessorPipelineRecord, self).__init__()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 self.out_dir = None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 self.process_time = None
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
13 self.processed_count = 0
753
9a92e2804562 bake: Add the list of deleted files to the bake/processing records.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
14 self.deleted = []
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 191
diff changeset
15 self.success = False
133
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
16
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 FLAG_NONE = 0
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
19 FLAG_PREPARED = 2**0
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
20 FLAG_PROCESSED = 2**1
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
21 FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**3
550
6f216c1ab6b1 bake: Add a flag to know which record entries got collapsed from last run.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
22 FLAG_COLLAPSED_FROM_LAST_RUN = 2**4
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
25 def _get_transition_key(path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
26 return hashlib.md5(path.encode('utf8')).hexdigest()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
27
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
28
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 class ProcessorPipelineRecordEntry(object):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
30 def __init__(self, path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
31 self.path = path
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 self.flags = FLAG_NONE
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 self.rel_outputs = []
191
308d5180bf81 processing: Add more information to the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 133
diff changeset
35 self.proc_tree = None
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self.errors = []
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 @property
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
39 def was_prepared(self):
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
40 return bool(self.flags & FLAG_PREPARED)
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
41
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
42 @property
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 def was_processed(self):
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
44 return (self.was_prepared and
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
45 (bool(self.flags & FLAG_PROCESSED) or len(self.errors) > 0))
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 @property
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 def was_processed_successfully(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 return self.was_processed and not self.errors
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
550
6f216c1ab6b1 bake: Add a flag to know which record entries got collapsed from last run.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
51 @property
6f216c1ab6b1 bake: Add a flag to know which record entries got collapsed from last run.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
52 def was_collapsed_from_last_run(self):
6f216c1ab6b1 bake: Add a flag to know which record entries got collapsed from last run.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
53 return self.flags & FLAG_COLLAPSED_FROM_LAST_RUN
6f216c1ab6b1 bake: Add a flag to know which record entries got collapsed from last run.
Ludovic Chabant <ludovic@chabant.com>
parents: 414
diff changeset
54
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 class TransitionalProcessorPipelineRecord(TransitionalRecord):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 def __init__(self, previous_path=None):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 super(TransitionalProcessorPipelineRecord, self).__init__(
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 ProcessorPipelineRecord, previous_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 def getTransitionKey(self, entry):
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
62 return _get_transition_key(entry.path)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
64 def getCurrentEntry(self, path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
65 key = _get_transition_key(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
66 pair = self.transitions.get(key)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
67 if pair is not None:
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
68 return pair[1]
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
69 return None
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
70
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
71 def getPreviousEntry(self, path):
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
72 key = _get_transition_key(path)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 221
diff changeset
73 pair = self.transitions.get(key)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 if pair is not None:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 return pair[0]
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 return None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 def collapseRecords(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 for prev, cur in self.transitions.values():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 if prev and cur and not cur.was_processed:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 # This asset wasn't processed, so the information from
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 # last time is still valid.
753
9a92e2804562 bake: Add the list of deleted files to the bake/processing records.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
83 cur.flags = (prev.flags
9a92e2804562 bake: Add the list of deleted files to the bake/processing records.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
84 & ~FLAG_PROCESSED
9a92e2804562 bake: Add the list of deleted files to the bake/processing records.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
85 | FLAG_COLLAPSED_FROM_LAST_RUN)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 cur.rel_outputs = list(prev.rel_outputs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 cur.errors = list(prev.errors)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 def getDeletions(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 for prev, cur in self.transitions.values():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 if prev and not cur:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92 for p in prev.rel_outputs:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 abs_p = os.path.join(self.previous.out_dir, p)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 yield (abs_p, 'previous asset was removed')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 elif prev and cur and cur.was_processed_successfully:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 diff = set(prev.rel_outputs) - set(cur.rel_outputs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 for p in diff:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 abs_p = os.path.join(self.previous.out_dir, p)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 yield (abs_p, 'asset changed outputs')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100