Mercurial > piecrust2
annotate piecrust/processing/records.py @ 415:0e9a94b7fdfa
bake: Improve bake record information.
* Store things in the bake record that require less interaction between the
master process and the workers. For instance, don't store the paginator
object in the render pass info -- instead, just store whether pagination
was used, and whether it had more items.
* Simplify information passing between workers and bake passes by saving the
rendering info to the JSON cache. This means the "render first sub" job
doesn't have to return anything except errors now.
* Add more performance counter info.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 20 Jun 2015 19:23:16 -0700 |
parents | c4b3a7fd2f87 |
children | 6f216c1ab6b1 |
rev | line source |
---|---|
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import os.path |
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
2 import hashlib |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 from piecrust.records import Record, TransitionalRecord |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 class ProcessorPipelineRecord(Record): |
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
7 RECORD_VERSION = 5 |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 def __init__(self): |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 super(ProcessorPipelineRecord, self).__init__() |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 self.out_dir = None |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 self.process_time = None |
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
13 self.processed_count = 0 |
215
a47580a0955b
bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents:
191
diff
changeset
|
14 self.success = False |
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
15 self.timers = None |
133
9e4c2e68a129
Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents:
120
diff
changeset
|
16 |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 FLAG_NONE = 0 |
221
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
19 FLAG_PREPARED = 2**0 |
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
20 FLAG_PROCESSED = 2**1 |
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
21 FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**3 |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 |
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
24 def _get_transition_key(path): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
25 return hashlib.md5(path.encode('utf8')).hexdigest() |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
26 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
27 |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 class ProcessorPipelineRecordEntry(object): |
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
29 def __init__(self, path): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
30 self.path = path |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 self.flags = FLAG_NONE |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 self.rel_outputs = [] |
191
308d5180bf81
processing: Add more information to the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents:
133
diff
changeset
|
34 self.proc_tree = None |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 self.errors = [] |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 @property |
221
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
38 def was_prepared(self): |
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
39 return bool(self.flags & FLAG_PREPARED) |
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
40 |
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
41 @property |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 def was_processed(self): |
221
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
43 return (self.was_prepared and |
f82262f59600
bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents:
216
diff
changeset
|
44 (bool(self.flags & FLAG_PROCESSED) or len(self.errors) > 0)) |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 @property |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 def was_processed_successfully(self): |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 return self.was_processed and not self.errors |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 class TransitionalProcessorPipelineRecord(TransitionalRecord): |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 def __init__(self, previous_path=None): |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 super(TransitionalProcessorPipelineRecord, self).__init__( |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 ProcessorPipelineRecord, previous_path) |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 def getTransitionKey(self, entry): |
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
57 return _get_transition_key(entry.path) |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 |
414
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
59 def getCurrentEntry(self, path): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
60 key = _get_transition_key(path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
61 pair = self.transitions.get(key) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
62 if pair is not None: |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
63 return pair[1] |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
64 return None |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
65 |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
66 def getPreviousEntry(self, path): |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
67 key = _get_transition_key(path) |
c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents:
221
diff
changeset
|
68 pair = self.transitions.get(key) |
120
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 if pair is not None: |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 return pair[0] |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 return None |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 def collapseRecords(self): |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 for prev, cur in self.transitions.values(): |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 if prev and cur and not cur.was_processed: |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 # This asset wasn't processed, so the information from |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 # last time is still valid. |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
78 cur.flags = prev.flags |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
79 cur.rel_outputs = list(prev.rel_outputs) |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
80 cur.errors = list(prev.errors) |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
81 |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 def getDeletions(self): |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
83 for prev, cur in self.transitions.values(): |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
84 if prev and not cur: |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
85 for p in prev.rel_outputs: |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
86 abs_p = os.path.join(self.previous.out_dir, p) |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
87 yield (abs_p, 'previous asset was removed') |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
88 elif prev and cur and cur.was_processed_successfully: |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
89 diff = set(prev.rel_outputs) - set(cur.rel_outputs) |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
90 for p in diff: |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
91 abs_p = os.path.join(self.previous.out_dir, p) |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
92 yield (abs_p, 'asset changed outputs') |
133845647083
Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
93 |