annotate piecrust/processing/records.py @ 411:e7b865f8f335

bake: Enable multiprocess baking. Baking is now done by running a worker per CPU, and sending jobs to them. This changes several things across the codebase: * Ability to not cache things related to pages other than the 'main' page (i.e. the page at the bottom of the execution stack). * Decouple the baking process from the bake records, so only the main process keeps track (and modifies) the bake record. * Remove the need for 'batch page getters' and loading a page directly from the page factories. There are various smaller changes too included here, including support for scope performance timers that are saved with the bake record and can be printed out to the console. Yes I got carried away. For testing, the in-memory 'mock' file-system doesn't work anymore, since we're spawning processes, so this is replaced by a 'tmpfs' file-system which is saved in temporary files on disk and deleted after tests have run.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 Jun 2015 17:09:19 -0700
parents f82262f59600
children c4b3a7fd2f87
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os.path
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 from piecrust.records import Record, TransitionalRecord
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 class ProcessorPipelineRecord(Record):
216
c5ada46b281a internal: Bump the processing record version.
Ludovic Chabant <ludovic@chabant.com>
parents: 215
diff changeset
6 RECORD_VERSION = 4
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 def __init__(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 super(ProcessorPipelineRecord, self).__init__()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 self.out_dir = None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 self.process_time = None
215
a47580a0955b bake: Better error handling for the processing pipeline.
Ludovic Chabant <ludovic@chabant.com>
parents: 191
diff changeset
12 self.success = False
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 def hasOverrideEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 return self.findEntry(rel_path) is not None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 def findEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 rel_path = rel_path.lower()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 for entry in self.entries:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 for out_path in entry.rel_outputs:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 if out_path.lower() == rel_path:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 return entry
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 return None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
133
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
25 def replaceEntry(self, new_entry):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
26 for e in self.entries:
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
27 if (e.base_dir == new_entry.base_dir and
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
28 e.rel_input == new_entry.rel_input):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
29 e.flags = new_entry.flags
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
30 e.rel_outputs = list(new_entry.rel_outputs)
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
31 e.errors = list(new_entry.errors)
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
32 break
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
33
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 FLAG_NONE = 0
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
36 FLAG_PREPARED = 2**0
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
37 FLAG_PROCESSED = 2**1
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
38 FLAG_OVERRIDEN = 2**2
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
39 FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**3
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 class ProcessorPipelineRecordEntry(object):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 def __init__(self, base_dir, rel_input):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 self.base_dir = base_dir
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 self.rel_input = rel_input
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 self.flags = FLAG_NONE
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 self.rel_outputs = []
191
308d5180bf81 processing: Add more information to the pipeline record.
Ludovic Chabant <ludovic@chabant.com>
parents: 133
diff changeset
49 self.proc_tree = None
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 self.errors = []
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 @property
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 def path(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 return os.path.join(self.base_dir, self.rel_input)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 @property
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
57 def was_prepared(self):
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
58 return bool(self.flags & FLAG_PREPARED)
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
59
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
60 @property
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 def was_processed(self):
221
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
62 return (self.was_prepared and
f82262f59600 bake: Fix processing record bugs and error logging for external processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 216
diff changeset
63 (bool(self.flags & FLAG_PROCESSED) or len(self.errors) > 0))
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 @property
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 def was_processed_successfully(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 return self.was_processed and not self.errors
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 class TransitionalProcessorPipelineRecord(TransitionalRecord):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 def __init__(self, previous_path=None):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 super(TransitionalProcessorPipelineRecord, self).__init__(
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 ProcessorPipelineRecord, previous_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 def getTransitionKey(self, entry):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 return entry.rel_input
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 def getPreviousEntry(self, rel_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 pair = self.transitions.get(rel_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 if pair is not None:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 return pair[0]
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 return None
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 def collapseRecords(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 for prev, cur in self.transitions.values():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 if prev and cur and not cur.was_processed:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 # This asset wasn't processed, so the information from
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 # last time is still valid.
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 cur.flags = prev.flags
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 cur.rel_outputs = list(prev.rel_outputs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 cur.errors = list(prev.errors)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 def getDeletions(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 for prev, cur in self.transitions.values():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 if prev and not cur:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 for p in prev.rel_outputs:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 abs_p = os.path.join(self.previous.out_dir, p)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 yield (abs_p, 'previous asset was removed')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 elif prev and cur and cur.was_processed_successfully:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 diff = set(prev.rel_outputs) - set(cur.rel_outputs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 for p in diff:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 abs_p = os.path.join(self.previous.out_dir, p)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 yield (abs_p, 'asset changed outputs')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104