Mercurial > piecrust2

comparison piecrust/processing/records.py @ 414:c4b3a7fd2f87

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Make pipeline processing multi-process. Not many changes here, as it's pretty straightforward, but an API change for processors so they know if they're being initialized/disposed from the main process or from one of the workers. This makes it possible to do global stuff that has side-effects (e.g. create a directory) vs. doing in-memory stuff.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:20:30 -0700
parents f82262f59600
children 6f216c1ab6b1
comparison
equal deleted inserted replaced
413:eacf0a3afd0c 414:c4b3a7fd2f87
1 import os.path 1 import os.path
2 import hashlib
2 from piecrust.records import Record, TransitionalRecord 3 from piecrust.records import Record, TransitionalRecord
3 4
4 5
5 class ProcessorPipelineRecord(Record): 6 class ProcessorPipelineRecord(Record):
6 RECORD_VERSION = 4 7 RECORD_VERSION = 5
7 8
8 def __init__(self): 9 def __init__(self):
9 super(ProcessorPipelineRecord, self).__init__() 10 super(ProcessorPipelineRecord, self).__init__()
10 self.out_dir = None 11 self.out_dir = None
11 self.process_time = None 12 self.process_time = None
13 self.processed_count = 0
12 self.success = False 14 self.success = False
13 15 self.timers = None
14 def hasOverrideEntry(self, rel_path):
15 return self.findEntry(rel_path) is not None
16
17 def findEntry(self, rel_path):
18 rel_path = rel_path.lower()
19 for entry in self.entries:
20 for out_path in entry.rel_outputs:
21 if out_path.lower() == rel_path:
22 return entry
23 return None
24
25 def replaceEntry(self, new_entry):
26 for e in self.entries:
27 if (e.base_dir == new_entry.base_dir and
28 e.rel_input == new_entry.rel_input):
29 e.flags = new_entry.flags
30 e.rel_outputs = list(new_entry.rel_outputs)
31 e.errors = list(new_entry.errors)
32 break
33 16
34 17
35 FLAG_NONE = 0 18 FLAG_NONE = 0
36 FLAG_PREPARED = 2**0 19 FLAG_PREPARED = 2**0
37 FLAG_PROCESSED = 2**1 20 FLAG_PROCESSED = 2**1
38 FLAG_OVERRIDEN = 2**2
39 FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**3 21 FLAG_BYPASSED_STRUCTURED_PROCESSING = 2**3
40 22
41 23
24 def _get_transition_key(path):
25 return hashlib.md5(path.encode('utf8')).hexdigest()
26
27
42 class ProcessorPipelineRecordEntry(object): 28 class ProcessorPipelineRecordEntry(object):
43 def __init__(self, base_dir, rel_input): 29 def __init__(self, path):
44 self.base_dir = base_dir 30 self.path = path
45 self.rel_input = rel_input
46 31
47 self.flags = FLAG_NONE 32 self.flags = FLAG_NONE
48 self.rel_outputs = [] 33 self.rel_outputs = []
49 self.proc_tree = None 34 self.proc_tree = None
50 self.errors = [] 35 self.errors = []
51
52 @property
53 def path(self):
54 return os.path.join(self.base_dir, self.rel_input)
55 36
56 @property 37 @property
57 def was_prepared(self): 38 def was_prepared(self):
58 return bool(self.flags & FLAG_PREPARED) 39 return bool(self.flags & FLAG_PREPARED)
59 40
71 def __init__(self, previous_path=None): 52 def __init__(self, previous_path=None):
72 super(TransitionalProcessorPipelineRecord, self).__init__( 53 super(TransitionalProcessorPipelineRecord, self).__init__(
73 ProcessorPipelineRecord, previous_path) 54 ProcessorPipelineRecord, previous_path)
74 55
75 def getTransitionKey(self, entry): 56 def getTransitionKey(self, entry):
76 return entry.rel_input 57 return _get_transition_key(entry.path)
77 58
78 def getPreviousEntry(self, rel_path): 59 def getCurrentEntry(self, path):
79 pair = self.transitions.get(rel_path) 60 key = _get_transition_key(path)
61 pair = self.transitions.get(key)
62 if pair is not None:
63 return pair[1]
64 return None
65
66 def getPreviousEntry(self, path):
67 key = _get_transition_key(path)
68 pair = self.transitions.get(key)
80 if pair is not None: 69 if pair is not None:
81 return pair[0] 70 return pair[0]
82 return None 71 return None
83 72
84 def collapseRecords(self): 73 def collapseRecords(self):