Mercurial > piecrust2

comparison piecrust/processing/tree.py @ 3:f485ba500df3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Gigantic change to basically make PieCrust 2 vaguely functional. - Serving works, with debug window. - Baking works, multi-threading, with dependency handling. - Various things not implemented yet.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 10 Aug 2014 23:43:16 -0700
parents
children 474c9882decf
comparison
equal deleted inserted replaced
2:40fa08b261b9 3:f485ba500df3
1 import os
2 import time
3 import os.path
4 import logging
5
6
7 logger = logging.getLogger(__name__)
8
9
10 STATE_UNKNOWN = 0
11 STATE_DIRTY = 1
12 STATE_CLEAN = 2
13
14
15 FORCE_BUILD = object()
16
17
18 class ProcessingTreeError(Exception):
19 pass
20
21
22 class ProcessorNotFoundError(ProcessingTreeError):
23 pass
24
25
26 class ProcessingTreeNode(object):
27 def __init__(self, path, available_procs, level=0):
28 self.path = path
29 self.available_procs = available_procs
30 self.outputs = []
31 self.level = level
32 self.state = STATE_UNKNOWN
33 self._processor = None
34
35 def getProcessor(self):
36 if self._processor is None:
37 _, ext = os.path.splitext(self.path)
38 for p in self.available_procs:
39 if p.supportsExtension(ext):
40 self._processor = p
41 self.available_procs.remove(p)
42 break
43 else:
44 raise ProcessorNotFoundError()
45 return self._processor
46
47 def setState(self, state, recursive=True):
48 self.state = state
49 if recursive:
50 for o in self.outputs:
51 o.setState(state, True)
52
53 @property
54 def is_leaf(self):
55 return len(self.outputs) == 0
56
57 def getLeaves(self):
58 if self.is_leaf:
59 return [self]
60 leaves = []
61 for o in self.outputs:
62 for l in o.getLeaves():
63 leaves.append(l)
64 return leaves
65
66
67 class ProcessingTreeBuilder(object):
68 def __init__(self, processors):
69 self.processors = processors
70
71 def build(self, path):
72 start_time = time.clock()
73 tree_root = ProcessingTreeNode(path, list(self.processors))
74
75 loop_guard = 100
76 walk_stack = [tree_root]
77 while len(walk_stack) > 0:
78 loop_guard -= 1
79 if loop_guard <= 0:
80 raise ProcessingTreeError("Infinite loop detected!")
81
82 cur_node = walk_stack.pop()
83 proc = cur_node.getProcessor()
84
85 # If the root tree node (and only that one) wants to bypass this
86 # whole tree business, so be it.
87 if proc.is_bypassing_structured_processing:
88 if proc != tree_root:
89 raise ProcessingTreeError("Only root processors can "
90 "bypass structured processing.")
91 break
92
93 # Get the destination directory and output files.
94 rel_dir, basename = os.path.split(cur_node.path)
95 out_names = proc.getOutputFilenames(basename)
96 if out_names is None:
97 continue
98
99 for n in out_names:
100 out_node = ProcessingTreeNode(
101 os.path.join(rel_dir, n),
102 list(cur_node.available_procs),
103 cur_node.level + 1)
104 cur_node.outputs.append(out_node)
105
106 if proc.PROCESSOR_NAME != 'copy':
107 walk_stack.append(out_node)
108
109 logger.debug(format_timed(start_time, "Built processing tree for: %s" % path))
110 return tree_root
111
112
113 class ProcessingTreeRunner(object):
114 def __init__(self, base_dir, tmp_dir, out_dir, lock=None):
115 self.base_dir = base_dir
116 self.tmp_dir = tmp_dir
117 self.out_dir = out_dir
118 self.lock = lock
119
120 def processSubTree(self, tree_root):
121 did_process = False
122 walk_stack = [tree_root]
123 while len(walk_stack) > 0:
124 cur_node = walk_stack.pop()
125
126 self._computeNodeState(cur_node)
127 if cur_node.state == STATE_DIRTY:
128 did_process_this_node = self.processNode(cur_node)
129 did_process |= did_process_this_node
130
131 if did_process_this_node:
132 for o in cur_node.outputs:
133 if not o.is_leaf:
134 walk_stack.append(o)
135 else:
136 for o in cur_node.outputs:
137 if not o.is_leaf:
138 walk_stack.append(o)
139 return did_process
140
141 def processNode(self, node):
142 full_path = self._getNodePath(node)
143 proc = node.getProcessor()
144 if proc.is_bypassing_structured_processing:
145 try:
146 start_time = time.clock()
147 proc.process(full_path, self.out_dir)
148 print_node(format_timed(start_time, "(bypassing structured processing)"))
149 return True
150 except Exception as e:
151 import sys
152 _, __, traceback = sys.exc_info()
153 raise Exception("Error processing: %s" % node.path, e), None, traceback
154
155 # All outputs of a node must go to the same directory, so we can get
156 # the output directory off of the first output.
157 base_out_dir = self._getNodeBaseDir(node.outputs[0])
158 rel_out_dir = os.path.dirname(node.path)
159 out_dir = os.path.join(base_out_dir, rel_out_dir)
160 if not os.path.isdir(out_dir):
161 if self.lock:
162 with self.lock:
163 if not os.path.isdir(out_dir):
164 os.makedirs(out_dir, 0755)
165 else:
166 os.makedirs(out_dir, 0755)
167
168 try:
169 start_time = time.clock()
170 proc_res = proc.process(full_path, out_dir)
171 if proc_res is None:
172 raise Exception("Processor '%s' didn't return a boolean "
173 "result value." % proc)
174 if proc_res:
175 print_node(node, "-> %s" % out_dir)
176 return True
177 else:
178 print_node(node, "-> %s [clean]" % out_dir)
179 return False
180 except Exception as e:
181 import sys
182 _, __, traceback = sys.exc_info()
183 raise Exception("Error processing: %s" % node.path, e), None, traceback
184
185 def _computeNodeState(self, node):
186 if node.state != STATE_UNKNOWN:
187 return
188
189 proc = node.getProcessor()
190 if (proc.is_bypassing_structured_processing or
191 not proc.is_delegating_dependency_check):
192 # This processor wants to handle things on its own...
193 node.setState(STATE_DIRTY, False)
194 return
195
196 start_time = time.clock()
197
198 # Get paths and modification times for the input path and
199 # all dependencies (if any).
200 base_dir = self._getNodeBaseDir(node)
201 full_path = os.path.join(base_dir, node.path)
202 in_mtime = (full_path, os.path.getmtime(full_path))
203 force_build = False
204 try:
205 deps = proc.getDependencies(full_path)
206 if deps == FORCE_BUILD:
207 force_build = True
208 elif deps is not None:
209 for dep in deps:
210 dep_mtime = os.path.getmtime(dep)
211 if dep_mtime > in_mtime[1]:
212 in_mtime = (dep, dep_mtime)
213 except Exception as e:
214 logger.warning("%s -- Will force-bake: %s" % (e, node.path))
215 node.setState(STATE_DIRTY, True)
216 return
217
218 if force_build:
219 # Just do what the processor told us to do.
220 node.setState(STATE_DIRTY, True)
221 message = "Processor requested a forced build."
222 print_node(node, message)
223 else:
224 # Get paths and modification times for the outputs.
225 message = None
226 for o in node.outputs:
227 full_out_path = self._getNodePath(o)
228 if not os.path.isfile(full_out_path):
229 message = "Output '%s' doesn't exist." % o.path
230 break
231 o_mtime = os.path.getmtime(full_out_path)
232 if o_mtime < in_mtime[1]:
233 message = "Input '%s' is newer than output '%s'." % (
234 in_mtime[0], o.path)
235 break
236 if message is not None:
237 node.setState(STATE_DIRTY, True)
238 message += " Re-processing sub-tree."
239 print_node(node, message)
240 else:
241 node.setState(STATE_CLEAN, False)
242
243 state = "dirty" if node.state == STATE_DIRTY else "clean"
244 logger.debug(format_timed(start_time, "Computed node dirtyness: %s" % state, node.level))
245
246 def _getNodeBaseDir(self, node):
247 if node.level == 0:
248 return self.base_dir
249 if node.is_leaf:
250 return self.out_dir
251 return os.path.join(self.tmp_dir, str(node.level))
252
253 def _getNodePath(self, node):
254 base_dir = self._getNodeBaseDir(node)
255 return os.path.join(base_dir, node.path)
256
257
258 def print_node(node, message=None, recursive=False):
259 indent = ' ' * node.level
260 try:
261 proc_name = node.getProcessor().PROCESSOR_NAME
262 except ProcessorNotFoundError:
263 proc_name = 'n/a'
264
265 message = message or ''
266 logger.debug('%s%s [%s] %s' % (indent, node.path, proc_name, message))
267
268 if recursive:
269 for o in node.outputs:
270 print_node(o, None, True)
271
272
273 def format_timed(start_time, message, indent_level=0):
274 end_time = time.clock()
275 indent = indent_level * ' '
276 build_time = '{0:8.1f} ms'.format((end_time - start_time) / 1000.0)
277 return "%s[%s] %s" % (indent, build_time, message)
278