Mercurial > piecrust2
diff piecrust/processing/tree.py @ 414:c4b3a7fd2f87
bake: Make pipeline processing multi-process.
Not many changes here, as it's pretty straightforward, but an API change for
processors so they know if they're being initialized/disposed from the main
process or from one of the workers. This makes it possible to do global stuff
that has side-effects (e.g. create a directory) vs. doing in-memory stuff.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Sat, 20 Jun 2015 19:20:30 -0700 |
| parents | a47580a0955b |
| children | 7147b06670fd |
line wrap: on
line diff
--- a/piecrust/processing/tree.py Sat Jun 20 19:16:38 2015 -0700 +++ b/piecrust/processing/tree.py Sat Jun 20 19:20:30 2015 -0700 @@ -79,7 +79,6 @@ self.processors = processors def build(self, path): - start_time = time.clock() tree_root = ProcessingTreeNode(path, list(self.processors)) loop_guard = 100 @@ -97,7 +96,7 @@ if proc.is_bypassing_structured_processing: if cur_node != tree_root: raise ProcessingTreeError("Only root processors can " - "bypass structured processing.") + "bypass structured processing.") break # Get the destination directory and output files. @@ -116,18 +115,14 @@ if proc.PROCESSOR_NAME != 'copy': walk_stack.append(out_node) - logger.debug(format_timed( - start_time, "Built processing tree for: %s" % path, - colored=False)) return tree_root class ProcessingTreeRunner(object): - def __init__(self, base_dir, tmp_dir, out_dir, lock=None): + def __init__(self, base_dir, tmp_dir, out_dir): self.base_dir = base_dir self.tmp_dir = tmp_dir self.out_dir = out_dir - self.lock = lock def processSubTree(self, tree_root): did_process = False @@ -155,8 +150,9 @@ proc = node.getProcessor() if proc.is_bypassing_structured_processing: try: - start_time = time.clock() - proc.process(full_path, self.out_dir) + start_time = time.perf_counter() + with proc.app.env.timerScope(proc.__class__.__name__): + proc.process(full_path, self.out_dir) print_node( node, format_timed( @@ -172,16 +168,15 @@ rel_out_dir = os.path.dirname(node.path) out_dir = os.path.join(base_out_dir, rel_out_dir) if not os.path.isdir(out_dir): - if self.lock: - with self.lock: - if not os.path.isdir(out_dir): - os.makedirs(out_dir, 0o755) - else: - os.makedirs(out_dir, 0o755) + try: + os.makedirs(out_dir, 0o755, exist_ok=True) + except OSError: + pass try: - start_time = time.clock() - proc_res = proc.process(full_path, out_dir) + start_time = time.perf_counter() + with proc.app.env.timerScope(proc.__class__.__name__): + proc_res = proc.process(full_path, out_dir) if proc_res is None: raise Exception("Processor '%s' didn't return a boolean " "result value." % proc) @@ -200,12 +195,12 @@ proc = node.getProcessor() if (proc.is_bypassing_structured_processing or - not proc.is_delegating_dependency_check): + not proc.is_delegating_dependency_check): # This processor wants to handle things on its own... node.setState(STATE_DIRTY, False) return - start_time = time.clock() + start_time = time.perf_counter() # Get paths and modification times for the input path and # all dependencies (if any).