changeset 117:6827dcc9d3fb

Changes to the asset processing pipeline: * Add semi-functional RequireJS processor. * Processors now match on the relative path. * Support for processors that add more processors of their own. * A couple of related fixes.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 28 Oct 2014 08:20:38 -0700
parents 1c13f3389fcb
children e5f048799d61
files piecrust/plugins/builtin.py piecrust/processing/base.py piecrust/processing/requirejs.py piecrust/processing/tree.py
diffstat 4 files changed, 103 insertions(+), 13 deletions(-) [+]
line wrap: on
line diff
--- a/piecrust/plugins/builtin.py	Mon Oct 27 08:18:12 2014 -0700
+++ b/piecrust/plugins/builtin.py	Tue Oct 28 08:20:38 2014 -0700
@@ -16,6 +16,7 @@
 from piecrust.plugins.base import PieCrustPlugin
 from piecrust.processing.base import CopyFileProcessor
 from piecrust.processing.less import LessProcessor
+from piecrust.processing.requirejs import RequireJSProcessor
 from piecrust.processing.sitemap import SitemapProcessor
 from piecrust.sources.base import DefaultPageSource
 from piecrust.sources.posts import (FlatPostsSource, ShallowPostsSource,
@@ -74,6 +75,7 @@
         return [
                 CopyFileProcessor(),
                 LessProcessor(),
+                RequireJSProcessor(),
                 SitemapProcessor()]
 
     def getImporters(self):
--- a/piecrust/processing/base.py	Mon Oct 27 08:18:12 2014 -0700
+++ b/piecrust/processing/base.py	Tue Oct 28 08:20:38 2014 -0700
@@ -36,7 +36,7 @@
     def onPipelineEnd(self, pipeline):
         pass
 
-    def matches(self, filename):
+    def matches(self, path):
         return False
 
     def getDependencies(self, path):
@@ -56,7 +56,7 @@
         super(CopyFileProcessor, self).__init__()
         self.priority = PRIORITY_LAST
 
-    def matches(self, filename):
+    def matches(self, path):
         return True
 
     def getOutputFilenames(self, filename):
@@ -74,9 +74,9 @@
         super(SimpleFileProcessor, self).__init__()
         self.extensions = extensions or {}
 
-    def matches(self, filename):
+    def matches(self, path):
         for ext in self.extensions:
-            if filename.endswith('.' + ext):
+            if path.endswith('.' + ext):
                 return True
         return False
 
@@ -169,13 +169,20 @@
             self.processors))
 
     def run(self, src_dir_or_file=None):
-        record = ProcessorPipelineRecord()
+        # Invoke pre-processors.
+        for proc in self.processors:
+            proc.onPipelineStart(self)
+
+        # Sort our processors again in case the pre-process step involved
+        # patching the processors with some new ones.
+        self.processors.sort(key=lambda p: p.priority)
 
         # Create the workers.
         pool = []
         queue = Queue()
         abort = threading.Event()
         pipeline_lock = threading.Lock()
+        record = ProcessorPipelineRecord()
         for i in range(self.num_workers):
             ctx = ProcessingWorkerContext(self, record, queue, abort,
                     pipeline_lock)
@@ -183,10 +190,6 @@
             worker.start()
             pool.append(worker)
 
-        # Invoke pre-processors.
-        for proc in self.processors:
-            proc.onPipelineStart(self)
-
         if src_dir_or_file is not None:
             # Process only the given path.
             # Find out what mount point this is in.
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/piecrust/processing/requirejs.py	Tue Oct 28 08:20:38 2014 -0700
@@ -0,0 +1,83 @@
+import os
+import os.path
+import json
+import hashlib
+import logging
+import platform
+import subprocess
+from piecrust.processing.base import Processor, PRIORITY_FIRST
+from piecrust.processing.tree import FORCE_BUILD
+
+
+logger = logging.getLogger(__name__)
+
+
+class RequireJSProcessor(Processor):
+    PROCESSOR_NAME = 'requirejs'
+
+    def __init__(self):
+        super(RequireJSProcessor, self).__init__()
+        self.is_bypassing_structured_processing = True
+        self._conf = None
+
+    def initialize(self, app):
+        super(RequireJSProcessor, self).initialize(app)
+
+        self._conf = app.config.get('requirejs')
+        if self._conf is None:
+            return
+
+        if 'build_path' not in self._conf:
+            raise Exception("You need to specify `requirejs/build_path` "
+                            "for RequireJS.")
+        self._conf.setdefault('bin', 'r.js')
+        self._conf.setdefault('out_path', self._conf['build_path'])
+
+    def onPipelineStart(self, pipeline):
+        super(RequireJSProcessor, self).onPipelineStart(pipeline)
+
+        logger.debug("Adding Javascript suppressor to build pipeline.")
+        skip = _JavascriptSkipProcessor(self._conf['build_path'])
+        pipeline.processors.append(skip)
+
+    def matches(self, path):
+        return path == self._conf['build_path']
+
+    def getDependencies(self, path):
+        return FORCE_BUILD
+
+    def process(self, path, out_dir):
+        args = [self._conf['bin'], '-o', path]
+        shell = (platform.system() == 'Windows')
+        cwd = self.app.root_dir
+        logger.debug("Running RequireJS: %s" % ' '.join(args))
+        try:
+            retcode = subprocess.call(args, shell=shell, cwd=cwd)
+        except FileNotFoundError as ex:
+            logger.error("Tried running RequireJS processor "
+                         "with command: %s" % args)
+            raise Exception("Error running RequireJS. "
+                            "Did you install it?") from ex
+        if retcode != 0:
+            raise Exception("Error occured in RequireJS compiler. "
+                            "Please check log messages above for "
+                            "more information.")
+        return True
+
+
+class _JavascriptSkipProcessor(Processor):
+    PROCESSOR_NAME = 'requirejs_javascript_skip'
+
+    def __init__(self, except_path=None):
+        super(_JavascriptSkipProcessor, self).__init__()
+        self.priority = PRIORITY_FIRST
+        self.is_bypassing_structured_processing = True
+        self._except_path = except_path
+
+    def matches(self, path):
+        _, ext = os.path.splitext(path)
+        return ext == '.js' and path != self._except_path
+
+    def process(self, in_path, out_path):
+        return False
+
--- a/piecrust/processing/tree.py	Mon Oct 27 08:18:12 2014 -0700
+++ b/piecrust/processing/tree.py	Tue Oct 28 08:20:38 2014 -0700
@@ -34,9 +34,8 @@
 
     def getProcessor(self):
         if self._processor is None:
-            _, filename = os.path.split(self.path)
             for p in self.available_procs:
-                if p.matches(filename):
+                if p.matches(self.path):
                     self._processor = p
                     self.available_procs.remove(p)
                     break
@@ -85,7 +84,7 @@
             # If the root tree node (and only that one) wants to bypass this
             # whole tree business, so be it.
             if proc.is_bypassing_structured_processing:
-                if proc != tree_root:
+                if cur_node != tree_root:
                     raise ProcessingTreeError("Only root processors can "
                             "bypass structured processing.")
                 break
@@ -145,7 +144,10 @@
             try:
                 start_time = time.clock()
                 proc.process(full_path, self.out_dir)
-                print_node(format_timed(start_time, "(bypassing structured processing)"))
+                print_node(
+                        node,
+                        format_timed(
+                            start_time, "(bypassing structured processing)"))
                 return True
             except Exception as e:
                 raise Exception("Error processing: %s" % node.path) from e