Mercurial > piecrust2
annotate piecrust/processing/compass.py @ 411:e7b865f8f335
bake: Enable multiprocess baking.
Baking is now done by running a worker per CPU, and sending jobs to them.
This changes several things across the codebase:
* Ability to not cache things related to pages other than the 'main' page
(i.e. the page at the bottom of the execution stack).
* Decouple the baking process from the bake records, so only the main process
keeps track (and modifies) the bake record.
* Remove the need for 'batch page getters' and loading a page directly from
the page factories.
There are various smaller changes too included here, including support for
scope performance timers that are saved with the bake record and can be
printed out to the console. Yes I got carried away.
For testing, the in-memory 'mock' file-system doesn't work anymore, since
we're spawning processes, so this is replaced by a 'tmpfs' file-system which
is saved in temporary files on disk and deleted after tests have run.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Fri, 12 Jun 2015 17:09:19 -0700 |
parents | 154b8df04829 |
children | c4b3a7fd2f87 |
rev | line source |
---|---|
196
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import os |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import os.path |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 import logging |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 import platform |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 import subprocess |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 from piecrust.processing.base import Processor, PRIORITY_FIRST |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 from piecrust.uriutil import multi_replace |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 logger = logging.getLogger(__name__) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
13 class CompassProcessor(Processor): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
14 PROCESSOR_NAME = 'compass' |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
15 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
16 STATE_UNKNOWN = 0 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 STATE_INACTIVE = 1 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 STATE_ACTIVE = 2 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 def __init__(self): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 super(CompassProcessor, self).__init__() |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
22 # Using a high priority is needed to get to the `.scss` files before |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 # the Sass processor. |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 self.priority = PRIORITY_FIRST |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 self.is_bypassing_structured_processing = True |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 self.is_delegating_dependency_check = False |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 self._state = self.STATE_UNKNOWN |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
29 def initialize(self, app): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 super(CompassProcessor, self).initialize(app) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 def onPipelineStart(self, pipeline): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 super(CompassProcessor, self).onPipelineStart(pipeline) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 self._maybeActivate(pipeline) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 def onPipelineEnd(self, pipeline): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 super(CompassProcessor, self).onPipelineEnd(pipeline) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 self._maybeRunCompass(pipeline) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
39 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
40 def matches(self, path): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
41 if self._state != self.STATE_ACTIVE: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 return False |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
43 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
44 _, ext = os.path.splitext(path) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 return ext == '.scss' or ext == '.sass' |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 def getDependencies(self, path): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 raise Exception("Compass processor should handle dependencies by " |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 "itself.") |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 def getOutputFilenames(self, filename): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 raise Exception("Compass processor should handle outputs by itself.") |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 def process(self, path, out_dir): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 if path.startswith(self.app.theme_dir): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 if not self._runInTheme: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
57 logger.debug("Scheduling Compass execution in theme directory " |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 "after the pipeline is done.") |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
59 self._runInTheme = True |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
60 else: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
61 if not self._runInSite: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
62 logger.debug("Scheduling Compass execution after the pipeline " |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
63 "is done.") |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
64 self._runInSite = True |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
65 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
66 def _maybeActivate(self, pipeline): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
67 if self._state != self.STATE_UNKNOWN: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
68 return |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 config = self.app.config.get('compass') |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 if config is None or not config.get('enable'): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 logger.debug("Compass processing is disabled (set " |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 "`compass/enable` to `true` to enable it).") |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 self._state = self.STATE_INACTIVE |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 return |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 logger.debug("Activating Compass processing for SCSS/SASS files.") |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
78 self._state = self.STATE_ACTIVE |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
79 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
80 bin_path = config.get('bin', 'compass') |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
81 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 config_path = config.get('config_path', 'config.rb') |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
83 config_path = os.path.join(self.app.root_dir, config_path) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
84 if not os.path.exists(config_path): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
85 raise Exception("Can't find Compass configuration file: %s" % |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
86 config_path) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
87 self._args = '%s compile --config "%s"' % (bin_path, config_path) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
88 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
89 frameworks = config.get('frameworks', []) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
90 if not isinstance(frameworks, list): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
91 frameworks = frameworks.split(',') |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
92 for f in frameworks: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
93 self._args += ' --load %s' % f |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
94 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
95 custom_args = config.get('options') |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
96 if custom_args: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
97 self._args += ' ' + custom_args |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
98 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
99 out_dir = pipeline.out_dir |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
100 tmp_dir = os.path.join(pipeline.tmp_dir, 'compass') |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
101 self._args = multi_replace( |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
102 self._args, |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
103 {'%out_dir%': out_dir, |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
104 '%tmp_dir%': tmp_dir}) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
105 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
106 self._runInSite = False |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
107 self._runInTheme = False |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
108 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
109 def _maybeRunCompass(self, pipeline): |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
110 if self._state != self.STATE_ACTIVE: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
111 return |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
112 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
113 logger.debug("Running Compass with:") |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
114 logger.debug(self._args) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
115 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
116 prev_cwd = os.getcwd() |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
117 os.chdir(self.app.root_dir) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
118 try: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
119 retcode = subprocess.call(self._args, shell=True) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
120 except FileNotFoundError as ex: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
121 logger.error("Tried running Compass with command: %s" % |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
122 self._args) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
123 raise Exception("Error running Compass. " |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
124 "Did you install it?") from ex |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
125 finally: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
126 os.chdir(prev_cwd) |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
127 |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
128 if retcode != 0: |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
129 raise Exception("Error occured in Compass. Please check " |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
130 "log messages above for more information.") |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
131 return True |
154b8df04829
processing: Add Compass and Sass processors.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
132 |