Mercurial > piecrust2

annotate piecrust/processing/browserify.py @ 1188:a7c43131d871

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Fix file write flushing problem with Python 3.8+ Writing the cache files fails in Python 3.8 because it looks like flushing behaviour has changed. We need to explicitly flush. And even then, in very rare occurrences, it looks like it can still run into racing conditions, so we do a very hacky and ugly "retry" loop when fetching cached data :(
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 15 Jun 2021 22:36:23 -0700
parents a3dec0fbd9ce
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
3 import hashlib
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import platform
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 import subprocess
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from piecrust.processing.base import Processor, PRIORITY_FIRST, FORCE_BUILD
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 logger = logging.getLogger(__name__)
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 class BrowserifyProcessor(Processor):
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 PROCESSOR_NAME = 'browserify'
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 def __init__(self):
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 super(BrowserifyProcessor, self).__init__()
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 self.priority = PRIORITY_FIRST
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
19 self._tmp_dir = None
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 self._conf = None
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 def initialize(self, app):
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 super(BrowserifyProcessor, self).initialize(app)
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 self._conf = app.config.get('browserify')
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 if self._conf is None:
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 return
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 if self._conf is True:
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 self._conf = {}
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31
1112
a3dec0fbd9ce bake: Fix bug on Windows where shim scripts of NodeJS tools couldn't run.
Ludovic Chabant <ludovic@chabant.com>
parents: 1049
diff changeset
32 bin_name = 'browserify'
a3dec0fbd9ce bake: Fix bug on Windows where shim scripts of NodeJS tools couldn't run.
Ludovic Chabant <ludovic@chabant.com>
parents: 1049
diff changeset
33 if platform.system() == 'Windows':
a3dec0fbd9ce bake: Fix bug on Windows where shim scripts of NodeJS tools couldn't run.
Ludovic Chabant <ludovic@chabant.com>
parents: 1049
diff changeset
34 bin_name += '.cmd'
a3dec0fbd9ce bake: Fix bug on Windows where shim scripts of NodeJS tools couldn't run.
Ludovic Chabant <ludovic@chabant.com>
parents: 1049
diff changeset
35 self._conf.setdefault('bin', bin_name)
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
37 def onPipelineStart(self, ctx):
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
38 self._tmp_dir = ctx.tmp_dir
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
39
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 def matches(self, path):
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 return self._conf is not None and os.path.splitext(path)[1] == '.js'
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 def getDependencies(self, path):
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
44 deps_path = self._getDepListPath(path)
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
45 try:
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
46 with open(deps_path, 'r', encoding='utf8') as f:
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
47 deps_list = f.read()
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
48 except OSError:
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
49 logger.debug("No dependency list found for Browserify target '%s' "
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
50 "at '%s'. Rebuilding" % (path, deps_path))
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
51 return FORCE_BUILD
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
52
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
53 deps_list = [d.strip() for d in deps_list.split('\n')]
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
54 return filter(lambda d: d, deps_list)
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
1041
717ac3c4ee77 bake: Fix where the Browserify processor outputs things.
Ludovic Chabant <ludovic@chabant.com>
parents: 1038
diff changeset
56 def getOutputFilenames(self, filename):
717ac3c4ee77 bake: Fix where the Browserify processor outputs things.
Ludovic Chabant <ludovic@chabant.com>
parents: 1038
diff changeset
57 return [filename]
717ac3c4ee77 bake: Fix where the Browserify processor outputs things.
Ludovic Chabant <ludovic@chabant.com>
parents: 1038
diff changeset
58
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 def process(self, path, out_dir):
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
60 # Update the dependency list file.
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
61 # Sadly there doesn't seem to be a way to get the list at the same
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
62 # time as compiling the bundle so we need to run the process twice :(
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
63 deps_list = self._runBrowserify([path, '--list'])
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
64 deps_list = deps_list.decode('utf8')
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
65 deps_path = self._getDepListPath(path)
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
66 with open(deps_path, 'w', encoding='utf8') as f:
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
67 f.write(deps_list)
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
69 # Actually compile the JS bundle.
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
70 out_path = os.path.join(out_dir, os.path.basename(path))
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
71 self._runBrowserify([path, '-o', out_path])
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
72
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
73 return True
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
74
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
75 def _runBrowserify(self, args):
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
76 args = [self._conf['bin']] + args
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 cwd = self.app.root_dir
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 logger.debug("Running Browserify: %s" % ' '.join(args))
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 try:
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
80 return subprocess.check_output(
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
81 args,
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
82 cwd=cwd,
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
83 stderr=subprocess.STDOUT)
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 except FileNotFoundError as ex:
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
85 logger.error("Tried running Browserify with command: %s" % args)
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 raise Exception("Error running Browserify. "
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 "Did you install it?") from ex
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
88 except subprocess.CalledProcessError as ex:
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
89 logger.error("Error occured while running Browserify:")
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
90 logger.info(ex.stdout)
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
91 logger.error(ex.stderr)
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
92 raise Exception("Error occured while running Browserify. "
1038
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 "Please check log messages above for "
7487e2df8a56 bake: Add support for Browserify.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 "more information.")
1049
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
95
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
96 def _getDepListPath(self, path):
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
97 deps_name = "%s_%s.deps" % (
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
98 os.path.basename(path),
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
99 hashlib.md5(path.encode('utf8')).hexdigest())
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
100 deps_path = os.path.join(self._tmp_dir, deps_name)
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
101 return deps_path
c2bfa1869f0d bake: Optimize Browserify dependency handling.
Ludovic Chabant <ludovic@chabant.com>
parents: 1041
diff changeset
102