Mercurial > piecrust2

annotate piecrust/publishing/base.py @ 702:c62d83e17abf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Some more optimizations. Write the output files in a background thread to get away from the GIL.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 16 Apr 2016 22:50:07 -0700
parents 8f9c0bdb3724
children 6abb436fea5b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
1 import os.path
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
2 import shlex
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
3 import logging
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
4 import threading
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
5 import subprocess
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
6
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
7
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
8 logger = logging.getLogger(__name__)
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 class PublishingContext(object):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 def __init__(self):
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
13 self.bake_out_dir = None
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
14 self.preview = False
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 class Publisher(object):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 def __init__(self, app, target):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 self.app = app
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 self.target = target
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
21 self.parsed_url = None
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 self.log_file_path = None
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
24 @property
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
25 def has_url_config(self):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
26 return self.parsed_url is not None
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
27
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
28 @property
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
29 def url_config(self):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
30 if self.parsed_url is not None:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
31 return self.getConfig()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
32 raise Exception("This publisher has a full configuration.")
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
33
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 def getConfig(self):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 return self.app.config.get('publish/%s' % self.target)
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 def getConfigValue(self, name):
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
38 if self.has_url_config:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
39 raise Exception("This publisher only has a URL configuration.")
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 return self.app.config.get('publish/%s/%s' % (self.target, name))
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 def run(self, ctx):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 raise NotImplementedError()
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
45
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
46 class ShellCommandPublisherBase(Publisher):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
47 def __init__(self, app, target):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
48 super(ShellCommandPublisherBase, self).__init__(app, target)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
49 self.expand_user_args = True
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
50
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
51 def run(self, ctx):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
52 args = self._getCommandArgs(ctx)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
53 if self.expand_user_args:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
54 args = [os.path.expanduser(i) for i in args]
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
55
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
56 if ctx.preview:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
57 preview_args = ' '.join([shlex.quote(i) for i in args])
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
58 logger.info(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
59 "Would run shell command: %s" % preview_args)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
60 return True
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
61
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
62 logger.debug(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
63 "Running shell command: %s" % args)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
64
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
65 proc = subprocess.Popen(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
66 args, cwd=self.app.root_dir, bufsize=0,
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
67 stdout=subprocess.PIPE)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
68
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
69 logger.debug("Running publishing monitor for PID %d" % proc.pid)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
70 thread = _PublishThread(proc)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
71 thread.start()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
72 proc.wait()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
73 thread.join()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
74
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
75 if proc.returncode != 0:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
76 logger.error(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
77 "Publish process returned code %d" % proc.returncode)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
78 else:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
79 logger.debug("Publish process returned successfully.")
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
80
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
81 return proc.returncode == 0
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
82
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
83 def _getCommandArgs(self, ctx):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
84 raise NotImplementedError()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
85
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
86
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
87 class _PublishThread(threading.Thread):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
88 def __init__(self, proc):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
89 super(_PublishThread, self).__init__(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
90 name='publish_monitor', daemon=True)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
91 self.proc = proc
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
92 self.root_logger = logging.getLogger()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
93
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
94 def run(self):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
95 for line in iter(self.proc.stdout.readline, b''):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
96 line_str = line.decode('utf8')
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
97 logger.info(line_str.rstrip('\r\n'))
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
98 for h in self.root_logger.handlers:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
99 h.flush()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
100
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
101 self.proc.communicate()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
102 logger.debug("Publish monitor exiting.")
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
103