Mercurial > piecrust2

annotate piecrust/publishing/base.py @ 758:6abb436fea5b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
publish: Make publisher more powerful and better exposed on the command line. * Make the `chef publish` command have one sub-command per publish target. * Add custom argument parsing per publisher to have strong extra arguments available per publish target. * Make publish targets a first class citizen of the `PieCrust` app class.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 25 Jun 2016 17:03:29 -0700
parents 8f9c0bdb3724
children f69fdc601845
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
1 import os.path
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
2 import shlex
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
3 import urllib.parse
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
4 import logging
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
5 import threading
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
6 import subprocess
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
7 from piecrust.configuration import try_get_dict_value
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
8
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
9
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
10 logger = logging.getLogger(__name__)
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
13 FILE_MODIFIED = 1
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
14 FILE_DELETED = 2
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
15
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
16
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
17 class PublisherConfigurationError(Exception):
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
18 pass
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
19
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
20
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 class PublishingContext(object):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 def __init__(self):
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
23 self.bake_out_dir = None
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
24 self.bake_record = None
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
25 self.processing_record = None
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
26 self.was_baked = False
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
27 self.preview = False
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
28 self.args = None
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 class Publisher(object):
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
32 PUBLISHER_NAME = 'undefined'
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
33 PUBLISHER_SCHEME = None
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
34
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
35 def __init__(self, app, target, config):
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self.app = app
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 self.target = target
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
38 self.config = config
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
39 self.has_url_config = isinstance(config, urllib.parse.ParseResult)
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 self.log_file_path = None
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
42 def setupPublishParser(self, parser, app):
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
43 return
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
44
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
45 def getConfigValue(self, name, default_value=None):
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
46 if self.has_url_config:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
47 raise Exception("This publisher only has a URL configuration.")
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
48 return try_get_dict_value(self.config, name, default_value)
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 def run(self, ctx):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 raise NotImplementedError()
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
53 def getBakedFiles(self, ctx):
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
54 for e in ctx.bake_record.entries:
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
55 for sub in e.subs:
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
56 if sub.was_baked:
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
57 yield sub.out_path
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
58 for e in ctx.processing_record.entries:
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
59 if e.was_processed:
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
60 yield from [os.path.join(ctx.processing_record.out_dir, p)
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
61 for p in e.rel_outputs]
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
62
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
63 def getDeletedFiles(self, ctx):
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
64 yield from ctx.bake_record.deleted
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
65 yield from ctx.processing_record.deleted
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
66
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
67
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
68 class ShellCommandPublisherBase(Publisher):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
69 def __init__(self, app, target):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
70 super(ShellCommandPublisherBase, self).__init__(app, target)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
71 self.expand_user_args = True
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
72
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
73 def run(self, ctx):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
74 args = self._getCommandArgs(ctx)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
75 if self.expand_user_args:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
76 args = [os.path.expanduser(i) for i in args]
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
77
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
78 if ctx.preview:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
79 preview_args = ' '.join([shlex.quote(i) for i in args])
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
80 logger.info(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
81 "Would run shell command: %s" % preview_args)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
82 return True
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
83
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
84 logger.debug(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
85 "Running shell command: %s" % args)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
86
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
87 proc = subprocess.Popen(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
88 args, cwd=self.app.root_dir, bufsize=0,
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
89 stdout=subprocess.PIPE)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
90
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
91 logger.debug("Running publishing monitor for PID %d" % proc.pid)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
92 thread = _PublishThread(proc)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
93 thread.start()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
94 proc.wait()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
95 thread.join()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
96
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
97 if proc.returncode != 0:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
98 logger.error(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
99 "Publish process returned code %d" % proc.returncode)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
100 else:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
101 logger.debug("Publish process returned successfully.")
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
102
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
103 return proc.returncode == 0
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
104
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
105 def _getCommandArgs(self, ctx):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
106 raise NotImplementedError()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
107
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
108
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
109 class _PublishThread(threading.Thread):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
110 def __init__(self, proc):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
111 super(_PublishThread, self).__init__(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
112 name='publish_monitor', daemon=True)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
113 self.proc = proc
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
114 self.root_logger = logging.getLogger()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
115
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
116 def run(self):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
117 for line in iter(self.proc.stdout.readline, b''):
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
118 line_str = line.decode('utf8')
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
119 logger.info(line_str.rstrip('\r\n'))
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
120 for h in self.root_logger.handlers:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
121 h.flush()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
122
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
123 self.proc.communicate()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
124 logger.debug("Publish monitor exiting.")
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
125