Mercurial > piecrust2

annotate piecrust/publishing/shell.py @ 1188:a7c43131d871

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Fix file write flushing problem with Python 3.8+ Writing the cache files fails in Python 3.8 because it looks like flushing behaviour has changed. We need to explicitly flush. And even then, in very rare occurrences, it looks like it can still run into racing conditions, so we do a very hacky and ugly "retry" loop when fetching cached data :(
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 15 Jun 2021 22:36:23 -0700
parents 8ca228956cc6
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
885
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
1 import os.path
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import shlex
885
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
3 import logging
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
4 import threading
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
5 import subprocess
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
6 from piecrust.publishing.base import Publisher
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
7
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
8
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
9 logger = logging.getLogger(__name__)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
10
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
11
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
12 class ShellCommandPublisherBase(Publisher):
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
13 def __init__(self, app, target, config):
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
14 super(ShellCommandPublisherBase, self).__init__(app, target, config)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
15 self.expand_user_args = True
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
16
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
17 def run(self, ctx):
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
18 args = self._getCommandArgs(ctx)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
19 if self.expand_user_args:
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
20 args = [os.path.expanduser(i) for i in args]
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
21
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
22 if ctx.preview:
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
23 preview_args = ' '.join([shlex.quote(i) for i in args])
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
24 logger.info(
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
25 "Would run shell command: %s" % preview_args)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
26 return True
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
27
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
28 logger.debug(
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
29 "Running shell command: %s" % args)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
30
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
31 proc = subprocess.Popen(
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
32 args, cwd=self.app.root_dir, bufsize=0,
946
8ca228956cc6 publish: Fix shell publisher crash, log `stderr` too.
Ludovic Chabant <ludovic@chabant.com>
parents: 885
diff changeset
33 stdout=subprocess.PIPE,
8ca228956cc6 publish: Fix shell publisher crash, log `stderr` too.
Ludovic Chabant <ludovic@chabant.com>
parents: 885
diff changeset
34 stderr=subprocess.STDOUT)
885
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
35
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
36 logger.debug("Running publishing monitor for PID %d" % proc.pid)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
37 thread = _PublishThread(proc)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
38 thread.start()
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
39 proc.wait()
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
40 thread.join()
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
41
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
42 if proc.returncode != 0:
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
43 logger.error(
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
44 "Publish process returned code %d" % proc.returncode)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
45 else:
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
46 logger.debug("Publish process returned successfully.")
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
47
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
48 return proc.returncode == 0
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
49
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
50 def _getCommandArgs(self, ctx):
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
51 raise NotImplementedError()
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
52
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
53
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
54 class _PublishThread(threading.Thread):
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
55 def __init__(self, proc):
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
56 super(_PublishThread, self).__init__(
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
57 name='publish_monitor', daemon=True)
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
58 self.proc = proc
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
59 self.root_logger = logging.getLogger()
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
60
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
61 def run(self):
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
62 for line in iter(self.proc.stdout.readline, b''):
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
63 line_str = line.decode('utf8')
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
64 logger.info(line_str.rstrip('\r\n'))
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
65 for h in self.root_logger.handlers:
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
66 h.flush()
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
67
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
68 self.proc.communicate()
13e8b50a2113 publish: Fix publishers API and add a simple "copy" publisher.
Ludovic Chabant <ludovic@chabant.com>
parents: 623
diff changeset
69 logger.debug("Publish monitor exiting.")
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 614
diff changeset
72 class ShellCommandPublisher(ShellCommandPublisherBase):
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 PUBLISHER_NAME = 'shell'
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 614
diff changeset
75 def _getCommandArgs(self, ctx):
946
8ca228956cc6 publish: Fix shell publisher crash, log `stderr` too.
Ludovic Chabant <ludovic@chabant.com>
parents: 885
diff changeset
76 target_cmd = self.config.get('command')
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 if not target_cmd:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 raise Exception("No command specified for publish target: %s" %
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 self.target)
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 args = shlex.split(target_cmd)
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 614
diff changeset
81 return args
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82