Mercurial > piecrust2

annotate piecrust/publishing/sftp.py @ 759:dd03385adb62

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
publish: Add SFTP publisher.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 25 Jun 2016 17:03:43 -0700
parents
children 3b33d9fb007c
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
759
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import socket
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import urllib.parse
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import getpass
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 import logging
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 import paramiko
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 from piecrust.publishing.base import Publisher, PublisherConfigurationError
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 logger = logging.getLogger(__name__)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 class SftpPublisher(Publisher):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 PUBLISHER_NAME = 'sftp'
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 PUBLISHER_SCHEME = 'sftp'
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 def setupPublishParser(self, parser, app):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 parser.add_argument(
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 '--force',
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 action='store_true',
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 help=("Upload the entire bake directory instead of only "
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 "the files changed by the last bake."))
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 def run(self, ctx):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 remote = self.config
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 if not self.has_url_config:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 host = self.getConfigValue('host')
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 if not host:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 raise PublisherConfigurationError(
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 "Publish target '%s' doesn't specify a 'host'." %
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 self.target)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 remote = urllib.parse.urlparse(host)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 hostname = remote.hostname
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 port = remote.port or 22
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 path = remote.path
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 if not hostname:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 hostname = path
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 path = ''
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 username = remote.username
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 pkey_path = None
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 if not self.has_url_config:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 if not username:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 username = self.getConfigValue('username')
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 if not path:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 path = self.getConfigValue('path')
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 pkey_path = self.getConfigValue('key')
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 password = None
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 if username:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 password = getpass.getpass("Password for '%s': " % username)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 logger.debug("Connecting to %s:%s..." % (hostname, port))
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 lfk = (not username and not pkey_path)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 sshc = paramiko.SSHClient()
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 sshc.load_system_host_keys()
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 sshc.set_missing_host_key_policy(paramiko.WarningPolicy())
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 sshc.connect(
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 hostname, port=port,
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 username=username, password=password,
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 key_filename=pkey_path,
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 look_for_keys=lfk)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 try:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 logger.info("Connected as %s" %
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 sshc.get_transport().get_username())
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 client = sshc.open_sftp()
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 try:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 self._upload(sshc, client, ctx, path)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 finally:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 client.close()
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 finally:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 sshc.close()
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 def _upload(self, session, client, ctx, dest_dir):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 if dest_dir:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 if dest_dir.startswith('~/'):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 _, out_chan, _ = session.exec_command("echo $HOME")
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 home_dir = out_chan.read().decode('utf8').strip()
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 dest_dir = home_dir + dest_dir[1:]
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 logger.debug("CHDIR %s" % dest_dir)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 try:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 client.chdir(dest_dir)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 except IOError:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 client.mkdir(dest_dir)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 client.chdir(dest_dir)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 known_dirs = {}
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92 if ctx.was_baked and not ctx.args.force:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 to_upload = list(self.getBakedFiles(ctx))
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 to_delete = list(self.getDeletedFiles(ctx))
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 if to_upload or to_delete:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
96 logger.info("Uploading new/changed files...")
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 for path in self.getBakedFiles(ctx):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 rel_path = os.path.relpath(path, ctx.bake_out_dir)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 logger.info(rel_path)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 if not ctx.preview:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 self._putFile(client, path, rel_path, known_dirs)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 logger.info("Deleting removed files...")
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 for path in self.getDeletedFiles(ctx):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
104 rel_path = os.path.relpath(path, ctx.bake_out_dir)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105 logger.info("%s [DELETE]" % rel_path)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106 if not ctx.preview:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
107 try:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 client.remove(rel_path)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109 except OSError:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 pass
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 else:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112 logger.info("Nothing to upload or delete on the remote server.")
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 logger.info("If you want to force uploading the entire website, "
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 "use the `--force` flag.")
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 else:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116 logger.info("Uploading entire website...")
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 for dirpath, dirnames, filenames in os.walk(ctx.bake_out_dir):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 for f in filenames:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
119 abs_f = os.path.join(dirpath, f)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120 rel_f = os.path.relpath(abs_f, ctx.bake_out_dir)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 logger.info(rel_f)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 if not ctx.preview:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 self._putFile(client, abs_f, rel_f, known_dirs)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125 def _putFile(self, client, local_path, remote_path, known_dirs):
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 # Split the remote path in bits.
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 remote_path = os.path.normpath(remote_path)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 if os.sep != '/':
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129 remote_path = remote_path.sub(os.sep, '/')
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 # Make sure each directory in the remote path exists... to prevent
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
132 # testing the same directories several times, we keep a cache of
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 # `known_dirs` which we know exist.
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134 remote_bits = remote_path.split('/')
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135 cur = ''
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136 for b in remote_bits[:-1]:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
137 cur = os.path.join(cur, b)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
138 if cur not in known_dirs:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
139 try:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140 info = client.stat(cur)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141 except FileNotFoundError:
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142 logger.debug("Creating remote dir: %s" % cur)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143 client.mkdir(cur)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
144 known_dirs[cur] = True
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
145
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
146 # Should be all good! Upload the file.
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
147 client.put(local_path, remote_path)
dd03385adb62 publish: Add SFTP publisher.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
148