Mercurial > piecrust2
changeset 885:13e8b50a2113
publish: Fix publishers API and add a simple "copy" publisher.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Tue, 20 Jun 2017 21:12:35 -0700 |
| parents | 18b3e2acd069 |
| children | dcdec4b951a1 |
| files | piecrust/app.py piecrust/commands/builtin/publishing.py piecrust/plugins/builtin.py piecrust/publishing/base.py piecrust/publishing/copy.py piecrust/publishing/publisher.py piecrust/publishing/rsync.py piecrust/publishing/sftp.py piecrust/publishing/shell.py piecrust/sources/mixins.py |
| diffstat | 10 files changed, 279 insertions(+), 252 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/app.py Tue Jun 20 21:10:39 2017 -0700 +++ b/piecrust/app.py Tue Jun 20 21:12:35 2017 -0700 @@ -178,20 +178,25 @@ publish_config = self.config.get('publish') if publish_config is None: return tgts + for n, t in publish_config.items(): - pub_type = None - is_scheme = False + pub_class = None if isinstance(t, dict): pub_type = t.get('type') + pub_class = defs_by_name[pub_type] + pub_cfg = t elif isinstance(t, str): comps = urllib.parse.urlparse(t) pub_type = comps.scheme - is_scheme = True - cls = (defs_by_scheme.get(pub_type) if is_scheme - else defs_by_name.get(pub_type)) - if cls is None: + pub_class = defs_by_scheme[pub_type] + pub_cfg = None + if pub_class is None: raise ConfigurationError("No such publisher: %s" % pub_type) - tgt = cls(self, n, t) + + tgt = pub_class(self, n, pub_cfg) + if pub_cfg is None: + tgt.parseUrlTarget(comps) + tgts.append(tgt) return tgts
--- a/piecrust/commands/builtin/publishing.py Tue Jun 20 21:10:39 2017 -0700 +++ b/piecrust/commands/builtin/publishing.py Tue Jun 20 21:12:35 2017 -0700 @@ -55,14 +55,12 @@ ctx.args.sub_func(ctx) def _doPublish(self, ctx): - from piecrust.publishing.publisher import Publisher + from piecrust.publishing.base import PublishingManager - pub = Publisher(ctx.app) + pub = PublishingManager(ctx.appfactory, ctx.app) pub.run( ctx.args.target, preview=ctx.args.preview, extra_args=ctx.args, - log_file=ctx.args.log_publisher, - applied_config_variant=ctx.config_variant, - applied_config_values=ctx.config_values) + log_file=ctx.args.log_publisher)
--- a/piecrust/plugins/builtin.py Tue Jun 20 21:10:39 2017 -0700 +++ b/piecrust/plugins/builtin.py Tue Jun 20 21:12:35 2017 -0700 @@ -152,11 +152,13 @@ WordpressXmlImporter()] def getPublishers(self): + from piecrust.publishing.copy import CopyPublisher from piecrust.publishing.sftp import SftpPublisher from piecrust.publishing.shell import ShellCommandPublisher from piecrust.publishing.rsync import RsyncPublisher return [ + CopyPublisher, ShellCommandPublisher, SftpPublisher, RsyncPublisher]
--- a/piecrust/publishing/base.py Tue Jun 20 21:10:39 2017 -0700 +++ b/piecrust/publishing/base.py Tue Jun 20 21:12:35 2017 -0700 @@ -1,10 +1,7 @@ import os.path -import shlex -import urllib.parse +import time import logging -import threading -import subprocess -from piecrust.configuration import try_get_dict_value +from piecrust.chefutil import format_timed logger = logging.getLogger(__name__) @@ -18,17 +15,17 @@ pass -class PublishingContext(object): +class PublishingContext: def __init__(self): self.bake_out_dir = None - self.bake_record = None + self.bake_records = None self.processing_record = None self.was_baked = False self.preview = False self.args = None -class Publisher(object): +class Publisher: PUBLISHER_NAME = 'undefined' PUBLISHER_SCHEME = None @@ -36,90 +33,129 @@ self.app = app self.target = target self.config = config - self.has_url_config = isinstance(config, urllib.parse.ParseResult) self.log_file_path = None def setupPublishParser(self, parser, app): return - def getConfigValue(self, name, default_value=None): - if self.has_url_config: - raise Exception("This publisher only has a URL configuration.") - return try_get_dict_value(self.config, name, default=default_value) + def parseUrlTarget(self, url): + raise NotImplementedError() def run(self, ctx): raise NotImplementedError() def getBakedFiles(self, ctx): - for e in ctx.bake_record.entries: - for sub in e.subs: - if sub.was_baked: - yield sub.out_path - for e in ctx.processing_record.entries: - if e.was_processed: - yield from [os.path.join(ctx.processing_record.out_dir, p) - for p in e.rel_outputs] + for rec in ctx.bake_records.records: + for e in rec.getEntries(): + paths = e.getAllOutputPaths() + if paths is not None: + yield from paths def getDeletedFiles(self, ctx): - yield from ctx.bake_record.deleted - yield from ctx.processing_record.deleted + for rec in ctx.bake_records.records: + yield from rec.deleted_out_paths + + +class InvalidPublishTargetError(Exception): + pass + + +class PublishingError(Exception): + pass -class ShellCommandPublisherBase(Publisher): - def __init__(self, app, target, config): - super(ShellCommandPublisherBase, self).__init__(app, target, config) - self.expand_user_args = True +class PublishingManager: + def __init__(self, appfactory, app): + self.appfactory = appfactory + self.app = app + + def run(self, target, + force=False, preview=False, extra_args=None, log_file=None): + start_time = time.perf_counter() + + # Get publisher for this target. + pub = self.app.getPublisher(target) + if pub is None: + raise InvalidPublishTargetError( + "No such publish target: %s" % target) + + # Will we need to bake first? + bake_first = pub.config.get('bake', True) - def run(self, ctx): - args = self._getCommandArgs(ctx) - if self.expand_user_args: - args = [os.path.expanduser(i) for i in args] + # Setup logging stuff. + hdlr = None + root_logger = logging.getLogger() + if log_file and not preview: + logger.debug("Adding file handler for: %s" % log_file) + hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8') + root_logger.addHandler(hdlr) + if not preview: + logger.info("Deploying to %s" % target) + else: + logger.info("Previewing deployment to %s" % target) - if ctx.preview: - preview_args = ' '.join([shlex.quote(i) for i in args]) - logger.info( - "Would run shell command: %s" % preview_args) - return True - - logger.debug( - "Running shell command: %s" % args) + # Bake first is necessary. + records = None + was_baked = False + bake_out_dir = os.path.join(self.app.root_dir, '_pub', target) + if bake_first: + if not preview: + bake_start_time = time.perf_counter() + logger.debug("Baking first to: %s" % bake_out_dir) - proc = subprocess.Popen( - args, cwd=self.app.root_dir, bufsize=0, - stdout=subprocess.PIPE) + from piecrust.baking.baker import Baker + baker = Baker( + self.appfactory, self.app, bake_out_dir, force=force) + records = baker.bake() + was_baked = True - logger.debug("Running publishing monitor for PID %d" % proc.pid) - thread = _PublishThread(proc) - thread.start() - proc.wait() - thread.join() + if not records.success: + raise Exception( + "Error during baking, aborting publishing.") + logger.info(format_timed(bake_start_time, "Baked website.")) + else: + logger.info("Would bake to: %s" % bake_out_dir) + + # Publish! + logger.debug( + "Running publish target '%s' with publisher: %s" % + (target, pub.PUBLISHER_NAME)) + pub_start_time = time.perf_counter() - if proc.returncode != 0: - logger.error( - "Publish process returned code %d" % proc.returncode) - else: - logger.debug("Publish process returned successfully.") + ctx = PublishingContext() + ctx.bake_out_dir = bake_out_dir + ctx.bake_records = records + ctx.was_baked = was_baked + ctx.preview = preview + ctx.args = extra_args + try: + pub.run(ctx) + except Exception as ex: + raise PublishingError( + "Error publishing to target: %s" % target) from ex + finally: + if hdlr: + root_logger.removeHandler(hdlr) + hdlr.close() - return proc.returncode == 0 + logger.info(format_timed( + pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME)) - def _getCommandArgs(self, ctx): - raise NotImplementedError() + logger.info(format_timed(start_time, 'Deployed to %s' % target)) -class _PublishThread(threading.Thread): - def __init__(self, proc): - super(_PublishThread, self).__init__( - name='publish_monitor', daemon=True) - self.proc = proc - self.root_logger = logging.getLogger() +def find_publisher_class(app, name, is_scheme=False): + attr_name = 'PUBLISHER_SCHEME' if is_scheme else 'PUBLISHER_NAME' + for pub_cls in app.plugin_loader.getPublishers(): + pub_sch = getattr(pub_cls, attr_name, None) + if pub_sch == name: + return pub_cls + return None - def run(self): - for line in iter(self.proc.stdout.readline, b''): - line_str = line.decode('utf8') - logger.info(line_str.rstrip('\r\n')) - for h in self.root_logger.handlers: - h.flush() - self.proc.communicate() - logger.debug("Publish monitor exiting.") +def find_publisher_name(app, scheme): + pub_cls = find_publisher_class(app, scheme, True) + if pub_cls: + return pub_cls.PUBLISHER_NAME + return None
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/publishing/copy.py Tue Jun 20 21:12:35 2017 -0700 @@ -0,0 +1,51 @@ +import os +import os.path +import shutil +import logging +from piecrust.publishing.base import Publisher + + +logger = logging.getLogger(__name__) + + +class CopyPublisher(Publisher): + PUBLISHER_NAME = 'copy' + PUBLISHER_SCHEME = 'file' + + def parseUrlTarget(self, url): + self.config = {'output': (url.netloc + url.path)} + + def run(self, ctx): + dest = self.config.get('output') + + if ctx.was_baked: + to_upload = list(self.getBakedFiles(ctx)) + to_delete = list(self.getDeletedFiles(ctx)) + if to_upload or to_delete: + logger.info("Copying new/changed files...") + for path in to_upload: + rel_path = os.path.relpath(path, ctx.bake_out_dir) + dest_path = os.path.join(dest, rel_path) + dest_dir = os.path.dirname(dest_path) + os.makedirs(dest_dir, exist_ok=True) + try: + dest_mtime = os.path.getmtime(dest_path) + except OSError: + dest_mtime = 0 + if os.path.getmtime(path) >= dest_mtime: + logger.info(rel_path) + if not ctx.preview: + shutil.copyfile(path, dest_path) + + logger.info("Deleting removed files...") + for path in self.getDeletedFiles(ctx): + rel_path = os.path.relpath(path, ctx.bake_out_dir) + logger.info("%s [DELETE]" % rel_path) + if not ctx.preview: + try: + os.remove(path) + except OSError: + pass + else: + logger.info("Nothing to copy to the output folder.") +
--- a/piecrust/publishing/publisher.py Tue Jun 20 21:10:39 2017 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,127 +0,0 @@ -import os.path -import time -import logging -from piecrust.chefutil import format_timed -from piecrust.publishing.base import PublishingContext - - -logger = logging.getLogger(__name__) - - -class InvalidPublishTargetError(Exception): - pass - - -class PublishingError(Exception): - pass - - -class Publisher(object): - def __init__(self, app): - self.app = app - - def run(self, target, - force=False, preview=False, extra_args=None, log_file=None, - applied_config_variant=None, applied_config_values=None): - start_time = time.perf_counter() - - # Get publisher for this target. - pub = self.app.getPublisher(target) - if pub is None: - raise InvalidPublishTargetError( - "No such publish target: %s" % target) - - # Will we need to bake first? - bake_first = True - if not pub.has_url_config: - bake_first = pub.getConfigValue('bake', True) - - # Setup logging stuff. - hdlr = None - root_logger = logging.getLogger() - if log_file and not preview: - logger.debug("Adding file handler for: %s" % log_file) - hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8') - root_logger.addHandler(hdlr) - if not preview: - logger.info("Deploying to %s" % target) - else: - logger.info("Previewing deployment to %s" % target) - - # Bake first is necessary. - rec1 = None - rec2 = None - was_baked = False - bake_out_dir = os.path.join(self.app.root_dir, '_pub', target) - if bake_first: - if not preview: - bake_start_time = time.perf_counter() - logger.debug("Baking first to: %s" % bake_out_dir) - - from piecrust.baking.baker import Baker - baker = Baker( - self.app, bake_out_dir, - applied_config_variant=applied_config_variant, - applied_config_values=applied_config_values) - rec1 = baker.bake() - - from piecrust.processing.pipeline import ProcessorPipeline - proc = ProcessorPipeline( - self.app, bake_out_dir, - applied_config_variant=applied_config_variant, - applied_config_values=applied_config_values) - rec2 = proc.run() - - was_baked = True - - if not rec1.success or not rec2.success: - raise Exception( - "Error during baking, aborting publishing.") - logger.info(format_timed(bake_start_time, "Baked website.")) - else: - logger.info("Would bake to: %s" % bake_out_dir) - - # Publish! - logger.debug( - "Running publish target '%s' with publisher: %s" % - (target, pub.PUBLISHER_NAME)) - pub_start_time = time.perf_counter() - - ctx = PublishingContext() - ctx.bake_out_dir = bake_out_dir - ctx.bake_record = rec1 - ctx.processing_record = rec2 - ctx.was_baked = was_baked - ctx.preview = preview - ctx.args = extra_args - try: - pub.run(ctx) - except Exception as ex: - raise PublishingError( - "Error publishing to target: %s" % target) from ex - finally: - if hdlr: - root_logger.removeHandler(hdlr) - hdlr.close() - - logger.info(format_timed( - pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME)) - - logger.info(format_timed(start_time, 'Deployed to %s' % target)) - - -def find_publisher_class(app, name, is_scheme=False): - attr_name = 'PUBLISHER_SCHEME' if is_scheme else 'PUBLISHER_NAME' - for pub_cls in app.plugin_loader.getPublishers(): - pub_sch = getattr(pub_cls, attr_name, None) - if pub_sch == name: - return pub_cls - return None - - -def find_publisher_name(app, scheme): - pub_cls = find_publisher_class(app, scheme, True) - if pub_cls: - return pub_cls.PUBLISHER_NAME - return None -
--- a/piecrust/publishing/rsync.py Tue Jun 20 21:10:39 2017 -0700 +++ b/piecrust/publishing/rsync.py Tue Jun 20 21:12:35 2017 -0700 @@ -1,21 +1,22 @@ -from piecrust.publishing.base import ShellCommandPublisherBase +from piecrust.publishing.shell import ShellCommandPublisherBase class RsyncPublisher(ShellCommandPublisherBase): PUBLISHER_NAME = 'rsync' PUBLISHER_SCHEME = 'rsync' + def parseUrlTarget(self, url): + self.config = { + 'destination': (url.netloc + url.path) + } + def _getCommandArgs(self, ctx): - if self.has_url_config: - orig = ctx.bake_out_dir - dest = self.config.netloc + self.config.path - else: - orig = self.getConfigValue('source', ctx.bake_out_dir) - dest = self.getConfigValue('destination') + orig = self.config.get('source', ctx.bake_out_dir) + dest = self.config.get('destination') + if not dest: + raise Exception("No destination specified.") - rsync_options = None - if not self.has_url_config: - rsync_options = self.getConfigValue('options') + rsync_options = self.config.get('options') if rsync_options is None: rsync_options = ['-avc', '--delete']
--- a/piecrust/publishing/sftp.py Tue Jun 20 21:10:39 2017 -0700 +++ b/piecrust/publishing/sftp.py Tue Jun 20 21:12:35 2017 -0700 @@ -1,6 +1,5 @@ import os import os.path -import socket import urllib.parse import getpass import logging @@ -17,20 +16,21 @@ def setupPublishParser(self, parser, app): parser.add_argument( - '--force', - action='store_true', - help=("Upload the entire bake directory instead of only " - "the files changed by the last bake.")) + '--force', + action='store_true', + help=("Upload the entire bake directory instead of only " + "the files changed by the last bake.")) + + def parseUrlTarget(self, url): + self.config = {'host': str(url)} def run(self, ctx): - remote = self.config - if not self.has_url_config: - host = self.getConfigValue('host') - if not host: - raise PublisherConfigurationError( - "Publish target '%s' doesn't specify a 'host'." % - self.target) - remote = urllib.parse.urlparse(host) + host = self.config.get('host') + if not host: + raise PublisherConfigurationError( + "Publish target '%s' doesn't specify a 'host'." % + self.target) + remote = urllib.parse.urlparse(host) hostname = remote.hostname port = remote.port or 22 @@ -39,16 +39,9 @@ hostname = path path = '' - username = remote.username - pkey_path = None - - if not self.has_url_config: - if not username: - username = self.getConfigValue('username') - if not path: - path = self.getConfigValue('path') - - pkey_path = self.getConfigValue('key') + username = self.config.get('username', remote.username) + path = self.config.get('path', path) + pkey_path = self.config.get('key') password = None if username and not ctx.preview: @@ -65,10 +58,10 @@ sshc.load_system_host_keys() sshc.set_missing_host_key_policy(paramiko.WarningPolicy()) sshc.connect( - hostname, port=port, - username=username, password=password, - key_filename=pkey_path, - look_for_keys=lfk) + hostname, port=port, + username=username, password=password, + key_filename=pkey_path, + look_for_keys=lfk) try: logger.info("Connected as %s" % sshc.get_transport().get_username()) @@ -120,9 +113,11 @@ except OSError: pass else: - logger.info("Nothing to upload or delete on the remote server.") - logger.info("If you want to force uploading the entire website, " - "use the `--force` flag.") + logger.info( + "Nothing to upload or delete on the remote server.") + logger.info( + "If you want to force uploading the entire website, " + "use the `--force` flag.") else: logger.info("Uploading entire website...") for dirpath, dirnames, filenames in os.walk(ctx.bake_out_dir): @@ -148,7 +143,7 @@ cur = os.path.join(cur, b) if cur not in known_dirs: try: - info = client.stat(cur) + client.stat(cur) except FileNotFoundError: logger.debug("Creating remote dir: %s" % cur) client.mkdir(cur)
--- a/piecrust/publishing/shell.py Tue Jun 20 21:10:39 2017 -0700 +++ b/piecrust/publishing/shell.py Tue Jun 20 21:12:35 2017 -0700 @@ -1,5 +1,71 @@ +import os.path import shlex -from piecrust.publishing.base import ShellCommandPublisherBase +import logging +import threading +import subprocess +from piecrust.publishing.base import Publisher + + +logger = logging.getLogger(__name__) + + +class ShellCommandPublisherBase(Publisher): + def __init__(self, app, target, config): + super(ShellCommandPublisherBase, self).__init__(app, target, config) + self.expand_user_args = True + + def run(self, ctx): + args = self._getCommandArgs(ctx) + if self.expand_user_args: + args = [os.path.expanduser(i) for i in args] + + if ctx.preview: + preview_args = ' '.join([shlex.quote(i) for i in args]) + logger.info( + "Would run shell command: %s" % preview_args) + return True + + logger.debug( + "Running shell command: %s" % args) + + proc = subprocess.Popen( + args, cwd=self.app.root_dir, bufsize=0, + stdout=subprocess.PIPE) + + logger.debug("Running publishing monitor for PID %d" % proc.pid) + thread = _PublishThread(proc) + thread.start() + proc.wait() + thread.join() + + if proc.returncode != 0: + logger.error( + "Publish process returned code %d" % proc.returncode) + else: + logger.debug("Publish process returned successfully.") + + return proc.returncode == 0 + + def _getCommandArgs(self, ctx): + raise NotImplementedError() + + +class _PublishThread(threading.Thread): + def __init__(self, proc): + super(_PublishThread, self).__init__( + name='publish_monitor', daemon=True) + self.proc = proc + self.root_logger = logging.getLogger() + + def run(self): + for line in iter(self.proc.stdout.readline, b''): + line_str = line.decode('utf8') + logger.info(line_str.rstrip('\r\n')) + for h in self.root_logger.handlers: + h.flush() + + self.proc.communicate() + logger.debug("Publish monitor exiting.") class ShellCommandPublisher(ShellCommandPublisherBase):
--- a/piecrust/sources/mixins.py Tue Jun 20 21:10:39 2017 -0700 +++ b/piecrust/sources/mixins.py Tue Jun 20 21:12:35 2017 -0700 @@ -21,8 +21,8 @@ spec_no_ext, _ = os.path.splitext(item.spec) assets_dir = spec_no_ext + assets_suffix try: - asset_files = osutil.listdir(assets_dir) - except OSError: + asset_files = list(osutil.listdir(assets_dir)) + except (OSError, FileNotFoundError): return None assets = []