Mercurial > piecrust2
changeset 758:6abb436fea5b
publish: Make publisher more powerful and better exposed on the command line.
* Make the `chef publish` command have one sub-command per publish target.
* Add custom argument parsing per publisher to have strong extra arguments
available per publish target.
* Make publish targets a first class citizen of the `PieCrust` app class.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Sat, 25 Jun 2016 17:03:29 -0700 |
| parents | 7147b06670fd |
| children | dd03385adb62 |
| files | piecrust/app.py piecrust/commands/builtin/publishing.py piecrust/plugins/builtin.py piecrust/publishing/base.py piecrust/publishing/publisher.py piecrust/publishing/rsync.py |
| diffstat | 6 files changed, 141 insertions(+), 109 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/app.py Sat Jun 25 17:01:08 2016 -0700 +++ b/piecrust/app.py Sat Jun 25 17:03:29 2016 -0700 @@ -2,6 +2,7 @@ import os.path import hashlib import logging +import urllib.parse from werkzeug.utils import cached_property from piecrust import ( RESOURCES_DIR, @@ -10,9 +11,9 @@ CONFIG_PATH, THEME_CONFIG_PATH) from piecrust.appconfig import PieCrustConfiguration from piecrust.cache import ExtensibleCache, NullExtensibleCache -from piecrust.plugins.base import PluginLoader +from piecrust.configuration import ConfigurationError, merge_dicts from piecrust.environment import StandardEnvironment -from piecrust.configuration import ConfigurationError, merge_dicts +from piecrust.plugins.base import PluginLoader from piecrust.routing import Route from piecrust.sources.base import REALM_THEME @@ -165,6 +166,36 @@ gens.append(gen) return gens + @cached_property + def publishers(self): + defs_by_name = {} + defs_by_scheme = {} + for cls in self.plugin_loader.getPublishers(): + defs_by_name[cls.PUBLISHER_NAME] = cls + if cls.PUBLISHER_SCHEME: + defs_by_scheme[cls.PUBLISHER_SCHEME] = cls + + tgts = [] + publish_config = self.config.get('publish') + if publish_config is None: + return tgts + for n, t in publish_config.items(): + pub_type = None + is_scheme = False + if isinstance(t, dict): + pub_type = t.get('type') + elif isinstance(t, str): + comps = urllib.parse.urlparse(t) + pub_type = comps.scheme + is_scheme = True + cls = (defs_by_scheme.get(pub_type) if is_scheme + else defs_by_name.get(pub_type)) + if cls is None: + raise ConfigurationError("No such publisher: %s" % pub_type) + tgt = cls(self, n, t) + tgts.append(tgt) + return tgts + def getSource(self, source_name): for source in self.sources: if source.name == source_name: @@ -195,6 +226,12 @@ return route return None + def getPublisher(self, target_name): + for pub in self.publishers: + if pub.target == target_name: + return pub + return None + def _get_dir(self, default_rel_dir): abs_dir = os.path.join(self.root_dir, default_rel_dir) if os.path.isdir(abs_dir):
--- a/piecrust/commands/builtin/publishing.py Sat Jun 25 17:01:08 2016 -0700 +++ b/piecrust/commands/builtin/publishing.py Sat Jun 25 17:03:29 2016 -0700 @@ -1,6 +1,7 @@ import logging import urllib.parse from piecrust.commands.base import ChefCommand +from piecrust.pathutil import SiteNotFoundError from piecrust.publishing.publisher import Publisher, find_publisher_name @@ -17,10 +18,6 @@ def setupParser(self, parser, app): parser.add_argument( - '-l', '--list', - action='store_true', - help="List available publish targets for the current site.") - parser.add_argument( '--log-publisher', metavar='LOG_FILE', help="Log the publisher's output to a given file.") @@ -28,51 +25,39 @@ '--preview', action='store_true', help="Only preview what the publisher would do.") - parser.add_argument( - 'target', - nargs='?', - default='default', - help="The publish target to use.") - def run(self, ctx): - if ctx.args.list: - pub_cfg = ctx.app.config.get('publish') - if not pub_cfg: - logger.info("No available publish targets.") - return + subparsers = parser.add_subparsers() + for pub in app.publishers: + p = subparsers.add_parser( + pub.target, + help="Publish using target '%s'." % pub.target) + pub.setupPublishParser(p, app) + p.set_defaults(sub_func=self._doPublish) + p.set_defaults(target=pub.target) - for name, cfg in pub_cfg.items(): - if isinstance(cfg, dict): - pub_type = cfg.get('type') - if pub_type: - desc = cfg.get('description') - bake_first = cfg.get('bake', True) - msg = '%s (%s)' % (name, pub_type) - if not bake_first: - msg += ' (no local baking)' - if desc: - msg += ': ' + desc - logger.info(msg) - else: - logger.error( - "%s (unknown type '%s')" % (name, pub_type)) - elif isinstance(cfg, str): - comps = urllib.parse.urlparse(str(cfg)) - pub_name = find_publisher_name(ctx.app, comps.scheme) - if pub_name: - logger.info("%s (%s)" % (name, pub_name)) - else: - logger.error( - "%s (unknown scheme '%s')" % - (name, comps.scheme)) - else: - logger.error( - "%s (incorrect configuration)" % name) + if not app.publishers: + parser.epilog = ( + "No publishers have been defined. You can define publishers " + "through the `publish` configuration settings. " + "For more information see: " + "https://bolt80.com/piecrust/en/latest/docs/publishing/") + + def checkedRun(self, ctx): + if ctx.app.root_dir is None: + raise SiteNotFoundError(theme=ctx.app.theme_site) + + if not hasattr(ctx.args, 'sub_func'): + ctx.parser.parse_args(['publish', '--help']) return + ctx.args.sub_func(ctx) + def _doPublish(self, ctx): pub = Publisher(ctx.app) pub.run( ctx.args.target, preview=ctx.args.preview, - log_file=ctx.args.log_publisher) + extra_args=ctx.args, + log_file=ctx.args.log_publisher, + applied_config_variant=ctx.config_variant, + applied_config_values=ctx.config_values)
--- a/piecrust/plugins/builtin.py Sat Jun 25 17:01:08 2016 -0700 +++ b/piecrust/plugins/builtin.py Sat Jun 25 17:03:29 2016 -0700 @@ -37,6 +37,7 @@ from piecrust.processing.sass import SassProcessor from piecrust.processing.sitemap import SitemapProcessor from piecrust.processing.util import ConcatProcessor +from piecrust.publishing.sftp import SftpPublisher from piecrust.publishing.shell import ShellCommandPublisher from piecrust.publishing.rsync import RsyncPublisher from piecrust.sources.default import DefaultPageSource @@ -133,5 +134,6 @@ def getPublishers(self): return [ ShellCommandPublisher, + SftpPublisher, RsyncPublisher]
--- a/piecrust/publishing/base.py Sat Jun 25 17:01:08 2016 -0700 +++ b/piecrust/publishing/base.py Sat Jun 25 17:03:29 2016 -0700 @@ -1,47 +1,69 @@ import os.path import shlex +import urllib.parse import logging import threading import subprocess +from piecrust.configuration import try_get_dict_value logger = logging.getLogger(__name__) +FILE_MODIFIED = 1 +FILE_DELETED = 2 + + +class PublisherConfigurationError(Exception): + pass + + class PublishingContext(object): def __init__(self): self.bake_out_dir = None + self.bake_record = None + self.processing_record = None + self.was_baked = False self.preview = False + self.args = None class Publisher(object): - def __init__(self, app, target): + PUBLISHER_NAME = 'undefined' + PUBLISHER_SCHEME = None + + def __init__(self, app, target, config): self.app = app self.target = target - self.parsed_url = None + self.config = config + self.has_url_config = isinstance(config, urllib.parse.ParseResult) self.log_file_path = None - @property - def has_url_config(self): - return self.parsed_url is not None + def setupPublishParser(self, parser, app): + return - @property - def url_config(self): - if self.parsed_url is not None: - return self.getConfig() - raise Exception("This publisher has a full configuration.") - - def getConfig(self): - return self.app.config.get('publish/%s' % self.target) - - def getConfigValue(self, name): + def getConfigValue(self, name, default_value=None): if self.has_url_config: raise Exception("This publisher only has a URL configuration.") - return self.app.config.get('publish/%s/%s' % (self.target, name)) + return try_get_dict_value(self.config, name, default_value) def run(self, ctx): raise NotImplementedError() + def getBakedFiles(self, ctx): + for e in ctx.bake_record.entries: + for sub in e.subs: + if sub.was_baked: + yield sub.out_path + for e in ctx.processing_record.entries: + if e.was_processed: + yield from [os.path.join(ctx.processing_record.out_dir, p) + for p in e.rel_outputs] + + def getDeletedFiles(self, ctx): + yield from ctx.bake_record.deleted + yield from ctx.processing_record.deleted + class ShellCommandPublisherBase(Publisher): def __init__(self, app, target):
--- a/piecrust/publishing/publisher.py Sat Jun 25 17:01:08 2016 -0700 +++ b/piecrust/publishing/publisher.py Sat Jun 25 17:03:29 2016 -0700 @@ -21,35 +21,21 @@ def __init__(self, app): self.app = app - def run(self, target, preview=False, log_file=None): + def run(self, target, + force=False, preview=False, extra_args=None, log_file=None, + applied_config_variant=None, applied_config_values=None): start_time = time.perf_counter() - # Get the configuration for this target. - target_cfg = self.app.config.get('publish/%s' % target) - if not target_cfg: + # Get publisher for this target. + pub = self.app.getPublisher(target) + if pub is None: raise InvalidPublishTargetError( "No such publish target: %s" % target) - target_type = None + # Will we need to bake first? bake_first = True - parsed_url = None - if isinstance(target_cfg, dict): - target_type = target_cfg.get('type') - if not target_type: - raise InvalidPublishTargetError( - "Publish target '%s' doesn't specify a type." % target) - bake_first = target_cfg.get('bake', True) - elif isinstance(target_cfg, str): - comps = urllib.parse.urlparse(target_cfg) - if not comps.scheme: - raise InvalidPublishTargetError( - "Publish target '%s' has an invalid target URL." % - target) - parsed_url = comps - target_type = find_publisher_name(self.app, comps.scheme) - if target_type is None: - raise InvalidPublishTargetError( - "No such publish target scheme: %s" % comps.scheme) + if not pub.has_url_config: + bake_first = pub.getConfigValue('bake', True) # Setup logging stuff. hdlr = None @@ -64,21 +50,31 @@ logger.info("Previewing deployment to %s" % target) # Bake first is necessary. - bake_out_dir = None + rec1 = None + rec2 = None + was_baked = False + bake_out_dir = os.path.join(self.app.root_dir, '_pub', target) if bake_first: - bake_out_dir = os.path.join(self.app.cache_dir, 'pub', target) if not preview: bake_start_time = time.perf_counter() logger.debug("Baking first to: %s" % bake_out_dir) from piecrust.baking.baker import Baker - baker = Baker(self.app, bake_out_dir) + baker = Baker( + self.app, bake_out_dir, + applied_config_variant=applied_config_variant, + applied_config_values=applied_config_values) rec1 = baker.bake() from piecrust.processing.pipeline import ProcessorPipeline - proc = ProcessorPipeline(self.app, bake_out_dir) + proc = ProcessorPipeline( + self.app, bake_out_dir, + applied_config_variant=applied_config_variant, + applied_config_values=applied_config_values) rec2 = proc.run() + was_baked = True + if not rec1.success or not rec2.success: raise Exception( "Error during baking, aborting publishing.") @@ -86,18 +82,6 @@ else: logger.info("Would bake to: %s" % bake_out_dir) - # Create the appropriate publisher. - pub = None - for pub_cls in self.app.plugin_loader.getPublishers(): - if pub_cls.PUBLISHER_NAME == target_type: - pub = pub_cls(self.app, target) - break - if pub is None: - raise InvalidPublishTargetError( - "Publish target '%s' has invalid type: %s" % - (target, target_type)) - pub.parsed_url = parsed_url - # Publish! logger.debug( "Running publish target '%s' with publisher: %s" % @@ -106,9 +90,13 @@ ctx = PublishingContext() ctx.bake_out_dir = bake_out_dir + ctx.bake_record = rec1 + ctx.processing_record = rec2 + ctx.was_baked = was_baked ctx.preview = preview + ctx.args = extra_args try: - success = pub.run(ctx) + pub.run(ctx) except Exception as ex: raise PublishingError( "Error publishing to target: %s" % target) from ex @@ -117,25 +105,23 @@ root_logger.removeHandler(hdlr) hdlr.close() - if not success: - raise PublishingError( - "Unknown error publishing to target: %s" % target) logger.info(format_timed( pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME)) logger.info(format_timed(start_time, 'Deployed to %s' % target)) -def find_publisher_class(app, scheme): +def find_publisher_class(app, name, is_scheme=False): + attr_name = 'PUBLISHER_SCHEME' if is_scheme else 'PUBLISHER_NAME' for pub_cls in app.plugin_loader.getPublishers(): - pub_sch = getattr(pub_cls, 'PUBLISHER_SCHEME', None) - if ('bake+%s' % pub_sch) == scheme: + pub_sch = getattr(pub_cls, attr_name, None) + if pub_sch == name: return pub_cls return None def find_publisher_name(app, scheme): - pub_cls = find_publisher_class(app, scheme) + pub_cls = find_publisher_class(app, scheme, True) if pub_cls: return pub_cls.PUBLISHER_NAME return None
--- a/piecrust/publishing/rsync.py Sat Jun 25 17:01:08 2016 -0700 +++ b/piecrust/publishing/rsync.py Sat Jun 25 17:03:29 2016 -0700 @@ -8,7 +8,7 @@ def _getCommandArgs(self, ctx): if self.has_url_config: orig = ctx.bake_out_dir - dest = self.parsed_url.netloc + self.parsed_url.path + dest = self.config.netloc + self.config.path else: orig = self.getConfigValue('source', ctx.bake_our_dir) dest = self.getConfigValue('destination')