Mercurial > piecrust2

changeset 621:8f9c0bdb3724

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
publish: Polish/refactor the publishing workflows. * Add a `--preview` option. * The `--list` option gives a nicer output, and prints warnings/errors for incorrect configuration. * Moved most of the `shell` code into a base class that's reusable. * Simplified the code to log publishing to a file. * Nicer output overall, with times.
author Ludovic Chabant <ludovic@chabant.com>
date Mon, 08 Feb 2016 20:44:26 -0800
parents c2708f20a87b
children 5d8e0c8cdb5f
files piecrust/commands/builtin/publishing.py piecrust/publishing/base.py piecrust/publishing/publisher.py piecrust/publishing/shell.py
diffstat 4 files changed, 214 insertions(+), 90 deletions(-) [+]
line wrap: on
line diff
--- a/piecrust/commands/builtin/publishing.py	Sat Feb 06 21:49:50 2016 -0800
+++ b/piecrust/commands/builtin/publishing.py	Mon Feb 08 20:44:26 2016 -0800
@@ -1,6 +1,7 @@
 import logging
+import urllib.parse
 from piecrust.commands.base import ChefCommand
-from piecrust.publishing.publisher import Publisher
+from piecrust.publishing.publisher import Publisher, find_publisher_name
 
 
 logger = logging.getLogger(__name__)
@@ -24,6 +25,10 @@
                 metavar='LOG_FILE',
                 help="Log the publisher's output to a given file.")
         parser.add_argument(
+                '--preview',
+                action='store_true',
+                help="Only preview what the publisher would do.")
+        parser.add_argument(
                 'target',
                 nargs='?',
                 default='default',
@@ -37,13 +42,37 @@
                 return
 
             for name, cfg in pub_cfg.items():
-                desc = cfg.get('description')
-                if not desc:
-                    logger.info(name)
+                if isinstance(cfg, dict):
+                    pub_type = cfg.get('type')
+                    if pub_type:
+                        desc = cfg.get('description')
+                        bake_first = cfg.get('bake', True)
+                        msg = '%s (%s)' % (name, pub_type)
+                        if not bake_first:
+                            msg += ' (no local baking)'
+                        if desc:
+                            msg += ': ' + desc
+                        logger.info(msg)
+                    else:
+                        logger.error(
+                                "%s (unknown type '%s')" % (name, pub_type))
+                elif isinstance(cfg, str):
+                    comps = urllib.parse.urlparse(str(cfg))
+                    pub_name = find_publisher_name(ctx.app, comps.scheme)
+                    if pub_name:
+                        logger.info("%s (%s)" % (name, pub_name))
+                    else:
+                        logger.error(
+                                "%s (unknown scheme '%s')" %
+                                (name, comps.scheme))
                 else:
-                    logger.info("%s: %s" % (name, desc))
+                    logger.error(
+                            "%s (incorrect configuration)" % name)
             return
 
         pub = Publisher(ctx.app)
-        pub.run(ctx.args.target, log_file=ctx.args.log_publisher)
+        pub.run(
+                ctx.args.target,
+                preview=ctx.args.preview,
+                log_file=ctx.args.log_publisher)
 
--- a/piecrust/publishing/base.py	Sat Feb 06 21:49:50 2016 -0800
+++ b/piecrust/publishing/base.py	Mon Feb 08 20:44:26 2016 -0800
@@ -1,23 +1,103 @@
+import os.path
+import shlex
+import logging
+import threading
+import subprocess
+
+
+logger = logging.getLogger(__name__)
 
 
 class PublishingContext(object):
     def __init__(self):
-        self.custom_logging_file = None
+        self.bake_out_dir = None
+        self.preview = False
 
 
 class Publisher(object):
     def __init__(self, app, target):
         self.app = app
         self.target = target
-        self.is_using_custom_logging = False
+        self.parsed_url = None
         self.log_file_path = None
 
+    @property
+    def has_url_config(self):
+        return self.parsed_url is not None
+
+    @property
+    def url_config(self):
+        if self.parsed_url is not None:
+            return self.getConfig()
+        raise Exception("This publisher has a full configuration.")
+
     def getConfig(self):
         return self.app.config.get('publish/%s' % self.target)
 
     def getConfigValue(self, name):
+        if self.has_url_config:
+            raise Exception("This publisher only has a URL configuration.")
         return self.app.config.get('publish/%s/%s' % (self.target, name))
 
     def run(self, ctx):
         raise NotImplementedError()
 
+
+class ShellCommandPublisherBase(Publisher):
+    def __init__(self, app, target):
+        super(ShellCommandPublisherBase, self).__init__(app, target)
+        self.expand_user_args = True
+
+    def run(self, ctx):
+        args = self._getCommandArgs(ctx)
+        if self.expand_user_args:
+            args = [os.path.expanduser(i) for i in args]
+
+        if ctx.preview:
+            preview_args = ' '.join([shlex.quote(i) for i in args])
+            logger.info(
+                    "Would run shell command: %s" % preview_args)
+            return True
+
+        logger.debug(
+                "Running shell command: %s" % args)
+
+        proc = subprocess.Popen(
+                args, cwd=self.app.root_dir, bufsize=0,
+                stdout=subprocess.PIPE)
+
+        logger.debug("Running publishing monitor for PID %d" % proc.pid)
+        thread = _PublishThread(proc)
+        thread.start()
+        proc.wait()
+        thread.join()
+
+        if proc.returncode != 0:
+            logger.error(
+                    "Publish process returned code %d" % proc.returncode)
+        else:
+            logger.debug("Publish process returned successfully.")
+
+        return proc.returncode == 0
+
+    def _getCommandArgs(self, ctx):
+        raise NotImplementedError()
+
+
+class _PublishThread(threading.Thread):
+    def __init__(self, proc):
+        super(_PublishThread, self).__init__(
+                name='publish_monitor', daemon=True)
+        self.proc = proc
+        self.root_logger = logging.getLogger()
+
+    def run(self):
+        for line in iter(self.proc.stdout.readline, b''):
+            line_str = line.decode('utf8')
+            logger.info(line_str.rstrip('\r\n'))
+            for h in self.root_logger.handlers:
+                h.flush()
+
+        self.proc.communicate()
+        logger.debug("Publish monitor exiting.")
+
--- a/piecrust/publishing/publisher.py	Sat Feb 06 21:49:50 2016 -0800
+++ b/piecrust/publishing/publisher.py	Mon Feb 08 20:44:26 2016 -0800
@@ -1,4 +1,8 @@
+import os.path
+import time
 import logging
+import urllib.parse
+from piecrust.chefutil import format_timed
 from piecrust.publishing.base import PublishingContext
 
 
@@ -17,17 +21,72 @@
     def __init__(self, app):
         self.app = app
 
-    def run(self, target, log_file=None):
+    def run(self, target, preview=False, log_file=None):
+        start_time = time.perf_counter()
+
+        # Get the configuration for this target.
         target_cfg = self.app.config.get('publish/%s' % target)
         if not target_cfg:
             raise InvalidPublishTargetError(
                     "No such publish target: %s" % target)
 
-        target_type = target_cfg.get('type')
-        if not target_type:
-            raise InvalidPublishTargetError(
-                    "Publish target '%s' doesn't specify a type." % target)
+        target_type = None
+        bake_first = True
+        parsed_url = None
+        if isinstance(target_cfg, dict):
+            target_type = target_cfg.get('type')
+            if not target_type:
+                raise InvalidPublishTargetError(
+                        "Publish target '%s' doesn't specify a type." % target)
+            bake_first = target_cfg.get('bake', True)
+        elif isinstance(target_cfg, str):
+            comps = urllib.parse.urlparse(target_cfg)
+            if not comps.scheme:
+                raise InvalidPublishTargetError(
+                        "Publish target '%s' has an invalid target URL." %
+                        target)
+            parsed_url = comps
+            target_type = find_publisher_name(self.app, comps.scheme)
+            if target_type is None:
+                raise InvalidPublishTargetError(
+                        "No such publish target scheme: %s" % comps.scheme)
 
+        # Setup logging stuff.
+        hdlr = None
+        root_logger = logging.getLogger()
+        if log_file and not preview:
+            logger.debug("Adding file handler for: %s" % log_file)
+            hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8')
+            root_logger.addHandler(hdlr)
+        if not preview:
+            logger.info("Deploying to %s" % target)
+        else:
+            logger.info("Previewing deployment to %s" % target)
+
+        # Bake first is necessary.
+        bake_out_dir = None
+        if bake_first:
+            bake_out_dir = os.path.join(self.app.cache_dir, 'pub', target)
+            if not preview:
+                bake_start_time = time.perf_counter()
+                logger.debug("Baking first to: %s" % bake_out_dir)
+
+                from piecrust.baking.baker import Baker
+                baker = Baker(self.app, bake_out_dir)
+                rec1 = baker.bake()
+
+                from piecrust.processing.pipeline import ProcessorPipeline
+                proc = ProcessorPipeline(self.app, bake_out_dir)
+                rec2 = proc.run()
+
+                if not rec1.success or not rec2.success:
+                    raise Exception(
+                            "Error during baking, aborting publishing.")
+                logger.info(format_timed(bake_start_time, "Baked website."))
+            else:
+                logger.info("Would bake to: %s" % bake_out_dir)
+
+        # Create the appropriate publisher.
         pub = None
         for pub_cls in self.app.plugin_loader.getPublishers():
             if pub_cls.PUBLISHER_NAME == target_type:
@@ -37,39 +96,47 @@
             raise InvalidPublishTargetError(
                     "Publish target '%s' has invalid type: %s" %
                     (target, target_type))
+        pub.parsed_url = parsed_url
+
+        # Publish!
+        logger.debug(
+                "Running publish target '%s' with publisher: %s" %
+                (target, pub.PUBLISHER_NAME))
+        pub_start_time = time.perf_counter()
 
         ctx = PublishingContext()
-
-        hdlr = None
-        if log_file:
-            if not pub.is_using_custom_logging:
-                logger.debug("Adding file handler for: %s" % log_file)
-                hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8')
-                logger.addHandler(hdlr)
-            else:
-                logger.debug("Creating custom log file: %s" % log_file)
-                ctx.custom_logging_file = open(
-                        log_file, mode='w', encoding='utf8')
-
-        intro_msg = ("Running publish target '%s' with publisher: %s" %
-                     (target, pub.PUBLISHER_NAME))
-        logger.debug(intro_msg)
-        if ctx.custom_logging_file:
-            ctx.custom_logging_file.write(intro_msg + "\n")
-
+        ctx.bake_out_dir = bake_out_dir
+        ctx.preview = preview
         try:
             success = pub.run(ctx)
         except Exception as ex:
             raise PublishingError(
                     "Error publishing to target: %s" % target) from ex
         finally:
-            if ctx.custom_logging_file:
-                ctx.custom_logging_file.close()
             if hdlr:
-                logger.removeHandler(hdlr)
+                root_logger.removeHandler(hdlr)
                 hdlr.close()
 
         if not success:
             raise PublishingError(
                     "Unknown error publishing to target: %s" % target)
+        logger.info(format_timed(
+            pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME))
 
+        logger.info(format_timed(start_time, 'Deployed to %s' % target))
+
+
+def find_publisher_class(app, scheme):
+    for pub_cls in app.plugin_loader.getPublishers():
+        pub_sch = getattr(pub_cls, 'PUBLISHER_SCHEME', None)
+        if ('bake+%s' % pub_sch) == scheme:
+            return pub_cls
+    return None
+
+
+def find_publisher_name(app, scheme):
+    pub_cls = find_publisher_class(app, scheme)
+    if pub_cls:
+        return pub_cls.PUBLISHER_NAME
+    return None
+
--- a/piecrust/publishing/shell.py	Sat Feb 06 21:49:50 2016 -0800
+++ b/piecrust/publishing/shell.py	Mon Feb 08 20:44:26 2016 -0800
@@ -1,67 +1,15 @@
-import sys
 import shlex
-import logging
-import threading
-import subprocess
-from piecrust.publishing.base import Publisher
+from piecrust.publishing.base import ShellCommandPublisherBase
 
 
-logger = logging.getLogger(__name__)
-
-
-class ShellCommandPublisher(Publisher):
+class ShellCommandPublisher(ShellCommandPublisherBase):
     PUBLISHER_NAME = 'shell'
 
-    def __init__(self, app, target):
-        super(ShellCommandPublisher, self).__init__(app, target)
-        self.is_using_custom_logging = True
-
-    def run(self, ctx):
+    def _getCommandArgs(self, ctx):
         target_cmd = self.getConfigValue('cmd')
         if not target_cmd:
             raise Exception("No command specified for publish target: %s" %
                             self.target)
         args = shlex.split(target_cmd)
-
-        logger.debug(
-                "Running shell command: %s" % args)
-
-        proc = subprocess.Popen(
-                args, cwd=self.app.root_dir, bufsize=0,
-                stdout=subprocess.PIPE,
-                universal_newlines=False)
-
-        logger.debug("Running publishing monitor for PID %d" % proc.pid)
-        thread = _PublishThread(proc, ctx.custom_logging_file)
-        thread.start()
-        proc.wait()
-        thread.join()
-
-        if proc.returncode != 0:
-            logger.error(
-                    "Publish process returned code %d" % proc.returncode)
-        else:
-            logger.debug("Publish process returned successfully.")
+        return args
 
-        return proc.returncode == 0
-
-
-class _PublishThread(threading.Thread):
-    def __init__(self, proc, log_fp):
-        super(_PublishThread, self).__init__(
-                name='publish_monitor', daemon=True)
-        self.proc = proc
-        self.log_fp = log_fp
-
-    def run(self):
-        for line in iter(self.proc.stdout.readline, b''):
-            line_str = line.decode('utf8')
-            sys.stdout.write(line_str)
-            sys.stdout.flush()
-            if self.log_fp:
-                self.log_fp.write(line_str)
-                self.log_fp.flush()
-
-        self.proc.communicate()
-        logger.debug("Publish monitor exiting.")
-