# HG changeset patch
# User Ludovic Chabant <ludovic@chabant.com>
# Date 1454601903 28800
# Node ID e2e955a3bb259743b22cc14cf8aea4db7298d732
# Parent  2edaefcb82cda04e99835897a7219310aff11398
publish: Add publish command.

* Add `shell` publisher.
* Refactor admin panel's publishing backend to use that, along with the new
  PID file support.

diff -r 2edaefcb82cd -r e2e955a3bb25 foodtruck/pubutil.py
--- a/foodtruck/pubutil.py	Thu Feb 04 08:03:52 2016 -0800
+++ b/foodtruck/pubutil.py	Thu Feb 04 08:05:03 2016 -0800
@@ -1,6 +1,7 @@
 import os
 import os.path
 import time
+import errno
 import signal
 import logging
 from .web import app
@@ -27,9 +28,37 @@
                   lambda *args: _shutdown_server_and_raise_sigint())
 
 
+def _pid_exists(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError as ex:
+        if ex.errno == errno.ESRCH:
+            # No such process.
+            return False
+        elif ex.errno == errno.EPERM:
+            # No permission, so process exists.
+            return True
+        else:
+            raise
+    else:
+        return True
+
+
+def _read_pid_file(pid_file):
+    logger.debug("Reading PID file: %s" % pid_file)
+    try:
+        with open(pid_file, 'r') as fp:
+            pid_str = fp.read()
+
+        return int(pid_str.strip())
+    except Exception:
+        logger.error("Error reading PID file.")
+        raise
+
+
 class PublishLogReader(object):
     _pub_max_time = 10 * 60   # Don't bother about pubs older than 10mins.
-    _poll_interval = 1        # Check the PID file every 1 seconds.
+    _poll_interval = 1        # Check the process every 1 seconds.
     _ping_interval = 30       # Send a ping message every 30 seconds.
 
     def __init__(self, pid_path, log_path):
@@ -41,7 +70,8 @@
 
     def run(self):
         logger.debug("Opening publish log...")
-
+        pid = None
+        is_running = False
         try:
             while not server_shutdown:
                 # PING!
@@ -51,19 +81,38 @@
                     self._last_ping_time = time.time()
                     yield bytes("event: ping\ndata: 1\n\n", 'utf8')
 
-                # Check pid file.
-                prev_mtime = self._pub_pid_mtime
+                # Check if the PID file has changed.
                 try:
-                    self._pub_pid_mtime = os.path.getmtime(self.pid_path)
-                    if time.time() - self._pub_pid_mtime > \
-                            self._pub_max_time:
-                        self._pub_pid_mtime = 0
+                    new_mtime = os.path.getmtime(self.pid_path)
                 except OSError:
-                    self._pub_pid_mtime = 0
+                    new_mtime = 0
+
+                if (new_mtime > 0 and
+                        time.time() - new_mtime > self._pub_max_time):
+                    new_mtime = 0
+
+                # Re-read the PID file.
+                prev_mtime = self._pub_pid_mtime
+                if new_mtime > 0 and new_mtime != prev_mtime:
+                    self._pub_pid_mtime = new_mtime
+                    pid = _read_pid_file(self.pid_path)
+                    if pid:
+                        logger.debug("Monitoring new process, PID: %d" % pid)
+
+                was_running = is_running
+                if pid:
+                    is_running = _pid_exists(pid)
+                    logger.debug(
+                            "Process %d is %s" %
+                            (pid, 'running' if is_running else 'not running'))
+                    if not is_running:
+                        pid = None
+                else:
+                    is_running = False
 
                 # Send data.
                 new_data = None
-                if self._pub_pid_mtime > 0 or prev_mtime > 0:
+                if is_running or was_running:
                     if self._last_seek < 0:
                         outstr = 'event: message\ndata: Publish started.\n\n'
                         yield bytes(outstr, 'utf8')
@@ -76,11 +125,11 @@
                             self._last_seek = fp.tell()
                     except OSError:
                         pass
-                if self._pub_pid_mtime == 0:
+                if not is_running:
                     self._last_seek = 0
 
                 if new_data:
-                    logger.debug("SSE: %s" % outstr)
+                    logger.debug("SSE: %s" % new_data)
                     for line in new_data.split('\n'):
                         outstr = 'event: message\ndata: %s\n\n' % line
                         yield bytes(outstr, 'utf8')
diff -r 2edaefcb82cd -r e2e955a3bb25 foodtruck/sites.py
--- a/foodtruck/sites.py	Thu Feb 04 08:03:52 2016 -0800
+++ b/foodtruck/sites.py	Thu Feb 04 08:05:03 2016 -0800
@@ -1,7 +1,6 @@
 import os
 import os.path
 import copy
-import shlex
 import logging
 import threading
 import subprocess
@@ -27,7 +26,6 @@
         self._global_config = config
         self._piecrust_app = None
         self._scm = None
-        self._publish_thread = None
         logger.debug("Creating site object for %s" % self.name)
 
     @property
@@ -56,70 +54,26 @@
         return self._scm
 
     @property
-    def is_publish_running(self):
-        return (self._publish_thread is not None and
-                self._publish_thread.is_alive())
+    def publish_pid_file(self):
+        return os.path.join(self.piecrust_app.cache_dir, 'publish.pid')
 
     @property
-    def publish_thread(self):
-        return self._publish_thread
+    def publish_log_file(self):
+        return os.path.join(self.piecrust_app.cache_dir, 'publish.log')
 
     def publish(self, target):
-        target_cfg = self.piecrust_app.config.get('publish/%s' % target)
-        if not target_cfg:
-            raise Exception("No such publish target: %s" % target)
-
-        target_cmd = target_cfg.get('cmd')
-        if not target_cmd:
-            raise Exception("No command specified for publish target: %s" %
-                            target)
-        publish_args = shlex.split(target_cmd)
-
-        logger.debug(
-                "Executing publish target '%s': %s" % (target, publish_args))
-        proc = subprocess.Popen(publish_args, cwd=self.root_dir,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
-
-        pid_file_path = os.path.join(self.root_dir, '.ft_pub.pid')
-        with open(pid_file_path, 'w') as fp:
-            fp.write(str(proc.pid))
-
-        logger.debug("Running publishing monitor for PID %d" % proc.pid)
-        self._publish_thread = _PublishThread(
-                self.name, self.root_dir, proc, self._onPublishEnd)
-        self._publish_thread.start()
+        args = [
+                'chef',
+                '--pid-file', self.publish_pid_file,
+                'publish', target,
+                '--log-publisher', self.publish_log_file]
+        proc = subprocess.Popen(args, cwd=self.root_dir)
 
-    def _onPublishEnd(self):
-        os.unlink(os.path.join(self.root_dir, '.ft_pub.pid'))
-        self._publish_thread = None
-
-
-class _PublishThread(threading.Thread):
-    def __init__(self, sitename, siteroot, proc, callback):
-        super(_PublishThread, self).__init__(
-                name='%s_publish' % sitename, daemon=True)
-        self.sitename = sitename
-        self.siteroot = siteroot
-        self.proc = proc
-        self.callback = callback
+        def _comm():
+            proc.communicate()
 
-        log_file_path = os.path.join(self.siteroot, '.ft_pub.log')
-        self.log_fp = open(log_file_path, 'w', encoding='utf8')
-
-    def run(self):
-        for line in self.proc.stdout:
-            self.log_fp.write(line.decode('utf8'))
-        for line in self.proc.stderr:
-            self.log_fp.write(line.decode('utf8'))
-        self.proc.communicate()
-        if self.proc.returncode != 0:
-            self.log_fp.write("Error, publish process returned code %d" %
-                              self.proc.returncode)
-        self.log_fp.close()
-
-        logger.debug("Publish ended for %s." % self.sitename)
-        self.callback()
+        t = threading.Thread(target=_comm, daemon=True)
+        t.start()
 
 
 class FoodTruckSites():
diff -r 2edaefcb82cd -r e2e955a3bb25 foodtruck/views/publish.py
--- a/foodtruck/views/publish.py	Thu Feb 04 08:03:52 2016 -0800
+++ b/foodtruck/views/publish.py	Thu Feb 04 08:05:03 2016 -0800
@@ -1,5 +1,3 @@
-import os
-import os.path
 import copy
 import logging
 from flask import request, g, url_for, render_template, Response
@@ -49,9 +47,8 @@
 @app.route('/publish-log')
 @login_required
 def stream_publish_log():
-    site = g.site
-    pid_path = os.path.join(site.root_dir, '.ft_pub.pid')
-    log_path = os.path.join(site.root_dir, '.ft_pub.log')
+    pid_path = g.site.publish_pid_file
+    log_path = g.site.publish_log_file
     rdr = PublishLogReader(pid_path, log_path)
 
     response = Response(rdr.run(), mimetype='text/event-stream')
diff -r 2edaefcb82cd -r e2e955a3bb25 piecrust/commands/builtin/publishing.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/piecrust/commands/builtin/publishing.py	Thu Feb 04 08:05:03 2016 -0800
@@ -0,0 +1,49 @@
+import logging
+from piecrust.commands.base import ChefCommand
+from piecrust.publishing.publisher import Publisher
+
+
+logger = logging.getLogger(__name__)
+
+
+class PublishCommand(ChefCommand):
+    """ Command for running publish targets for the current site.
+    """
+    def __init__(self):
+        super(PublishCommand, self).__init__()
+        self.name = 'publish'
+        self.description = "Publishes you website to a specific target."
+
+    def setupParser(self, parser, app):
+        parser.add_argument(
+                '-l', '--list',
+                action='store_true',
+                help="List available publish targets for the current site.")
+        parser.add_argument(
+                '--log-publisher',
+                metavar='LOG_FILE',
+                help="Log the publisher's output to a given file.")
+        parser.add_argument(
+                'target',
+                nargs='?',
+                default='default',
+                help="The publish target to use.")
+
+    def run(self, ctx):
+        if ctx.args.list:
+            pub_cfg = ctx.app.config.get('publish')
+            if not pub_cfg:
+                logger.info("No available publish targets.")
+                return
+
+            for name, cfg in pub_cfg.items():
+                desc = cfg.get('description')
+                if not desc:
+                    logger.info(name)
+                else:
+                    logger.info("%s: %s" % (name, desc))
+            return
+
+        pub = Publisher(ctx.app)
+        pub.run(ctx.args.target, log_file=ctx.args.log_publisher)
+
diff -r 2edaefcb82cd -r e2e955a3bb25 piecrust/plugins/base.py
--- a/piecrust/plugins/base.py	Thu Feb 04 08:03:52 2016 -0800
+++ b/piecrust/plugins/base.py	Thu Feb 04 08:05:03 2016 -0800
@@ -33,6 +33,9 @@
     def getSources(self):
         return []
 
+    def getPublishers(self):
+        return []
+
     def initialize(self, app):
         pass
 
@@ -83,6 +86,9 @@
     def getSources(self):
         return self._getPluginComponents('getSources')
 
+    def getPublishers(self):
+        return self._getPluginComponents('getPublishers')
+
     def _ensureLoaded(self):
         if self._plugins is not None:
             return
diff -r 2edaefcb82cd -r e2e955a3bb25 piecrust/plugins/builtin.py
--- a/piecrust/plugins/builtin.py	Thu Feb 04 08:03:52 2016 -0800
+++ b/piecrust/plugins/builtin.py	Thu Feb 04 08:05:03 2016 -0800
@@ -6,6 +6,7 @@
         RootCommand, ShowConfigCommand,
         FindCommand, ShowSourcesCommand, ShowRoutesCommand, ShowPathsCommand)
 from piecrust.commands.builtin.plugins import PluginsCommand
+from piecrust.commands.builtin.publishing import PublishCommand
 from piecrust.commands.builtin.scaffolding import (
         PrepareCommand,
         DefaultPrepareTemplatesCommandExtension,
@@ -33,6 +34,7 @@
 from piecrust.processing.sass import SassProcessor
 from piecrust.processing.sitemap import SitemapProcessor
 from piecrust.processing.util import ConcatProcessor
+from piecrust.publishing.shell import ShellCommandPublisher
 from piecrust.sources.default import DefaultPageSource
 from piecrust.sources.posts import (
         FlatPostsSource, ShallowPostsSource, HierarchyPostsSource)
@@ -64,7 +66,8 @@
                 BakeCommand(),
                 ShowRecordCommand(),
                 ServeCommand(),
-                AdministrationPanelCommand()]
+                AdministrationPanelCommand(),
+                PublishCommand()]
 
     def getCommandExtensions(self):
         return [
@@ -117,3 +120,7 @@
                 JekyllImporter(),
                 WordpressXmlImporter()]
 
+    def getPublishers(self):
+        return [
+                ShellCommandPublisher]
+
diff -r 2edaefcb82cd -r e2e955a3bb25 piecrust/publishing/__init__.py
diff -r 2edaefcb82cd -r e2e955a3bb25 piecrust/publishing/base.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/piecrust/publishing/base.py	Thu Feb 04 08:05:03 2016 -0800
@@ -0,0 +1,23 @@
+
+
+class PublishingContext(object):
+    def __init__(self):
+        self.custom_logging_file = None
+
+
+class Publisher(object):
+    def __init__(self, app, target):
+        self.app = app
+        self.target = target
+        self.is_using_custom_logging = False
+        self.log_file_path = None
+
+    def getConfig(self):
+        return self.app.config.get('publish/%s' % self.target)
+
+    def getConfigValue(self, name):
+        return self.app.config.get('publish/%s/%s' % (self.target, name))
+
+    def run(self, ctx):
+        raise NotImplementedError()
+
diff -r 2edaefcb82cd -r e2e955a3bb25 piecrust/publishing/publisher.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/piecrust/publishing/publisher.py	Thu Feb 04 08:05:03 2016 -0800
@@ -0,0 +1,75 @@
+import logging
+from piecrust.publishing.base import PublishingContext
+
+
+logger = logging.getLogger(__name__)
+
+
+class InvalidPublishTargetError(Exception):
+    pass
+
+
+class PublishingError(Exception):
+    pass
+
+
+class Publisher(object):
+    def __init__(self, app):
+        self.app = app
+
+    def run(self, target, log_file=None):
+        target_cfg = self.app.config.get('publish/%s' % target)
+        if not target_cfg:
+            raise InvalidPublishTargetError(
+                    "No such publish target: %s" % target)
+
+        target_type = target_cfg.get('type')
+        if not target_type:
+            raise InvalidPublishTargetError(
+                    "Publish target '%s' doesn't specify a type." % target)
+
+        pub = None
+        for pub_cls in self.app.plugin_loader.getPublishers():
+            if pub_cls.PUBLISHER_NAME == target_type:
+                pub = pub_cls(self.app, target)
+                break
+        if pub is None:
+            raise InvalidPublishTargetError(
+                    "Publish target '%s' has invalid type: %s" %
+                    (target, target_type))
+
+        ctx = PublishingContext()
+
+        hdlr = None
+        if log_file:
+            if not pub.is_using_custom_logging:
+                logger.debug("Adding file handler for: %s" % log_file)
+                hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8')
+                logger.addHandler(hdlr)
+            else:
+                logger.debug("Creating custom log file: %s" % log_file)
+                ctx.custom_logging_file = open(
+                        log_file, mode='w', encoding='utf8')
+
+        intro_msg = ("Running publish target '%s' with publisher: %s" %
+                     (target, pub.PUBLISHER_NAME))
+        logger.debug(intro_msg)
+        if ctx.custom_logging_file:
+            ctx.custom_logging_file.write(intro_msg + "\n")
+
+        try:
+            success = pub.run(ctx)
+        except Exception as ex:
+            raise PublishingError(
+                    "Error publishing to target: %s" % target) from ex
+        finally:
+            if ctx.custom_logging_file:
+                ctx.custom_logging_file.close()
+            if hdlr:
+                logger.removeHandler(hdlr)
+                hdlr.close()
+
+        if not success:
+            raise PublishingError(
+                    "Unknown error publishing to target: %s" % target)
+
diff -r 2edaefcb82cd -r e2e955a3bb25 piecrust/publishing/shell.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/piecrust/publishing/shell.py	Thu Feb 04 08:05:03 2016 -0800
@@ -0,0 +1,66 @@
+import sys
+import shlex
+import logging
+import threading
+import subprocess
+from piecrust.publishing.base import Publisher
+
+
+logger = logging.getLogger(__name__)
+
+
+class ShellCommandPublisher(Publisher):
+    PUBLISHER_NAME = 'shell'
+
+    def __init__(self, app, target):
+        super(ShellCommandPublisher, self).__init__(app, target)
+        self.is_using_custom_logging = True
+
+    def run(self, ctx):
+        target_cmd = self.getConfigValue('cmd')
+        if not target_cmd:
+            raise Exception("No command specified for publish target: %s" %
+                            self.target)
+        args = shlex.split(target_cmd)
+
+        logger.debug(
+                "Running shell command: %s" % args)
+
+        proc = subprocess.Popen(
+                args, cwd=self.app.root_dir, bufsize=0,
+                stdout=subprocess.PIPE,
+                universal_newlines=False)
+
+        logger.debug("Running publishing monitor for PID %d" % proc.pid)
+        thread = _PublishThread(proc, ctx.custom_logging_file)
+        thread.start()
+        proc.wait()
+        thread.join()
+
+        if proc.returncode != 0:
+            logger.error(
+                    "Publish process returned code %d" % proc.returncode)
+        else:
+            logger.debug("Publish process returned successfully.")
+
+        return proc.returncode == 0
+
+
+class _PublishThread(threading.Thread):
+    def __init__(self, proc, log_fp):
+        super(_PublishThread, self).__init__(
+                name='publish_monitor', daemon=True)
+        self.proc = proc
+        self.log_fp = log_fp
+
+    def run(self):
+        for line in iter(self.proc.stdout.readline, b''):
+            line_str = line.decode('utf8')
+            sys.stdout.write(line_str)
+            sys.stdout.flush()
+            if self.log_fp:
+                self.log_fp.write(line_str)
+
+        self.proc.communicate()
+        logger.debug("Publish monitor exiting.")
+