Mercurial > piecrust2
changeset 613:e2e955a3bb25
publish: Add publish command.
* Add `shell` publisher.
* Refactor admin panel's publishing backend to use that, along with the new
PID file support.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Thu, 04 Feb 2016 08:05:03 -0800 |
| parents | 2edaefcb82cd |
| children | a2d9ef307a08 |
| files | foodtruck/pubutil.py foodtruck/sites.py foodtruck/views/publish.py piecrust/commands/builtin/publishing.py piecrust/plugins/base.py piecrust/plugins/builtin.py piecrust/publishing/__init__.py piecrust/publishing/base.py piecrust/publishing/publisher.py piecrust/publishing/shell.py |
| diffstat | 9 files changed, 304 insertions(+), 78 deletions(-) [+] |
line wrap: on
line diff
--- a/foodtruck/pubutil.py Thu Feb 04 08:03:52 2016 -0800 +++ b/foodtruck/pubutil.py Thu Feb 04 08:05:03 2016 -0800 @@ -1,6 +1,7 @@ import os import os.path import time +import errno import signal import logging from .web import app @@ -27,9 +28,37 @@ lambda *args: _shutdown_server_and_raise_sigint()) +def _pid_exists(pid): + try: + os.kill(pid, 0) + except OSError as ex: + if ex.errno == errno.ESRCH: + # No such process. + return False + elif ex.errno == errno.EPERM: + # No permission, so process exists. + return True + else: + raise + else: + return True + + +def _read_pid_file(pid_file): + logger.debug("Reading PID file: %s" % pid_file) + try: + with open(pid_file, 'r') as fp: + pid_str = fp.read() + + return int(pid_str.strip()) + except Exception: + logger.error("Error reading PID file.") + raise + + class PublishLogReader(object): _pub_max_time = 10 * 60 # Don't bother about pubs older than 10mins. - _poll_interval = 1 # Check the PID file every 1 seconds. + _poll_interval = 1 # Check the process every 1 seconds. _ping_interval = 30 # Send a ping message every 30 seconds. def __init__(self, pid_path, log_path): @@ -41,7 +70,8 @@ def run(self): logger.debug("Opening publish log...") - + pid = None + is_running = False try: while not server_shutdown: # PING! @@ -51,19 +81,38 @@ self._last_ping_time = time.time() yield bytes("event: ping\ndata: 1\n\n", 'utf8') - # Check pid file. - prev_mtime = self._pub_pid_mtime + # Check if the PID file has changed. try: - self._pub_pid_mtime = os.path.getmtime(self.pid_path) - if time.time() - self._pub_pid_mtime > \ - self._pub_max_time: - self._pub_pid_mtime = 0 + new_mtime = os.path.getmtime(self.pid_path) except OSError: - self._pub_pid_mtime = 0 + new_mtime = 0 + + if (new_mtime > 0 and + time.time() - new_mtime > self._pub_max_time): + new_mtime = 0 + + # Re-read the PID file. + prev_mtime = self._pub_pid_mtime + if new_mtime > 0 and new_mtime != prev_mtime: + self._pub_pid_mtime = new_mtime + pid = _read_pid_file(self.pid_path) + if pid: + logger.debug("Monitoring new process, PID: %d" % pid) + + was_running = is_running + if pid: + is_running = _pid_exists(pid) + logger.debug( + "Process %d is %s" % + (pid, 'running' if is_running else 'not running')) + if not is_running: + pid = None + else: + is_running = False # Send data. new_data = None - if self._pub_pid_mtime > 0 or prev_mtime > 0: + if is_running or was_running: if self._last_seek < 0: outstr = 'event: message\ndata: Publish started.\n\n' yield bytes(outstr, 'utf8') @@ -76,11 +125,11 @@ self._last_seek = fp.tell() except OSError: pass - if self._pub_pid_mtime == 0: + if not is_running: self._last_seek = 0 if new_data: - logger.debug("SSE: %s" % outstr) + logger.debug("SSE: %s" % new_data) for line in new_data.split('\n'): outstr = 'event: message\ndata: %s\n\n' % line yield bytes(outstr, 'utf8')
--- a/foodtruck/sites.py Thu Feb 04 08:03:52 2016 -0800 +++ b/foodtruck/sites.py Thu Feb 04 08:05:03 2016 -0800 @@ -1,7 +1,6 @@ import os import os.path import copy -import shlex import logging import threading import subprocess @@ -27,7 +26,6 @@ self._global_config = config self._piecrust_app = None self._scm = None - self._publish_thread = None logger.debug("Creating site object for %s" % self.name) @property @@ -56,70 +54,26 @@ return self._scm @property - def is_publish_running(self): - return (self._publish_thread is not None and - self._publish_thread.is_alive()) + def publish_pid_file(self): + return os.path.join(self.piecrust_app.cache_dir, 'publish.pid') @property - def publish_thread(self): - return self._publish_thread + def publish_log_file(self): + return os.path.join(self.piecrust_app.cache_dir, 'publish.log') def publish(self, target): - target_cfg = self.piecrust_app.config.get('publish/%s' % target) - if not target_cfg: - raise Exception("No such publish target: %s" % target) - - target_cmd = target_cfg.get('cmd') - if not target_cmd: - raise Exception("No command specified for publish target: %s" % - target) - publish_args = shlex.split(target_cmd) - - logger.debug( - "Executing publish target '%s': %s" % (target, publish_args)) - proc = subprocess.Popen(publish_args, cwd=self.root_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - pid_file_path = os.path.join(self.root_dir, '.ft_pub.pid') - with open(pid_file_path, 'w') as fp: - fp.write(str(proc.pid)) - - logger.debug("Running publishing monitor for PID %d" % proc.pid) - self._publish_thread = _PublishThread( - self.name, self.root_dir, proc, self._onPublishEnd) - self._publish_thread.start() + args = [ + 'chef', + '--pid-file', self.publish_pid_file, + 'publish', target, + '--log-publisher', self.publish_log_file] + proc = subprocess.Popen(args, cwd=self.root_dir) - def _onPublishEnd(self): - os.unlink(os.path.join(self.root_dir, '.ft_pub.pid')) - self._publish_thread = None - - -class _PublishThread(threading.Thread): - def __init__(self, sitename, siteroot, proc, callback): - super(_PublishThread, self).__init__( - name='%s_publish' % sitename, daemon=True) - self.sitename = sitename - self.siteroot = siteroot - self.proc = proc - self.callback = callback + def _comm(): + proc.communicate() - log_file_path = os.path.join(self.siteroot, '.ft_pub.log') - self.log_fp = open(log_file_path, 'w', encoding='utf8') - - def run(self): - for line in self.proc.stdout: - self.log_fp.write(line.decode('utf8')) - for line in self.proc.stderr: - self.log_fp.write(line.decode('utf8')) - self.proc.communicate() - if self.proc.returncode != 0: - self.log_fp.write("Error, publish process returned code %d" % - self.proc.returncode) - self.log_fp.close() - - logger.debug("Publish ended for %s." % self.sitename) - self.callback() + t = threading.Thread(target=_comm, daemon=True) + t.start() class FoodTruckSites():
--- a/foodtruck/views/publish.py Thu Feb 04 08:03:52 2016 -0800 +++ b/foodtruck/views/publish.py Thu Feb 04 08:05:03 2016 -0800 @@ -1,5 +1,3 @@ -import os -import os.path import copy import logging from flask import request, g, url_for, render_template, Response @@ -49,9 +47,8 @@ @app.route('/publish-log') @login_required def stream_publish_log(): - site = g.site - pid_path = os.path.join(site.root_dir, '.ft_pub.pid') - log_path = os.path.join(site.root_dir, '.ft_pub.log') + pid_path = g.site.publish_pid_file + log_path = g.site.publish_log_file rdr = PublishLogReader(pid_path, log_path) response = Response(rdr.run(), mimetype='text/event-stream')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/commands/builtin/publishing.py Thu Feb 04 08:05:03 2016 -0800 @@ -0,0 +1,49 @@ +import logging +from piecrust.commands.base import ChefCommand +from piecrust.publishing.publisher import Publisher + + +logger = logging.getLogger(__name__) + + +class PublishCommand(ChefCommand): + """ Command for running publish targets for the current site. + """ + def __init__(self): + super(PublishCommand, self).__init__() + self.name = 'publish' + self.description = "Publishes you website to a specific target." + + def setupParser(self, parser, app): + parser.add_argument( + '-l', '--list', + action='store_true', + help="List available publish targets for the current site.") + parser.add_argument( + '--log-publisher', + metavar='LOG_FILE', + help="Log the publisher's output to a given file.") + parser.add_argument( + 'target', + nargs='?', + default='default', + help="The publish target to use.") + + def run(self, ctx): + if ctx.args.list: + pub_cfg = ctx.app.config.get('publish') + if not pub_cfg: + logger.info("No available publish targets.") + return + + for name, cfg in pub_cfg.items(): + desc = cfg.get('description') + if not desc: + logger.info(name) + else: + logger.info("%s: %s" % (name, desc)) + return + + pub = Publisher(ctx.app) + pub.run(ctx.args.target, log_file=ctx.args.log_publisher) +
--- a/piecrust/plugins/base.py Thu Feb 04 08:03:52 2016 -0800 +++ b/piecrust/plugins/base.py Thu Feb 04 08:05:03 2016 -0800 @@ -33,6 +33,9 @@ def getSources(self): return [] + def getPublishers(self): + return [] + def initialize(self, app): pass @@ -83,6 +86,9 @@ def getSources(self): return self._getPluginComponents('getSources') + def getPublishers(self): + return self._getPluginComponents('getPublishers') + def _ensureLoaded(self): if self._plugins is not None: return
--- a/piecrust/plugins/builtin.py Thu Feb 04 08:03:52 2016 -0800 +++ b/piecrust/plugins/builtin.py Thu Feb 04 08:05:03 2016 -0800 @@ -6,6 +6,7 @@ RootCommand, ShowConfigCommand, FindCommand, ShowSourcesCommand, ShowRoutesCommand, ShowPathsCommand) from piecrust.commands.builtin.plugins import PluginsCommand +from piecrust.commands.builtin.publishing import PublishCommand from piecrust.commands.builtin.scaffolding import ( PrepareCommand, DefaultPrepareTemplatesCommandExtension, @@ -33,6 +34,7 @@ from piecrust.processing.sass import SassProcessor from piecrust.processing.sitemap import SitemapProcessor from piecrust.processing.util import ConcatProcessor +from piecrust.publishing.shell import ShellCommandPublisher from piecrust.sources.default import DefaultPageSource from piecrust.sources.posts import ( FlatPostsSource, ShallowPostsSource, HierarchyPostsSource) @@ -64,7 +66,8 @@ BakeCommand(), ShowRecordCommand(), ServeCommand(), - AdministrationPanelCommand()] + AdministrationPanelCommand(), + PublishCommand()] def getCommandExtensions(self): return [ @@ -117,3 +120,7 @@ JekyllImporter(), WordpressXmlImporter()] + def getPublishers(self): + return [ + ShellCommandPublisher] +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/publishing/base.py Thu Feb 04 08:05:03 2016 -0800 @@ -0,0 +1,23 @@ + + +class PublishingContext(object): + def __init__(self): + self.custom_logging_file = None + + +class Publisher(object): + def __init__(self, app, target): + self.app = app + self.target = target + self.is_using_custom_logging = False + self.log_file_path = None + + def getConfig(self): + return self.app.config.get('publish/%s' % self.target) + + def getConfigValue(self, name): + return self.app.config.get('publish/%s/%s' % (self.target, name)) + + def run(self, ctx): + raise NotImplementedError() +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/publishing/publisher.py Thu Feb 04 08:05:03 2016 -0800 @@ -0,0 +1,75 @@ +import logging +from piecrust.publishing.base import PublishingContext + + +logger = logging.getLogger(__name__) + + +class InvalidPublishTargetError(Exception): + pass + + +class PublishingError(Exception): + pass + + +class Publisher(object): + def __init__(self, app): + self.app = app + + def run(self, target, log_file=None): + target_cfg = self.app.config.get('publish/%s' % target) + if not target_cfg: + raise InvalidPublishTargetError( + "No such publish target: %s" % target) + + target_type = target_cfg.get('type') + if not target_type: + raise InvalidPublishTargetError( + "Publish target '%s' doesn't specify a type." % target) + + pub = None + for pub_cls in self.app.plugin_loader.getPublishers(): + if pub_cls.PUBLISHER_NAME == target_type: + pub = pub_cls(self.app, target) + break + if pub is None: + raise InvalidPublishTargetError( + "Publish target '%s' has invalid type: %s" % + (target, target_type)) + + ctx = PublishingContext() + + hdlr = None + if log_file: + if not pub.is_using_custom_logging: + logger.debug("Adding file handler for: %s" % log_file) + hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8') + logger.addHandler(hdlr) + else: + logger.debug("Creating custom log file: %s" % log_file) + ctx.custom_logging_file = open( + log_file, mode='w', encoding='utf8') + + intro_msg = ("Running publish target '%s' with publisher: %s" % + (target, pub.PUBLISHER_NAME)) + logger.debug(intro_msg) + if ctx.custom_logging_file: + ctx.custom_logging_file.write(intro_msg + "\n") + + try: + success = pub.run(ctx) + except Exception as ex: + raise PublishingError( + "Error publishing to target: %s" % target) from ex + finally: + if ctx.custom_logging_file: + ctx.custom_logging_file.close() + if hdlr: + logger.removeHandler(hdlr) + hdlr.close() + + if not success: + raise PublishingError( + "Unknown error publishing to target: %s" % target) +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/publishing/shell.py Thu Feb 04 08:05:03 2016 -0800 @@ -0,0 +1,66 @@ +import sys +import shlex +import logging +import threading +import subprocess +from piecrust.publishing.base import Publisher + + +logger = logging.getLogger(__name__) + + +class ShellCommandPublisher(Publisher): + PUBLISHER_NAME = 'shell' + + def __init__(self, app, target): + super(ShellCommandPublisher, self).__init__(app, target) + self.is_using_custom_logging = True + + def run(self, ctx): + target_cmd = self.getConfigValue('cmd') + if not target_cmd: + raise Exception("No command specified for publish target: %s" % + self.target) + args = shlex.split(target_cmd) + + logger.debug( + "Running shell command: %s" % args) + + proc = subprocess.Popen( + args, cwd=self.app.root_dir, bufsize=0, + stdout=subprocess.PIPE, + universal_newlines=False) + + logger.debug("Running publishing monitor for PID %d" % proc.pid) + thread = _PublishThread(proc, ctx.custom_logging_file) + thread.start() + proc.wait() + thread.join() + + if proc.returncode != 0: + logger.error( + "Publish process returned code %d" % proc.returncode) + else: + logger.debug("Publish process returned successfully.") + + return proc.returncode == 0 + + +class _PublishThread(threading.Thread): + def __init__(self, proc, log_fp): + super(_PublishThread, self).__init__( + name='publish_monitor', daemon=True) + self.proc = proc + self.log_fp = log_fp + + def run(self): + for line in iter(self.proc.stdout.readline, b''): + line_str = line.decode('utf8') + sys.stdout.write(line_str) + sys.stdout.flush() + if self.log_fp: + self.log_fp.write(line_str) + + self.proc.communicate() + logger.debug("Publish monitor exiting.") +