Mercurial > piecrust2

annotate foodtruck/pubutil.py @ 613:e2e955a3bb25

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
publish: Add publish command. * Add `shell` publisher. * Refactor admin panel's publishing backend to use that, along with the new PID file support.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 04 Feb 2016 08:05:03 -0800
parents 978d8bca9fb3
children cbb170d9c894
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import time
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
4 import errno
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import signal
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 import logging
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from .web import app
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 logger = logging.getLogger(__name__)
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 server_shutdown = False
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 def _shutdown_server_and_raise_sigint():
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 # This is needed when hitting CTRL+C to shutdown the Werkzeug server,
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 # otherwise SSE generators will keep it alive.
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 logger.debug("Shutting down SSE generators...")
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 global server_shutdown
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 server_shutdown = True
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 raise KeyboardInterrupt()
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 if app.config['FOODTRUCK_CMDLINE_MODE']:
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 # Make sure CTRL+C works correctly.
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 signal.signal(signal.SIGINT,
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 lambda *args: _shutdown_server_and_raise_sigint())
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
31 def _pid_exists(pid):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
32 try:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
33 os.kill(pid, 0)
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
34 except OSError as ex:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
35 if ex.errno == errno.ESRCH:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
36 # No such process.
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
37 return False
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
38 elif ex.errno == errno.EPERM:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
39 # No permission, so process exists.
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
40 return True
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
41 else:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
42 raise
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
43 else:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
44 return True
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
45
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
46
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
47 def _read_pid_file(pid_file):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
48 logger.debug("Reading PID file: %s" % pid_file)
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
49 try:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
50 with open(pid_file, 'r') as fp:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
51 pid_str = fp.read()
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
52
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
53 return int(pid_str.strip())
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
54 except Exception:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
55 logger.error("Error reading PID file.")
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
56 raise
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
57
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
58
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 class PublishLogReader(object):
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 _pub_max_time = 10 * 60 # Don't bother about pubs older than 10mins.
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
61 _poll_interval = 1 # Check the process every 1 seconds.
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 _ping_interval = 30 # Send a ping message every 30 seconds.
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 def __init__(self, pid_path, log_path):
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 self.pid_path = pid_path
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 self.log_path = log_path
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 self._pub_pid_mtime = 0
609
978d8bca9fb3 admin: Fix "Publish started" message showing up multiple times.
Ludovic Chabant <ludovic@chabant.com>
parents: 602
diff changeset
68 self._last_seek = -1
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 self._last_ping_time = 0
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 def run(self):
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 logger.debug("Opening publish log...")
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
73 pid = None
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
74 is_running = False
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 try:
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 while not server_shutdown:
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 # PING!
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 interval = time.time() - self._last_ping_time
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 if interval > self._ping_interval:
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 logger.debug("Sending ping...")
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 self._last_ping_time = time.time()
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 yield bytes("event: ping\ndata: 1\n\n", 'utf8')
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
84 # Check if the PID file has changed.
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 try:
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
86 new_mtime = os.path.getmtime(self.pid_path)
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 except OSError:
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
88 new_mtime = 0
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
89
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
90 if (new_mtime > 0 and
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
91 time.time() - new_mtime > self._pub_max_time):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
92 new_mtime = 0
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
93
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
94 # Re-read the PID file.
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
95 prev_mtime = self._pub_pid_mtime
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
96 if new_mtime > 0 and new_mtime != prev_mtime:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
97 self._pub_pid_mtime = new_mtime
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
98 pid = _read_pid_file(self.pid_path)
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
99 if pid:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
100 logger.debug("Monitoring new process, PID: %d" % pid)
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
101
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
102 was_running = is_running
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
103 if pid:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
104 is_running = _pid_exists(pid)
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
105 logger.debug(
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
106 "Process %d is %s" %
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
107 (pid, 'running' if is_running else 'not running'))
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
108 if not is_running:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
109 pid = None
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
110 else:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
111 is_running = False
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113 # Send data.
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 new_data = None
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
115 if is_running or was_running:
609
978d8bca9fb3 admin: Fix "Publish started" message showing up multiple times.
Ludovic Chabant <ludovic@chabant.com>
parents: 602
diff changeset
116 if self._last_seek < 0:
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 outstr = 'event: message\ndata: Publish started.\n\n'
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 yield bytes(outstr, 'utf8')
609
978d8bca9fb3 admin: Fix "Publish started" message showing up multiple times.
Ludovic Chabant <ludovic@chabant.com>
parents: 602
diff changeset
119 self._last_seek = 0
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 try:
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 with open(self.log_path, 'r', encoding='utf8') as fp:
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 fp.seek(self._last_seek)
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 new_data = fp.read()
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
125 self._last_seek = fp.tell()
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
126 except OSError:
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
127 pass
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
128 if not is_running:
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129 self._last_seek = 0
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
131 if new_data:
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents: 609
diff changeset
132 logger.debug("SSE: %s" % new_data)
602
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
133 for line in new_data.split('\n'):
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134 outstr = 'event: message\ndata: %s\n\n' % line
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
135 yield bytes(outstr, 'utf8')
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
136
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
137 time.sleep(self._poll_interval)
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
138
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
139 except GeneratorExit:
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
140 pass
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142 logger.debug("Closing publish log...")
c6bc0ef03f82 admin: Better UI for publishing websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143