Mercurial > piecrust2
changeset 553:cc6f3dbe3048
serve: Extract some of the server's functionality into WSGI middlewares.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 08 Aug 2015 22:01:47 -0700 |
parents | 9612cfc6455a |
children | 155c7e20414f |
files | piecrust/serving/middlewares.py piecrust/serving/procloop.py piecrust/serving/server.py piecrust/serving/util.py piecrust/serving/wrappers.py piecrust/wsgiutil/__init__.py tests/conftest.py |
diffstat | 7 files changed, 194 insertions(+), 149 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/serving/middlewares.py Sat Aug 08 22:01:47 2015 -0700 @@ -0,0 +1,94 @@ +import os.path +from werkzeug.wrappers import Request, Response +from werkzeug.wsgi import ClosingIterator +from piecrust import RESOURCES_DIR, CACHE_DIR +from piecrust.serving.util import make_wrapped_file_response + + +class StaticResourcesMiddleware(object): + """ WSGI middleware that serves static files from the `resources/server` + directory in the PieCrust package. + """ + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + static_mount = '/__piecrust_static/' + + request = Request(environ) + if request.path.startswith(static_mount): + rel_req_path = request.path[len(static_mount):] + mount = os.path.join(RESOURCES_DIR, 'server') + full_path = os.path.join(mount, rel_req_path) + try: + response = make_wrapped_file_response( + environ, request, full_path) + return response(environ, start_response) + except OSError: + pass + + return self.app(environ, start_response) + + +class PieCrustDebugMiddleware(object): + """ WSGI middleware that handles debugging of PieCrust stuff. + """ + def __init__(self, app, root_dir, debug=False, + sub_cache_dir=None, run_sse_check=None): + self.app = app + self.root_dir = root_dir + self.debug = debug + self.run_sse_check = run_sse_check + self._proc_loop = None + self._out_dir = os.path.join(root_dir, CACHE_DIR, 'server') + if sub_cache_dir: + self._out_dir = os.path.join(sub_cache_dir, 'server') + self._handlers = { + 'werkzeug_shutdown': self._shutdownWerkzeug, + 'pipeline_status': self._startSSEProvider} + + if not self.run_sse_check or self.run_sse_check(): + # When using a server with code reloading, some implementations + # use process forking and we end up going here twice. We only want + # to start the pipeline loop in the inner process most of the + # time so we let the implementation tell us if this is OK. + from piecrust.serving.procloop import ProcessingLoop + self._proc_loop = ProcessingLoop(root_dir, self._out_dir, + sub_cache_dir=sub_cache_dir, + debug=debug) + self._proc_loop.start() + + def __call__(self, environ, start_response): + debug_mount = '/__piecrust_debug/' + + request = Request(environ) + if request.path.startswith(debug_mount): + rel_req_path = request.path[len(debug_mount):] + handler = self._handlers.get(rel_req_path) + if handler is not None: + return handler(request, start_response) + + return self.app(environ, start_response) + + def _shutdownWerkzeug(self, request, start_response): + shutdown_func = request.environ.get('werkzeug.server.shutdown') + if shutdown_func is None: + raise RuntimeError('Not running with the Werkzeug Server') + shutdown_func() + response = Response("Server shutting down...") + return response(request.environ, start_response) + + def _startSSEProvider(self, request, start_response): + from piecrust.serving.procloop import ( + PipelineStatusServerSentEventProducer) + provider = PipelineStatusServerSentEventProducer( + self._proc_loop) + it = provider.run() + response = Response(it, mimetype='text/event-stream') + response.headers['Cache-Control'] = 'no-cache' + response.headers['Last-Event-ID'] = \ + self._proc_loop.last_status_id + return ClosingIterator( + response(request.environ, start_response), + [provider.close]) +
--- a/piecrust/serving/procloop.py Sat Aug 08 16:12:04 2015 -0700 +++ b/piecrust/serving/procloop.py Sat Aug 08 22:01:47 2015 -0700 @@ -6,6 +6,8 @@ import logging import itertools import threading +from piecrust.app import PieCrust +from piecrust.processing.pipeline import ProcessorPipeline logger = logging.getLogger(__name__) @@ -72,10 +74,14 @@ class ProcessingLoop(threading.Thread): - def __init__(self, pipeline): + def __init__(self, root_dir, out_dir, sub_cache_dir=None, debug=False): super(ProcessingLoop, self).__init__( name='pipeline-reloader', daemon=True) - self.pipeline = pipeline + # TODO: re-create the app when `config.yml` is changed. + self.app = PieCrust(root_dir=root_dir, debug=debug) + if sub_cache_dir: + self.app._useSubCacheDir(sub_cache_dir) + self.pipeline = ProcessorPipeline(self.app, out_dir) self.last_status_id = 0 self.interval = 1 self._paths = set() @@ -94,8 +100,7 @@ def run(self): # Build the first list of known files and run the pipeline once. - app = self.pipeline.app - roots = [os.path.join(app.root_dir, r) + roots = [os.path.join(self.app.root_dir, r) for r in self.pipeline.mounts.keys()] for root in roots: for dirpath, dirnames, filenames in os.walk(root):
--- a/piecrust/serving/server.py Sat Aug 08 16:12:04 2015 -0700 +++ b/piecrust/serving/server.py Sat Aug 08 22:01:47 2015 -0700 @@ -1,20 +1,18 @@ import io import os -import re import gzip import time import os.path import hashlib import logging -import datetime from werkzeug.exceptions import ( NotFound, MethodNotAllowed, InternalServerError, HTTPException) from werkzeug.wrappers import Request, Response -from werkzeug.wsgi import ClosingIterator, wrap_file from jinja2 import FileSystemLoader, Environment from piecrust import CACHE_DIR, RESOURCES_DIR from piecrust.app import PieCrust from piecrust.rendering import QualifiedPage, PageRenderingContext, render_page +from piecrust.serving.util import content_type_map, make_wrapped_file_response from piecrust.sources.base import MODE_PARSING from piecrust.uriutil import split_sub_uri @@ -22,6 +20,14 @@ logger = logging.getLogger(__name__) +class WsgiServer(object): + def __init__(self, root_dir, **kwargs): + self.server = Server(root_dir, **kwargs) + + def __call__(self, environ, start_response): + return self.server._run_request(environ, start_response) + + class ServeRecord(object): def __init__(self): self.entries = {} @@ -45,14 +51,6 @@ self.used_source_names = set() -class WsgiServerWrapper(object): - def __init__(self, server): - self.server = server - - def __call__(self, environ, start_response): - return self.server._run_request(environ, start_response) - - class MultipleNotFound(HTTPException): code = 404 @@ -73,50 +71,21 @@ class Server(object): def __init__(self, root_dir, debug=False, sub_cache_dir=None, enable_debug_info=True, - static_preview=True, run_sse_check=None): + static_preview=True): self.root_dir = root_dir self.debug = debug self.sub_cache_dir = sub_cache_dir self.enable_debug_info = enable_debug_info - self.run_sse_check = run_sse_check self.static_preview = static_preview self._page_record = ServeRecord() self._out_dir = os.path.join(root_dir, CACHE_DIR, 'server') - self._proc_loop = None - self._mimetype_map = load_mimetype_map() - - def getWsgiApp(self): - # Bake all the assets so we know what we have, and so we can serve - # them to the client. We need a temp app for this. - app = PieCrust(root_dir=self.root_dir, debug=self.debug) - if self.sub_cache_dir: - app._useSubCacheDir(self.sub_cache_dir) - self._out_dir = os.path.join(app.sub_cache_dir, 'server') - - if not self.run_sse_check or self.run_sse_check(): - # When using a server with code reloading, some implementations - # use process forking and we end up going here twice. We only want - # to start the pipeline loop in the inner process most of the - # time so we let the implementation tell us if this is OK. - from piecrust.processing.pipeline import ProcessorPipeline - from piecrust.serving.procloop import ProcessingLoop - pipeline = ProcessorPipeline(app, self._out_dir) - self._proc_loop = ProcessingLoop(pipeline) - self._proc_loop.start() - - # Run the WSGI app. - wsgi_wrapper = WsgiServerWrapper(self) - return wsgi_wrapper + if sub_cache_dir: + self._out_dir = os.path.join(sub_cache_dir, 'server') def _run_request(self, environ, start_response): try: response = self._try_run_request(environ) - if isinstance(response, tuple): - response, close_func = response - return ClosingIterator(response(environ, start_response), - [close_func]) - else: - return response(environ, start_response) + return response(environ, start_response) except Exception as ex: if self.debug: raise @@ -132,11 +101,6 @@ request.method) raise MethodNotAllowed() - # Handle special requests right away. - response = self._try_special_request(environ, request) - if response is not None: - return response - # Also handle requests to a pipeline-built asset right away. response = self._try_serve_asset(environ, request) if response is not None: @@ -177,43 +141,6 @@ msg = "There was an error trying to serve: %s" % request.path raise InternalServerError(msg) from ex - def _try_special_request(self, environ, request): - static_mount = '/__piecrust_static/' - if request.path.startswith(static_mount): - rel_req_path = request.path[len(static_mount):] - mount = os.path.join(RESOURCES_DIR, 'server') - full_path = os.path.join(mount, rel_req_path) - try: - response = self._make_wrapped_file_response( - environ, request, full_path) - return response - except OSError: - pass - - debug_mount = '/__piecrust_debug/' - if request.path.startswith(debug_mount): - rel_req_path = request.path[len(debug_mount):] - if rel_req_path == 'werkzeug_shutdown': - shutdown_func = environ.get('werkzeug.server.shutdown') - if shutdown_func is None: - raise RuntimeError('Not running with the Werkzeug Server') - shutdown_func() - return Response("Server shutting down...") - - if rel_req_path == 'pipeline_status': - from piecrust.serving.procloop import ( - PipelineStatusServerSentEventProducer) - provider = PipelineStatusServerSentEventProducer( - self._proc_loop) - it = provider.run() - response = Response(it, mimetype='text/event-stream') - response.headers['Cache-Control'] = 'no-cache' - response.headers['Last-Event-ID'] = \ - self._proc_loop.last_status_id - return response, provider.close - - return None - def _try_serve_asset(self, environ, request): rel_req_path = request.path.lstrip('/').replace('/', os.sep) if request.path.startswith('/_cache/'): @@ -224,8 +151,7 @@ full_path = os.path.join(self._out_dir, rel_req_path) try: - response = self._make_wrapped_file_response( - environ, request, full_path) + response = make_wrapped_file_response(environ, request, full_path) return response except OSError: pass @@ -239,7 +165,7 @@ if not os.path.isfile(full_path): return None - return self._make_wrapped_file_response(environ, request, full_path) + return make_wrapped_file_response(environ, request, full_path) def _try_serve_page(self, app, environ, request): # Try to find what matches the requested URL. @@ -393,28 +319,6 @@ # Ok all good. return rendered_page - def _make_wrapped_file_response(self, environ, request, path): - logger.debug("Serving %s" % path) - - # Check if we can return a 304 status code. - mtime = os.path.getmtime(path) - etag_str = '%s$$%s' % (path, mtime) - etag = hashlib.md5(etag_str.encode('utf8')).hexdigest() - if etag in request.if_none_match: - response = Response() - response.status_code = 304 - return response - - wrapper = wrap_file(environ, open(path, 'rb')) - response = Response(wrapper) - _, ext = os.path.splitext(path) - response.set_etag(etag) - response.last_modified = datetime.datetime.fromtimestamp(mtime) - response.mimetype = self._mimetype_map.get( - ext.lstrip('.'), 'text/plain') - response.direct_passthrough = True - return response - def _handle_error(self, exception, environ, start_response): code = 500 if isinstance(exception, HTTPException): @@ -458,18 +362,6 @@ pass -content_type_map = { - 'html': 'text/html', - 'xml': 'text/xml', - 'txt': 'text/plain', - 'text': 'text/plain', - 'css': 'text/css', - 'xhtml': 'application/xhtml+xml', - 'atom': 'application/atom+xml', # or 'text/xml'? - 'rss': 'application/rss+xml', # or 'text/xml'? - 'json': 'application/json'} - - def find_routes(routes, uri): res = [] tax_res = [] @@ -493,15 +385,3 @@ return super(ErrorMessageLoader, self).get_source(env, template) -def load_mimetype_map(): - mimetype_map = {} - sep_re = re.compile(r'\s+') - path = os.path.join(os.path.dirname(__file__), 'mime.types') - with open(path, 'r') as f: - for line in f: - tokens = sep_re.split(line) - if len(tokens) > 1: - for t in tokens[1:]: - mimetype_map[t] = tokens[0] - return mimetype_map -
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/serving/util.py Sat Aug 08 22:01:47 2015 -0700 @@ -0,0 +1,60 @@ +import re +import os.path +import hashlib +import logging +import datetime +from werkzeug.wrappers import Response +from werkzeug.wsgi import wrap_file + + +logger = logging.getLogger(__name__) + + +def load_mimetype_map(): + mimetype_map = {} + sep_re = re.compile(r'\s+') + path = os.path.join(os.path.dirname(__file__), 'mime.types') + with open(path, 'r') as f: + for line in f: + tokens = sep_re.split(line) + if len(tokens) > 1: + for t in tokens[1:]: + mimetype_map[t] = tokens[0] + return mimetype_map + + +def make_wrapped_file_response(environ, request, path): + logger.debug("Serving %s" % path) + + # Check if we can return a 304 status code. + mtime = os.path.getmtime(path) + etag_str = '%s$$%s' % (path, mtime) + etag = hashlib.md5(etag_str.encode('utf8')).hexdigest() + if etag in request.if_none_match: + response = Response() + response.status_code = 304 + return response + + wrapper = wrap_file(environ, open(path, 'rb')) + response = Response(wrapper) + _, ext = os.path.splitext(path) + response.set_etag(etag) + response.last_modified = datetime.datetime.fromtimestamp(mtime) + response.mimetype = mimetype_map.get( + ext.lstrip('.'), 'text/plain') + response.direct_passthrough = True + return response + + +mimetype_map = load_mimetype_map() +content_type_map = { + 'html': 'text/html', + 'xml': 'text/xml', + 'txt': 'text/plain', + 'text': 'text/plain', + 'css': 'text/css', + 'xhtml': 'application/xhtml+xml', + 'atom': 'application/atom+xml', # or 'text/xml'? + 'rss': 'application/rss+xml', # or 'text/xml'? + 'json': 'application/json'} +
--- a/piecrust/serving/wrappers.py Sat Aug 08 16:12:04 2015 -0700 +++ b/piecrust/serving/wrappers.py Sat Aug 08 22:01:47 2015 -0700 @@ -2,7 +2,6 @@ import logging import threading import urllib.request -from piecrust.serving.server import Server logger = logging.getLogger(__name__) @@ -90,8 +89,15 @@ app_wrapper.run() -def _get_piecrust_server(root_dir, **kwargs): - server = Server(root_dir, **kwargs) - app = server.getWsgiApp() +def _get_piecrust_server(root_dir, debug=False, sub_cache_dir=None, + run_sse_check=None): + from piecrust.serving.middlewares import ( + StaticResourcesMiddleware, PieCrustDebugMiddleware) + from piecrust.serving.server import WsgiServer + app = WsgiServer(root_dir, debug=debug, sub_cache_dir=sub_cache_dir) + app = StaticResourcesMiddleware(app) + app = PieCrustDebugMiddleware(app, root_dir, + sub_cache_dir=sub_cache_dir, + run_sse_check=run_sse_check) return app
--- a/piecrust/wsgiutil/__init__.py Sat Aug 08 16:12:04 2015 -0700 +++ b/piecrust/wsgiutil/__init__.py Sat Aug 08 22:01:47 2015 -0700 @@ -1,10 +1,9 @@ -from piecrust.serving.server import Server +from piecrust.serving.server import WsgiServer def get_app(root_dir, sub_cache_dir='prod', enable_debug_info=False): - server = Server(root_dir, - sub_cache_dir=sub_cache_dir, - enable_debug_info=enable_debug_info) - app = server.getWsgiApp() + app = WsgiServer(root_dir, + sub_cache_dir=sub_cache_dir, + enable_debug_info=enable_debug_info) return app
--- a/tests/conftest.py Sat Aug 08 16:12:04 2015 -0700 +++ b/tests/conftest.py Sat Aug 08 22:01:47 2015 -0700 @@ -286,7 +286,8 @@ self.server = server def __call__(self, environ, start_response): - return self.server._try_run_request(environ, start_response) + response = self.server._try_run_request(environ) + return response(environ, start_response) def runtest(self): fs = self._prepareMockFs()