changeset 553:cc6f3dbe3048

serve: Extract some of the server's functionality into WSGI middlewares.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 08 Aug 2015 22:01:47 -0700
parents 9612cfc6455a
children 155c7e20414f
files piecrust/serving/middlewares.py piecrust/serving/procloop.py piecrust/serving/server.py piecrust/serving/util.py piecrust/serving/wrappers.py piecrust/wsgiutil/__init__.py tests/conftest.py
diffstat 7 files changed, 194 insertions(+), 149 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/piecrust/serving/middlewares.py	Sat Aug 08 22:01:47 2015 -0700
@@ -0,0 +1,94 @@
+import os.path
+from werkzeug.wrappers import Request, Response
+from werkzeug.wsgi import ClosingIterator
+from piecrust import RESOURCES_DIR, CACHE_DIR
+from piecrust.serving.util import make_wrapped_file_response
+
+
+class StaticResourcesMiddleware(object):
+    """ WSGI middleware that serves static files from the `resources/server`
+        directory in the PieCrust package.
+    """
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        static_mount = '/__piecrust_static/'
+
+        request = Request(environ)
+        if request.path.startswith(static_mount):
+            rel_req_path = request.path[len(static_mount):]
+            mount = os.path.join(RESOURCES_DIR, 'server')
+            full_path = os.path.join(mount, rel_req_path)
+            try:
+                response = make_wrapped_file_response(
+                        environ, request, full_path)
+                return response(environ, start_response)
+            except OSError:
+                pass
+
+        return self.app(environ, start_response)
+
+
+class PieCrustDebugMiddleware(object):
+    """ WSGI middleware that handles debugging of PieCrust stuff.
+    """
+    def __init__(self, app, root_dir, debug=False,
+                 sub_cache_dir=None, run_sse_check=None):
+        self.app = app
+        self.root_dir = root_dir
+        self.debug = debug
+        self.run_sse_check = run_sse_check
+        self._proc_loop = None
+        self._out_dir = os.path.join(root_dir, CACHE_DIR, 'server')
+        if sub_cache_dir:
+            self._out_dir = os.path.join(sub_cache_dir, 'server')
+        self._handlers = {
+                'werkzeug_shutdown': self._shutdownWerkzeug,
+                'pipeline_status': self._startSSEProvider}
+
+        if not self.run_sse_check or self.run_sse_check():
+            # When using a server with code reloading, some implementations
+            # use process forking and we end up going here twice. We only want
+            # to start the pipeline loop in the inner process most of the
+            # time so we let the implementation tell us if this is OK.
+            from piecrust.serving.procloop import ProcessingLoop
+            self._proc_loop = ProcessingLoop(root_dir, self._out_dir,
+                                             sub_cache_dir=sub_cache_dir,
+                                             debug=debug)
+            self._proc_loop.start()
+
+    def __call__(self, environ, start_response):
+        debug_mount = '/__piecrust_debug/'
+
+        request = Request(environ)
+        if request.path.startswith(debug_mount):
+            rel_req_path = request.path[len(debug_mount):]
+            handler = self._handlers.get(rel_req_path)
+            if handler is not None:
+                return handler(request, start_response)
+
+        return self.app(environ, start_response)
+
+    def _shutdownWerkzeug(self, request, start_response):
+        shutdown_func = request.environ.get('werkzeug.server.shutdown')
+        if shutdown_func is None:
+            raise RuntimeError('Not running with the Werkzeug Server')
+        shutdown_func()
+        response = Response("Server shutting down...")
+        return response(request.environ, start_response)
+
+    def _startSSEProvider(self, request, start_response):
+        from piecrust.serving.procloop import (
+                PipelineStatusServerSentEventProducer)
+        provider = PipelineStatusServerSentEventProducer(
+                self._proc_loop)
+        it = provider.run()
+        response = Response(it, mimetype='text/event-stream')
+        response.headers['Cache-Control'] = 'no-cache'
+        response.headers['Last-Event-ID'] = \
+            self._proc_loop.last_status_id
+        return ClosingIterator(
+                response(request.environ, start_response),
+                [provider.close])
+
--- a/piecrust/serving/procloop.py	Sat Aug 08 16:12:04 2015 -0700
+++ b/piecrust/serving/procloop.py	Sat Aug 08 22:01:47 2015 -0700
@@ -6,6 +6,8 @@
 import logging
 import itertools
 import threading
+from piecrust.app import PieCrust
+from piecrust.processing.pipeline import ProcessorPipeline
 
 
 logger = logging.getLogger(__name__)
@@ -72,10 +74,14 @@
 
 
 class ProcessingLoop(threading.Thread):
-    def __init__(self, pipeline):
+    def __init__(self, root_dir, out_dir, sub_cache_dir=None, debug=False):
         super(ProcessingLoop, self).__init__(
                 name='pipeline-reloader', daemon=True)
-        self.pipeline = pipeline
+        # TODO: re-create the app when `config.yml` is changed.
+        self.app = PieCrust(root_dir=root_dir, debug=debug)
+        if sub_cache_dir:
+            self.app._useSubCacheDir(sub_cache_dir)
+        self.pipeline = ProcessorPipeline(self.app, out_dir)
         self.last_status_id = 0
         self.interval = 1
         self._paths = set()
@@ -94,8 +100,7 @@
 
     def run(self):
         # Build the first list of known files and run the pipeline once.
-        app = self.pipeline.app
-        roots = [os.path.join(app.root_dir, r)
+        roots = [os.path.join(self.app.root_dir, r)
                  for r in self.pipeline.mounts.keys()]
         for root in roots:
             for dirpath, dirnames, filenames in os.walk(root):
--- a/piecrust/serving/server.py	Sat Aug 08 16:12:04 2015 -0700
+++ b/piecrust/serving/server.py	Sat Aug 08 22:01:47 2015 -0700
@@ -1,20 +1,18 @@
 import io
 import os
-import re
 import gzip
 import time
 import os.path
 import hashlib
 import logging
-import datetime
 from werkzeug.exceptions import (
         NotFound, MethodNotAllowed, InternalServerError, HTTPException)
 from werkzeug.wrappers import Request, Response
-from werkzeug.wsgi import ClosingIterator, wrap_file
 from jinja2 import FileSystemLoader, Environment
 from piecrust import CACHE_DIR, RESOURCES_DIR
 from piecrust.app import PieCrust
 from piecrust.rendering import QualifiedPage, PageRenderingContext, render_page
+from piecrust.serving.util import content_type_map, make_wrapped_file_response
 from piecrust.sources.base import MODE_PARSING
 from piecrust.uriutil import split_sub_uri
 
@@ -22,6 +20,14 @@
 logger = logging.getLogger(__name__)
 
 
+class WsgiServer(object):
+    def __init__(self, root_dir, **kwargs):
+        self.server = Server(root_dir, **kwargs)
+
+    def __call__(self, environ, start_response):
+        return self.server._run_request(environ, start_response)
+
+
 class ServeRecord(object):
     def __init__(self):
         self.entries = {}
@@ -45,14 +51,6 @@
         self.used_source_names = set()
 
 
-class WsgiServerWrapper(object):
-    def __init__(self, server):
-        self.server = server
-
-    def __call__(self, environ, start_response):
-        return self.server._run_request(environ, start_response)
-
-
 class MultipleNotFound(HTTPException):
     code = 404
 
@@ -73,50 +71,21 @@
 class Server(object):
     def __init__(self, root_dir,
                  debug=False, sub_cache_dir=None, enable_debug_info=True,
-                 static_preview=True, run_sse_check=None):
+                 static_preview=True):
         self.root_dir = root_dir
         self.debug = debug
         self.sub_cache_dir = sub_cache_dir
         self.enable_debug_info = enable_debug_info
-        self.run_sse_check = run_sse_check
         self.static_preview = static_preview
         self._page_record = ServeRecord()
         self._out_dir = os.path.join(root_dir, CACHE_DIR, 'server')
-        self._proc_loop = None
-        self._mimetype_map = load_mimetype_map()
-
-    def getWsgiApp(self):
-        # Bake all the assets so we know what we have, and so we can serve
-        # them to the client. We need a temp app for this.
-        app = PieCrust(root_dir=self.root_dir, debug=self.debug)
-        if self.sub_cache_dir:
-            app._useSubCacheDir(self.sub_cache_dir)
-        self._out_dir = os.path.join(app.sub_cache_dir, 'server')
-
-        if not self.run_sse_check or self.run_sse_check():
-            # When using a server with code reloading, some implementations
-            # use process forking and we end up going here twice. We only want
-            # to start the pipeline loop in the inner process most of the
-            # time so we let the implementation tell us if this is OK.
-            from piecrust.processing.pipeline import ProcessorPipeline
-            from piecrust.serving.procloop import ProcessingLoop
-            pipeline = ProcessorPipeline(app, self._out_dir)
-            self._proc_loop = ProcessingLoop(pipeline)
-            self._proc_loop.start()
-
-        # Run the WSGI app.
-        wsgi_wrapper = WsgiServerWrapper(self)
-        return wsgi_wrapper
+        if sub_cache_dir:
+            self._out_dir = os.path.join(sub_cache_dir, 'server')
 
     def _run_request(self, environ, start_response):
         try:
             response = self._try_run_request(environ)
-            if isinstance(response, tuple):
-                response, close_func = response
-                return ClosingIterator(response(environ, start_response),
-                                       [close_func])
-            else:
-                return response(environ, start_response)
+            return response(environ, start_response)
         except Exception as ex:
             if self.debug:
                 raise
@@ -132,11 +101,6 @@
                          request.method)
             raise MethodNotAllowed()
 
-        # Handle special requests right away.
-        response = self._try_special_request(environ, request)
-        if response is not None:
-            return response
-
         # Also handle requests to a pipeline-built asset right away.
         response = self._try_serve_asset(environ, request)
         if response is not None:
@@ -177,43 +141,6 @@
             msg = "There was an error trying to serve: %s" % request.path
             raise InternalServerError(msg) from ex
 
-    def _try_special_request(self, environ, request):
-        static_mount = '/__piecrust_static/'
-        if request.path.startswith(static_mount):
-            rel_req_path = request.path[len(static_mount):]
-            mount = os.path.join(RESOURCES_DIR, 'server')
-            full_path = os.path.join(mount, rel_req_path)
-            try:
-                response = self._make_wrapped_file_response(
-                        environ, request, full_path)
-                return response
-            except OSError:
-                pass
-
-        debug_mount = '/__piecrust_debug/'
-        if request.path.startswith(debug_mount):
-            rel_req_path = request.path[len(debug_mount):]
-            if rel_req_path == 'werkzeug_shutdown':
-                shutdown_func = environ.get('werkzeug.server.shutdown')
-                if shutdown_func is None:
-                    raise RuntimeError('Not running with the Werkzeug Server')
-                shutdown_func()
-                return Response("Server shutting down...")
-
-            if rel_req_path == 'pipeline_status':
-                from piecrust.serving.procloop import (
-                        PipelineStatusServerSentEventProducer)
-                provider = PipelineStatusServerSentEventProducer(
-                        self._proc_loop)
-                it = provider.run()
-                response = Response(it, mimetype='text/event-stream')
-                response.headers['Cache-Control'] = 'no-cache'
-                response.headers['Last-Event-ID'] = \
-                    self._proc_loop.last_status_id
-                return response, provider.close
-
-        return None
-
     def _try_serve_asset(self, environ, request):
         rel_req_path = request.path.lstrip('/').replace('/', os.sep)
         if request.path.startswith('/_cache/'):
@@ -224,8 +151,7 @@
             full_path = os.path.join(self._out_dir, rel_req_path)
 
         try:
-            response = self._make_wrapped_file_response(
-                    environ, request, full_path)
+            response = make_wrapped_file_response(environ, request, full_path)
             return response
         except OSError:
             pass
@@ -239,7 +165,7 @@
         if not os.path.isfile(full_path):
             return None
 
-        return self._make_wrapped_file_response(environ, request, full_path)
+        return make_wrapped_file_response(environ, request, full_path)
 
     def _try_serve_page(self, app, environ, request):
         # Try to find what matches the requested URL.
@@ -393,28 +319,6 @@
         # Ok all good.
         return rendered_page
 
-    def _make_wrapped_file_response(self, environ, request, path):
-        logger.debug("Serving %s" % path)
-
-        # Check if we can return a 304 status code.
-        mtime = os.path.getmtime(path)
-        etag_str = '%s$$%s' % (path, mtime)
-        etag = hashlib.md5(etag_str.encode('utf8')).hexdigest()
-        if etag in request.if_none_match:
-            response = Response()
-            response.status_code = 304
-            return response
-
-        wrapper = wrap_file(environ, open(path, 'rb'))
-        response = Response(wrapper)
-        _, ext = os.path.splitext(path)
-        response.set_etag(etag)
-        response.last_modified = datetime.datetime.fromtimestamp(mtime)
-        response.mimetype = self._mimetype_map.get(
-                ext.lstrip('.'), 'text/plain')
-        response.direct_passthrough = True
-        return response
-
     def _handle_error(self, exception, environ, start_response):
         code = 500
         if isinstance(exception, HTTPException):
@@ -458,18 +362,6 @@
     pass
 
 
-content_type_map = {
-        'html': 'text/html',
-        'xml': 'text/xml',
-        'txt': 'text/plain',
-        'text': 'text/plain',
-        'css': 'text/css',
-        'xhtml': 'application/xhtml+xml',
-        'atom': 'application/atom+xml',  # or 'text/xml'?
-        'rss': 'application/rss+xml',    # or 'text/xml'?
-        'json': 'application/json'}
-
-
 def find_routes(routes, uri):
     res = []
     tax_res = []
@@ -493,15 +385,3 @@
         return super(ErrorMessageLoader, self).get_source(env, template)
 
 
-def load_mimetype_map():
-    mimetype_map = {}
-    sep_re = re.compile(r'\s+')
-    path = os.path.join(os.path.dirname(__file__), 'mime.types')
-    with open(path, 'r') as f:
-        for line in f:
-            tokens = sep_re.split(line)
-            if len(tokens) > 1:
-                for t in tokens[1:]:
-                    mimetype_map[t] = tokens[0]
-    return mimetype_map
-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/piecrust/serving/util.py	Sat Aug 08 22:01:47 2015 -0700
@@ -0,0 +1,60 @@
+import re
+import os.path
+import hashlib
+import logging
+import datetime
+from werkzeug.wrappers import Response
+from werkzeug.wsgi import wrap_file
+
+
+logger = logging.getLogger(__name__)
+
+
+def load_mimetype_map():
+    mimetype_map = {}
+    sep_re = re.compile(r'\s+')
+    path = os.path.join(os.path.dirname(__file__), 'mime.types')
+    with open(path, 'r') as f:
+        for line in f:
+            tokens = sep_re.split(line)
+            if len(tokens) > 1:
+                for t in tokens[1:]:
+                    mimetype_map[t] = tokens[0]
+    return mimetype_map
+
+
+def make_wrapped_file_response(environ, request, path):
+    logger.debug("Serving %s" % path)
+
+    # Check if we can return a 304 status code.
+    mtime = os.path.getmtime(path)
+    etag_str = '%s$$%s' % (path, mtime)
+    etag = hashlib.md5(etag_str.encode('utf8')).hexdigest()
+    if etag in request.if_none_match:
+        response = Response()
+        response.status_code = 304
+        return response
+
+    wrapper = wrap_file(environ, open(path, 'rb'))
+    response = Response(wrapper)
+    _, ext = os.path.splitext(path)
+    response.set_etag(etag)
+    response.last_modified = datetime.datetime.fromtimestamp(mtime)
+    response.mimetype = mimetype_map.get(
+            ext.lstrip('.'), 'text/plain')
+    response.direct_passthrough = True
+    return response
+
+
+mimetype_map = load_mimetype_map()
+content_type_map = {
+        'html': 'text/html',
+        'xml': 'text/xml',
+        'txt': 'text/plain',
+        'text': 'text/plain',
+        'css': 'text/css',
+        'xhtml': 'application/xhtml+xml',
+        'atom': 'application/atom+xml',  # or 'text/xml'?
+        'rss': 'application/rss+xml',    # or 'text/xml'?
+        'json': 'application/json'}
+
--- a/piecrust/serving/wrappers.py	Sat Aug 08 16:12:04 2015 -0700
+++ b/piecrust/serving/wrappers.py	Sat Aug 08 22:01:47 2015 -0700
@@ -2,7 +2,6 @@
 import logging
 import threading
 import urllib.request
-from piecrust.serving.server import Server
 
 
 logger = logging.getLogger(__name__)
@@ -90,8 +89,15 @@
     app_wrapper.run()
 
 
-def _get_piecrust_server(root_dir, **kwargs):
-    server = Server(root_dir, **kwargs)
-    app = server.getWsgiApp()
+def _get_piecrust_server(root_dir, debug=False, sub_cache_dir=None,
+                         run_sse_check=None):
+    from piecrust.serving.middlewares import (
+            StaticResourcesMiddleware, PieCrustDebugMiddleware)
+    from piecrust.serving.server import WsgiServer
+    app = WsgiServer(root_dir, debug=debug, sub_cache_dir=sub_cache_dir)
+    app = StaticResourcesMiddleware(app)
+    app = PieCrustDebugMiddleware(app, root_dir,
+                                  sub_cache_dir=sub_cache_dir,
+                                  run_sse_check=run_sse_check)
     return app
 
--- a/piecrust/wsgiutil/__init__.py	Sat Aug 08 16:12:04 2015 -0700
+++ b/piecrust/wsgiutil/__init__.py	Sat Aug 08 22:01:47 2015 -0700
@@ -1,10 +1,9 @@
-from piecrust.serving.server import Server
+from piecrust.serving.server import WsgiServer
 
 
 def get_app(root_dir, sub_cache_dir='prod', enable_debug_info=False):
-    server = Server(root_dir,
-                    sub_cache_dir=sub_cache_dir,
-                    enable_debug_info=enable_debug_info)
-    app = server.getWsgiApp()
+    app = WsgiServer(root_dir,
+                     sub_cache_dir=sub_cache_dir,
+                     enable_debug_info=enable_debug_info)
     return app
 
--- a/tests/conftest.py	Sat Aug 08 16:12:04 2015 -0700
+++ b/tests/conftest.py	Sat Aug 08 22:01:47 2015 -0700
@@ -286,7 +286,8 @@
             self.server = server
 
         def __call__(self, environ, start_response):
-            return self.server._try_run_request(environ, start_response)
+            response = self.server._try_run_request(environ)
+            return response(environ, start_response)
 
     def runtest(self):
         fs = self._prepareMockFs()