Mercurial > piecrust2

comparison piecrust/serving/server.py @ 374:fa3ee8a8ee2d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
serve: Split the server code in a couple modules inside a `serving` package. This makes the `serve` command's code a bit more removed from implementation details, and paves the way for the CMS mode.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 07 May 2015 21:37:38 -0700
parents
children aade4ea57e7f
comparison
equal deleted inserted replaced
373:9fb7c4921d75 374:fa3ee8a8ee2d
1 import io
2 import os
3 import re
4 import gzip
5 import time
6 import os.path
7 import hashlib
8 import logging
9 import datetime
10 from werkzeug.exceptions import (
11 NotFound, MethodNotAllowed, InternalServerError, HTTPException)
12 from werkzeug.wrappers import Request, Response
13 from werkzeug.wsgi import ClosingIterator, wrap_file
14 from jinja2 import FileSystemLoader, Environment
15 from piecrust.app import PieCrust
16 from piecrust.rendering import QualifiedPage, PageRenderingContext, render_page
17 from piecrust.sources.base import MODE_PARSING
18 from piecrust.uriutil import split_sub_uri
19
20
21 logger = logging.getLogger(__name__)
22
23
24 class ServeRecord(object):
25 def __init__(self):
26 self.entries = {}
27
28 def addEntry(self, entry):
29 key = self._makeKey(entry.uri, entry.sub_num)
30 self.entries[key] = entry
31
32 def getEntry(self, uri, sub_num):
33 key = self._makeKey(uri, sub_num)
34 return self.entries.get(key)
35
36 def _makeKey(self, uri, sub_num):
37 return "%s:%s" % (uri, sub_num)
38
39
40 class ServeRecordPageEntry(object):
41 def __init__(self, uri, sub_num):
42 self.uri = uri
43 self.sub_num = sub_num
44 self.used_source_names = set()
45
46
47 class WsgiServerWrapper(object):
48 def __init__(self, server):
49 self.server = server
50
51 def __call__(self, environ, start_response):
52 return self.server._run_request(environ, start_response)
53
54
55 class Server(object):
56 def __init__(self, root_dir,
57 debug=False, sub_cache_dir=None,
58 static_preview=True, run_sse_check=None):
59 self.root_dir = root_dir
60 self.debug = debug
61 self.sub_cache_dir = sub_cache_dir
62 self.run_sse_check = run_sse_check
63 self.static_preview = static_preview
64 self._out_dir = None
65 self._page_record = None
66 self._proc_loop = None
67 self._mimetype_map = load_mimetype_map()
68
69 def getWsgiApp(self):
70 # Bake all the assets so we know what we have, and so we can serve
71 # them to the client. We need a temp app for this.
72 app = PieCrust(root_dir=self.root_dir, debug=self.debug)
73 app._useSubCacheDir(self.sub_cache_dir)
74 self._out_dir = os.path.join(app.sub_cache_dir, 'server')
75 self._page_record = ServeRecord()
76
77 if not self.run_sse_check or self.run_sse_check():
78 # When using a server with code reloading, some implementations
79 # use process forking and we end up going here twice. We only want
80 # to start the pipeline loop in the inner process most of the
81 # time so we let the implementation tell us if this is OK.
82 from piecrust.processing.base import ProcessorPipeline
83 from piecrust.serving.procloop import ProcessingLoop
84 pipeline = ProcessorPipeline(app, self._out_dir)
85 self._proc_loop = ProcessingLoop(pipeline)
86 self._proc_loop.start()
87
88 # Run the WSGI app.
89 wsgi_wrapper = WsgiServerWrapper(self)
90 return wsgi_wrapper
91
92 def _run_request(self, environ, start_response):
93 try:
94 return self._try_run_request(environ, start_response)
95 except Exception as ex:
96 if self.debug:
97 raise
98 return self._handle_error(ex, environ, start_response)
99
100 def _try_run_request(self, environ, start_response):
101 request = Request(environ)
102
103 # We don't support anything else than GET requests since we're
104 # previewing something that will be static later.
105 if self.static_preview and request.method != 'GET':
106 logger.error("Only GET requests are allowed, got %s" %
107 request.method)
108 raise MethodNotAllowed()
109
110 # Handle special requests right away.
111 response = self._try_special_request(environ, request)
112 if response is not None:
113 return response(environ, start_response)
114
115 # Also handle requests to a pipeline-built asset right away.
116 response = self._try_serve_asset(environ, request)
117 if response is not None:
118 return response(environ, start_response)
119
120 # Create the app for this request.
121 app = PieCrust(root_dir=self.root_dir, debug=self.debug)
122 app._useSubCacheDir(self.sub_cache_dir)
123 app.config.set('site/root', '/')
124 app.config.set('server/is_serving', True)
125 if (app.config.get('site/enable_debug_info') and
126 '!debug' in request.args):
127 app.config.set('site/show_debug_info', True)
128
129 # We'll serve page assets directly from where they are.
130 app.env.base_asset_url_format = '/_asset/%path%'
131
132 # Let's see if it can be a page asset.
133 response = self._try_serve_page_asset(app, environ, request)
134 if response is not None:
135 return response(environ, start_response)
136
137 # Nope. Let's see if it's an actual page.
138 try:
139 response = self._try_serve_page(app, environ, request)
140 return response(environ, start_response)
141 except (RouteNotFoundError, SourceNotFoundError) as ex:
142 raise NotFound(str(ex)) from ex
143 except HTTPException:
144 raise
145 except Exception as ex:
146 if app.debug:
147 logger.exception(ex)
148 raise
149 msg = str(ex)
150 logger.error(msg)
151 raise InternalServerError(msg) from ex
152
153 def _try_special_request(self, environ, request):
154 static_mount = '/__piecrust_static/'
155 if request.path.startswith(static_mount):
156 rel_req_path = request.path[len(static_mount):]
157 mount = os.path.join(
158 os.path.dirname(__file__),
159 'resources', 'server')
160 full_path = os.path.join(mount, rel_req_path)
161 try:
162 response = self._make_wrapped_file_response(
163 environ, request, full_path)
164 return response
165 except OSError:
166 pass
167
168 debug_mount = '/__piecrust_debug/'
169 if request.path.startswith(debug_mount):
170 rel_req_path = request.path[len(debug_mount):]
171 if rel_req_path == 'pipeline_status':
172 from piecrust.server.procloop import (
173 PipelineStatusServerSideEventProducer)
174 provider = PipelineStatusServerSideEventProducer(
175 self._proc_loop.status_queue)
176 it = ClosingIterator(provider.run(), [provider.close])
177 response = Response(it)
178 response.headers['Cache-Control'] = 'no-cache'
179 if 'text/event-stream' in request.accept_mimetypes:
180 response.mimetype = 'text/event-stream'
181 response.direct_passthrough = True
182 response.implicit_sequence_conversion = False
183 return response
184
185 return None
186
187 def _try_serve_asset(self, environ, request):
188 rel_req_path = request.path.lstrip('/').replace('/', os.sep)
189 if request.path.startswith('/_cache/'):
190 # Some stuff needs to be served directly from the cache directory,
191 # like LESS CSS map files.
192 full_path = os.path.join(self.root_dir, rel_req_path)
193 else:
194 full_path = os.path.join(self._out_dir, rel_req_path)
195
196 try:
197 response = self._make_wrapped_file_response(
198 environ, request, full_path)
199 return response
200 except OSError:
201 pass
202 return None
203
204 def _try_serve_page_asset(self, app, environ, request):
205 if not request.path.startswith('/_asset/'):
206 return None
207
208 full_path = os.path.join(app.root_dir, request.path[len('/_asset/'):])
209 if not os.path.isfile(full_path):
210 return None
211
212 return self._make_wrapped_file_response(environ, request, full_path)
213
214 def _try_serve_page(self, app, environ, request):
215 # Try to find what matches the requested URL.
216 req_path, page_num = split_sub_uri(app, request.path)
217
218 routes = find_routes(app.routes, req_path)
219 if len(routes) == 0:
220 raise RouteNotFoundError("Can't find route for: %s" % req_path)
221
222 rendered_page = None
223 first_not_found = None
224 for route, route_metadata in routes:
225 try:
226 logger.debug("Trying to render match from source '%s'." %
227 route.source_name)
228 rendered_page = self._try_render_page(
229 app, route, route_metadata, page_num, req_path)
230 if rendered_page is not None:
231 break
232 except NotFound as nfe:
233 if first_not_found is None:
234 first_not_found = nfe
235 else:
236 raise SourceNotFoundError(
237 "Can't find path for: %s (looked in: %s)" %
238 (req_path, [r.source_name for r, _ in routes]))
239
240 # If we haven't found any good match, raise whatever exception we
241 # first got. Otherwise, raise a generic exception.
242 if rendered_page is None:
243 first_not_found = first_not_found or NotFound(
244 "This page couldn't be found.")
245 raise first_not_found
246
247 # Start doing stuff.
248 page = rendered_page.page
249 rp_content = rendered_page.content
250
251 # Profiling.
252 if app.config.get('site/show_debug_info'):
253 now_time = time.clock()
254 timing_info = (
255 '%8.1f ms' %
256 ((now_time - app.env.start_time) * 1000.0))
257 rp_content = rp_content.replace(
258 '__PIECRUST_TIMING_INFORMATION__', timing_info)
259
260 # Build the response.
261 response = Response()
262
263 etag = hashlib.md5(rp_content.encode('utf8')).hexdigest()
264 if not app.debug and etag in request.if_none_match:
265 response.status_code = 304
266 return response
267
268 response.set_etag(etag)
269 response.content_md5 = etag
270
271 cache_control = response.cache_control
272 if app.debug:
273 cache_control.no_cache = True
274 cache_control.must_revalidate = True
275 else:
276 cache_time = (page.config.get('cache_time') or
277 app.config.get('site/cache_time'))
278 if cache_time:
279 cache_control.public = True
280 cache_control.max_age = cache_time
281
282 content_type = page.config.get('content_type')
283 if content_type and '/' not in content_type:
284 mimetype = content_type_map.get(content_type, content_type)
285 else:
286 mimetype = content_type
287 if mimetype:
288 response.mimetype = mimetype
289
290 if ('gzip' in request.accept_encodings and
291 app.config.get('site/enable_gzip')):
292 try:
293 with io.BytesIO() as gzip_buffer:
294 with gzip.open(gzip_buffer, mode='wt',
295 encoding='utf8') as gzip_file:
296 gzip_file.write(rp_content)
297 rp_content = gzip_buffer.getvalue()
298 response.content_encoding = 'gzip'
299 except Exception:
300 logger.exception("Error compressing response, "
301 "falling back to uncompressed.")
302 response.set_data(rp_content)
303
304 return response
305
306 def _try_render_page(self, app, route, route_metadata, page_num, req_path):
307 # Match the route to an actual factory.
308 taxonomy_info = None
309 source = app.getSource(route.source_name)
310 if route.taxonomy_name is None:
311 factory = source.findPageFactory(route_metadata, MODE_PARSING)
312 if factory is None:
313 return None
314 else:
315 taxonomy = app.getTaxonomy(route.taxonomy_name)
316 route_terms = route_metadata.get(taxonomy.term_name)
317 if route_terms is None:
318 return None
319
320 tax_page_ref = taxonomy.getPageRef(source.name)
321 factory = tax_page_ref.getFactory()
322 tax_terms = route.unslugifyTaxonomyTerm(route_terms)
323 route_metadata[taxonomy.term_name] = tax_terms
324 taxonomy_info = (taxonomy, tax_terms)
325
326 # Build the page.
327 page = factory.buildPage()
328 # We force the rendering of the page because it could not have
329 # changed, but include pages that did change.
330 qp = QualifiedPage(page, route, route_metadata)
331 render_ctx = PageRenderingContext(qp,
332 page_num=page_num,
333 force_render=True)
334 if taxonomy_info is not None:
335 taxonomy, tax_terms = taxonomy_info
336 render_ctx.setTaxonomyFilter(taxonomy, tax_terms)
337
338 # See if this page is known to use sources. If that's the case,
339 # just don't use cached rendered segments for that page (but still
340 # use them for pages that are included in it).
341 uri = qp.getUri()
342 assert uri == req_path
343 entry = self._page_record.getEntry(uri, page_num)
344 if (taxonomy_info is not None or entry is None or
345 entry.used_source_names):
346 cache_key = '%s:%s' % (uri, page_num)
347 app.env.rendered_segments_repository.invalidate(cache_key)
348
349 # Render the page.
350 rendered_page = render_page(render_ctx)
351
352 # Check if this page is a taxonomy page that actually doesn't match
353 # anything.
354 if taxonomy_info is not None:
355 paginator = rendered_page.data.get('pagination')
356 if (paginator and paginator.is_loaded and
357 len(paginator.items) == 0):
358 taxonomy = taxonomy_info[0]
359 message = ("This URL matched a route for taxonomy '%s' but "
360 "no pages have been found to have it. This page "
361 "won't be generated by a bake." % taxonomy.name)
362 raise NotFound(message)
363
364 # Remember stuff for next time.
365 if entry is None:
366 entry = ServeRecordPageEntry(req_path, page_num)
367 self._page_record.addEntry(entry)
368 for p, pinfo in render_ctx.render_passes.items():
369 entry.used_source_names |= pinfo.used_source_names
370
371 # Ok all good.
372 return rendered_page
373
374 def _make_wrapped_file_response(self, environ, request, path):
375 logger.debug("Serving %s" % path)
376
377 # Check if we can return a 304 status code.
378 mtime = os.path.getmtime(path)
379 etag_str = '%s$$%s' % (path, mtime)
380 etag = hashlib.md5(etag_str.encode('utf8')).hexdigest()
381 if etag in request.if_none_match:
382 response = Response()
383 response.status_code = 304
384 return response
385
386 wrapper = wrap_file(environ, open(path, 'rb'))
387 response = Response(wrapper)
388 _, ext = os.path.splitext(path)
389 response.set_etag(etag)
390 response.last_modified = datetime.datetime.fromtimestamp(mtime)
391 response.mimetype = self._mimetype_map.get(
392 ext.lstrip('.'), 'text/plain')
393 return response
394
395 def _handle_error(self, exception, environ, start_response):
396 code = 500
397 if isinstance(exception, HTTPException):
398 code = exception.code
399
400 path = 'error'
401 if isinstance(exception, NotFound):
402 path += '404'
403
404 descriptions = self._get_exception_descriptions(exception)
405
406 env = Environment(loader=ErrorMessageLoader())
407 template = env.get_template(path)
408 context = {'details': descriptions}
409 response = Response(template.render(context), mimetype='text/html')
410 response.status_code = code
411 return response(environ, start_response)
412
413 def _get_exception_descriptions(self, exception):
414 desc = []
415 while exception is not None:
416 if isinstance(exception, HTTPException):
417 desc.append(exception.description)
418 else:
419 desc.append(str(exception))
420
421 inner_ex = exception.__cause__
422 if inner_ex is None:
423 inner_ex = exception.__context__
424 exception = inner_ex
425 return desc
426
427
428 class RouteNotFoundError(Exception):
429 pass
430
431
432 class SourceNotFoundError(Exception):
433 pass
434
435
436 content_type_map = {
437 'html': 'text/html',
438 'xml': 'text/xml',
439 'txt': 'text/plain',
440 'text': 'text/plain',
441 'css': 'text/css',
442 'xhtml': 'application/xhtml+xml',
443 'atom': 'application/atom+xml', # or 'text/xml'?
444 'rss': 'application/rss+xml', # or 'text/xml'?
445 'json': 'application/json'}
446
447
448 def find_routes(routes, uri):
449 res = []
450 for route in routes:
451 metadata = route.matchUri(uri)
452 if metadata is not None:
453 res.append((route, metadata))
454 return res
455
456
457 class ErrorMessageLoader(FileSystemLoader):
458 def __init__(self):
459 base_dir = os.path.join(os.path.dirname(__file__), 'resources',
460 'messages')
461 super(ErrorMessageLoader, self).__init__(base_dir)
462
463 def get_source(self, env, template):
464 template += '.html'
465 return super(ErrorMessageLoader, self).get_source(env, template)
466
467
468 def load_mimetype_map():
469 mimetype_map = {}
470 sep_re = re.compile(r'\s+')
471 path = os.path.join(os.path.dirname(__file__), 'mime.types')
472 with open(path, 'r') as f:
473 for line in f:
474 tokens = sep_re.split(line)
475 if len(tokens) > 1:
476 for t in tokens[1:]:
477 mimetype_map[t] = tokens[0]
478 return mimetype_map
479