comparison piecrust/serving.py @ 113:de257cc40ce1

Re-enable proper caching of rendered segments in server. The server keeps records on files that are processed while the server is running. Disk caching is simply disabled for files that are known to use other pages -- because unlike the baker, there's no cheap way to know which files are up to date or not, and rendering is faster enough anyway.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 19 Oct 2014 00:30:44 -0700
parents 208c652551a3
children 28444014ce7d
comparison
equal deleted inserted replaced
112:d31cbbdb4ecc 113:de257cc40ce1
24 24
25 logger = logging.getLogger(__name__) 25 logger = logging.getLogger(__name__)
26 26
27 27
28 class ServingEnvironment(StandardEnvironment): 28 class ServingEnvironment(StandardEnvironment):
29 pass
30
31
32 class ServeRecord(object):
29 def __init__(self): 33 def __init__(self):
30 super(ServingEnvironment, self).__init__() 34 self.entries = {}
31 del self.fs_caches['renders'] 35
36 def addEntry(self, entry):
37 key = self._makeKey(entry.uri, entry.sub_num)
38 self.entries[key] = entry
39
40 def getEntry(self, uri, sub_num):
41 key = self._makeKey(uri, sub_num)
42 return self.entries.get(key)
43
44 def _makeKey(self, uri, sub_num):
45 return "%s:%s" % (uri, sub_num)
46
47
48 class ServeRecordPageEntry(object):
49 def __init__(self, uri, sub_num):
50 self.uri = uri
51 self.sub_num = sub_num
52 self.used_source_names = set()
32 53
33 54
34 class Server(object): 55 class Server(object):
35 def __init__(self, root_dir, host='localhost', port='8080', 56 def __init__(self, root_dir, host='localhost', port='8080',
36 debug=False, static_preview=True, 57 debug=False, static_preview=True,
42 self.static_preview = static_preview 63 self.static_preview = static_preview
43 self.synchronous_asset_pipeline = synchronous_asset_pipeline 64 self.synchronous_asset_pipeline = synchronous_asset_pipeline
44 self._out_dir = None 65 self._out_dir = None
45 self._skip_patterns = None 66 self._skip_patterns = None
46 self._force_patterns = None 67 self._force_patterns = None
47 self._record = None 68 self._asset_record = None
69 self._page_record = None
48 self._mimetype_map = load_mimetype_map() 70 self._mimetype_map = load_mimetype_map()
49 71
50 def run(self): 72 def run(self):
51 # Bake all the assets so we know what we have, and so we can serve 73 # Bake all the assets so we know what we have, and so we can serve
52 # them to the client. We need a temp app for this. 74 # them to the client. We need a temp app for this.
57 self._force_patterns = app.config.get('baker/force_patterns') 79 self._force_patterns = app.config.get('baker/force_patterns')
58 pipeline = ProcessorPipeline( 80 pipeline = ProcessorPipeline(
59 app, mounts, self._out_dir, 81 app, mounts, self._out_dir,
60 skip_patterns=self._skip_patterns, 82 skip_patterns=self._skip_patterns,
61 force_patterns=self._force_patterns) 83 force_patterns=self._force_patterns)
62 self._record = pipeline.run() 84 self._asset_record = pipeline.run()
85 self._page_record = ServeRecord()
63 86
64 # Run the WSGI app. 87 # Run the WSGI app.
65 wsgi_wrapper = WsgiServer(self) 88 wsgi_wrapper = WsgiServer(self)
66 run_simple(self.host, self.port, wsgi_wrapper, 89 run_simple(self.host, self.port, wsgi_wrapper,
67 use_debugger=True, use_reloader=True) 90 use_debugger=True, use_reloader=True)
82 if self.static_preview and request.method != 'GET': 105 if self.static_preview and request.method != 'GET':
83 logger.error("Only GET requests are allowed, got %s" % request.method) 106 logger.error("Only GET requests are allowed, got %s" % request.method)
84 raise MethodNotAllowed() 107 raise MethodNotAllowed()
85 108
86 # Create the app for this request. 109 # Create the app for this request.
87 env = ServingEnvironment() 110 app = PieCrust(root_dir=self.root_dir, debug=self.debug)
88 app = PieCrust(root_dir=self.root_dir, debug=self.debug, env=env)
89 app.config.set('site/root', '/') 111 app.config.set('site/root', '/')
90 app.config.set('site/pretty_urls', True) 112 app.config.set('site/pretty_urls', True)
91 app.config.set('server/is_serving', True) 113 app.config.set('server/is_serving', True)
92 114
93 # We'll serve page assets directly from where they are. 115 # We'll serve page assets directly from where they are.
117 raise InternalServerError() 139 raise InternalServerError()
118 140
119 def _try_serve_asset(self, app, environ, request): 141 def _try_serve_asset(self, app, environ, request):
120 logger.debug("Searching for asset with path: %s" % request.path) 142 logger.debug("Searching for asset with path: %s" % request.path)
121 rel_req_path = request.path.lstrip('/').replace('/', os.sep) 143 rel_req_path = request.path.lstrip('/').replace('/', os.sep)
122 entry = self._record.findEntry(rel_req_path) 144 entry = self._asset_record.findEntry(rel_req_path)
123 if entry is None: 145 if entry is None:
124 return None 146 return None
125 147
126 # Yep, we know about this URL because we processed an asset that 148 # Yep, we know about this URL because we processed an asset that
127 # maps to it... make sure it's up to date by re-processing it 149 # maps to it... make sure it's up to date by re-processing it
195 else: 217 else:
196 raise SourceNotFoundError("Can't find path for: %s " 218 raise SourceNotFoundError("Can't find path for: %s "
197 "(looked in: %s)" % 219 "(looked in: %s)" %
198 (req_path, [r.source_name for r, _ in routes])) 220 (req_path, [r.source_name for r, _ in routes]))
199 221
200 # Build the page and render it. 222 # Build the page.
201 fac = PageFactory(source, rel_path, fac_metadata) 223 fac = PageFactory(source, rel_path, fac_metadata)
202 page = fac.buildPage() 224 page = fac.buildPage()
203 render_ctx = PageRenderingContext(page, req_path, page_num) 225 render_ctx = PageRenderingContext(page, req_path, page_num)
204 if taxonomy is not None: 226 if taxonomy is not None:
205 flt = PaginationFilter() 227 flt = PaginationFilter()
206 if taxonomy.is_multiple: 228 if taxonomy.is_multiple:
207 flt.addClause(HasFilterClause(taxonomy.name, term_value)) 229 flt.addClause(HasFilterClause(taxonomy.name, term_value))
208 else: 230 else:
209 flt.addClause(IsFilterClause(taxonomy.name, term_value)) 231 flt.addClause(IsFilterClause(taxonomy.name, term_value))
210 render_ctx.pagination_filter = flt 232 render_ctx.pagination_filter = flt
211
212 render_ctx.custom_data = { 233 render_ctx.custom_data = {
213 taxonomy.term_name: term_value} 234 taxonomy.term_name: term_value}
235
236 # See if this page is known to use sources. If that's the case,
237 # just don't use cached rendered segments for that page (but still
238 # use them for pages that are included in it).
239 entry = self._page_record.getEntry(req_path, page_num)
240 if (taxonomy is not None or entry is None or
241 entry.used_source_names):
242 cache_key = '%s:%s' % (req_path, page_num)
243 app.env.rendered_segments_repository.invalidate(cache_key)
244
245 # Render the page.
214 rendered_page = render_page(render_ctx) 246 rendered_page = render_page(render_ctx)
215 rp_content = rendered_page.content 247 rp_content = rendered_page.content
216 248
249 if entry is None:
250 entry = ServeRecordPageEntry(req_path, page_num)
251 self._page_record.addEntry(entry)
252 entry.used_source_names = set(render_ctx.used_source_names)
253
254 # Profiling.
217 if app.debug: 255 if app.debug:
218 now_time = time.clock() 256 now_time = time.clock()
219 timing_info = ('%8.1f ms' % 257 timing_info = ('%8.1f ms' %
220 ((now_time - app.env.start_time) * 1000.0)) 258 ((now_time - app.env.start_time) * 1000.0))
221 rp_content = rp_content.replace('__PIECRUST_TIMING_INFORMATION__', 259 rp_content = rp_content.replace('__PIECRUST_TIMING_INFORMATION__',
222 timing_info) 260 timing_info)
223 261
224 # Start response. 262 # Build the response.
225 response = Response() 263 response = Response()
226 264
227 etag = hashlib.md5(rp_content.encode('utf8')).hexdigest() 265 etag = hashlib.md5(rp_content.encode('utf8')).hexdigest()
228 if not app.debug and etag in request.if_none_match: 266 if not app.debug and etag in request.if_none_match:
229 response.status_code = 304 267 response.status_code = 304