Mercurial > piecrust2

comparison piecrust/rendering.py @ 415:0e9a94b7fdfa

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Improve bake record information. * Store things in the bake record that require less interaction between the master process and the workers. For instance, don't store the paginator object in the render pass info -- instead, just store whether pagination was used, and whether it had more items. * Simplify information passing between workers and bake passes by saving the rendering info to the JSON cache. This means the "render first sub" job doesn't have to return anything except errors now. * Add more performance counter info.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:23:16 -0700
parents e7b865f8f335
children 6801ad5aa1d4
comparison
equal deleted inserted replaced
414:c4b3a7fd2f87 415:0e9a94b7fdfa
1 import re 1 import re
2 import os.path 2 import os.path
3 import copy
3 import logging 4 import logging
4 from werkzeug.utils import cached_property 5 from werkzeug.utils import cached_property
5 from piecrust.data.builder import ( 6 from piecrust.data.builder import (
6 DataBuildingContext, build_page_data, build_layout_data) 7 DataBuildingContext, build_page_data, build_layout_data)
7 from piecrust.data.filters import ( 8 from piecrust.data.filters import (
38 39
39 def __getattr__(self, name): 40 def __getattr__(self, name):
40 return getattr(self.page, name) 41 return getattr(self.page, name)
41 42
42 43
44 class RenderedSegments(object):
45 def __init__(self, segments, render_pass_info):
46 self.segments = segments
47 self.render_pass_info = render_pass_info
48
49
50 class RenderedLayout(object):
51 def __init__(self, content, render_pass_info):
52 self.content = content
53 self.render_pass_info = render_pass_info
54
55
43 class RenderedPage(object): 56 class RenderedPage(object):
44 def __init__(self, page, uri, num=1): 57 def __init__(self, page, uri, num=1):
45 self.page = page 58 self.page = page
46 self.uri = uri 59 self.uri = uri
47 self.num = num 60 self.num = num
48 self.data = None 61 self.data = None
49 self.content = None 62 self.content = None
63 self.render_info = None
50 64
51 @property 65 @property
52 def app(self): 66 def app(self):
53 return self.page.app 67 return self.page.app
68
69 def copyRenderInfo(self):
70 return copy.deepcopy(self.render_info)
54 71
55 72
56 PASS_NONE = 0 73 PASS_NONE = 0
57 PASS_FORMATTING = 1 74 PASS_FORMATTING = 1
58 PASS_RENDERING = 2 75 PASS_RENDERING = 2
63 80
64 class RenderPassInfo(object): 81 class RenderPassInfo(object):
65 def __init__(self): 82 def __init__(self):
66 self.used_source_names = set() 83 self.used_source_names = set()
67 self.used_taxonomy_terms = set() 84 self.used_taxonomy_terms = set()
85 self.used_pagination = False
86 self.pagination_has_more = False
87 self.used_assets = False
88
89 def merge(self, other):
90 self.used_source_names |= other.used_source_names
91 self.used_taxonomy_terms |= other.used_taxonomy_terms
92 self.used_pagination = self.used_pagination or other.used_pagination
93 self.pagination_has_more = (self.pagination_has_more or
94 other.pagination_has_more)
95 self.used_assets = self.used_assets or other.used_assets
96
97 def _toJson(self):
98 data = {
99 'used_source_names': list(self.used_source_names),
100 'used_taxonomy_terms': list(self.used_taxonomy_terms),
101 'used_pagination': self.used_pagination,
102 'pagination_has_more': self.pagination_has_more,
103 'used_assets': self.used_assets}
104 return data
105
106 @staticmethod
107 def _fromJson(data):
108 assert data is not None
109 rpi = RenderPassInfo()
110 rpi.used_source_names = set(data['used_source_names'])
111 for i in data['used_taxonomy_terms']:
112 terms = i[2]
113 if isinstance(terms, list):
114 terms = tuple(terms)
115 rpi.used_taxonomy_terms.add((i[0], i[1], terms))
116 rpi.used_pagination = data['used_pagination']
117 rpi.pagination_has_more = data['pagination_has_more']
118 rpi.used_assets = data['used_assets']
119 return rpi
68 120
69 121
70 class PageRenderingContext(object): 122 class PageRenderingContext(object):
71 def __init__(self, qualified_page, page_num=1, force_render=False): 123 def __init__(self, qualified_page, page_num=1, force_render=False):
72 self.page = qualified_page 124 self.page = qualified_page
76 self.pagination_filter = None 128 self.pagination_filter = None
77 self.custom_data = None 129 self.custom_data = None
78 self._current_pass = PASS_NONE 130 self._current_pass = PASS_NONE
79 131
80 self.render_passes = {} 132 self.render_passes = {}
81 self.used_pagination = None
82 self.used_assets = None
83 133
84 @property 134 @property
85 def app(self): 135 def app(self):
86 return self.page.app 136 return self.page.app
87 137
90 return self.page.source_metadata 140 return self.page.source_metadata
91 141
92 @cached_property 142 @cached_property
93 def uri(self): 143 def uri(self):
94 return self.page.getUri(self.page_num) 144 return self.page.getUri(self.page_num)
95
96 @property
97 def pagination_has_more(self):
98 if self.used_pagination is None:
99 return False
100 return self.used_pagination.has_more
101 145
102 @property 146 @property
103 def current_pass_info(self): 147 def current_pass_info(self):
104 return self.render_passes.get(self._current_pass) 148 return self.render_passes.get(self._current_pass)
105 149
108 self.render_passes.setdefault(rdr_pass, RenderPassInfo()) 152 self.render_passes.setdefault(rdr_pass, RenderPassInfo())
109 self._current_pass = rdr_pass 153 self._current_pass = rdr_pass
110 154
111 def setPagination(self, paginator): 155 def setPagination(self, paginator):
112 self._raiseIfNoCurrentPass() 156 self._raiseIfNoCurrentPass()
113 if self.used_pagination is not None: 157 pass_info = self.current_pass_info
158 if pass_info.used_pagination:
114 raise Exception("Pagination has already been used.") 159 raise Exception("Pagination has already been used.")
115 self.used_pagination = paginator 160 assert paginator.is_loaded
161 pass_info.used_pagination = True
162 pass_info.pagination_has_more = paginator.has_more
116 self.addUsedSource(paginator._source) 163 self.addUsedSource(paginator._source)
117 164
118 def addUsedSource(self, source): 165 def addUsedSource(self, source):
119 self._raiseIfNoCurrentPass() 166 self._raiseIfNoCurrentPass()
120 if isinstance(source, PageSource): 167 if isinstance(source, PageSource):
121 pass_info = self.render_passes[self._current_pass] 168 pass_info = self.current_pass_info
122 pass_info.used_source_names.add(source.name) 169 pass_info.used_source_names.add(source.name)
123 170
124 def setTaxonomyFilter(self, taxonomy, term_value): 171 def setTaxonomyFilter(self, taxonomy, term_value):
125 is_combination = isinstance(term_value, tuple) 172 is_combination = isinstance(term_value, tuple)
126 flt = PaginationFilter(value_accessor=page_value_accessor) 173 flt = PaginationFilter(value_accessor=page_value_accessor)
148 195
149 def render_page(ctx): 196 def render_page(ctx):
150 eis = ctx.app.env.exec_info_stack 197 eis = ctx.app.env.exec_info_stack
151 eis.pushPage(ctx.page, ctx) 198 eis.pushPage(ctx.page, ctx)
152 try: 199 try:
153 page = ctx.page
154
155 # Build the data for both segment and layout rendering. 200 # Build the data for both segment and layout rendering.
156 data_ctx = DataBuildingContext(page, page_num=ctx.page_num) 201 page_data = _build_render_data(ctx)
157 data_ctx.pagination_source = ctx.pagination_source
158 data_ctx.pagination_filter = ctx.pagination_filter
159 page_data = build_page_data(data_ctx)
160 if ctx.custom_data:
161 page_data.update(ctx.custom_data)
162 202
163 # Render content segments. 203 # Render content segments.
164 ctx.setCurrentPass(PASS_FORMATTING) 204 ctx.setCurrentPass(PASS_FORMATTING)
165 repo = ctx.app.env.rendered_segments_repository 205 repo = ctx.app.env.rendered_segments_repository
166 save_to_fs = True 206 save_to_fs = True
167 if ctx.app.env.fs_cache_only_for_main_page and not eis.is_main_page: 207 if ctx.app.env.fs_cache_only_for_main_page and not eis.is_main_page:
168 save_to_fs = False 208 save_to_fs = False
169 if repo and not ctx.force_render: 209 if repo and not ctx.force_render:
170 contents = repo.get( 210 render_result = repo.get(
171 ctx.uri, 211 ctx.uri,
172 lambda: _do_render_page_segments(page, page_data), 212 lambda: _do_render_page_segments(ctx.page, page_data),
173 fs_cache_time=page.path_mtime, 213 fs_cache_time=ctx.page.path_mtime,
174 save_to_fs=save_to_fs) 214 save_to_fs=save_to_fs)
175 else: 215 else:
176 contents = _do_render_page_segments(page, page_data) 216 render_result = _do_render_page_segments(ctx.page, page_data)
177 if repo: 217 if repo:
178 repo.put(ctx.uri, contents, save_to_fs) 218 repo.put(ctx.uri, render_result, save_to_fs)
179 219
180 # Render layout. 220 # Render layout.
221 page = ctx.page
181 ctx.setCurrentPass(PASS_RENDERING) 222 ctx.setCurrentPass(PASS_RENDERING)
182 layout_name = page.config.get('layout') 223 layout_name = page.config.get('layout')
183 if layout_name is None: 224 if layout_name is None:
184 layout_name = page.source.config.get('default_layout', 'default') 225 layout_name = page.source.config.get('default_layout', 'default')
185 null_names = ['', 'none', 'nil'] 226 null_names = ['', 'none', 'nil']
186 if layout_name not in null_names: 227 if layout_name not in null_names:
187 build_layout_data(page, page_data, contents) 228 build_layout_data(page, page_data, render_result['segments'])
188 output = render_layout(layout_name, page, page_data) 229 layout_result = _do_render_layout(layout_name, page, page_data)
189 else: 230 else:
190 output = contents['content'] 231 layout_result = {
232 'content': render_result['segments']['content'],
233 'pass_info': None}
191 234
192 rp = RenderedPage(page, ctx.uri, ctx.page_num) 235 rp = RenderedPage(page, ctx.uri, ctx.page_num)
193 rp.data = page_data 236 rp.data = page_data
194 rp.content = output 237 rp.content = layout_result['content']
238 rp.render_info = {
239 PASS_FORMATTING: RenderPassInfo._fromJson(
240 render_result['pass_info'])}
241 if layout_result['pass_info'] is not None:
242 rp.render_info[PASS_RENDERING] = RenderPassInfo._fromJson(
243 layout_result['pass_info'])
195 return rp 244 return rp
196 finally: 245 finally:
197 ctx.setCurrentPass(PASS_NONE) 246 ctx.setCurrentPass(PASS_NONE)
198 eis.popPage() 247 eis.popPage()
199 248
200 249
201 def render_page_segments(ctx): 250 def render_page_segments(ctx):
202 repo = ctx.app.env.rendered_segments_repository
203 if repo:
204 cache_key = ctx.uri
205 return repo.get(
206 cache_key,
207 lambda: _do_render_page_segments_from_ctx(ctx),
208 fs_cache_time=ctx.page.path_mtime)
209
210 return _do_render_page_segments_from_ctx(ctx)
211
212
213 def _do_render_page_segments_from_ctx(ctx):
214 eis = ctx.app.env.exec_info_stack 251 eis = ctx.app.env.exec_info_stack
215 eis.pushPage(ctx.page, ctx) 252 eis.pushPage(ctx.page, ctx)
216 ctx.setCurrentPass(PASS_FORMATTING)
217 try: 253 try:
218 data_ctx = DataBuildingContext(ctx.page, page_num=ctx.page_num) 254 page_data = _build_render_data(ctx)
219 page_data = build_page_data(data_ctx) 255
220 return _do_render_page_segments(ctx.page, page_data) 256 ctx.setCurrentPass(PASS_FORMATTING)
257 repo = ctx.app.env.rendered_segments_repository
258 save_to_fs = True
259 if ctx.app.env.fs_cache_only_for_main_page and not eis.is_main_page:
260 save_to_fs = False
261 if repo and not ctx.force_render:
262 render_result = repo.get(
263 ctx.uri,
264 lambda: _do_render_page_segments(ctx.page, page_data),
265 fs_cache_time=ctx.page.path_mtime,
266 save_to_fs=save_to_fs)
267 else:
268 render_result = _do_render_page_segments(ctx.page, page_data)
269 if repo:
270 repo.put(ctx.uri, render_result, save_to_fs)
221 finally: 271 finally:
222 ctx.setCurrentPass(PASS_NONE) 272 ctx.setCurrentPass(PASS_NONE)
223 eis.popPage() 273 eis.popPage()
224 274
275 rs = RenderedSegments(
276 render_result['segments'],
277 RenderPassInfo._fromJson(render_result['pass_info']))
278 return rs
279
280
281 def _build_render_data(ctx):
282 data_ctx = DataBuildingContext(ctx.page, page_num=ctx.page_num)
283 data_ctx.pagination_source = ctx.pagination_source
284 data_ctx.pagination_filter = ctx.pagination_filter
285 page_data = build_page_data(data_ctx)
286 if ctx.custom_data:
287 page_data.update(ctx.custom_data)
288 return page_data
289
225 290
226 def _do_render_page_segments(page, page_data): 291 def _do_render_page_segments(page, page_data):
227 app = page.app 292 app = page.app
293
294 cpi = app.env.exec_info_stack.current_page_info
295 assert cpi is not None
296 assert cpi.page == page
297
228 engine_name = page.config.get('template_engine') 298 engine_name = page.config.get('template_engine')
229 format_name = page.config.get('format') 299 format_name = page.config.get('format')
230 300
231 engine = get_template_engine(app, engine_name) 301 engine = get_template_engine(app, engine_name)
232 302
233 formatted_content = {} 303 formatted_segments = {}
234 for seg_name, seg in page.raw_content.items(): 304 for seg_name, seg in page.raw_content.items():
235 seg_text = '' 305 seg_text = ''
236 for seg_part in seg.parts: 306 for seg_part in seg.parts:
237 part_format = seg_part.fmt or format_name 307 part_format = seg_part.fmt or format_name
238 try: 308 try:
244 err.lineno += seg_part.line 314 err.lineno += seg_part.line
245 raise err 315 raise err
246 316
247 part_text = format_text(app, part_format, part_text) 317 part_text = format_text(app, part_format, part_text)
248 seg_text += part_text 318 seg_text += part_text
249 formatted_content[seg_name] = seg_text 319 formatted_segments[seg_name] = seg_text
250 320
251 if seg_name == 'content': 321 if seg_name == 'content':
252 m = content_abstract_re.search(seg_text) 322 m = content_abstract_re.search(seg_text)
253 if m: 323 if m:
254 offset = m.start() 324 offset = m.start()
255 content_abstract = seg_text[:offset] 325 content_abstract = seg_text[:offset]
256 formatted_content['content.abstract'] = content_abstract 326 formatted_segments['content.abstract'] = content_abstract
257 327
258 return formatted_content 328 pass_info = cpi.render_ctx.render_passes.get(PASS_FORMATTING)
259 329 res = {
260 330 'segments': formatted_segments,
261 def render_layout(layout_name, page, layout_data): 331 'pass_info': pass_info._toJson()}
332 return res
333
334
335 def _do_render_layout(layout_name, page, layout_data):
336 cpi = page.app.env.exec_info_stack.current_page_info
337 assert cpi is not None
338 assert cpi.page == page
339
262 names = layout_name.split(',') 340 names = layout_name.split(',')
263 default_template_engine = get_template_engine(page.app, None) 341 default_template_engine = get_template_engine(page.app, None)
264 default_exts = ['.' + e.lstrip('.') 342 default_exts = ['.' + e.lstrip('.')
265 for e in default_template_engine.EXTENSIONS] 343 for e in default_template_engine.EXTENSIONS]
266 full_names = [] 344 full_names = []
279 output = engine.renderFile(full_names, layout_data) 357 output = engine.renderFile(full_names, layout_data)
280 except TemplateNotFoundError as ex: 358 except TemplateNotFoundError as ex:
281 msg = "Can't find template for page: %s\n" % page.path 359 msg = "Can't find template for page: %s\n" % page.path
282 msg += "Looked for: %s" % ', '.join(full_names) 360 msg += "Looked for: %s" % ', '.join(full_names)
283 raise Exception(msg) from ex 361 raise Exception(msg) from ex
284 return output 362
363 pass_info = cpi.render_ctx.render_passes.get(PASS_RENDERING)
364 res = {'content': output, 'pass_info': pass_info._toJson()}
365 return res
285 366
286 367
287 def get_template_engine(app, engine_name): 368 def get_template_engine(app, engine_name):
288 if engine_name == 'html': 369 if engine_name == 'html':
289 engine_name = None 370 engine_name = None