Mercurial > piecrust2

comparison piecrust/rendering.py @ 852:4850f8c21b6e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
core: Start of the big refactor for PieCrust 3.0. * Everything is a `ContentSource`, including assets directories. * Most content sources are subclasses of the base file-system source. * A source is processed by a "pipeline", and there are 2 built-in pipelines, one for assets and one for pages. The asset pipeline is vaguely functional, but the page pipeline is completely broken right now. * Rewrite the baking process as just running appropriate pipelines on each content item. This should allow for better parallelization.
author Ludovic Chabant <ludovic@chabant.com>
date Wed, 17 May 2017 00:11:48 -0700
parents dca51cd8147a
children f070a4fc033c
comparison
equal deleted inserted replaced
851:2c7e57d80bba 852:4850f8c21b6e
1 import re 1 import re
2 import os.path 2 import os.path
3 import copy 3 import copy
4 import logging 4 import logging
5 from werkzeug.utils import cached_property
6 from piecrust.data.builder import ( 5 from piecrust.data.builder import (
7 DataBuildingContext, build_page_data, build_layout_data) 6 DataBuildingContext, build_page_data, add_layout_data)
8 from piecrust.data.filters import (
9 PaginationFilter, SettingFilterClause, page_value_accessor)
10 from piecrust.fastpickle import _pickle_object, _unpickle_object 7 from piecrust.fastpickle import _pickle_object, _unpickle_object
11 from piecrust.sources.base import PageSource 8 from piecrust.sources.base import ContentSource
12 from piecrust.templating.base import TemplateNotFoundError, TemplatingError 9 from piecrust.templating.base import TemplateNotFoundError, TemplatingError
13 10
14 11
15 logger = logging.getLogger(__name__) 12 logger = logging.getLogger(__name__)
16 13
17 14
18 content_abstract_re = re.compile(r'^<!--\s*(more|(page)?break)\s*-->\s*$', 15 content_abstract_re = re.compile(r'^<!--\s*(more|(page)?break)\s*-->\s*$',
19 re.MULTILINE) 16 re.MULTILINE)
20 17
21 18
22 class PageRenderingError(Exception): 19 class RenderingError(Exception):
23 pass 20 pass
24 21
25 22
26 class TemplateEngineNotFound(Exception): 23 class TemplateEngineNotFound(Exception):
27 pass 24 pass
28
29
30 class QualifiedPage(object):
31 def __init__(self, page, route, route_metadata):
32 self.page = page
33 self.route = route
34 self.route_metadata = route_metadata
35
36 def getUri(self, sub_num=1):
37 return self.route.getUri(self.route_metadata, sub_num=sub_num)
38
39 def __getattr__(self, name):
40 return getattr(self.page, name)
41 25
42 26
43 class RenderedSegments(object): 27 class RenderedSegments(object):
44 def __init__(self, segments, render_pass_info): 28 def __init__(self, segments, render_pass_info):
45 self.segments = segments 29 self.segments = segments
51 self.content = content 35 self.content = content
52 self.render_pass_info = render_pass_info 36 self.render_pass_info = render_pass_info
53 37
54 38
55 class RenderedPage(object): 39 class RenderedPage(object):
56 def __init__(self, page, uri, num=1): 40 def __init__(self, qualified_page):
57 self.page = page 41 self.qualified_page = qualified_page
58 self.uri = uri
59 self.num = num
60 self.data = None 42 self.data = None
61 self.content = None 43 self.content = None
62 self.render_info = [None, None] 44 self.render_info = [None, None]
63 45
64 @property 46 @property
65 def app(self): 47 def app(self):
66 return self.page.app 48 return self.qualified_page.app
67 49
68 def copyRenderInfo(self): 50 def copyRenderInfo(self):
69 return copy.deepcopy(self.render_info) 51 return copy.deepcopy(self.render_info)
70 52
71 53
92 if create_if_missing: 74 if create_if_missing:
93 return self._custom_info.setdefault(key, default) 75 return self._custom_info.setdefault(key, default)
94 return self._custom_info.get(key, default) 76 return self._custom_info.get(key, default)
95 77
96 78
97 class PageRenderingContext(object): 79 class RenderingContext(object):
98 def __init__(self, qualified_page, page_num=1, 80 def __init__(self, qualified_page, force_render=False):
99 force_render=False, is_from_request=False): 81 self.qualified_page = qualified_page
100 self.page = qualified_page
101 self.page_num = page_num
102 self.force_render = force_render 82 self.force_render = force_render
103 self.is_from_request = is_from_request
104 self.pagination_source = None 83 self.pagination_source = None
105 self.pagination_filter = None 84 self.pagination_filter = None
106 self.custom_data = {} 85 self.custom_data = {}
107 self.render_passes = [None, None] # Same length as RENDER_PASSES 86 self.render_passes = [None, None] # Same length as RENDER_PASSES
108 self._current_pass = PASS_NONE 87 self._current_pass = PASS_NONE
109 88
110 @property 89 @property
111 def app(self): 90 def app(self):
112 return self.page.app 91 return self.qualified_page.app
113
114 @property
115 def source_metadata(self):
116 return self.page.source_metadata
117
118 @cached_property
119 def uri(self):
120 return self.page.getUri(self.page_num)
121 92
122 @property 93 @property
123 def current_pass_info(self): 94 def current_pass_info(self):
124 if self._current_pass != PASS_NONE: 95 if self._current_pass != PASS_NONE:
125 return self.render_passes[self._current_pass] 96 return self.render_passes[self._current_pass]
140 pass_info.pagination_has_more = paginator.has_more 111 pass_info.pagination_has_more = paginator.has_more
141 self.addUsedSource(paginator._source) 112 self.addUsedSource(paginator._source)
142 113
143 def addUsedSource(self, source): 114 def addUsedSource(self, source):
144 self._raiseIfNoCurrentPass() 115 self._raiseIfNoCurrentPass()
145 if isinstance(source, PageSource): 116 if isinstance(source, ContentSource):
146 pass_info = self.current_pass_info 117 pass_info = self.current_pass_info
147 pass_info.used_source_names.add(source.name) 118 pass_info.used_source_names.add(source.name)
148 119
149 def _raiseIfNoCurrentPass(self): 120 def _raiseIfNoCurrentPass(self):
150 if self._current_pass == PASS_NONE: 121 if self._current_pass == PASS_NONE:
151 raise Exception("No rendering pass is currently active.") 122 raise Exception("No rendering pass is currently active.")
152 123
153 124
125 class RenderingContextStack(object):
126 def __init__(self):
127 self._ctx_stack = []
128
129 @property
130 def current_ctx(self):
131 if len(self._ctx_stack) == 0:
132 return None
133 return self._ctx_stack[-1]
134
135 @property
136 def is_main_ctx(self):
137 return len(self._ctx_stack) == 1
138
139 def hasPage(self, page):
140 for ei in self._ctx_stack:
141 if ei.qualified_page.page == page:
142 return True
143 return False
144
145 def pushCtx(self, render_ctx):
146 for ctx in self._ctx_stack:
147 if ctx.qualified_page.page == render_ctx.qualified_page.page:
148 raise Exception("Loop detected during rendering!")
149 self._ctx_stack.append(render_ctx)
150
151 def popCtx(self):
152 del self._ctx_stack[-1]
153
154 def clear(self):
155 self._ctx_stack = []
156
157
154 def render_page(ctx): 158 def render_page(ctx):
155 eis = ctx.app.env.exec_info_stack 159 env = ctx.app.env
156 eis.pushPage(ctx.page, ctx) 160
161 stack = env.render_ctx_stack
162 stack.pushCtx(ctx)
163
164 qpage = ctx.qualified_page
165
157 try: 166 try:
158 # Build the data for both segment and layout rendering. 167 # Build the data for both segment and layout rendering.
159 with ctx.app.env.timerScope("BuildRenderData"): 168 with env.timerScope("BuildRenderData"):
160 page_data = _build_render_data(ctx) 169 page_data = _build_render_data(ctx)
161 170
162 # Render content segments. 171 # Render content segments.
163 ctx.setCurrentPass(PASS_FORMATTING) 172 ctx.setCurrentPass(PASS_FORMATTING)
164 repo = ctx.app.env.rendered_segments_repository 173 repo = env.rendered_segments_repository
165 save_to_fs = True 174 save_to_fs = True
166 if ctx.app.env.fs_cache_only_for_main_page and not eis.is_main_page: 175 if env.fs_cache_only_for_main_page and not stack.is_main_ctx:
167 save_to_fs = False 176 save_to_fs = False
168 with ctx.app.env.timerScope("PageRenderSegments"): 177 with env.timerScope("PageRenderSegments"):
169 if repo and not ctx.force_render: 178 if repo is not None and not ctx.force_render:
170 render_result = repo.get( 179 render_result = repo.get(
171 ctx.uri, 180 qpage.uri,
172 lambda: _do_render_page_segments(ctx.page, page_data), 181 lambda: _do_render_page_segments(ctx, page_data),
173 fs_cache_time=ctx.page.path_mtime, 182 fs_cache_time=qpage.page.content_mtime,
174 save_to_fs=save_to_fs) 183 save_to_fs=save_to_fs)
175 else: 184 else:
176 render_result = _do_render_page_segments(ctx.page, page_data) 185 render_result = _do_render_page_segments(ctx, page_data)
177 if repo: 186 if repo:
178 repo.put(ctx.uri, render_result, save_to_fs) 187 repo.put(qpage.uri, render_result, save_to_fs)
179 188
180 # Render layout. 189 # Render layout.
181 page = ctx.page
182 ctx.setCurrentPass(PASS_RENDERING) 190 ctx.setCurrentPass(PASS_RENDERING)
183 layout_name = page.config.get('layout') 191 layout_name = qpage.page.config.get('layout')
184 if layout_name is None: 192 if layout_name is None:
185 layout_name = page.source.config.get('default_layout', 'default') 193 layout_name = qpage.page.source.config.get(
194 'default_layout', 'default')
186 null_names = ['', 'none', 'nil'] 195 null_names = ['', 'none', 'nil']
187 if layout_name not in null_names: 196 if layout_name not in null_names:
188 with ctx.app.env.timerScope("BuildRenderData"): 197 with ctx.app.env.timerScope("BuildRenderData"):
189 build_layout_data(page, page_data, render_result['segments']) 198 add_layout_data(page_data, render_result['segments'])
190 199
191 with ctx.app.env.timerScope("PageRenderLayout"): 200 with ctx.app.env.timerScope("PageRenderLayout"):
192 layout_result = _do_render_layout(layout_name, page, page_data) 201 layout_result = _do_render_layout(
202 layout_name, qpage, page_data)
193 else: 203 else:
194 layout_result = { 204 layout_result = {
195 'content': render_result['segments']['content'], 205 'content': render_result['segments']['content'],
196 'pass_info': None} 206 'pass_info': None}
197 207
198 rp = RenderedPage(page, ctx.uri, ctx.page_num) 208 rp = RenderedPage(qpage)
199 rp.data = page_data 209 rp.data = page_data
200 rp.content = layout_result['content'] 210 rp.content = layout_result['content']
201 rp.render_info[PASS_FORMATTING] = _unpickle_object( 211 rp.render_info[PASS_FORMATTING] = _unpickle_object(
202 render_result['pass_info']) 212 render_result['pass_info'])
203 if layout_result['pass_info'] is not None: 213 if layout_result['pass_info'] is not None:
204 rp.render_info[PASS_RENDERING] = _unpickle_object( 214 rp.render_info[PASS_RENDERING] = _unpickle_object(
205 layout_result['pass_info']) 215 layout_result['pass_info'])
206 return rp 216 return rp
217
207 except Exception as ex: 218 except Exception as ex:
208 if ctx.app.debug: 219 if ctx.app.debug:
209 raise 220 raise
210 logger.exception(ex) 221 logger.exception(ex)
211 page_rel_path = os.path.relpath(ctx.page.path, ctx.app.root_dir) 222 page_rel_path = os.path.relpath(ctx.page.path, ctx.app.root_dir)
212 raise Exception("Error rendering page: %s" % page_rel_path) from ex 223 raise Exception("Error rendering page: %s" % page_rel_path) from ex
224
213 finally: 225 finally:
214 ctx.setCurrentPass(PASS_NONE) 226 ctx.setCurrentPass(PASS_NONE)
215 eis.popPage() 227 stack.popCtx()
216 228
217 229
218 def render_page_segments(ctx): 230 def render_page_segments(ctx):
219 eis = ctx.app.env.exec_info_stack 231 env = ctx.app.env
220 eis.pushPage(ctx.page, ctx) 232
233 stack = env.render_ctx_stack
234 stack.pushCtx(ctx)
235
236 qpage = ctx.qualified_page
237
221 try: 238 try:
222 ctx.setCurrentPass(PASS_FORMATTING) 239 ctx.setCurrentPass(PASS_FORMATTING)
223 repo = ctx.app.env.rendered_segments_repository 240 repo = ctx.app.env.rendered_segments_repository
224 save_to_fs = True 241 save_to_fs = True
225 if ctx.app.env.fs_cache_only_for_main_page and not eis.is_main_page: 242 if ctx.app.env.fs_cache_only_for_main_page and not stack.is_main_ctx:
226 save_to_fs = False 243 save_to_fs = False
227 with ctx.app.env.timerScope("PageRenderSegments"): 244 with ctx.app.env.timerScope("PageRenderSegments"):
228 if repo and not ctx.force_render: 245 if repo is not None and not ctx.force_render:
229 render_result = repo.get( 246 render_result = repo.get(
230 ctx.uri, 247 qpage.uri,
231 lambda: _do_render_page_segments_from_ctx(ctx), 248 lambda: _do_render_page_segments_from_ctx(ctx),
232 fs_cache_time=ctx.page.path_mtime, 249 fs_cache_time=qpage.page.content_mtime,
233 save_to_fs=save_to_fs) 250 save_to_fs=save_to_fs)
234 else: 251 else:
235 render_result = _do_render_page_segments_from_ctx(ctx) 252 render_result = _do_render_page_segments_from_ctx(ctx)
236 if repo: 253 if repo:
237 repo.put(ctx.uri, render_result, save_to_fs) 254 repo.put(qpage.uri, render_result, save_to_fs)
238 finally: 255 finally:
239 ctx.setCurrentPass(PASS_NONE) 256 ctx.setCurrentPass(PASS_NONE)
240 eis.popPage() 257 stack.popCtx()
241 258
242 rs = RenderedSegments( 259 rs = RenderedSegments(
243 render_result['segments'], 260 render_result['segments'],
244 _unpickle_object(render_result['pass_info'])) 261 _unpickle_object(render_result['pass_info']))
245 return rs 262 return rs
246 263
247 264
248 def _build_render_data(ctx): 265 def _build_render_data(ctx):
249 with ctx.app.env.timerScope("PageDataBuild"): 266 with ctx.app.env.timerScope("PageDataBuild"):
250 data_ctx = DataBuildingContext(ctx.page, page_num=ctx.page_num) 267 data_ctx = DataBuildingContext(ctx.qualified_page)
251 data_ctx.pagination_source = ctx.pagination_source 268 data_ctx.pagination_source = ctx.pagination_source
252 data_ctx.pagination_filter = ctx.pagination_filter 269 data_ctx.pagination_filter = ctx.pagination_filter
253 page_data = build_page_data(data_ctx) 270 page_data = build_page_data(data_ctx)
254 if ctx.custom_data: 271 if ctx.custom_data:
255 page_data._appendMapping(ctx.custom_data) 272 page_data._appendMapping(ctx.custom_data)
256 return page_data 273 return page_data
257 274
258 275
259 def _do_render_page_segments_from_ctx(ctx): 276 def _do_render_page_segments_from_ctx(ctx):
260 page_data = _build_render_data(ctx) 277 page_data = _build_render_data(ctx)
261 return _do_render_page_segments(ctx.page, page_data) 278 return _do_render_page_segments(ctx, page_data)
262 279
263 280
264 def _do_render_page_segments(page, page_data): 281 def _do_render_page_segments(ctx, page_data):
282 page = ctx.qualified_page.page
265 app = page.app 283 app = page.app
266
267 cpi = app.env.exec_info_stack.current_page_info
268 assert cpi is not None
269 assert cpi.page == page
270 284
271 engine_name = page.config.get('template_engine') 285 engine_name = page.config.get('template_engine')
272 format_name = page.config.get('format') 286 format_name = page.config.get('format')
273 287
274 engine = get_template_engine(app, engine_name) 288 engine = get_template_engine(app, engine_name)
280 part_format = seg_part.fmt or format_name 294 part_format = seg_part.fmt or format_name
281 try: 295 try:
282 with app.env.timerScope( 296 with app.env.timerScope(
283 engine.__class__.__name__ + '_segment'): 297 engine.__class__.__name__ + '_segment'):
284 part_text = engine.renderSegmentPart( 298 part_text = engine.renderSegmentPart(
285 page.path, seg_part, page_data) 299 page.path, seg_part, page_data)
286 except TemplatingError as err: 300 except TemplatingError as err:
287 err.lineno += seg_part.line 301 err.lineno += seg_part.line
288 raise err 302 raise err
289 303
290 part_text = format_text(app, part_format, part_text) 304 part_text = format_text(app, part_format, part_text)
296 if m: 310 if m:
297 offset = m.start() 311 offset = m.start()
298 content_abstract = seg_text[:offset] 312 content_abstract = seg_text[:offset]
299 formatted_segments['content.abstract'] = content_abstract 313 formatted_segments['content.abstract'] = content_abstract
300 314
301 pass_info = cpi.render_ctx.render_passes[PASS_FORMATTING] 315 pass_info = ctx.render_passes[PASS_FORMATTING]
302 res = { 316 res = {
303 'segments': formatted_segments, 317 'segments': formatted_segments,
304 'pass_info': _pickle_object(pass_info)} 318 'pass_info': _pickle_object(pass_info)}
305 return res 319 return res
306 320
307 321
308 def _do_render_layout(layout_name, page, layout_data): 322 def _do_render_layout(layout_name, page, layout_data):
309 cpi = page.app.env.exec_info_stack.current_page_info 323 cpi = page.app.env.exec_info_stack.current_page_info