Mercurial > piecrust2

comparison piecrust/baking/single.py @ 338:938be93215cb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Improve render context and bake record, fix incremental bake bugs. * Used sources and taxonomies are now stored on a per-render-pass basis. This fixes bugs where sources/taxonomies were used for one pass, but that pass is skipped on a later bake because its result is cached. * Bake records are now created for all pages even when they're not baked. Record collapsing is gone except for taxonomy index pages. * Bake records now also have sub-entries in order to store information about each sub-page, since some sub-pages could use sources/taxonomies differently than others, or be missing from the output. This lets PieCrust handle clean/dirty states on a sub-page level.
author Ludovic Chabant <ludovic@chabant.com>
date Mon, 06 Apr 2015 19:59:54 -0700
parents b034f6f15e22
children b8ff1780b491
comparison
equal deleted inserted replaced
337:49408002798e 338:938be93215cb
2 import shutil 2 import shutil
3 import codecs 3 import codecs
4 import logging 4 import logging
5 import urllib.parse 5 import urllib.parse
6 from piecrust.baking.records import ( 6 from piecrust.baking.records import (
7 FLAG_OVERRIDEN, FLAG_SOURCE_MODIFIED, FLAG_FORCED_BY_SOURCE) 7 BakeRecordPassInfo, BakeRecordPageEntry, BakeRecordSubPageEntry)
8 from piecrust.data.filters import (PaginationFilter, HasFilterClause, 8 from piecrust.data.filters import (
9 PaginationFilter, HasFilterClause,
9 IsFilterClause, AndBooleanClause, 10 IsFilterClause, AndBooleanClause,
10 page_value_accessor) 11 page_value_accessor)
11 from piecrust.rendering import (PageRenderingContext, render_page, 12 from piecrust.rendering import (
13 PageRenderingContext, render_page,
12 PASS_FORMATTING, PASS_RENDERING) 14 PASS_FORMATTING, PASS_RENDERING)
13 from piecrust.sources.base import (PageFactory, 15 from piecrust.sources.base import (
16 PageFactory,
14 REALM_NAMES, REALM_USER, REALM_THEME) 17 REALM_NAMES, REALM_USER, REALM_THEME)
15 from piecrust.uriutil import split_uri 18 from piecrust.uriutil import split_uri
16 19
17 20
18 logger = logging.getLogger(__name__) 21 logger = logging.getLogger(__name__)
58 return os.path.normpath(os.path.join(*bake_path)) 61 return os.path.normpath(os.path.join(*bake_path))
59 62
60 def bake(self, factory, route, record_entry): 63 def bake(self, factory, route, record_entry):
61 bake_taxonomy_info = None 64 bake_taxonomy_info = None
62 route_metadata = dict(factory.metadata) 65 route_metadata = dict(factory.metadata)
66
67 # Add taxonomy metadata for generating the URL if needed.
63 if record_entry.taxonomy_info: 68 if record_entry.taxonomy_info:
64 tax_name, tax_term, tax_source_name = record_entry.taxonomy_info 69 tax_name, tax_term, tax_source_name = record_entry.taxonomy_info
65 taxonomy = self.app.getTaxonomy(tax_name) 70 taxonomy = self.app.getTaxonomy(tax_name)
66 slugified_term = route.slugifyTaxonomyTerm(tax_term) 71 slugified_term = route.slugifyTaxonomyTerm(tax_term)
67 route_metadata[taxonomy.term_name] = slugified_term 72 route_metadata[taxonomy.term_name] = slugified_term
69 74
70 # Generate the URL using the route. 75 # Generate the URL using the route.
71 page = factory.buildPage() 76 page = factory.buildPage()
72 uri = route.getUri(route_metadata, provider=page) 77 uri = route.getUri(route_metadata, provider=page)
73 78
79 # See if this URL has been overriden by a previously baked page.
80 # If that page is from another realm (e.g. a user page vs. a theme
81 # page), we silently skip this page. If they're from the same realm,
82 # we don't allow overriding and raise an error (this is probably
83 # because of a misconfigured configuration that allows for ambiguous
84 # URLs between 2 routes or sources).
74 override = self.record.getOverrideEntry(factory, uri) 85 override = self.record.getOverrideEntry(factory, uri)
75 if override is not None: 86 if override is not None:
76 override_source = self.app.getSource(override.source_name) 87 override_source = self.app.getSource(override.source_name)
77 if override_source.realm == factory.source.realm: 88 if override_source.realm == factory.source.realm:
78 raise BakingError( 89 raise BakingError(
81 override.source_name, 92 override.source_name,
82 override.rel_path)) 93 override.rel_path))
83 logger.debug("'%s' [%s] is overriden by '%s:%s'. Skipping" % 94 logger.debug("'%s' [%s] is overriden by '%s:%s'. Skipping" %
84 (factory.ref_spec, uri, override.source_name, 95 (factory.ref_spec, uri, override.source_name,
85 override.rel_path)) 96 override.rel_path))
86 record_entry.flags |= FLAG_OVERRIDEN 97 record_entry.flags |= BakeRecordPageEntry.FLAG_OVERRIDEN
87 return 98 return
88 99
100 # Setup the record entry.
101 record_entry.config = copy_public_page_config(page.config)
102
103 # Start baking the sub-pages.
89 cur_sub = 1 104 cur_sub = 1
90 has_more_subs = True 105 has_more_subs = True
91 force_this = self.force 106 force_this = self.force
92 invalidate_formatting = False 107 invalidate_formatting = False
93 record_entry.config = copy_public_page_config(page.config)
94 prev_record_entry = self.record.getPreviousEntry( 108 prev_record_entry = self.record.getPreviousEntry(
95 factory.source.name, factory.rel_path, 109 factory.source.name, factory.rel_path,
96 record_entry.taxonomy_info) 110 record_entry.taxonomy_info)
97 111
98 logger.debug("Baking '%s'..." % uri) 112 logger.debug("Baking '%s'..." % uri)
99 113
100 # If the current page is known to use pages from other sources,
101 # see if any of those got baked, or are going to be baked for some
102 # reason. If so, we need to bake this one too.
103 # (this happens for instance with the main page of a blog).
104 if prev_record_entry and prev_record_entry.was_baked_successfully:
105 invalidated_render_passes = set()
106 used_src_names = list(prev_record_entry.used_source_names)
107 for src_name, rdr_pass in used_src_names:
108 entries = self.record.getCurrentEntries(src_name)
109 for e in entries:
110 if e.was_baked or e.flags & FLAG_SOURCE_MODIFIED:
111 invalidated_render_passes.add(rdr_pass)
112 break
113 if len(invalidated_render_passes) > 0:
114 logger.debug("'%s' is known to use sources %s, at least one "
115 "of which got baked. Will force bake this page. "
116 % (uri, used_src_names))
117 record_entry.flags |= FLAG_FORCED_BY_SOURCE
118 force_this = True
119
120 if PASS_FORMATTING in invalidated_render_passes:
121 logger.debug("Will invalidate cached formatting for '%s' "
122 "since sources were using during that pass."
123 % uri)
124 invalidate_formatting = True
125
126 while has_more_subs: 114 while has_more_subs:
115 # Get the URL and path for this sub-page.
127 sub_uri = route.getUri(route_metadata, sub_num=cur_sub, 116 sub_uri = route.getUri(route_metadata, sub_num=cur_sub,
128 provider=page) 117 provider=page)
129 out_path = self.getOutputPath(sub_uri) 118 out_path = self.getOutputPath(sub_uri)
119
120 # Create the sub-entry for the bake record.
121 record_sub_entry = BakeRecordSubPageEntry(sub_uri, out_path)
122 record_entry.subs.append(record_sub_entry)
123
124 # Find a corresponding sub-entry in the previous bake record.
125 prev_record_sub_entry = None
126 if prev_record_entry:
127 try:
128 prev_record_sub_entry = prev_record_entry.getSub(cur_sub)
129 except IndexError:
130 pass
131
132 # Figure out what to do with this page.
133 if (prev_record_sub_entry and
134 (prev_record_sub_entry.was_baked_successfully or
135 prev_record_sub_entry.was_clean)):
136 # If the current page is known to use pages from other sources,
137 # see if any of those got baked, or are going to be baked for
138 # some reason. If so, we need to bake this one too.
139 # (this happens for instance with the main page of a blog).
140 dirty_src_names, invalidated_render_passes = (
141 self._getDirtySourceNamesAndRenderPasses(
142 prev_record_sub_entry))
143 if len(invalidated_render_passes) > 0:
144 logger.debug(
145 "'%s' is known to use sources %s, which have "
146 "items that got (re)baked. Will force bake this "
147 "page. " % (uri, dirty_src_names))
148 record_sub_entry.flags |= \
149 BakeRecordSubPageEntry.FLAG_FORCED_BY_SOURCE
150 force_this = True
151
152 if PASS_FORMATTING in invalidated_render_passes:
153 logger.debug(
154 "Will invalidate cached formatting for '%s' "
155 "since sources were using during that pass."
156 % uri)
157 invalidate_formatting = True
158 elif (prev_record_sub_entry and
159 prev_record_sub_entry.errors):
160 # Previous bake failed. We'll have to bake it again.
161 logger.debug(
162 "Previous record entry indicates baking failed for "
163 "'%s'. Will bake it again." % uri)
164 record_sub_entry.flags |= \
165 BakeRecordSubPageEntry.FLAG_FORCED_BY_PREVIOUS_ERRORS
166 force_this = True
167 elif not prev_record_sub_entry:
168 # No previous record. We'll have to bake it.
169 logger.debug("No previous record entry found for '%s'. Will "
170 "force bake it." % uri)
171 record_sub_entry.flags |= \
172 BakeRecordSubPageEntry.FLAG_FORCED_BY_NO_PREVIOUS
173 force_this = True
130 174
131 # Check for up-to-date outputs. 175 # Check for up-to-date outputs.
132 do_bake = True 176 do_bake = True
133 if not force_this: 177 if not force_this:
134 try: 178 try:
141 pass 185 pass
142 186
143 # If this page didn't bake because it's already up-to-date. 187 # If this page didn't bake because it's already up-to-date.
144 # Keep trying for as many subs as we know this page has. 188 # Keep trying for as many subs as we know this page has.
145 if not do_bake: 189 if not do_bake:
146 if (prev_record_entry is not None and 190 prev_record_sub_entry.collapseRenderPasses(record_sub_entry)
147 prev_record_entry.num_subs < cur_sub): 191 record_sub_entry.flags = BakeRecordSubPageEntry.FLAG_NONE
148 logger.debug("") 192
193 if prev_record_entry.num_subs >= cur_sub + 1:
149 cur_sub += 1 194 cur_sub += 1
150 has_more_subs = True 195 has_more_subs = True
151 logger.debug(" %s is up to date, skipping to next " 196 logger.debug(" %s is up to date, skipping to next "
152 "sub-page." % out_path) 197 "sub-page." % out_path)
153 record_entry.clean_uris.append(sub_uri)
154 record_entry.clean_out_paths.append(out_path)
155 continue 198 continue
156 199
157 # We don't know how many subs to expect... just skip.
158 logger.debug(" %s is up to date, skipping bake." % out_path) 200 logger.debug(" %s is up to date, skipping bake." % out_path)
159 break 201 break
160 202
161 # All good, proceed. 203 # All good, proceed.
162 try: 204 try:
163 if invalidate_formatting: 205 if invalidate_formatting:
164 cache_key = sub_uri 206 cache_key = sub_uri
165 self.app.env.rendered_segments_repository.invalidate( 207 self.app.env.rendered_segments_repository.invalidate(
166 cache_key) 208 cache_key)
209 record_sub_entry.flags |= \
210 BakeRecordSubPageEntry.FLAG_FORMATTING_INVALIDATED
167 211
168 logger.debug(" p%d -> %s" % (cur_sub, out_path)) 212 logger.debug(" p%d -> %s" % (cur_sub, out_path))
169 ctx, rp = self._bakeSingle(page, sub_uri, cur_sub, out_path, 213 ctx, rp = self._bakeSingle(page, sub_uri, cur_sub, out_path,
170 bake_taxonomy_info) 214 bake_taxonomy_info)
171 except Exception as ex: 215 except Exception as ex:
172 if self.app.debug: 216 if self.app.debug:
173 logger.exception(ex) 217 logger.exception(ex)
174 page_rel_path = os.path.relpath(page.path, self.app.root_dir) 218 page_rel_path = os.path.relpath(page.path, self.app.root_dir)
175 raise BakingError("%s: error baking '%s'." % 219 raise BakingError("%s: error baking '%s'." %
176 (page_rel_path, uri)) from ex 220 (page_rel_path, uri)) from ex
221
222 # Record what we did.
223 record_sub_entry.flags |= BakeRecordSubPageEntry.FLAG_BAKED
224 self.record.dirty_source_names.add(record_entry.source_name)
225 for p, pinfo in ctx.render_passes.items():
226 brpi = BakeRecordPassInfo()
227 brpi.used_source_names = set(pinfo.used_source_names)
228 brpi.used_taxonomy_terms = set(pinfo.used_taxonomy_terms)
229 record_sub_entry.render_passes[p] = brpi
230 if prev_record_sub_entry:
231 prev_record_sub_entry.collapseRenderPasses(record_sub_entry)
177 232
178 # Copy page assets. 233 # Copy page assets.
179 if (cur_sub == 1 and self.copy_assets and 234 if (cur_sub == 1 and self.copy_assets and
180 ctx.used_assets is not None): 235 ctx.used_assets is not None):
181 if self.pretty_urls: 236 if self.pretty_urls:
188 243
189 logger.debug("Copying page assets to: %s" % out_assets_dir) 244 logger.debug("Copying page assets to: %s" % out_assets_dir)
190 if not os.path.isdir(out_assets_dir): 245 if not os.path.isdir(out_assets_dir):
191 os.makedirs(out_assets_dir, 0o755) 246 os.makedirs(out_assets_dir, 0o755)
192 for ap in ctx.used_assets: 247 for ap in ctx.used_assets:
193 dest_ap = os.path.join(out_assets_dir, os.path.basename(ap)) 248 dest_ap = os.path.join(out_assets_dir,
249 os.path.basename(ap))
194 logger.debug(" %s -> %s" % (ap, dest_ap)) 250 logger.debug(" %s -> %s" % (ap, dest_ap))
195 shutil.copy(ap, dest_ap) 251 shutil.copy(ap, dest_ap)
196 252 record_entry.assets.append(ap)
197 # Record what we did and figure out if we have more work. 253
198 record_entry.out_uris.append(sub_uri) 254 # Figure out if we have more work.
199 record_entry.out_paths.append(out_path)
200 record_entry.used_source_names |= ctx.used_source_names
201 record_entry.used_taxonomy_terms |= ctx.used_taxonomy_terms
202
203 has_more_subs = False 255 has_more_subs = False
204 if ctx.used_pagination is not None: 256 if ctx.used_pagination is not None:
205 if cur_sub == 1:
206 record_entry.used_pagination_item_count = \
207 ctx.used_pagination.total_item_count
208 if ctx.used_pagination.has_more: 257 if ctx.used_pagination.has_more:
209 cur_sub += 1 258 cur_sub += 1
210 has_more_subs = True 259 has_more_subs = True
211 260
212 def _bakeSingle(self, page, sub_uri, num, out_path, 261 def _bakeSingle(self, page, sub_uri, num, out_path,
225 with codecs.open(out_path, 'w', 'utf8') as fp: 274 with codecs.open(out_path, 'w', 'utf8') as fp:
226 fp.write(rp.content) 275 fp.write(rp.content)
227 276
228 return ctx, rp 277 return ctx, rp
229 278
279 def _getDirtySourceNamesAndRenderPasses(self, record_sub_entry):
280 dirty_src_names = set()
281 invalidated_render_passes = set()
282 for p, pinfo in record_sub_entry.render_passes.items():
283 for src_name in pinfo.used_source_names:
284 is_dirty = (src_name in self.record.dirty_source_names)
285 if is_dirty:
286 invalidated_render_passes.add(p)
287 dirty_src_names.add(src_name)
288 break
289 return dirty_src_names, invalidated_render_passes
290