Mercurial > piecrust2

comparison piecrust/baking/records.py @ 338:938be93215cb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Improve render context and bake record, fix incremental bake bugs. * Used sources and taxonomies are now stored on a per-render-pass basis. This fixes bugs where sources/taxonomies were used for one pass, but that pass is skipped on a later bake because its result is cached. * Bake records are now created for all pages even when they're not baked. Record collapsing is gone except for taxonomy index pages. * Bake records now also have sub-entries in order to store information about each sub-page, since some sub-pages could use sources/taxonomies differently than others, or be missing from the output. This lets PieCrust handle clean/dirty states on a sub-page level.
author Ludovic Chabant <ludovic@chabant.com>
date Mon, 06 Apr 2015 19:59:54 -0700
parents b034f6f15e22
children 2cd2b5d07129
comparison
equal deleted inserted replaced
337:49408002798e 338:938be93215cb
1 import copy
1 import os.path 2 import os.path
2 import logging 3 import logging
3 from piecrust.records import Record, TransitionalRecord 4 from piecrust.records import Record, TransitionalRecord
4 5
5 6
17 key += taxonomy_term 18 key += taxonomy_term
18 return key 19 return key
19 20
20 21
21 class BakeRecord(Record): 22 class BakeRecord(Record):
22 RECORD_VERSION = 11 23 RECORD_VERSION = 12
23 24
24 def __init__(self): 25 def __init__(self):
25 super(BakeRecord, self).__init__() 26 super(BakeRecord, self).__init__()
26 self.out_dir = None 27 self.out_dir = None
27 self.bake_time = None 28 self.bake_time = None
28 self.success = True 29 self.success = True
29 30
30 31
31 FLAG_NONE = 0 32 class BakeRecordPassInfo(object):
32 FLAG_SOURCE_MODIFIED = 2**0 33 def __init__(self):
33 FLAG_OVERRIDEN = 2**1 34 self.used_source_names = set()
34 FLAG_FORCED_BY_SOURCE = 2**2 35 self.used_taxonomy_terms = set()
36
37
38 class BakeRecordSubPageEntry(object):
39 FLAG_NONE = 0
40 FLAG_BAKED = 2**0
41 FLAG_FORCED_BY_SOURCE = 2**1
42 FLAG_FORCED_BY_NO_PREVIOUS = 2**2
43 FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3
44 FLAG_FORMATTING_INVALIDATED = 2**4
45
46 def __init__(self, out_uri, out_path):
47 self.out_uri = out_uri
48 self.out_path = out_path
49 self.flags = self.FLAG_NONE
50 self.errors = []
51 self.render_passes = {}
52
53 @property
54 def was_clean(self):
55 return (self.flags & self.FLAG_BAKED) == 0 and len(self.errors) == 0
56
57 @property
58 def was_baked(self):
59 return (self.flags & self.FLAG_BAKED) != 0
60
61 @property
62 def was_baked_successfully(self):
63 return self.was_baked and len(self.errors) == 0
64
65 def collapseRenderPasses(self, other):
66 for p, pinfo in self.render_passes.items():
67 if p not in other.render_passes:
68 other.render_passes[p] = copy.deepcopy(pinfo)
35 69
36 70
37 class BakeRecordPageEntry(object): 71 class BakeRecordPageEntry(object):
38 """ An entry in the bake record. 72 """ An entry in the bake record.
39 73
40 The `taxonomy_info` attribute should be a tuple of the form: 74 The `taxonomy_info` attribute should be a tuple of the form:
41 (taxonomy name, term, source name) 75 (taxonomy name, term, source name)
42 """ 76 """
77 FLAG_NONE = 0
78 FLAG_NEW = 2**0
79 FLAG_SOURCE_MODIFIED = 2**1
80 FLAG_OVERRIDEN = 2**2
81
43 def __init__(self, source_name, rel_path, path, taxonomy_info=None): 82 def __init__(self, source_name, rel_path, path, taxonomy_info=None):
44 self.source_name = source_name 83 self.source_name = source_name
45 self.rel_path = rel_path 84 self.rel_path = rel_path
46 self.path = path 85 self.path = path
47 self.taxonomy_info = taxonomy_info 86 self.taxonomy_info = taxonomy_info
48 87 self.flags = self.FLAG_NONE
49 self.flags = FLAG_NONE
50 self.config = None 88 self.config = None
51 self.errors = [] 89 self.subs = []
52 self.out_uris = [] 90 self.assets = []
53 self.out_paths = []
54 self.clean_uris = []
55 self.clean_out_paths = []
56 self.used_source_names = set()
57 self.used_taxonomy_terms = set()
58 self.used_pagination_item_count = 0
59 91
60 @property 92 @property
61 def path_mtime(self): 93 def path_mtime(self):
62 return os.path.getmtime(self.path) 94 return os.path.getmtime(self.path)
63 95
64 @property 96 @property
65 def was_baked(self): 97 def was_overriden(self):
66 return len(self.out_paths) > 0 or len(self.errors) > 0 98 return (self.flags & self.FLAG_OVERRIDEN) != 0
67
68 @property
69 def was_baked_successfully(self):
70 return len(self.out_paths) > 0 and len(self.errors) == 0
71 99
72 @property 100 @property
73 def num_subs(self): 101 def num_subs(self):
74 return len(self.out_paths) 102 return len(self.subs)
103
104 @property
105 def was_any_sub_baked(self):
106 for o in self.subs:
107 if o.was_baked:
108 return True
109 return False
110
111 def getSub(self, sub_index):
112 return self.subs[sub_index - 1]
113
114 def getAllErrors(self):
115 for o in self.subs:
116 yield from o.errors
117
118 def getAllUsedSourceNames(self):
119 res = set()
120 for o in self.subs:
121 for p, pinfo in o.render_passes.items():
122 res |= pinfo.used_source_names
123 return res
124
125 def getAllUsedTaxonomyTerms(self):
126 res = set()
127 for o in self.subs:
128 for p, pinfo in o.render_passes.items():
129 res |= pinfo.used_taxonomy_terms
130 return res
75 131
76 132
77 class TransitionalBakeRecord(TransitionalRecord): 133 class TransitionalBakeRecord(TransitionalRecord):
78 def __init__(self, previous_path=None): 134 def __init__(self, previous_path=None):
79 super(TransitionalBakeRecord, self).__init__(BakeRecord, 135 super(TransitionalBakeRecord, self).__init__(BakeRecord,
80 previous_path) 136 previous_path)
137 self.dirty_source_names = set()
81 138
82 def addEntry(self, entry): 139 def addEntry(self, entry):
83 if (self.previous.bake_time and 140 if (self.previous.bake_time and
84 entry.path_mtime >= self.previous.bake_time): 141 entry.path_mtime >= self.previous.bake_time):
85 entry.flags |= FLAG_SOURCE_MODIFIED 142 entry.flags |= BakeRecordPageEntry.FLAG_SOURCE_MODIFIED
143 self.dirty_source_names.add(entry.source_name)
86 super(TransitionalBakeRecord, self).addEntry(entry) 144 super(TransitionalBakeRecord, self).addEntry(entry)
87 145
88 def getTransitionKey(self, entry): 146 def getTransitionKey(self, entry):
89 return _get_transition_key(entry.source_name, entry.rel_path, 147 return _get_transition_key(entry.source_name, entry.rel_path,
90 entry.taxonomy_info) 148 entry.taxonomy_info)
91 149
92 def getOverrideEntry(self, factory, uri): 150 def getOverrideEntry(self, factory, uri):
93 for pair in self.transitions.values(): 151 for pair in self.transitions.values():
94 prev = pair[0]
95 cur = pair[1] 152 cur = pair[1]
96 if (cur and 153 if (cur and
97 (cur.source_name != factory.source.name or 154 (cur.source_name != factory.source.name or
98 cur.rel_path != factory.rel_path) and 155 cur.rel_path != factory.rel_path)):
99 len(cur.out_uris) > 0 and cur.out_uris[0] == uri): 156 for o in cur.subs:
100 return cur 157 if o.out_uri == uri:
101 if (prev and 158 return cur
102 (prev.source_name != factory.source.name or
103 prev.rel_path != factory.rel_path) and
104 len(prev.out_uris) > 0 and prev.out_uris[0] == uri):
105 return prev
106 return None 159 return None
107 160
108 def getPreviousEntry(self, source_name, rel_path, taxonomy_info=None): 161 def getPreviousEntry(self, source_name, rel_path, taxonomy_info=None):
109 key = _get_transition_key(source_name, rel_path, taxonomy_info) 162 key = _get_transition_key(source_name, rel_path, taxonomy_info)
110 pair = self.transitions.get(key) 163 pair = self.transitions.get(key)
111 if pair is not None: 164 if pair is not None:
112 return pair[0] 165 return pair[0]
113 return None 166 return None
114 167
115 def getCurrentEntries(self, source_name): 168 def collapseEntry(self, prev_entry):
116 return [e for e in self.current.entries 169 cur_entry = copy.deepcopy(prev_entry)
117 if e.source_name == source_name] 170 cur_entry.flags = BakeRecordPageEntry.FLAG_NONE
118 171 for o in cur_entry.subs:
119 def collapseRecords(self): 172 o.flags = BakeRecordSubPageEntry.FLAG_NONE
120 for prev, cur in self.transitions.values(): 173 self.addEntry(cur_entry)
121 if prev and cur and not cur.was_baked:
122 # This page wasn't baked, so the information from last
123 # time is still valid (we didn't get any information
124 # since we didn't bake).
125 cur.flags = prev.flags
126 if prev.config:
127 cur.config = prev.config.copy()
128 cur.out_uris = list(prev.out_uris)
129 cur.out_paths = list(prev.out_paths)
130 cur.errors = list(prev.errors)
131 cur.used_source_names = set(prev.used_source_names)
132 cur.used_taxonomy_terms = set(prev.used_taxonomy_terms)
133 174
134 def getDeletions(self): 175 def getDeletions(self):
135 for prev, cur in self.transitions.values(): 176 for prev, cur in self.transitions.values():
136 if prev and not cur: 177 if prev and not cur:
137 for p in prev.out_paths: 178 for sub in prev.subs:
138 yield (p, 'previous source file was removed') 179 yield (sub.out_path, 'previous source file was removed')
139 elif prev and cur and cur.was_baked_successfully: 180 elif prev and cur:
140 diff = set(prev.out_paths) - set(cur.out_paths) 181 prev_out_paths = [o.out_path for o in prev.subs]
182 cur_out_paths = [o.out_path for o in cur.subs]
183 diff = set(prev_out_paths) - set(cur_out_paths)
141 for p in diff: 184 for p in diff:
142 yield (p, 'source file changed outputs') 185 yield (p, 'source file changed outputs')
143 186
187 def _onNewEntryAdded(self, entry):
188 entry.flags |= BakeRecordPageEntry.FLAG_NEW
189