Mercurial > piecrust2

comparison piecrust/baking/records.py @ 120:133845647083

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Better error management and removal support in baking/processing. * Baker and processor pipeline now store errors in their records. * They also support deleting output files that are no longer valid. * The basic transitional record class implements more boilerplate code. * The processor pipeline is run from the `bake` command directly. * New unit tests. * Unit test mocking now mocks `os.remove` too.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 09 Nov 2014 14:46:23 -0800
parents 0445a2232de7
children 1f4c3dae1fe8
comparison
equal deleted inserted replaced
119:0811f92cbdc7 120:133845647083
1 import os.path 1 import os.path
2 import logging 2 import logging
3 from piecrust import APP_VERSION
4 from piecrust.sources.base import PageSource 3 from piecrust.sources.base import PageSource
5 from piecrust.records import Record 4 from piecrust.records import Record, TransitionalRecord
6 5
7 6
8 logger = logging.getLogger(__name__) 7 logger = logging.getLogger(__name__)
9
10
11 RECORD_VERSION = 6
12 8
13 9
14 def _get_transition_key(source_name, rel_path, taxonomy_name=None, 10 def _get_transition_key(source_name, rel_path, taxonomy_name=None,
15 taxonomy_term=None): 11 taxonomy_term=None):
16 key = '%s:%s' % (source_name, rel_path) 12 key = '%s:%s' % (source_name, rel_path)
22 key += taxonomy_term 18 key += taxonomy_term
23 return key 19 return key
24 20
25 21
26 class BakeRecord(Record): 22 class BakeRecord(Record):
23 RECORD_VERSION = 8
24
27 def __init__(self): 25 def __init__(self):
28 super(BakeRecord, self).__init__() 26 super(BakeRecord, self).__init__()
29 self.out_dir = None 27 self.out_dir = None
30 self.bake_time = None 28 self.bake_time = None
31 self.app_version = APP_VERSION
32 self.record_version = RECORD_VERSION
33
34 def hasLatestVersion(self):
35 return (self.app_version == APP_VERSION and
36 self.record_version == RECORD_VERSION)
37
38 def __setstate__(self, state):
39 state.setdefault('app_version', -1)
40 state.setdefault('record_version', -1)
41 super(BakeRecord, self).__setstate__(state)
42 29
43 30
44 FLAG_NONE = 0 31 FLAG_NONE = 0
45 FLAG_SOURCE_MODIFIED = 2**0 32 FLAG_SOURCE_MODIFIED = 2**0
46 FLAG_OVERRIDEN = 2**1 33 FLAG_OVERRIDEN = 2**1
55 self.taxonomy_term = taxonomy_term 42 self.taxonomy_term = taxonomy_term
56 self.path_mtime = os.path.getmtime(factory.path) 43 self.path_mtime = os.path.getmtime(factory.path)
57 44
58 self.flags = FLAG_NONE 45 self.flags = FLAG_NONE
59 self.config = None 46 self.config = None
47 self.errors = []
60 self.out_uris = [] 48 self.out_uris = []
61 self.out_paths = [] 49 self.out_paths = []
62 self.used_source_names = set() 50 self.used_source_names = set()
63 self.used_taxonomy_terms = set() 51 self.used_taxonomy_terms = set()
64 52
65 @property 53 @property
66 def was_baked(self): 54 def was_baked(self):
67 return len(self.out_paths) > 0 55 return len(self.out_paths) > 0 or len(self.errors) > 0
56
57 @property
58 def was_baked_successfully(self):
59 return len(self.out_paths) > 0 and len(self.errors) == 0
68 60
69 @property 61 @property
70 def num_subs(self): 62 def num_subs(self):
71 return len(self.out_paths) 63 return len(self.out_paths)
72
73 @property
74 def transition_key(self):
75 return _get_transition_key(self.source_name, self.rel_path,
76 self.taxonomy_name, self.taxonomy_term)
77 64
78 def __getstate__(self): 65 def __getstate__(self):
79 state = self.__dict__.copy() 66 state = self.__dict__.copy()
80 del state['path_mtime'] 67 del state['path_mtime']
81 return state 68 return state
82 69
83 70
84 class TransitionalBakeRecord(object): 71 class TransitionalBakeRecord(TransitionalRecord):
85 DELETION_MISSING = 1
86 DELETION_CHANGED = 2
87
88 def __init__(self, previous_path=None): 72 def __init__(self, previous_path=None):
89 self.previous = BakeRecord() 73 super(TransitionalBakeRecord, self).__init__(BakeRecord,
90 self.current = BakeRecord() 74 previous_path)
91 self.transitions = {}
92 self.incremental_count = 0
93 if previous_path:
94 self.loadPrevious(previous_path)
95 self.current.entry_added += self._onCurrentEntryAdded
96
97 def loadPrevious(self, previous_path):
98 try:
99 self.previous = BakeRecord.load(previous_path)
100 except Exception as ex:
101 logger.debug("Error loading previous record: %s" % ex)
102 logger.debug("Will reset to an empty one.")
103 self.previous = BakeRecord()
104 return
105
106 for e in self.previous.entries:
107 self.transitions[e.transition_key] = (e, None)
108
109 def clearPrevious(self):
110 self.previous = BakeRecord()
111
112 def saveCurrent(self, current_path):
113 self.current.save(current_path)
114 75
115 def addEntry(self, entry): 76 def addEntry(self, entry):
116 if (self.previous.bake_time and 77 if (self.previous.bake_time and
117 entry.path_mtime >= self.previous.bake_time): 78 entry.path_mtime >= self.previous.bake_time):
118 entry.flags |= FLAG_SOURCE_MODIFIED 79 entry.flags |= FLAG_SOURCE_MODIFIED
119 self.current.addEntry(entry) 80 super(TransitionalBakeRecord, self).addEntry(entry)
81
82 def getTransitionKey(self, entry):
83 return _get_transition_key(entry.source_name, entry.rel_path,
84 entry.taxonomy_name, entry.taxonomy_term)
120 85
121 def getOverrideEntry(self, factory, uri): 86 def getOverrideEntry(self, factory, uri):
122 for pair in self.transitions.values(): 87 for pair in self.transitions.values():
123 prev = pair[0] 88 prev = pair[0]
124 cur = pair[1] 89 cur = pair[1]
146 def getCurrentEntries(self, source_name): 111 def getCurrentEntries(self, source_name):
147 return [e for e in self.current.entries 112 return [e for e in self.current.entries
148 if e.source_name == source_name] 113 if e.source_name == source_name]
149 114
150 def collapseRecords(self): 115 def collapseRecords(self):
151 for pair in self.transitions.values(): 116 for prev, cur in self.transitions.values():
152 prev = pair[0]
153 cur = pair[1]
154
155 if prev and cur and not cur.was_baked: 117 if prev and cur and not cur.was_baked:
156 # This page wasn't baked, so the information from last 118 # This page wasn't baked, so the information from last
157 # time is still valid (we didn't get any information 119 # time is still valid (we didn't get any information
158 # since we didn't bake). 120 # since we didn't bake).
159 cur.flags = prev.flags 121 cur.flags = prev.flags
160 if prev.config: 122 if prev.config:
161 cur.config = prev.config.copy() 123 cur.config = prev.config.copy()
162 cur.out_uris = list(prev.out_uris) 124 cur.out_uris = list(prev.out_uris)
163 cur.out_paths = list(prev.out_paths) 125 cur.out_paths = list(prev.out_paths)
126 cur.errors = list(prev.errors)
164 cur.used_source_names = set(prev.used_source_names) 127 cur.used_source_names = set(prev.used_source_names)
165 cur.used_taxonomy_terms = set(prev.used_taxonomy_terms) 128 cur.used_taxonomy_terms = set(prev.used_taxonomy_terms)
166 129
167 def _onCurrentEntryAdded(self, entry): 130 def getDeletions(self):
168 key = entry.transition_key 131 for prev, cur in self.transitions.values():
169 te = self.transitions.get(key) 132 if prev and not cur:
170 if te is None: 133 for p in prev.out_paths:
171 logger.debug("Adding new record entry: %s" % key) 134 yield (p, 'previous source file was removed')
172 self.transitions[key] = (None, entry) 135 elif prev and cur and cur.was_baked_successfully:
173 return 136 diff = set(prev.out_paths) - set(cur.out_paths)
137 for p in diff:
138 yield (p, 'source file changed outputs')
174 139
175 if te[1] is not None:
176 raise Exception("A current entry already exists for: %s" %
177 key)
178 logger.debug("Setting current record entry: %s" % key)
179 self.transitions[key] = (te[0], entry)
180