Mercurial > piecrust2
comparison piecrust/pipelines/records.py @ 852:4850f8c21b6e
core: Start of the big refactor for PieCrust 3.0.
* Everything is a `ContentSource`, including assets directories.
* Most content sources are subclasses of the base file-system source.
* A source is processed by a "pipeline", and there are 2 built-in pipelines,
one for assets and one for pages. The asset pipeline is vaguely functional,
but the page pipeline is completely broken right now.
* Rewrite the baking process as just running appropriate pipelines on each
content item. This should allow for better parallelization.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Wed, 17 May 2017 00:11:48 -0700 |
parents | |
children | f070a4fc033c |
comparison
equal
deleted
inserted
replaced
851:2c7e57d80bba | 852:4850f8c21b6e |
---|---|
1 import os | |
2 import os.path | |
3 import pickle | |
4 import hashlib | |
5 import logging | |
6 from piecrust import APP_VERSION | |
7 | |
8 | |
9 logger = logging.getLogger(__name__) | |
10 | |
11 | |
12 class MultiRecord: | |
13 RECORD_VERSION = 12 | |
14 | |
15 def __init__(self): | |
16 self.records = [] | |
17 self.success = True | |
18 self.bake_time = 0 | |
19 self.incremental_count = 0 | |
20 self.invalidated = False | |
21 self.stats = None | |
22 self._app_version = APP_VERSION | |
23 self._record_version = self.RECORD_VERSION | |
24 | |
25 def getRecord(self, record_name, auto_create=True): | |
26 for r in self.records: | |
27 if r.name == record_name: | |
28 return r | |
29 if not auto_create: | |
30 return None | |
31 record = Record() | |
32 self.records.append(record) | |
33 return record | |
34 | |
35 def save(self, path): | |
36 path_dir = os.path.dirname(path) | |
37 if not os.path.isdir(path_dir): | |
38 os.makedirs(path_dir, 0o755) | |
39 | |
40 with open(path, 'wb') as fp: | |
41 pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL) | |
42 | |
43 @staticmethod | |
44 def load(path): | |
45 logger.debug("Loading bake records from: %s" % path) | |
46 with open(path, 'rb') as fp: | |
47 return pickle.load(fp) | |
48 | |
49 | |
50 class Record: | |
51 def __init__(self): | |
52 self.name = None | |
53 self.entries = [] | |
54 self.stats = {} | |
55 self.out_dir = None | |
56 self.success = True | |
57 | |
58 | |
59 class RecordEntry: | |
60 def __init__(self): | |
61 self.item_spec = None | |
62 self.errors = [] | |
63 | |
64 @property | |
65 def success(self): | |
66 return len(self.errors) == 0 | |
67 | |
68 | |
69 def _are_records_valid(multi_record): | |
70 return (multi_record._app_version == APP_VERSION and | |
71 multi_record._record_version == MultiRecord.RECORD_VERSION) | |
72 | |
73 | |
74 def load_records(path): | |
75 try: | |
76 multi_record = MultiRecord.load(path) | |
77 except Exception as ex: | |
78 logger.debug("Error loading records from: %s" % path) | |
79 logger.debug(ex) | |
80 logger.debug("Will use empty records.") | |
81 multi_record = None | |
82 | |
83 was_invalid = False | |
84 if multi_record is not None and not _are_records_valid(multi_record): | |
85 logger.debug( | |
86 "Records from '%s' have old version: %s/%s." % | |
87 (path, multi_record._app_version, multi_record._record_version)) | |
88 logger.debug("Will use empty records.") | |
89 multi_record = None | |
90 was_invalid = True | |
91 | |
92 if multi_record is None: | |
93 multi_record = MultiRecord() | |
94 multi_record.invalidated = was_invalid | |
95 | |
96 return multi_record | |
97 | |
98 | |
99 def _build_diff_key(item_spec): | |
100 return hashlib.md5(item_spec.encode('utf8')).hexdigest() | |
101 | |
102 | |
103 class MultiRecordHistory: | |
104 def __init__(self, previous, current): | |
105 if previous is None or current is None: | |
106 raise ValueError() | |
107 | |
108 self.previous = previous | |
109 self.current = current | |
110 self.histories = [] | |
111 self._buildHistories(previous, current) | |
112 | |
113 def getHistory(self, record_name): | |
114 for h in self.histories: | |
115 if h.name == record_name: | |
116 return h | |
117 return None | |
118 | |
119 def _buildHistories(self, previous, current): | |
120 pairs = {} | |
121 if previous: | |
122 for r in previous.records: | |
123 pairs[r.name] = (r, None) | |
124 if current: | |
125 for r in current.records: | |
126 p = pairs.get(r.name, (None, None)) | |
127 if p[1] is not None: | |
128 raise Exception("Got several records named: %s" % r.name) | |
129 pairs[r.name] = (p[0], r) | |
130 | |
131 for p, c in pairs.values(): | |
132 self.histories.append(RecordHistory(p, c)) | |
133 | |
134 | |
135 class RecordHistory: | |
136 def __init__(self, previous, current): | |
137 self._diffs = {} | |
138 self._previous = previous | |
139 self._current = current | |
140 | |
141 if previous and current and previous.name != current.name: | |
142 raise Exception("The two records must have the same name! " | |
143 "Got '%s' and '%s'." % | |
144 (previous.name, current.name)) | |
145 | |
146 self._buildDiffs() | |
147 | |
148 @property | |
149 def name(self): | |
150 return self._current.name | |
151 | |
152 @property | |
153 def current(self): | |
154 return self._current | |
155 | |
156 @property | |
157 def previous(self): | |
158 return self._previous | |
159 | |
160 @property | |
161 def diffs(self): | |
162 return self._diffs.values() | |
163 | |
164 def _buildDiffs(self): | |
165 if self._previous is not None: | |
166 for e in self._previous.entries: | |
167 key = _build_diff_key(e.item_spec) | |
168 self._diffs[key] = (e, None) | |
169 | |
170 if self._current is not None: | |
171 for e in self._current.entries: | |
172 key = _build_diff_key(e.item_spec) | |
173 diff = self._diffs.get(key) | |
174 if diff is None: | |
175 self._diffs[key] = (None, e) | |
176 elif diff[1] is None: | |
177 self._diffs[key] = (diff[0], e) | |
178 else: | |
179 raise Exception( | |
180 "A current record entry already exists for: %s" % key) | |
181 |