comparison piecrust/pipelines/records.py @ 852:4850f8c21b6e

core: Start of the big refactor for PieCrust 3.0. * Everything is a `ContentSource`, including assets directories. * Most content sources are subclasses of the base file-system source. * A source is processed by a "pipeline", and there are 2 built-in pipelines, one for assets and one for pages. The asset pipeline is vaguely functional, but the page pipeline is completely broken right now. * Rewrite the baking process as just running appropriate pipelines on each content item. This should allow for better parallelization.
author Ludovic Chabant <ludovic@chabant.com>
date Wed, 17 May 2017 00:11:48 -0700
parents
children f070a4fc033c
comparison
equal deleted inserted replaced
851:2c7e57d80bba 852:4850f8c21b6e
1 import os
2 import os.path
3 import pickle
4 import hashlib
5 import logging
6 from piecrust import APP_VERSION
7
8
9 logger = logging.getLogger(__name__)
10
11
12 class MultiRecord:
13 RECORD_VERSION = 12
14
15 def __init__(self):
16 self.records = []
17 self.success = True
18 self.bake_time = 0
19 self.incremental_count = 0
20 self.invalidated = False
21 self.stats = None
22 self._app_version = APP_VERSION
23 self._record_version = self.RECORD_VERSION
24
25 def getRecord(self, record_name, auto_create=True):
26 for r in self.records:
27 if r.name == record_name:
28 return r
29 if not auto_create:
30 return None
31 record = Record()
32 self.records.append(record)
33 return record
34
35 def save(self, path):
36 path_dir = os.path.dirname(path)
37 if not os.path.isdir(path_dir):
38 os.makedirs(path_dir, 0o755)
39
40 with open(path, 'wb') as fp:
41 pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL)
42
43 @staticmethod
44 def load(path):
45 logger.debug("Loading bake records from: %s" % path)
46 with open(path, 'rb') as fp:
47 return pickle.load(fp)
48
49
50 class Record:
51 def __init__(self):
52 self.name = None
53 self.entries = []
54 self.stats = {}
55 self.out_dir = None
56 self.success = True
57
58
59 class RecordEntry:
60 def __init__(self):
61 self.item_spec = None
62 self.errors = []
63
64 @property
65 def success(self):
66 return len(self.errors) == 0
67
68
69 def _are_records_valid(multi_record):
70 return (multi_record._app_version == APP_VERSION and
71 multi_record._record_version == MultiRecord.RECORD_VERSION)
72
73
74 def load_records(path):
75 try:
76 multi_record = MultiRecord.load(path)
77 except Exception as ex:
78 logger.debug("Error loading records from: %s" % path)
79 logger.debug(ex)
80 logger.debug("Will use empty records.")
81 multi_record = None
82
83 was_invalid = False
84 if multi_record is not None and not _are_records_valid(multi_record):
85 logger.debug(
86 "Records from '%s' have old version: %s/%s." %
87 (path, multi_record._app_version, multi_record._record_version))
88 logger.debug("Will use empty records.")
89 multi_record = None
90 was_invalid = True
91
92 if multi_record is None:
93 multi_record = MultiRecord()
94 multi_record.invalidated = was_invalid
95
96 return multi_record
97
98
99 def _build_diff_key(item_spec):
100 return hashlib.md5(item_spec.encode('utf8')).hexdigest()
101
102
103 class MultiRecordHistory:
104 def __init__(self, previous, current):
105 if previous is None or current is None:
106 raise ValueError()
107
108 self.previous = previous
109 self.current = current
110 self.histories = []
111 self._buildHistories(previous, current)
112
113 def getHistory(self, record_name):
114 for h in self.histories:
115 if h.name == record_name:
116 return h
117 return None
118
119 def _buildHistories(self, previous, current):
120 pairs = {}
121 if previous:
122 for r in previous.records:
123 pairs[r.name] = (r, None)
124 if current:
125 for r in current.records:
126 p = pairs.get(r.name, (None, None))
127 if p[1] is not None:
128 raise Exception("Got several records named: %s" % r.name)
129 pairs[r.name] = (p[0], r)
130
131 for p, c in pairs.values():
132 self.histories.append(RecordHistory(p, c))
133
134
135 class RecordHistory:
136 def __init__(self, previous, current):
137 self._diffs = {}
138 self._previous = previous
139 self._current = current
140
141 if previous and current and previous.name != current.name:
142 raise Exception("The two records must have the same name! "
143 "Got '%s' and '%s'." %
144 (previous.name, current.name))
145
146 self._buildDiffs()
147
148 @property
149 def name(self):
150 return self._current.name
151
152 @property
153 def current(self):
154 return self._current
155
156 @property
157 def previous(self):
158 return self._previous
159
160 @property
161 def diffs(self):
162 return self._diffs.values()
163
164 def _buildDiffs(self):
165 if self._previous is not None:
166 for e in self._previous.entries:
167 key = _build_diff_key(e.item_spec)
168 self._diffs[key] = (e, None)
169
170 if self._current is not None:
171 for e in self._current.entries:
172 key = _build_diff_key(e.item_spec)
173 diff = self._diffs.get(key)
174 if diff is None:
175 self._diffs[key] = (None, e)
176 elif diff[1] is None:
177 self._diffs[key] = (diff[0], e)
178 else:
179 raise Exception(
180 "A current record entry already exists for: %s" % key)
181