Mercurial > piecrust2
diff piecrust/pipelines/records.py @ 852:4850f8c21b6e
core: Start of the big refactor for PieCrust 3.0.
* Everything is a `ContentSource`, including assets directories.
* Most content sources are subclasses of the base file-system source.
* A source is processed by a "pipeline", and there are 2 built-in pipelines,
one for assets and one for pages. The asset pipeline is vaguely functional,
but the page pipeline is completely broken right now.
* Rewrite the baking process as just running appropriate pipelines on each
content item. This should allow for better parallelization.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Wed, 17 May 2017 00:11:48 -0700 |
parents | |
children | f070a4fc033c |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/pipelines/records.py Wed May 17 00:11:48 2017 -0700 @@ -0,0 +1,181 @@ +import os +import os.path +import pickle +import hashlib +import logging +from piecrust import APP_VERSION + + +logger = logging.getLogger(__name__) + + +class MultiRecord: + RECORD_VERSION = 12 + + def __init__(self): + self.records = [] + self.success = True + self.bake_time = 0 + self.incremental_count = 0 + self.invalidated = False + self.stats = None + self._app_version = APP_VERSION + self._record_version = self.RECORD_VERSION + + def getRecord(self, record_name, auto_create=True): + for r in self.records: + if r.name == record_name: + return r + if not auto_create: + return None + record = Record() + self.records.append(record) + return record + + def save(self, path): + path_dir = os.path.dirname(path) + if not os.path.isdir(path_dir): + os.makedirs(path_dir, 0o755) + + with open(path, 'wb') as fp: + pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL) + + @staticmethod + def load(path): + logger.debug("Loading bake records from: %s" % path) + with open(path, 'rb') as fp: + return pickle.load(fp) + + +class Record: + def __init__(self): + self.name = None + self.entries = [] + self.stats = {} + self.out_dir = None + self.success = True + + +class RecordEntry: + def __init__(self): + self.item_spec = None + self.errors = [] + + @property + def success(self): + return len(self.errors) == 0 + + +def _are_records_valid(multi_record): + return (multi_record._app_version == APP_VERSION and + multi_record._record_version == MultiRecord.RECORD_VERSION) + + +def load_records(path): + try: + multi_record = MultiRecord.load(path) + except Exception as ex: + logger.debug("Error loading records from: %s" % path) + logger.debug(ex) + logger.debug("Will use empty records.") + multi_record = None + + was_invalid = False + if multi_record is not None and not _are_records_valid(multi_record): + logger.debug( + "Records from '%s' have old version: %s/%s." % + (path, multi_record._app_version, multi_record._record_version)) + logger.debug("Will use empty records.") + multi_record = None + was_invalid = True + + if multi_record is None: + multi_record = MultiRecord() + multi_record.invalidated = was_invalid + + return multi_record + + +def _build_diff_key(item_spec): + return hashlib.md5(item_spec.encode('utf8')).hexdigest() + + +class MultiRecordHistory: + def __init__(self, previous, current): + if previous is None or current is None: + raise ValueError() + + self.previous = previous + self.current = current + self.histories = [] + self._buildHistories(previous, current) + + def getHistory(self, record_name): + for h in self.histories: + if h.name == record_name: + return h + return None + + def _buildHistories(self, previous, current): + pairs = {} + if previous: + for r in previous.records: + pairs[r.name] = (r, None) + if current: + for r in current.records: + p = pairs.get(r.name, (None, None)) + if p[1] is not None: + raise Exception("Got several records named: %s" % r.name) + pairs[r.name] = (p[0], r) + + for p, c in pairs.values(): + self.histories.append(RecordHistory(p, c)) + + +class RecordHistory: + def __init__(self, previous, current): + self._diffs = {} + self._previous = previous + self._current = current + + if previous and current and previous.name != current.name: + raise Exception("The two records must have the same name! " + "Got '%s' and '%s'." % + (previous.name, current.name)) + + self._buildDiffs() + + @property + def name(self): + return self._current.name + + @property + def current(self): + return self._current + + @property + def previous(self): + return self._previous + + @property + def diffs(self): + return self._diffs.values() + + def _buildDiffs(self): + if self._previous is not None: + for e in self._previous.entries: + key = _build_diff_key(e.item_spec) + self._diffs[key] = (e, None) + + if self._current is not None: + for e in self._current.entries: + key = _build_diff_key(e.item_spec) + diff = self._diffs.get(key) + if diff is None: + self._diffs[key] = (None, e) + elif diff[1] is None: + self._diffs[key] = (diff[0], e) + else: + raise Exception( + "A current record entry already exists for: %s" % key) +