diff piecrust/pipelines/records.py @ 852:4850f8c21b6e

core: Start of the big refactor for PieCrust 3.0. * Everything is a `ContentSource`, including assets directories. * Most content sources are subclasses of the base file-system source. * A source is processed by a "pipeline", and there are 2 built-in pipelines, one for assets and one for pages. The asset pipeline is vaguely functional, but the page pipeline is completely broken right now. * Rewrite the baking process as just running appropriate pipelines on each content item. This should allow for better parallelization.
author Ludovic Chabant <ludovic@chabant.com>
date Wed, 17 May 2017 00:11:48 -0700
parents
children f070a4fc033c
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/piecrust/pipelines/records.py	Wed May 17 00:11:48 2017 -0700
@@ -0,0 +1,181 @@
+import os
+import os.path
+import pickle
+import hashlib
+import logging
+from piecrust import APP_VERSION
+
+
+logger = logging.getLogger(__name__)
+
+
+class MultiRecord:
+    RECORD_VERSION = 12
+
+    def __init__(self):
+        self.records = []
+        self.success = True
+        self.bake_time = 0
+        self.incremental_count = 0
+        self.invalidated = False
+        self.stats = None
+        self._app_version = APP_VERSION
+        self._record_version = self.RECORD_VERSION
+
+    def getRecord(self, record_name, auto_create=True):
+        for r in self.records:
+            if r.name == record_name:
+                return r
+        if not auto_create:
+            return None
+        record = Record()
+        self.records.append(record)
+        return record
+
+    def save(self, path):
+        path_dir = os.path.dirname(path)
+        if not os.path.isdir(path_dir):
+            os.makedirs(path_dir, 0o755)
+
+        with open(path, 'wb') as fp:
+            pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL)
+
+    @staticmethod
+    def load(path):
+        logger.debug("Loading bake records from: %s" % path)
+        with open(path, 'rb') as fp:
+            return pickle.load(fp)
+
+
+class Record:
+    def __init__(self):
+        self.name = None
+        self.entries = []
+        self.stats = {}
+        self.out_dir = None
+        self.success = True
+
+
+class RecordEntry:
+    def __init__(self):
+        self.item_spec = None
+        self.errors = []
+
+    @property
+    def success(self):
+        return len(self.errors) == 0
+
+
+def _are_records_valid(multi_record):
+    return (multi_record._app_version == APP_VERSION and
+            multi_record._record_version == MultiRecord.RECORD_VERSION)
+
+
+def load_records(path):
+    try:
+        multi_record = MultiRecord.load(path)
+    except Exception as ex:
+        logger.debug("Error loading records from: %s" % path)
+        logger.debug(ex)
+        logger.debug("Will use empty records.")
+        multi_record = None
+
+    was_invalid = False
+    if multi_record is not None and not _are_records_valid(multi_record):
+        logger.debug(
+            "Records from '%s' have old version: %s/%s." %
+            (path, multi_record._app_version, multi_record._record_version))
+        logger.debug("Will use empty records.")
+        multi_record = None
+        was_invalid = True
+
+    if multi_record is None:
+        multi_record = MultiRecord()
+        multi_record.invalidated = was_invalid
+
+    return multi_record
+
+
+def _build_diff_key(item_spec):
+    return hashlib.md5(item_spec.encode('utf8')).hexdigest()
+
+
+class MultiRecordHistory:
+    def __init__(self, previous, current):
+        if previous is None or current is None:
+            raise ValueError()
+
+        self.previous = previous
+        self.current = current
+        self.histories = []
+        self._buildHistories(previous, current)
+
+    def getHistory(self, record_name):
+        for h in self.histories:
+            if h.name == record_name:
+                return h
+        return None
+
+    def _buildHistories(self, previous, current):
+        pairs = {}
+        if previous:
+            for r in previous.records:
+                pairs[r.name] = (r, None)
+        if current:
+            for r in current.records:
+                p = pairs.get(r.name, (None, None))
+                if p[1] is not None:
+                    raise Exception("Got several records named: %s" % r.name)
+                pairs[r.name] = (p[0], r)
+
+        for p, c in pairs.values():
+            self.histories.append(RecordHistory(p, c))
+
+
+class RecordHistory:
+    def __init__(self, previous, current):
+        self._diffs = {}
+        self._previous = previous
+        self._current = current
+
+        if previous and current and previous.name != current.name:
+            raise Exception("The two records must have the same name! "
+                            "Got '%s' and '%s'." %
+                            (previous.name, current.name))
+
+        self._buildDiffs()
+
+    @property
+    def name(self):
+        return self._current.name
+
+    @property
+    def current(self):
+        return self._current
+
+    @property
+    def previous(self):
+        return self._previous
+
+    @property
+    def diffs(self):
+        return self._diffs.values()
+
+    def _buildDiffs(self):
+        if self._previous is not None:
+            for e in self._previous.entries:
+                key = _build_diff_key(e.item_spec)
+                self._diffs[key] = (e, None)
+
+        if self._current is not None:
+            for e in self._current.entries:
+                key = _build_diff_key(e.item_spec)
+                diff = self._diffs.get(key)
+                if diff is None:
+                    self._diffs[key] = (None, e)
+                elif diff[1] is None:
+                    self._diffs[key] = (diff[0], e)
+                else:
+                    raise Exception(
+                        "A current record entry already exists for: %s" % key)
+