view piecrust/baking/records.py @ 415:0e9a94b7fdfa

bake: Improve bake record information. * Store things in the bake record that require less interaction between the master process and the workers. For instance, don't store the paginator object in the render pass info -- instead, just store whether pagination was used, and whether it had more items. * Simplify information passing between workers and bake passes by saving the rendering info to the JSON cache. This means the "render first sub" job doesn't have to return anything except errors now. * Add more performance counter info.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:23:16 -0700
parents e7b865f8f335
children 61d606fbc313
line wrap: on
line source

import copy
import os.path
import hashlib
import logging
from piecrust.records import Record, TransitionalRecord


logger = logging.getLogger(__name__)


def _get_transition_key(path, taxonomy_info=None):
    key = path
    if taxonomy_info:
        key += '+%s:%s=' % (taxonomy_info.source_name,
                            taxonomy_info.taxonomy_name)
        if isinstance(taxonomy_info.term, tuple):
            key += '/'.join(taxonomy_info.term)
        else:
            key += taxonomy_info.term
    return hashlib.md5(key.encode('utf8')).hexdigest()


class BakeRecord(Record):
    RECORD_VERSION = 14

    def __init__(self):
        super(BakeRecord, self).__init__()
        self.out_dir = None
        self.bake_time = None
        self.baked_count = {}
        self.timers = None
        self.success = True


class SubPageBakeInfo(object):
    FLAG_NONE = 0
    FLAG_BAKED = 2**0
    FLAG_FORCED_BY_SOURCE = 2**1
    FLAG_FORCED_BY_NO_PREVIOUS = 2**2
    FLAG_FORCED_BY_PREVIOUS_ERRORS = 2**3
    FLAG_FORMATTING_INVALIDATED = 2**4

    def __init__(self, out_uri, out_path):
        self.out_uri = out_uri
        self.out_path = out_path
        self.flags = self.FLAG_NONE
        self.errors = []
        self.render_info = None

    @property
    def was_clean(self):
        return (self.flags & self.FLAG_BAKED) == 0 and len(self.errors) == 0

    @property
    def was_baked(self):
        return (self.flags & self.FLAG_BAKED) != 0

    @property
    def was_baked_successfully(self):
        return self.was_baked and len(self.errors) == 0

    def anyPass(self, func):
        assert self.render_info is not None
        for p, pinfo in self.render_info.items():
            if func(pinfo):
                return True
        return False

    def copyRenderInfo(self):
        assert self.render_info
        return copy.deepcopy(self.render_info)


class TaxonomyInfo(object):
    def __init__(self, taxonomy_name, source_name, term):
        self.taxonomy_name = taxonomy_name
        self.source_name = source_name
        self.term = term


class BakeRecordEntry(object):
    """ An entry in the bake record.

        The `taxonomy_info` attribute should be a tuple of the form:
        (taxonomy name, term, source name)
    """
    FLAG_NONE = 0
    FLAG_NEW = 2**0
    FLAG_SOURCE_MODIFIED = 2**1
    FLAG_OVERRIDEN = 2**2

    def __init__(self, source_name, path, taxonomy_info=None):
        self.source_name = source_name
        self.path = path
        self.taxonomy_info = taxonomy_info
        self.flags = self.FLAG_NONE
        self.config = None
        self.errors = []
        self.subs = []

    @property
    def path_mtime(self):
        return os.path.getmtime(self.path)

    @property
    def was_overriden(self):
        return (self.flags & self.FLAG_OVERRIDEN) != 0

    @property
    def num_subs(self):
        return len(self.subs)

    @property
    def was_any_sub_baked(self):
        for o in self.subs:
            if o.was_baked:
                return True
        return False

    @property
    def all_assets(self):
        for sub in self.subs:
            yield from sub.assets

    @property
    def has_any_error(self):
        if len(self.errors) > 0:
            return True
        for o in self.subs:
            if len(o.errors) > 0:
                return True
        return False

    def getSub(self, sub_index):
        return self.subs[sub_index - 1]

    def getAllErrors(self):
        yield from self.errors
        for o in self.subs:
            yield from o.errors

    def getAllUsedSourceNames(self):
        res = set()
        for o in self.subs:
            if o.render_info is not None:
                for p, pinfo in o.render_info.items():
                    res |= pinfo.used_source_names
        return res

    def getAllUsedTaxonomyTerms(self):
        res = set()
        for o in self.subs:
            if o.render_info is not None:
                for p, pinfo in o.render_info.items():
                    res |= pinfo.used_taxonomy_terms
        return res


class TransitionalBakeRecord(TransitionalRecord):
    def __init__(self, previous_path=None):
        super(TransitionalBakeRecord, self).__init__(BakeRecord,
                                                     previous_path)
        self.dirty_source_names = set()

    def addEntry(self, entry):
        if (self.previous.bake_time and
                entry.path_mtime >= self.previous.bake_time):
            entry.flags |= BakeRecordEntry.FLAG_SOURCE_MODIFIED
            self.dirty_source_names.add(entry.source_name)
        super(TransitionalBakeRecord, self).addEntry(entry)

    def getTransitionKey(self, entry):
        return _get_transition_key(entry.path, entry.taxonomy_info)

    def getPreviousAndCurrentEntries(self, path, taxonomy_info=None):
        key = _get_transition_key(path, taxonomy_info)
        pair = self.transitions.get(key)
        return pair

    def getOverrideEntry(self, path, uri):
        for pair in self.transitions.values():
            cur = pair[1]
            if cur and cur.path != path:
                for o in cur.subs:
                    if o.out_uri == uri:
                        return cur
        return None

    def getPreviousEntry(self, path, taxonomy_info=None):
        pair = self.getPreviousAndCurrentEntries(path, taxonomy_info)
        if pair is not None:
            return pair[0]
        return None

    def getCurrentEntry(self, path, taxonomy_info=None):
        pair = self.getPreviousAndCurrentEntries(path, taxonomy_info)
        if pair is not None:
            return pair[1]
        return None

    def collapseEntry(self, prev_entry):
        cur_entry = copy.deepcopy(prev_entry)
        cur_entry.flags = BakeRecordEntry.FLAG_NONE
        for o in cur_entry.subs:
            o.flags = SubPageBakeInfo.FLAG_NONE
        self.addEntry(cur_entry)

    def getDeletions(self):
        for prev, cur in self.transitions.values():
            if prev and not cur:
                for sub in prev.subs:
                    yield (sub.out_path, 'previous source file was removed')
            elif prev and cur:
                prev_out_paths = [o.out_path for o in prev.subs]
                cur_out_paths = [o.out_path for o in cur.subs]
                diff = set(prev_out_paths) - set(cur_out_paths)
                for p in diff:
                    yield (p, 'source file changed outputs')

    def _onNewEntryAdded(self, entry):
        entry.flags |= BakeRecordEntry.FLAG_NEW