view piecrust/sources/pageref.py @ 411:e7b865f8f335

bake: Enable multiprocess baking. Baking is now done by running a worker per CPU, and sending jobs to them. This changes several things across the codebase: * Ability to not cache things related to pages other than the 'main' page (i.e. the page at the bottom of the execution stack). * Decouple the baking process from the bake records, so only the main process keeps track (and modifies) the bake record. * Remove the need for 'batch page getters' and loading a page directly from the page factories. There are various smaller changes too included here, including support for scope performance timers that are saved with the bake record and can be printed out to the console. Yes I got carried away. For testing, the in-memory 'mock' file-system doesn't work anymore, since we're spawning processes, so this is replaced by a 'tmpfs' file-system which is saved in temporary files on disk and deleted after tests have run.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 Jun 2015 17:09:19 -0700
parents 4b1019bb2533
children ab5c6a8ae90a
line wrap: on
line source

import re
import os.path
import copy
from piecrust.sources.base import PageFactory


page_ref_pattern = re.compile(r'(?P<src>[\w]+)\:(?P<path>.*?)(;|$)')


class PageNotFoundError(Exception):
    pass


class PageRef(object):
    """ A reference to a page, with support for looking a page in different
        realms.
    """
    _INDEX_NEEDS_LOADING = -2
    _INDEX_NOT_FOUND = -1

    class _HitInfo(object):
        def __init__(self, source_name, rel_path, path, metadata):
            self.source_name = source_name
            self.rel_path = rel_path
            self.path = path
            self.metadata = metadata

    def __init__(self, app, page_ref):
        self.app = app
        self._page_ref = page_ref
        self._hits = None
        self._first_valid_hit_index = self._INDEX_NEEDS_LOADING
        self._exts = list(app.config.get('site/auto_formats').keys())

    @property
    def exists(self):
        try:
            self._checkHits()
            return True
        except PageNotFoundError:
            return False

    @property
    def source_name(self):
        return self._first_valid_hit.source_name

    @property
    def source(self):
        return self.app.getSource(self.source_name)

    @property
    def rel_path(self):
        return self._first_valid_hit.rel_path

    @property
    def path(self):
        return self._first_valid_hit.path

    @property
    def metadata(self):
        return self._first_valid_hit.metadata

    @property
    def possible_ref_specs(self):
        self._load()
        return ['%s:%s' % (h.source_name, h.rel_path) for h in self._hits]

    @property
    def possible_split_ref_specs(self):
        self._load()
        return [(h.source_name, h.rel_path) for h in self._hits]

    @property
    def possible_paths(self):
        self._load()
        return [h.path for h in self._hits]

    def getFactory(self):
        return PageFactory(self.source, self.rel_path,
                           copy.deepcopy(self.metadata))

    @property
    def _first_valid_hit(self):
        self._checkHits()
        return self._hits[self._first_valid_hit_index]

    def _load(self):
        if self._hits is not None:
            return

        self._hits = []

        if self._page_ref is None:
            self._first_valid_hit_index = self._INDEX_NOT_FOUND
            return

        it = list(page_ref_pattern.finditer(self._page_ref))
        if len(it) == 0:
            raise Exception("Invalid page ref: %s" % self._page_ref)

        for m in it:
            source_name = m.group('src')
            source = self.app.getSource(source_name)
            if source is None:
                raise Exception("No such source: %s" % source_name)
            rel_path = m.group('path')
            if '%ext%' in rel_path:
                for e in self._exts:
                    cur_rel_path = rel_path.replace('%ext%', e)
                    path, metadata = source.resolveRef(cur_rel_path)
                    self._hits.append(self._HitInfo(
                            source_name, cur_rel_path, path, metadata))
            else:
                path, metadata = source.resolveRef(rel_path)
                self._hits.append(
                        self._HitInfo(source_name, rel_path, path, metadata))

    def _checkHits(self):
        if self._first_valid_hit_index >= 0:
            return

        if self._first_valid_hit_index == self._INDEX_NEEDS_LOADING:
            self._load()
            self._first_valid_hit_index = self._INDEX_NOT_FOUND
            for i, hit in enumerate(self._hits):
                if os.path.isfile(hit.path):
                    self._first_valid_hit_index = i
                    break

        if self._first_valid_hit_index == self._INDEX_NOT_FOUND:
            raise PageNotFoundError(
                    "No valid paths were found for page reference: %s" %
                    self._page_ref)