view piecrust/data/builder.py @ 411:e7b865f8f335

bake: Enable multiprocess baking. Baking is now done by running a worker per CPU, and sending jobs to them. This changes several things across the codebase: * Ability to not cache things related to pages other than the 'main' page (i.e. the page at the bottom of the execution stack). * Decouple the baking process from the bake records, so only the main process keeps track (and modifies) the bake record. * Remove the need for 'batch page getters' and loading a page directly from the page factories. There are various smaller changes too included here, including support for scope performance timers that are saved with the bake record and can be printed out to the console. Yes I got carried away. For testing, the in-memory 'mock' file-system doesn't work anymore, since we're spawning processes, so this is replaced by a 'tmpfs' file-system which is saved in temporary files on disk and deleted after tests have run.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 Jun 2015 17:09:19 -0700
parents 9caaf78a17db
children f1b759c188b0
line wrap: on
line source

import re
import time
import copy
import logging
from werkzeug.utils import cached_property
from piecrust import APP_VERSION
from piecrust.configuration import merge_dicts
from piecrust.data.assetor import Assetor
from piecrust.data.debug import build_debug_info
from piecrust.data.linker import PageLinkerData
from piecrust.data.paginator import Paginator
from piecrust.uriutil import split_sub_uri


logger = logging.getLogger(__name__)


class DataBuildingContext(object):
    def __init__(self, qualified_page, page_num=1):
        self.page = qualified_page
        self.page_num = page_num
        self.pagination_source = None
        self.pagination_filter = None

    @property
    def app(self):
        return self.page.app

    @cached_property
    def uri(self):
        return self.page.getUri(self.page_num)


def build_page_data(ctx):
    app = ctx.app
    page = ctx.page
    first_uri, _ = split_sub_uri(app, ctx.uri)

    pc_data = PieCrustData()
    pgn_source = ctx.pagination_source or get_default_pagination_source(page)
    paginator = Paginator(page, pgn_source,
                          page_num=ctx.page_num,
                          pgn_filter=ctx.pagination_filter)
    assetor = Assetor(page, first_uri)
    linker = PageLinkerData(page.source, page.rel_path)
    data = {
            'piecrust': pc_data,
            'page': {},
            'assets': assetor,
            'pagination': paginator,
            'family': linker
            }
    page_data = data['page']
    page_data.update(copy.deepcopy(page.source_metadata))
    page_data.update(page.config.getDeepcopy(app.debug))
    page_data['url'] = ctx.uri
    page_data['timestamp'] = time.mktime(page.datetime.timetuple())
    date_format = app.config.get('site/date_format')
    if date_format:
        page_data['date'] = page.datetime.strftime(date_format)

    #TODO: handle slugified taxonomy terms.

    site_data = build_site_data(page)
    merge_dicts(data, site_data)

    # Do this at the end because we want all the data to be ready to be
    # displayed in the debugger window.
    if (app.config.get('site/show_debug_info') and
            not app.config.get('baker/is_baking')):
        pc_data._enableDebugInfo(page, data)

    return data


def build_layout_data(page, page_data, contents):
    for name, txt in contents.items():
        if name in page_data:
            logger.warning("Content segment '%s' will hide existing data." %
                           name)
        page_data[name] = txt


class PieCrustData(object):
    debug_render = ['version', 'url', 'branding', 'debug_info']
    debug_render_invoke = ['version', 'url', 'branding', 'debug_info']
    debug_render_redirect = {'debug_info': '_debugRenderDebugInfo'}

    def __init__(self):
        self.version = APP_VERSION
        self.url = 'http://bolt80.com/piecrust/'
        self.branding = 'Baked with <em><a href="%s">PieCrust</a> %s</em>.' % (
                'http://bolt80.com/piecrust/', APP_VERSION)
        self._page = None
        self._data = None

    @property
    def debug_info(self):
        if self._page is not None and self._data is not None:
            try:
                return build_debug_info(self._page, self._data)
            except Exception as ex:
                logger.exception(ex)
                return ('An error occured while generating debug info. '
                        'Please check the logs.')
        return ''

    def _enableDebugInfo(self, page, data):
        self._page = page
        self._data = data

    def _debugRenderDebugInfo(self):
        return "The very thing you're looking at!"


re_endpoint_sep = re.compile(r'[\/\.]')


def build_site_data(page):
    app = page.app
    data = app.config.getDeepcopy(app.debug)
    for source in app.sources:
        endpoint_bits = re_endpoint_sep.split(source.data_endpoint)
        endpoint = data
        for e in endpoint_bits[:-1]:
            if e not in endpoint:
                endpoint[e] = {}
            endpoint = endpoint[e]
        user_data = endpoint.get(endpoint_bits[-1])
        provider = source.buildDataProvider(page, user_data)
        endpoint[endpoint_bits[-1]] = provider
    return data


def get_default_pagination_source(page):
    app = page.app
    source_name = page.config.get('source') or page.config.get('blog')
    if source_name is None:
        source_name = app.config.get('site/default_pagination_source')
    if source_name is None:
        blog_names = app.config.get('site/blogs')
        if blog_names is not None:
            source_name = blog_names[0]
        else:
            source_name = app.sources[0].name
    source = app.getSource(source_name)
    return source