view piecrust/environment.py @ 415:0e9a94b7fdfa

bake: Improve bake record information. * Store things in the bake record that require less interaction between the master process and the workers. For instance, don't store the paginator object in the render pass info -- instead, just store whether pagination was used, and whether it had more items. * Simplify information passing between workers and bake passes by saving the rendering info to the JSON cache. This means the "render first sub" job doesn't have to return anything except errors now. * Add more performance counter info.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:23:16 -0700
parents a1567766c83c
children ff6cc43fb40c
line wrap: on
line source

import time
import json
import logging
import hashlib
import contextlib
import collections
import repoze.lru


logger = logging.getLogger(__name__)


def _make_fs_cache_key(key):
    return hashlib.md5(key.encode('utf8')).hexdigest()


class MemCache(object):
    """ Simple memory cache. It can be backed by a simple file-system
        cache, but items need to be JSON-serializable to do this.
    """
    def __init__(self, size=2048):
        self.cache = repoze.lru.LRUCache(size)
        self.fs_cache = None
        self._last_access_hit = None
        self._invalidated_fs_items = set()

    @property
    def last_access_hit(self):
        return self._last_access_hit

    def invalidate(self, key):
        logger.debug("Invalidating cache item '%s'." % key)
        self.cache.invalidate(key)
        if self.fs_cache:
            logger.debug("Invalidating FS cache item '%s'." % key)
            fs_key = _make_fs_cache_key(key)
            self._invalidated_fs_items.add(fs_key)

    def put(self, key, item, save_to_fs=True):
        self.cache.put(key, item)
        if self.fs_cache and save_to_fs:
            fs_key = _make_fs_cache_key(key)
            item_raw = json.dumps(item)
            self.fs_cache.write(fs_key, item_raw)

    def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
        self._last_access_hit = True
        item = self.cache.get(key)
        if item is None:
            if (self.fs_cache is not None and
                    fs_cache_time is not None):
                # Try first from the file-system cache.
                fs_key = _make_fs_cache_key(key)
                if (fs_key not in self._invalidated_fs_items and
                        self.fs_cache.isValid(fs_key, fs_cache_time)):
                    logger.debug("'%s' found in file-system cache." %
                                 key)
                    item_raw = self.fs_cache.read(fs_key)
                    item = json.loads(
                            item_raw,
                            object_pairs_hook=collections.OrderedDict)
                    self.cache.put(key, item)
                    return item

            # Look into the mem-cache.
            logger.debug("'%s' not found in cache, must build." % key)
            item = item_maker()
            self.cache.put(key, item)
            self._last_access_hit = False

            # Save to the file-system if needed.
            if self.fs_cache is not None and save_to_fs:
                item_raw = json.dumps(item)
                self.fs_cache.write(fs_key, item_raw)

        return item


class ExecutionInfo(object):
    def __init__(self, page, render_ctx):
        self.page = page
        self.render_ctx = render_ctx
        self.was_cache_valid = False
        self.start_time = time.clock()


class ExecutionInfoStack(object):
    def __init__(self):
        self._page_stack = []

    @property
    def current_page_info(self):
        if len(self._page_stack) == 0:
            return None
        return self._page_stack[-1]

    @property
    def is_main_page(self):
        return len(self._page_stack) == 1

    def hasPage(self, page):
        for ei in self._page_stack:
            if ei.page == page:
                return True
        return False

    def pushPage(self, page, render_ctx):
        if len(self._page_stack) > 0:
            top = self._page_stack[-1]
            assert top.page is not page
        self._page_stack.append(ExecutionInfo(page, render_ctx))

    def popPage(self):
        del self._page_stack[-1]

    def clear(self):
        self._page_stack = []


class Environment(object):
    def __init__(self):
        self.start_time = None
        self.exec_info_stack = ExecutionInfoStack()
        self.was_cache_cleaned = False
        self.base_asset_url_format = '%uri%'
        self.page_repository = MemCache()
        self.rendered_segments_repository = MemCache()
        self.fs_caches = {
                'renders': self.rendered_segments_repository}
        self.fs_cache_only_for_main_page = False
        self._timers = {}

    def initialize(self, app):
        self.start_time = time.perf_counter()
        self.exec_info_stack.clear()
        self.was_cache_cleaned = False
        self.base_asset_url_format = '%uri%'
        self._onSubCacheDirChanged(app)

    def registerTimer(self, category, *, raise_if_registered=True):
        if raise_if_registered and category in self._timers:
            raise Exception("Timer '%s' has already been registered." %
                            category)
        self._timers[category] = 0

    @contextlib.contextmanager
    def timerScope(self, category):
        start = time.perf_counter()
        yield
        self._timers[category] += time.perf_counter() - start

    def stepTimer(self, category, value):
        self._timers[category] += value

    def _onSubCacheDirChanged(self, app):
        for name, repo in self.fs_caches.items():
            cache = app.cache.getCache(name)
            repo.fs_cache = cache


class StandardEnvironment(Environment):
    def __init__(self):
        super(StandardEnvironment, self).__init__()