view piecrust/cache.py @ 550:6f216c1ab6b1

bake: Add a flag to know which record entries got collapsed from last run. This makes it possible to find entries for things that were actually baked during the current run, as opposed to skipped because they were "clean".
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 04 Aug 2015 21:22:30 -0700
parents 5feb71d31a4f
children 61d606fbc313
line wrap: on
line source

import os
import os.path
import json
import shutil
import codecs
import hashlib
import logging
import collections
import repoze.lru


logger = logging.getLogger(__name__)


class ExtensibleCache(object):
    def __init__(self, base_dir):
        self.base_dir = base_dir
        self.caches = {}

    @property
    def enabled(self):
        return True

    def getCache(self, name):
        c = self.caches.get(name)
        if c is None:
            c_dir = os.path.join(self.base_dir, name)
            if not os.path.isdir(c_dir):
                os.makedirs(c_dir, 0o755)

            c = SimpleCache(c_dir)
            self.caches[name] = c
        return c

    def getCacheDir(self, name):
        return os.path.join(self.base_dir, name)

    def getCacheNames(self, except_names=None):
        _, dirnames, __ = next(os.walk(self.base_dir))
        if except_names is None:
            return dirnames
        return [dn for dn in dirnames if dn not in except_names]

    def clearCache(self, name):
        cache_dir = self.getCacheDir(name)
        if os.path.isdir(cache_dir):
            logger.debug("Cleaning cache: %s" % cache_dir)
            shutil.rmtree(cache_dir)

            # Re-create the cache-dir because now our Cache instance points
            # to a directory that doesn't exist anymore.
            os.makedirs(cache_dir, 0o755)

    def clearCaches(self, except_names=None):
        for name in self.getCacheNames(except_names=except_names):
            self.clearCache(name)


class SimpleCache(object):
    def __init__(self, base_dir):
        self.base_dir = base_dir
        if not os.path.isdir(base_dir):
            raise Exception("Cache directory doesn't exist: %s" % base_dir)

    def isValid(self, path, time):
        cache_time = self.getCacheTime(path)
        if cache_time is None:
            return False
        if isinstance(time, list):
            for t in time:
                if cache_time < t:
                    return False
            return True
        return cache_time >= time

    def getCacheTime(self, path):
        cache_path = self.getCachePath(path)
        try:
            return os.path.getmtime(cache_path)
        except os.error:
            return None

    def has(self, path):
        cache_path = self.getCachePath(path)
        return os.path.isfile(cache_path)

    def read(self, path):
        cache_path = self.getCachePath(path)
        logger.debug("Reading cache: %s" % cache_path)
        with codecs.open(cache_path, 'r', 'utf-8') as fp:
            return fp.read()

    def write(self, path, content):
        cache_path = self.getCachePath(path)
        cache_dir = os.path.dirname(cache_path)
        if not os.path.isdir(cache_dir):
            os.makedirs(cache_dir, 0o755)
        logger.debug("Writing cache: %s" % cache_path)
        with codecs.open(cache_path, 'w', 'utf-8') as fp:
            fp.write(content)

    def getCachePath(self, path):
        if path.startswith('.'):
            path = '__index__' + path
        return os.path.join(self.base_dir, path)


class NullCache(object):
    def isValid(self, path, time):
        return False

    def getCacheTime(self, path):
        return None

    def has(self, path):
        return False

    def read(self, path):
        raise Exception("Null cache has no data.")

    def write(self, path, content):
        pass

    def getCachePath(self, path):
        raise Exception("Null cache can't make paths.")


class NullExtensibleCache(object):
    def __init__(self):
        self.null_cache = NullCache()

    @property
    def enabled(self):
        return False

    def getCache(self, name):
        return self.null_cache

    def getCacheDir(self, name):
        raise NotImplementedError()

    def getCacheNames(self, except_names=None):
        return []

    def clearCache(self, name):
        pass

    def clearCaches(self, except_names=None):
        pass


def _make_fs_cache_key(key):
    return hashlib.md5(key.encode('utf8')).hexdigest()


class MemCache(object):
    """ Simple memory cache. It can be backed by a simple file-system
        cache, but items need to be JSON-serializable to do this.
    """
    def __init__(self, size=2048):
        self.cache = repoze.lru.LRUCache(size)
        self.fs_cache = None
        self._last_access_hit = None
        self._invalidated_fs_items = set()

    @property
    def last_access_hit(self):
        return self._last_access_hit

    def invalidate(self, key):
        logger.debug("Invalidating cache item '%s'." % key)
        self.cache.invalidate(key)
        if self.fs_cache:
            logger.debug("Invalidating FS cache item '%s'." % key)
            fs_key = _make_fs_cache_key(key)
            self._invalidated_fs_items.add(fs_key)

    def put(self, key, item, save_to_fs=True):
        self.cache.put(key, item)
        if self.fs_cache and save_to_fs:
            fs_key = _make_fs_cache_key(key)
            item_raw = json.dumps(item)
            self.fs_cache.write(fs_key, item_raw)

    def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
        self._last_access_hit = True
        item = self.cache.get(key)
        if item is None:
            if (self.fs_cache is not None and
                    fs_cache_time is not None):
                # Try first from the file-system cache.
                fs_key = _make_fs_cache_key(key)
                if (fs_key not in self._invalidated_fs_items and
                        self.fs_cache.isValid(fs_key, fs_cache_time)):
                    logger.debug("'%s' found in file-system cache." %
                                 key)
                    item_raw = self.fs_cache.read(fs_key)
                    item = json.loads(
                            item_raw,
                            object_pairs_hook=collections.OrderedDict)
                    self.cache.put(key, item)
                    return item

            # Look into the mem-cache.
            logger.debug("'%s' not found in cache, must build." % key)
            item = item_maker()
            self.cache.put(key, item)
            self._last_access_hit = False

            # Save to the file-system if needed.
            if self.fs_cache is not None and save_to_fs:
                item_raw = json.dumps(item)
                self.fs_cache.write(fs_key, item_raw)

        return item