view piecrust/cache.py @ 661:2f780b191541

internal: Fix a bug with registering taxonomy terms that are not strings. Some objects, like the blog data provider's taxnonomy entries, can render as strings, but are objects themselves. When registering them as "used terms", we need to use their string representation.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 01 Mar 2016 22:26:09 -0800
parents 5feb71d31a4f
children 61d606fbc313
line wrap: on
line source

import os
import os.path
import json
import shutil
import codecs
import hashlib
import logging
import collections
import repoze.lru


logger = logging.getLogger(__name__)


class ExtensibleCache(object):
    def __init__(self, base_dir):
        self.base_dir = base_dir
        self.caches = {}

    @property
    def enabled(self):
        return True

    def getCache(self, name):
        c = self.caches.get(name)
        if c is None:
            c_dir = os.path.join(self.base_dir, name)
            if not os.path.isdir(c_dir):
                os.makedirs(c_dir, 0o755)

            c = SimpleCache(c_dir)
            self.caches[name] = c
        return c

    def getCacheDir(self, name):
        return os.path.join(self.base_dir, name)

    def getCacheNames(self, except_names=None):
        _, dirnames, __ = next(os.walk(self.base_dir))
        if except_names is None:
            return dirnames
        return [dn for dn in dirnames if dn not in except_names]

    def clearCache(self, name):
        cache_dir = self.getCacheDir(name)
        if os.path.isdir(cache_dir):
            logger.debug("Cleaning cache: %s" % cache_dir)
            shutil.rmtree(cache_dir)

            # Re-create the cache-dir because now our Cache instance points
            # to a directory that doesn't exist anymore.
            os.makedirs(cache_dir, 0o755)

    def clearCaches(self, except_names=None):
        for name in self.getCacheNames(except_names=except_names):
            self.clearCache(name)


class SimpleCache(object):
    def __init__(self, base_dir):
        self.base_dir = base_dir
        if not os.path.isdir(base_dir):
            raise Exception("Cache directory doesn't exist: %s" % base_dir)

    def isValid(self, path, time):
        cache_time = self.getCacheTime(path)
        if cache_time is None:
            return False
        if isinstance(time, list):
            for t in time:
                if cache_time < t:
                    return False
            return True
        return cache_time >= time

    def getCacheTime(self, path):
        cache_path = self.getCachePath(path)
        try:
            return os.path.getmtime(cache_path)
        except os.error:
            return None

    def has(self, path):
        cache_path = self.getCachePath(path)
        return os.path.isfile(cache_path)

    def read(self, path):
        cache_path = self.getCachePath(path)
        logger.debug("Reading cache: %s" % cache_path)
        with codecs.open(cache_path, 'r', 'utf-8') as fp:
            return fp.read()

    def write(self, path, content):
        cache_path = self.getCachePath(path)
        cache_dir = os.path.dirname(cache_path)
        if not os.path.isdir(cache_dir):
            os.makedirs(cache_dir, 0o755)
        logger.debug("Writing cache: %s" % cache_path)
        with codecs.open(cache_path, 'w', 'utf-8') as fp:
            fp.write(content)

    def getCachePath(self, path):
        if path.startswith('.'):
            path = '__index__' + path
        return os.path.join(self.base_dir, path)


class NullCache(object):
    def isValid(self, path, time):
        return False

    def getCacheTime(self, path):
        return None

    def has(self, path):
        return False

    def read(self, path):
        raise Exception("Null cache has no data.")

    def write(self, path, content):
        pass

    def getCachePath(self, path):
        raise Exception("Null cache can't make paths.")


class NullExtensibleCache(object):
    def __init__(self):
        self.null_cache = NullCache()

    @property
    def enabled(self):
        return False

    def getCache(self, name):
        return self.null_cache

    def getCacheDir(self, name):
        raise NotImplementedError()

    def getCacheNames(self, except_names=None):
        return []

    def clearCache(self, name):
        pass

    def clearCaches(self, except_names=None):
        pass


def _make_fs_cache_key(key):
    return hashlib.md5(key.encode('utf8')).hexdigest()


class MemCache(object):
    """ Simple memory cache. It can be backed by a simple file-system
        cache, but items need to be JSON-serializable to do this.
    """
    def __init__(self, size=2048):
        self.cache = repoze.lru.LRUCache(size)
        self.fs_cache = None
        self._last_access_hit = None
        self._invalidated_fs_items = set()

    @property
    def last_access_hit(self):
        return self._last_access_hit

    def invalidate(self, key):
        logger.debug("Invalidating cache item '%s'." % key)
        self.cache.invalidate(key)
        if self.fs_cache:
            logger.debug("Invalidating FS cache item '%s'." % key)
            fs_key = _make_fs_cache_key(key)
            self._invalidated_fs_items.add(fs_key)

    def put(self, key, item, save_to_fs=True):
        self.cache.put(key, item)
        if self.fs_cache and save_to_fs:
            fs_key = _make_fs_cache_key(key)
            item_raw = json.dumps(item)
            self.fs_cache.write(fs_key, item_raw)

    def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
        self._last_access_hit = True
        item = self.cache.get(key)
        if item is None:
            if (self.fs_cache is not None and
                    fs_cache_time is not None):
                # Try first from the file-system cache.
                fs_key = _make_fs_cache_key(key)
                if (fs_key not in self._invalidated_fs_items and
                        self.fs_cache.isValid(fs_key, fs_cache_time)):
                    logger.debug("'%s' found in file-system cache." %
                                 key)
                    item_raw = self.fs_cache.read(fs_key)
                    item = json.loads(
                            item_raw,
                            object_pairs_hook=collections.OrderedDict)
                    self.cache.put(key, item)
                    return item

            # Look into the mem-cache.
            logger.debug("'%s' not found in cache, must build." % key)
            item = item_maker()
            self.cache.put(key, item)
            self._last_access_hit = False

            # Save to the file-system if needed.
            if self.fs_cache is not None and save_to_fs:
                item_raw = json.dumps(item)
                self.fs_cache.write(fs_key, item_raw)

        return item