Mercurial > piecrust2
view piecrust/cache.py @ 876:d1095774bfcf
refactor: Fix some issues with record/cache entry collisions, add counters.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Thu, 15 Jun 2017 07:33:40 -0700 |
parents | 4850f8c21b6e |
children | f2b75e4be981 |
line wrap: on
line source
import os import os.path import json import shutil import codecs import hashlib import logging import repoze.lru logger = logging.getLogger(__name__) class ExtensibleCache(object): def __init__(self, base_dir): self.base_dir = base_dir self.caches = {} @property def enabled(self): return True def getCache(self, name): c = self.caches.get(name) if c is None: c_dir = os.path.join(self.base_dir, name) if not os.path.isdir(c_dir): os.makedirs(c_dir, 0o755) c = SimpleCache(c_dir) self.caches[name] = c return c def getCacheDir(self, name): return os.path.join(self.base_dir, name) def getCacheNames(self, except_names=None): _, dirnames, __ = next(os.walk(self.base_dir)) if except_names is None: return dirnames return [dn for dn in dirnames if dn not in except_names] def clearCache(self, name): cache_dir = self.getCacheDir(name) if os.path.isdir(cache_dir): logger.debug("Cleaning cache: %s" % cache_dir) shutil.rmtree(cache_dir) # Re-create the cache-dir because now our Cache instance points # to a directory that doesn't exist anymore. os.makedirs(cache_dir, 0o755) def clearCaches(self, except_names=None): for name in self.getCacheNames(except_names=except_names): self.clearCache(name) class SimpleCache(object): def __init__(self, base_dir): self.base_dir = base_dir if not os.path.isdir(base_dir): raise Exception("Cache directory doesn't exist: %s" % base_dir) def isValid(self, path, time): cache_time = self.getCacheTime(path) if cache_time is None: return False if isinstance(time, list): for t in time: if cache_time < t: return False return True return cache_time >= time def getCacheTime(self, path): cache_path = self.getCachePath(path) try: return os.path.getmtime(cache_path) except os.error: return None def has(self, path): cache_path = self.getCachePath(path) return os.path.isfile(cache_path) def read(self, path): cache_path = self.getCachePath(path) logger.debug("Reading cache: %s" % cache_path) with codecs.open(cache_path, 'r', 'utf-8') as fp: return fp.read() def write(self, path, content): cache_path = self.getCachePath(path) cache_dir = os.path.dirname(cache_path) if not os.path.isdir(cache_dir): os.makedirs(cache_dir, 0o755) logger.debug("Writing cache: %s" % cache_path) with codecs.open(cache_path, 'w', 'utf-8') as fp: fp.write(content) def getCachePath(self, path): if path.startswith('.'): path = '__index__' + path return os.path.join(self.base_dir, path) class NullCache(object): def isValid(self, path, time): return False def getCacheTime(self, path): return None def has(self, path): return False def read(self, path): raise Exception("Null cache has no data.") def write(self, path, content): pass def getCachePath(self, path): raise Exception("Null cache can't make paths.") class NullExtensibleCache(object): def __init__(self): self.null_cache = NullCache() @property def enabled(self): return False def getCache(self, name): return self.null_cache def getCacheDir(self, name): raise NotImplementedError() def getCacheNames(self, except_names=None): return [] def clearCache(self, name): pass def clearCaches(self, except_names=None): pass def _make_fs_cache_key(key): return hashlib.md5(key.encode('utf8')).hexdigest() class MemCache(object): """ Simple memory cache. It can be backed by a simple file-system cache, but items need to be JSON-serializable to do this. """ def __init__(self, size=2048): self.cache = repoze.lru.LRUCache(size) self.fs_cache = None self._last_access_hit = None self._invalidated_fs_items = set() self._missed_keys = [] self._misses = 0 self._hits = 0 @property def last_access_hit(self): return self._last_access_hit def invalidate(self, key): logger.debug("Invalidating cache item '%s'." % key) self.cache.invalidate(key) if self.fs_cache: logger.debug("Invalidating FS cache item '%s'." % key) fs_key = _make_fs_cache_key(key) self._invalidated_fs_items.add(fs_key) def put(self, key, item, save_to_fs=True): self.cache.put(key, item) if self.fs_cache and save_to_fs: fs_key = _make_fs_cache_key(key) item_raw = json.dumps(item) self.fs_cache.write(fs_key, item_raw) def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True): self._last_access_hit = True item = self.cache.get(key) if item is not None: self._hits += 1 return item if self.fs_cache is not None: if fs_cache_time is None: raise ValueError( "No file-system cache time was given for '%s'. " "This would result in degraded performance." % key) # Try first from the file-system cache. fs_key = _make_fs_cache_key(key) if (fs_key not in self._invalidated_fs_items and self.fs_cache.isValid(fs_key, fs_cache_time)): logger.debug("'%s' found in file-system cache." % key) item_raw = self.fs_cache.read(fs_key) item = json.loads(item_raw) self.cache.put(key, item) self._hits += 1 return item # Look into the mem-cache. logger.debug("'%s' not found in cache, must build." % key) item = item_maker() self.cache.put(key, item) self._last_access_hit = False self._misses += 1 self._missed_keys.append(key) # Save to the file-system if needed. if self.fs_cache is not None and save_to_fs: item_raw = json.dumps(item) self.fs_cache.write(fs_key, item_raw) return item