Mercurial > piecrust2
diff piecrust/cache.py @ 416:ff6cc43fb40c
internal: Move `MemCache` to the `cache` module, remove threading locks.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 20 Jun 2015 21:05:25 -0700 |
parents | c2ca72fb7f0b |
children | 5feb71d31a4f |
line wrap: on
line diff
--- a/piecrust/cache.py Sat Jun 20 19:23:16 2015 -0700 +++ b/piecrust/cache.py Sat Jun 20 21:05:25 2015 -0700 @@ -1,9 +1,12 @@ import os import os.path +import json import shutil import codecs +import hashlib import logging -import threading +import collections +import repoze.lru logger = logging.getLogger(__name__) @@ -12,7 +15,6 @@ class ExtensibleCache(object): def __init__(self, base_dir): self.base_dir = base_dir - self.lock = threading.Lock() self.caches = {} @property @@ -22,15 +24,12 @@ def getCache(self, name): c = self.caches.get(name) if c is None: - with self.lock: - c = self.caches.get(name) - if c is None: - c_dir = os.path.join(self.base_dir, name) - if not os.path.isdir(c_dir): - os.makedirs(c_dir, 0o755) + c_dir = os.path.join(self.base_dir, name) + if not os.path.isdir(c_dir): + os.makedirs(c_dir, 0o755) - c = SimpleCache(c_dir) - self.caches[name] = c + c = SimpleCache(c_dir) + self.caches[name] = c return c def getCacheDir(self, name): @@ -145,3 +144,70 @@ def clearCaches(self, except_names=None): pass + +def _make_fs_cache_key(key): + return hashlib.md5(key.encode('utf8')).hexdigest() + + +class MemCache(object): + """ Simple memory cache. It can be backed by a simple file-system + cache, but items need to be JSON-serializable to do this. + """ + def __init__(self, size=2048): + self.cache = repoze.lru.LRUCache(size) + self.fs_cache = None + self._last_access_hit = None + self._invalidated_fs_items = set() + + @property + def last_access_hit(self): + return self._last_access_hit + + def invalidate(self, key): + logger.debug("Invalidating cache item '%s'." % key) + self.cache.invalidate(key) + if self.fs_cache: + logger.debug("Invalidating FS cache item '%s'." % key) + fs_key = _make_fs_cache_key(key) + self._invalidated_fs_items.add(fs_key) + + def put(self, key, item, save_to_fs=True): + self.cache.put(key, item) + if self.fs_cache and save_to_fs: + fs_key = _make_fs_cache_key(key) + item_raw = json.dumps(item) + self.fs_cache.write(fs_key, item_raw) + + def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True): + self._last_access_hit = True + item = self.cache.get(key) + if item is None: + if (self.fs_cache is not None and + fs_cache_time is not None): + # Try first from the file-system cache. + fs_key = _make_fs_cache_key(key) + if (fs_key not in self._invalidated_fs_items and + self.fs_cache.isValid(fs_key, fs_cache_time)): + logger.debug("'%s' found in file-system cache." % + key) + item_raw = self.fs_cache.read(fs_key) + item = json.loads( + item_raw, + object_pairs_hook=collections.OrderedDict) + self.cache.put(key, item) + return item + + # Look into the mem-cache. + logger.debug("'%s' not found in cache, must build." % key) + item = item_maker() + self.cache.put(key, item) + self._last_access_hit = False + + # Save to the file-system if needed. + if self.fs_cache is not None and save_to_fs: + item_raw = json.dumps(item) + self.fs_cache.write(fs_key, item_raw) + + return item + +