Mercurial > piecrust2
changeset 416:ff6cc43fb40c
internal: Move `MemCache` to the `cache` module, remove threading locks.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Sat, 20 Jun 2015 21:05:25 -0700 |
| parents | 0e9a94b7fdfa |
| children | eef887cec776 |
| files | piecrust/cache.py piecrust/environment.py |
| diffstat | 2 files changed, 78 insertions(+), 81 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/cache.py Sat Jun 20 19:23:16 2015 -0700 +++ b/piecrust/cache.py Sat Jun 20 21:05:25 2015 -0700 @@ -1,9 +1,12 @@ import os import os.path +import json import shutil import codecs +import hashlib import logging -import threading +import collections +import repoze.lru logger = logging.getLogger(__name__) @@ -12,7 +15,6 @@ class ExtensibleCache(object): def __init__(self, base_dir): self.base_dir = base_dir - self.lock = threading.Lock() self.caches = {} @property @@ -22,15 +24,12 @@ def getCache(self, name): c = self.caches.get(name) if c is None: - with self.lock: - c = self.caches.get(name) - if c is None: - c_dir = os.path.join(self.base_dir, name) - if not os.path.isdir(c_dir): - os.makedirs(c_dir, 0o755) + c_dir = os.path.join(self.base_dir, name) + if not os.path.isdir(c_dir): + os.makedirs(c_dir, 0o755) - c = SimpleCache(c_dir) - self.caches[name] = c + c = SimpleCache(c_dir) + self.caches[name] = c return c def getCacheDir(self, name): @@ -145,3 +144,70 @@ def clearCaches(self, except_names=None): pass + +def _make_fs_cache_key(key): + return hashlib.md5(key.encode('utf8')).hexdigest() + + +class MemCache(object): + """ Simple memory cache. It can be backed by a simple file-system + cache, but items need to be JSON-serializable to do this. + """ + def __init__(self, size=2048): + self.cache = repoze.lru.LRUCache(size) + self.fs_cache = None + self._last_access_hit = None + self._invalidated_fs_items = set() + + @property + def last_access_hit(self): + return self._last_access_hit + + def invalidate(self, key): + logger.debug("Invalidating cache item '%s'." % key) + self.cache.invalidate(key) + if self.fs_cache: + logger.debug("Invalidating FS cache item '%s'." % key) + fs_key = _make_fs_cache_key(key) + self._invalidated_fs_items.add(fs_key) + + def put(self, key, item, save_to_fs=True): + self.cache.put(key, item) + if self.fs_cache and save_to_fs: + fs_key = _make_fs_cache_key(key) + item_raw = json.dumps(item) + self.fs_cache.write(fs_key, item_raw) + + def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True): + self._last_access_hit = True + item = self.cache.get(key) + if item is None: + if (self.fs_cache is not None and + fs_cache_time is not None): + # Try first from the file-system cache. + fs_key = _make_fs_cache_key(key) + if (fs_key not in self._invalidated_fs_items and + self.fs_cache.isValid(fs_key, fs_cache_time)): + logger.debug("'%s' found in file-system cache." % + key) + item_raw = self.fs_cache.read(fs_key) + item = json.loads( + item_raw, + object_pairs_hook=collections.OrderedDict) + self.cache.put(key, item) + return item + + # Look into the mem-cache. + logger.debug("'%s' not found in cache, must build." % key) + item = item_maker() + self.cache.put(key, item) + self._last_access_hit = False + + # Save to the file-system if needed. + if self.fs_cache is not None and save_to_fs: + item_raw = json.dumps(item) + self.fs_cache.write(fs_key, item_raw) + + return item + +
--- a/piecrust/environment.py Sat Jun 20 19:23:16 2015 -0700 +++ b/piecrust/environment.py Sat Jun 20 21:05:25 2015 -0700 @@ -1,87 +1,18 @@ import time -import json import logging -import hashlib import contextlib -import collections -import repoze.lru +from piecrust.cache import MemCache logger = logging.getLogger(__name__) -def _make_fs_cache_key(key): - return hashlib.md5(key.encode('utf8')).hexdigest() - - -class MemCache(object): - """ Simple memory cache. It can be backed by a simple file-system - cache, but items need to be JSON-serializable to do this. - """ - def __init__(self, size=2048): - self.cache = repoze.lru.LRUCache(size) - self.fs_cache = None - self._last_access_hit = None - self._invalidated_fs_items = set() - - @property - def last_access_hit(self): - return self._last_access_hit - - def invalidate(self, key): - logger.debug("Invalidating cache item '%s'." % key) - self.cache.invalidate(key) - if self.fs_cache: - logger.debug("Invalidating FS cache item '%s'." % key) - fs_key = _make_fs_cache_key(key) - self._invalidated_fs_items.add(fs_key) - - def put(self, key, item, save_to_fs=True): - self.cache.put(key, item) - if self.fs_cache and save_to_fs: - fs_key = _make_fs_cache_key(key) - item_raw = json.dumps(item) - self.fs_cache.write(fs_key, item_raw) - - def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True): - self._last_access_hit = True - item = self.cache.get(key) - if item is None: - if (self.fs_cache is not None and - fs_cache_time is not None): - # Try first from the file-system cache. - fs_key = _make_fs_cache_key(key) - if (fs_key not in self._invalidated_fs_items and - self.fs_cache.isValid(fs_key, fs_cache_time)): - logger.debug("'%s' found in file-system cache." % - key) - item_raw = self.fs_cache.read(fs_key) - item = json.loads( - item_raw, - object_pairs_hook=collections.OrderedDict) - self.cache.put(key, item) - return item - - # Look into the mem-cache. - logger.debug("'%s' not found in cache, must build." % key) - item = item_maker() - self.cache.put(key, item) - self._last_access_hit = False - - # Save to the file-system if needed. - if self.fs_cache is not None and save_to_fs: - item_raw = json.dumps(item) - self.fs_cache.write(fs_key, item_raw) - - return item - - class ExecutionInfo(object): def __init__(self, page, render_ctx): self.page = page self.render_ctx = render_ctx self.was_cache_valid = False - self.start_time = time.clock() + self.start_time = time.perf_counter() class ExecutionInfoStack(object):