diff piecrust/cache.py @ 416:ff6cc43fb40c

internal: Move `MemCache` to the `cache` module, remove threading locks.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 21:05:25 -0700
parents c2ca72fb7f0b
children 5feb71d31a4f
line wrap: on
line diff
--- a/piecrust/cache.py	Sat Jun 20 19:23:16 2015 -0700
+++ b/piecrust/cache.py	Sat Jun 20 21:05:25 2015 -0700
@@ -1,9 +1,12 @@
 import os
 import os.path
+import json
 import shutil
 import codecs
+import hashlib
 import logging
-import threading
+import collections
+import repoze.lru
 
 
 logger = logging.getLogger(__name__)
@@ -12,7 +15,6 @@
 class ExtensibleCache(object):
     def __init__(self, base_dir):
         self.base_dir = base_dir
-        self.lock = threading.Lock()
         self.caches = {}
 
     @property
@@ -22,15 +24,12 @@
     def getCache(self, name):
         c = self.caches.get(name)
         if c is None:
-            with self.lock:
-                c = self.caches.get(name)
-                if c is None:
-                    c_dir = os.path.join(self.base_dir, name)
-                    if not os.path.isdir(c_dir):
-                        os.makedirs(c_dir, 0o755)
+            c_dir = os.path.join(self.base_dir, name)
+            if not os.path.isdir(c_dir):
+                os.makedirs(c_dir, 0o755)
 
-                    c = SimpleCache(c_dir)
-                    self.caches[name] = c
+            c = SimpleCache(c_dir)
+            self.caches[name] = c
         return c
 
     def getCacheDir(self, name):
@@ -145,3 +144,70 @@
     def clearCaches(self, except_names=None):
         pass
 
+
+def _make_fs_cache_key(key):
+    return hashlib.md5(key.encode('utf8')).hexdigest()
+
+
+class MemCache(object):
+    """ Simple memory cache. It can be backed by a simple file-system
+        cache, but items need to be JSON-serializable to do this.
+    """
+    def __init__(self, size=2048):
+        self.cache = repoze.lru.LRUCache(size)
+        self.fs_cache = None
+        self._last_access_hit = None
+        self._invalidated_fs_items = set()
+
+    @property
+    def last_access_hit(self):
+        return self._last_access_hit
+
+    def invalidate(self, key):
+        logger.debug("Invalidating cache item '%s'." % key)
+        self.cache.invalidate(key)
+        if self.fs_cache:
+            logger.debug("Invalidating FS cache item '%s'." % key)
+            fs_key = _make_fs_cache_key(key)
+            self._invalidated_fs_items.add(fs_key)
+
+    def put(self, key, item, save_to_fs=True):
+        self.cache.put(key, item)
+        if self.fs_cache and save_to_fs:
+            fs_key = _make_fs_cache_key(key)
+            item_raw = json.dumps(item)
+            self.fs_cache.write(fs_key, item_raw)
+
+    def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
+        self._last_access_hit = True
+        item = self.cache.get(key)
+        if item is None:
+            if (self.fs_cache is not None and
+                    fs_cache_time is not None):
+                # Try first from the file-system cache.
+                fs_key = _make_fs_cache_key(key)
+                if (fs_key not in self._invalidated_fs_items and
+                        self.fs_cache.isValid(fs_key, fs_cache_time)):
+                    logger.debug("'%s' found in file-system cache." %
+                                 key)
+                    item_raw = self.fs_cache.read(fs_key)
+                    item = json.loads(
+                            item_raw,
+                            object_pairs_hook=collections.OrderedDict)
+                    self.cache.put(key, item)
+                    return item
+
+            # Look into the mem-cache.
+            logger.debug("'%s' not found in cache, must build." % key)
+            item = item_maker()
+            self.cache.put(key, item)
+            self._last_access_hit = False
+
+            # Save to the file-system if needed.
+            if self.fs_cache is not None and save_to_fs:
+                item_raw = json.dumps(item)
+                self.fs_cache.write(fs_key, item_raw)
+
+        return item
+
+