Mercurial > piecrust2

comparison piecrust/cache.py @ 416:ff6cc43fb40c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
internal: Move `MemCache` to the `cache` module, remove threading locks.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 21:05:25 -0700
parents c2ca72fb7f0b
children 5feb71d31a4f
comparison
equal deleted inserted replaced
415:0e9a94b7fdfa 416:ff6cc43fb40c
1 import os 1 import os
2 import os.path 2 import os.path
3 import json
3 import shutil 4 import shutil
4 import codecs 5 import codecs
6 import hashlib
5 import logging 7 import logging
6 import threading 8 import collections
9 import repoze.lru
7 10
8 11
9 logger = logging.getLogger(__name__) 12 logger = logging.getLogger(__name__)
10 13
11 14
12 class ExtensibleCache(object): 15 class ExtensibleCache(object):
13 def __init__(self, base_dir): 16 def __init__(self, base_dir):
14 self.base_dir = base_dir 17 self.base_dir = base_dir
15 self.lock = threading.Lock()
16 self.caches = {} 18 self.caches = {}
17 19
18 @property 20 @property
19 def enabled(self): 21 def enabled(self):
20 return True 22 return True
21 23
22 def getCache(self, name): 24 def getCache(self, name):
23 c = self.caches.get(name) 25 c = self.caches.get(name)
24 if c is None: 26 if c is None:
25 with self.lock: 27 c_dir = os.path.join(self.base_dir, name)
26 c = self.caches.get(name) 28 if not os.path.isdir(c_dir):
27 if c is None: 29 os.makedirs(c_dir, 0o755)
28 c_dir = os.path.join(self.base_dir, name) 30
29 if not os.path.isdir(c_dir): 31 c = SimpleCache(c_dir)
30 os.makedirs(c_dir, 0o755) 32 self.caches[name] = c
31
32 c = SimpleCache(c_dir)
33 self.caches[name] = c
34 return c 33 return c
35 34
36 def getCacheDir(self, name): 35 def getCacheDir(self, name):
37 return os.path.join(self.base_dir, name) 36 return os.path.join(self.base_dir, name)
38 37
143 pass 142 pass
144 143
145 def clearCaches(self, except_names=None): 144 def clearCaches(self, except_names=None):
146 pass 145 pass
147 146
147
148 def _make_fs_cache_key(key):
149 return hashlib.md5(key.encode('utf8')).hexdigest()
150
151
152 class MemCache(object):
153 """ Simple memory cache. It can be backed by a simple file-system
154 cache, but items need to be JSON-serializable to do this.
155 """
156 def __init__(self, size=2048):
157 self.cache = repoze.lru.LRUCache(size)
158 self.fs_cache = None
159 self._last_access_hit = None
160 self._invalidated_fs_items = set()
161
162 @property
163 def last_access_hit(self):
164 return self._last_access_hit
165
166 def invalidate(self, key):
167 logger.debug("Invalidating cache item '%s'." % key)
168 self.cache.invalidate(key)
169 if self.fs_cache:
170 logger.debug("Invalidating FS cache item '%s'." % key)
171 fs_key = _make_fs_cache_key(key)
172 self._invalidated_fs_items.add(fs_key)
173
174 def put(self, key, item, save_to_fs=True):
175 self.cache.put(key, item)
176 if self.fs_cache and save_to_fs:
177 fs_key = _make_fs_cache_key(key)
178 item_raw = json.dumps(item)
179 self.fs_cache.write(fs_key, item_raw)
180
181 def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
182 self._last_access_hit = True
183 item = self.cache.get(key)
184 if item is None:
185 if (self.fs_cache is not None and
186 fs_cache_time is not None):
187 # Try first from the file-system cache.
188 fs_key = _make_fs_cache_key(key)
189 if (fs_key not in self._invalidated_fs_items and
190 self.fs_cache.isValid(fs_key, fs_cache_time)):
191 logger.debug("'%s' found in file-system cache." %
192 key)
193 item_raw = self.fs_cache.read(fs_key)
194 item = json.loads(
195 item_raw,
196 object_pairs_hook=collections.OrderedDict)
197 self.cache.put(key, item)
198 return item
199
200 # Look into the mem-cache.
201 logger.debug("'%s' not found in cache, must build." % key)
202 item = item_maker()
203 self.cache.put(key, item)
204 self._last_access_hit = False
205
206 # Save to the file-system if needed.
207 if self.fs_cache is not None and save_to_fs:
208 item_raw = json.dumps(item)
209 self.fs_cache.write(fs_key, item_raw)
210
211 return item
212
213