Mercurial > piecrust2

comparison piecrust/cache.py @ 911:f2b75e4be981

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
internal: Use pickle for caching things on disk. This is just easier and lets us use proper classes instead of converting to/from dictionaries.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 23 Jul 2017 18:03:21 -0700
parents d1095774bfcf
children cd0345e4001e
comparison
equal deleted inserted replaced
910:371731b555ec 911:f2b75e4be981
1 import os 1 import os
2 import os.path 2 import os.path
3 import json
4 import shutil 3 import shutil
5 import codecs 4 import pickle
6 import hashlib 5 import hashlib
7 import logging 6 import logging
8 import repoze.lru 7 import repoze.lru
9 8
10 9
82 def has(self, path): 81 def has(self, path):
83 cache_path = self.getCachePath(path) 82 cache_path = self.getCachePath(path)
84 return os.path.isfile(cache_path) 83 return os.path.isfile(cache_path)
85 84
86 def read(self, path): 85 def read(self, path):
87 cache_path = self.getCachePath(path) 86 with self.openRead(path, mode='r', encoding='utf8') as fp:
88 logger.debug("Reading cache: %s" % cache_path)
89 with codecs.open(cache_path, 'r', 'utf-8') as fp:
90 return fp.read() 87 return fp.read()
91 88
89 def openRead(self, path, mode='r', encoding=None):
90 cache_path = self.getCachePath(path)
91 return open(cache_path, mode=mode, encoding=encoding)
92
92 def write(self, path, content): 93 def write(self, path, content):
94 with self.openWrite(path, mode='w', encoding='utf8') as fp:
95 fp.write(content)
96
97 def openWrite(self, path, mode='w', encoding=None):
93 cache_path = self.getCachePath(path) 98 cache_path = self.getCachePath(path)
94 cache_dir = os.path.dirname(cache_path) 99 cache_dir = os.path.dirname(cache_path)
95 if not os.path.isdir(cache_dir): 100 if not os.path.isdir(cache_dir):
96 os.makedirs(cache_dir, 0o755) 101 os.makedirs(cache_dir, 0o755)
97 logger.debug("Writing cache: %s" % cache_path) 102 return open(cache_path, mode=mode, encoding=encoding)
98 with codecs.open(cache_path, 'w', 'utf-8') as fp:
99 fp.write(content)
100 103
101 def getCachePath(self, path): 104 def getCachePath(self, path):
102 if path.startswith('.'): 105 if path.startswith('.'):
103 path = '__index__' + path 106 path = '__index__' + path
104 return os.path.join(self.base_dir, path) 107 return os.path.join(self.base_dir, path)
152 return hashlib.md5(key.encode('utf8')).hexdigest() 155 return hashlib.md5(key.encode('utf8')).hexdigest()
153 156
154 157
155 class MemCache(object): 158 class MemCache(object):
156 """ Simple memory cache. It can be backed by a simple file-system 159 """ Simple memory cache. It can be backed by a simple file-system
157 cache, but items need to be JSON-serializable to do this. 160 cache, but items need to be pickle-able to do this.
158 """ 161 """
159 def __init__(self, size=2048): 162 def __init__(self, size=2048):
160 self.cache = repoze.lru.LRUCache(size) 163 self.cache = repoze.lru.LRUCache(size)
161 self.fs_cache = None 164 self.fs_cache = None
162 self._last_access_hit = None 165 self._last_access_hit = None
179 182
180 def put(self, key, item, save_to_fs=True): 183 def put(self, key, item, save_to_fs=True):
181 self.cache.put(key, item) 184 self.cache.put(key, item)
182 if self.fs_cache and save_to_fs: 185 if self.fs_cache and save_to_fs:
183 fs_key = _make_fs_cache_key(key) 186 fs_key = _make_fs_cache_key(key)
184 item_raw = json.dumps(item) 187 with self.fs_cache.openWrite(fs_key, mode='wb') as fp:
185 self.fs_cache.write(fs_key, item_raw) 188 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL)
186 189
187 def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True): 190 def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
188 self._last_access_hit = True 191 self._last_access_hit = True
189 item = self.cache.get(key) 192 item = self.cache.get(key)
190 if item is not None: 193 if item is not None:
199 202
200 # Try first from the file-system cache. 203 # Try first from the file-system cache.
201 fs_key = _make_fs_cache_key(key) 204 fs_key = _make_fs_cache_key(key)
202 if (fs_key not in self._invalidated_fs_items and 205 if (fs_key not in self._invalidated_fs_items and
203 self.fs_cache.isValid(fs_key, fs_cache_time)): 206 self.fs_cache.isValid(fs_key, fs_cache_time)):
204 logger.debug("'%s' found in file-system cache." % 207 logger.debug("'%s' found in file-system cache." % key)
205 key) 208 with self.fs_cache.openRead(fs_key, mode='rb') as fp:
206 item_raw = self.fs_cache.read(fs_key) 209 item = pickle.load(fp)
207 item = json.loads(item_raw)
208 self.cache.put(key, item) 210 self.cache.put(key, item)
209 self._hits += 1 211 self._hits += 1
210 return item 212 return item
211 213
212 # Look into the mem-cache. 214 # Look into the mem-cache.
217 self._misses += 1 219 self._misses += 1
218 self._missed_keys.append(key) 220 self._missed_keys.append(key)
219 221
220 # Save to the file-system if needed. 222 # Save to the file-system if needed.
221 if self.fs_cache is not None and save_to_fs: 223 if self.fs_cache is not None and save_to_fs:
222 item_raw = json.dumps(item) 224 with self.fs_cache.openWrite(fs_key, mode='wb') as fp:
223 self.fs_cache.write(fs_key, item_raw) 225 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL)
224 226
225 return item 227 return item
226 228
227