Mercurial > piecrust2

annotate piecrust/cache.py @ 1145:e94737572542

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
serve: Fix an issue where false positive matches were rendered as the requested page. Now we try to render the page, but also try to detect for the most common "empty" pages.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 05 Jun 2018 22:08:51 -0700
parents cd0345e4001e
children a7c43131d871
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
371
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
3 import shutil
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
4 import pickle
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
5 import hashlib
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
6 import logging
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
7 import repoze.lru
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
8
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
9
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
10 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
11
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
12
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
13 class ExtensibleCache(object):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
14 def __init__(self, base_dir):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
15 self.base_dir = base_dir
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
16 self.caches = {}
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
17
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
18 @property
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
19 def enabled(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
20 return True
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
21
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
22 def getCache(self, name):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
23 c = self.caches.get(name)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
24 if c is None:
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
25 c_dir = os.path.join(self.base_dir, name)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
26 if not os.path.isdir(c_dir):
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
27 os.makedirs(c_dir, 0o755)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
28
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
29 c = SimpleCache(c_dir)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
30 self.caches[name] = c
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
31 return c
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32
45
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
33 def getCacheDir(self, name):
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
34 return os.path.join(self.base_dir, name)
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
35
105
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
36 def getCacheNames(self, except_names=None):
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
37 _, dirnames, __ = next(os.walk(self.base_dir))
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
38 if except_names is None:
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
39 return dirnames
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
40 return [dn for dn in dirnames if dn not in except_names]
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
41
371
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
42 def clearCache(self, name):
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
43 cache_dir = self.getCacheDir(name)
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
44 if os.path.isdir(cache_dir):
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
45 logger.debug("Cleaning cache: %s" % cache_dir)
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
46 shutil.rmtree(cache_dir)
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
47
424
5feb71d31a4f internal: Fix caches being orphaned from their directory.
Ludovic Chabant <ludovic@chabant.com>
parents: 416
diff changeset
48 # Re-create the cache-dir because now our Cache instance points
5feb71d31a4f internal: Fix caches being orphaned from their directory.
Ludovic Chabant <ludovic@chabant.com>
parents: 416
diff changeset
49 # to a directory that doesn't exist anymore.
5feb71d31a4f internal: Fix caches being orphaned from their directory.
Ludovic Chabant <ludovic@chabant.com>
parents: 416
diff changeset
50 os.makedirs(cache_dir, 0o755)
5feb71d31a4f internal: Fix caches being orphaned from their directory.
Ludovic Chabant <ludovic@chabant.com>
parents: 416
diff changeset
51
371
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
52 def clearCaches(self, except_names=None):
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
53 for name in self.getCacheNames(except_names=except_names):
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
54 self.clearCache(name)
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
55
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 class SimpleCache(object):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 def __init__(self, base_dir):
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
59 self.base_dir = base_dir
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 if not os.path.isdir(base_dir):
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
61 raise Exception("Cache directory doesn't exist: %s" % base_dir)
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 def isValid(self, path, time):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 cache_time = self.getCacheTime(path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 if cache_time is None:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 return False
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 if isinstance(time, list):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 for t in time:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 if cache_time < t:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 return False
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 return True
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 return cache_time >= time
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 def getCacheTime(self, path):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 cache_path = self.getCachePath(path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 try:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 return os.path.getmtime(cache_path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 except os.error:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 return None
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 def has(self, path):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 cache_path = self.getCachePath(path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 return os.path.isfile(cache_path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 def read(self, path):
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
86 with self.openRead(path, mode='r', encoding='utf8') as fp:
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 return fp.read()
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
89 def openRead(self, path, mode='r', encoding=None):
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
90 cache_path = self.getCachePath(path)
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
91 return open(cache_path, mode=mode, encoding=encoding)
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
92
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 def write(self, path, content):
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
94 with self.openWrite(path, mode='w', encoding='utf8') as fp:
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
95 fp.write(content)
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
96
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
97 def openWrite(self, path, mode='w', encoding=None):
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 cache_path = self.getCachePath(path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 cache_dir = os.path.dirname(cache_path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 if not os.path.isdir(cache_dir):
5
474c9882decf Upgrade to Python 3.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
101 os.makedirs(cache_dir, 0o755)
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
102 return open(cache_path, mode=mode, encoding=encoding)
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
104 def getCachePath(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
105 if path.startswith('.'):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
106 path = '__index__' + path
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
107 return os.path.join(self.base_dir, path)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
108
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
109
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
110 class NullCache(object):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
111 def isValid(self, path, time):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
112 return False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
113
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
114 def getCacheTime(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
115 return None
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
116
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
117 def has(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
118 return False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
119
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
120 def read(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
121 raise Exception("Null cache has no data.")
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
122
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
123 def write(self, path, content):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
124 pass
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
125
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
126 def getCachePath(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
127 raise Exception("Null cache can't make paths.")
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
128
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
129
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
130 class NullExtensibleCache(object):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
131 def __init__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
132 self.null_cache = NullCache()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
133
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
134 @property
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
135 def enabled(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
136 return False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
137
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
138 def getCache(self, name):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
139 return self.null_cache
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
140
371
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
141 def getCacheDir(self, name):
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
142 raise NotImplementedError()
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
143
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
144 def getCacheNames(self, except_names=None):
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
145 return []
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
146
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
147 def clearCache(self, name):
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
148 pass
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
149
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
150 def clearCaches(self, except_names=None):
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
151 pass
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 105
diff changeset
152
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
153
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
154 def _make_fs_cache_key(key):
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
155 return hashlib.md5(key.encode('utf8')).hexdigest()
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
156
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
157
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
158 class MemCache(object):
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
159 """ Simple memory cache. It can be backed by a simple file-system
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
160 cache, but items need to be pickle-able to do this.
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
161 """
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
162 def __init__(self, size=2048):
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
163 self.cache = repoze.lru.LRUCache(size)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
164 self.fs_cache = None
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
165 self._last_access_hit = None
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
166 self._invalidated_fs_items = set()
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
167 self._missed_keys = []
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
168 self._misses = 0
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
169 self._hits = 0
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
170
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
171 @property
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
172 def last_access_hit(self):
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
173 return self._last_access_hit
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
174
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
175 def invalidate(self, key):
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
176 logger.debug("Invalidating cache item '%s'." % key)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
177 self.cache.invalidate(key)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
178 if self.fs_cache:
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
179 logger.debug("Invalidating FS cache item '%s'." % key)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
180 fs_key = _make_fs_cache_key(key)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
181 self._invalidated_fs_items.add(fs_key)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
182
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
183 def put(self, key, item, save_to_fs=True):
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
184 self.cache.put(key, item)
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
185 if self.fs_cache and save_to_fs:
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
186 fs_key = _make_fs_cache_key(key)
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
187 with self.fs_cache.openWrite(fs_key, mode='wb') as fp:
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
188 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL)
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
189
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
190 def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True):
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
191 self._last_access_hit = True
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
192 item = self.cache.get(key)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
193 if item is not None:
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
194 self._hits += 1
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
195 return item
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
196
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
197 if self.fs_cache is not None:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
198 if fs_cache_time is None:
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
199 raise ValueError(
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
200 "No file-system cache time was given for '%s'. "
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
201 "This would result in degraded performance." % key)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 852
diff changeset
202
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
203 # Try first from the file-system cache.
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
204 fs_key = _make_fs_cache_key(key)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
205 if (fs_key not in self._invalidated_fs_items and
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
206 self.fs_cache.isValid(fs_key, fs_cache_time)):
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
207 with self.fs_cache.openRead(fs_key, mode='rb') as fp:
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
208 item = pickle.load(fp)
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
209 self.cache.put(key, item)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
210 self._hits += 1
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
211 return item
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
212
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
213 # Look into the mem-cache.
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
214 item = item_maker()
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
215 self.cache.put(key, item)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
216 self._last_access_hit = False
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
217 self._misses += 1
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
218 self._missed_keys.append(key)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
219
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
220 # Save to the file-system if needed.
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 424
diff changeset
221 if self.fs_cache is not None and save_to_fs:
911
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
222 with self.fs_cache.openWrite(fs_key, mode='wb') as fp:
f2b75e4be981 internal: Use pickle for caching things on disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
223 pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL)
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
224
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
225 return item
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
226