Mercurial > piecrust2

annotate piecrust/cache.py @ 182:a54d3c0b5f4a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
tests: Patch `os.path.exists` and improve patching for `open`. You can specify additional modules for which to patch `open`. Also, it was incorrectly updating the opened file, even when it was opened for read only. Now it only updates the contents if the file was opened for write, and supports appending to the end. Last, it supports opening text files in binary mode.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 04 Jan 2015 14:55:41 -0800
parents 7d2fdf43d7ca
children c2ca72fb7f0b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import codecs
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
4 import logging
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
5 import threading
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
6
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
7
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
8 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
9
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
10
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
11 class ExtensibleCache(object):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
12 def __init__(self, base_dir):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
13 self.base_dir = base_dir
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
14 self.lock = threading.Lock()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
15 self.caches = {}
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
16
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
17 @property
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
18 def enabled(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
19 return True
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
20
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
21 def getCache(self, name):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
22 c = self.caches.get(name)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
23 if c is None:
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
24 with self.lock:
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
25 c = self.caches.get(name)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
26 if c is None:
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
27 c_dir = os.path.join(self.base_dir, name)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
28 if not os.path.isdir(c_dir):
5
474c9882decf Upgrade to Python 3.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
29 os.makedirs(c_dir, 0o755)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
30
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
31 c = SimpleCache(c_dir)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
32 self.caches[name] = c
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
33 return c
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34
45
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
35 def getCacheDir(self, name):
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
36 return os.path.join(self.base_dir, name)
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
37
105
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
38 def getCacheNames(self, except_names=None):
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
39 _, dirnames, __ = next(os.walk(self.base_dir))
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
40 if except_names is None:
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
41 return dirnames
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
42 return [dn for dn in dirnames if dn not in except_names]
7d2fdf43d7ca Property clean all caches when force baking, except the `app` cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 45
diff changeset
43
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 class SimpleCache(object):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 def __init__(self, base_dir):
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
47 self.base_dir = base_dir
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 if not os.path.isdir(base_dir):
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
49 raise Exception("Cache directory doesn't exist: %s" % base_dir)
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 def isValid(self, path, time):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 cache_time = self.getCacheTime(path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 if cache_time is None:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 return False
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 if isinstance(time, list):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 for t in time:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 if cache_time < t:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 return False
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 return True
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 return cache_time >= time
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 def getCacheTime(self, path):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 cache_path = self.getCachePath(path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 try:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 return os.path.getmtime(cache_path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 except os.error:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 return None
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 def has(self, path):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 cache_path = self.getCachePath(path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 return os.path.isfile(cache_path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 def read(self, path):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 cache_path = self.getCachePath(path)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
75 logger.debug("Reading cache: %s" % cache_path)
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 with codecs.open(cache_path, 'r', 'utf-8') as fp:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 return fp.read()
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79 def write(self, path, content):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 cache_path = self.getCachePath(path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 cache_dir = os.path.dirname(cache_path)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 if not os.path.isdir(cache_dir):
5
474c9882decf Upgrade to Python 3.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
83 os.makedirs(cache_dir, 0o755)
45
efd0d3bacc9e Don't recursively clean the cache.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
84 logger.debug("Writing cache: %s" % cache_path)
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 with codecs.open(cache_path, 'w', 'utf-8') as fp:
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 fp.write(content)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
88 def getCachePath(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
89 if path.startswith('.'):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
90 path = '__index__' + path
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
91 return os.path.join(self.base_dir, path)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
92
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
93
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
94 class NullCache(object):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
95 def isValid(self, path, time):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
96 return False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
97
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
98 def getCacheTime(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
99 return None
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
100
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
101 def has(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
102 return False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
103
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
104 def read(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
105 raise Exception("Null cache has no data.")
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
106
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
107 def write(self, path, content):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
108 pass
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
109
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
110 def getCachePath(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
111 raise Exception("Null cache can't make paths.")
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
112
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
113
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
114 class NullExtensibleCache(object):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
115 def __init__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
116 self.null_cache = NullCache()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
117
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
118 @property
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
119 def enabled(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
120 return False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
121
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
122 def getCache(self, name):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
123 return self.null_cache
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
124