Mercurial > piecrust2

annotate piecrust/environment.py @ 47:558e3602be40

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Cache rendered segments to disk.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 22 Aug 2014 12:25:35 -0700
parents 43091c9837bf
children fce061f8c2ed
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
1 import re
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
2 import time
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
3 import json
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
5 import threading
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
6 import repoze.lru
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 logger = logging.getLogger(__name__)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
12 re_fs_cache_key = re.compile(r'[^\d\w\-\._]+')
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
13
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
14
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
15 def _make_fs_cache_key(key):
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
16 return re_fs_cache_key.sub('_', key)
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
17
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
18
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
19 class MemCache(object):
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
20 """ Simple memory cache. It can be backed by a simple file-system
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
21 cache, but items need to be JSON-serializable to do this.
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
22 """
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
23 def __init__(self, size=2048):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
24 self.cache = repoze.lru.LRUCache(size)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
25 self.lock = threading.RLock()
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
26 self.fs_cache = None
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
27
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
28 def get(self, key, item_maker):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
29 item = self.cache.get(key)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
30 if item is None:
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
31 logger.debug("Acquiring lock for: %s" % key)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
32 with self.lock:
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
33 item = self.cache.get(key)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
34 if item is None:
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
35 if self.fs_cache is not None:
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
36 # Try first from the file-system cache.
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
37 fs_key = _make_fs_cache_key(key)
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
38 logger.debug("'%s' not found in cache, trying the "
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
39 "file-system: %s" % (key, fs_key))
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
40 try:
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
41 item_raw = self.fs_cache.read(fs_key)
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
42 item = json.loads(item_raw)
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
43 self.cache.put(key, item)
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
44 return item
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
45 except:
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
46 pass
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
48 # Look into the mem-cache.
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
49 logger.debug("'%s' not found in cache, must build." % key)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
50 item = item_maker()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
51 self.cache.put(key, item)
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
52
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
53 # Save to the file-system if needed.
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
54 if self.fs_cache is not None:
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
55 item_raw = json.dumps(item)
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
56 self.fs_cache.write(fs_key, item_raw)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
57 return item
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
58
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
59
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
60 PHASE_PAGE_PARSING = 0
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
61 PHASE_PAGE_FORMATTING = 1
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
62 PHASE_PAGE_RENDERING = 2
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
65 class ExecutionInfo(object):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
66 def __init__(self, page, phase, render_ctx):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
67 self.page = page
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
68 self.phase = phase
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
69 self.render_ctx = render_ctx
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
70 self.was_cache_valid = False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
71 self.start_time = time.clock()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
72
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
73
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
74 class ExecutionInfoStack(threading.local):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
75 def __init__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
76 self._page_stack = []
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
77
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
78 @property
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
79 def current_page_info(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
80 if len(self._page_stack) == 0:
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
81 return None
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
82 return self._page_stack[-1]
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
83
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
84 @property
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
85 def is_main_page(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
86 return len(self._page_stack) == 1
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
87
12
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
88 def hasPage(self, page):
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
89 for ei in self._page_stack:
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
90 if ei.page == page:
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
91 return True
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
92 return False
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
93
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
94 def pushPage(self, page, phase, render_ctx):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
95 self._page_stack.append(ExecutionInfo(page, phase, render_ctx))
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
96
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
97 def popPage(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
98 del self._page_stack[-1]
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 class Environment(object):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 def __init__(self):
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
103 self.start_time = time.clock()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
104 self.exec_info_stack = ExecutionInfoStack()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
105 self.was_cache_cleaned = False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
106 self.page_repository = MemCache()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
107 self.rendered_segments_repository = MemCache()
32
43091c9837bf Fix problems with asset URLs.
Ludovic Chabant <ludovic@chabant.com>
parents: 12
diff changeset
108 self.base_asset_url_format = '%uri%'
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 def initialize(self, app):
47
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
111 cache = app.cache.getCache('renders')
558e3602be40 Cache rendered segments to disk.
Ludovic Chabant <ludovic@chabant.com>
parents: 32
diff changeset
112 self.rendered_segments_repository.fs_cache = cache
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
113
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 class StandardEnvironment(Environment):
1
aaa8fb7c8918 Re-arranged modules to reduce dependencies to builtin stuff.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
116 def __init__(self):
aaa8fb7c8918 Re-arranged modules to reduce dependencies to builtin stuff.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
117 super(StandardEnvironment, self).__init__()
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118