Mercurial > piecrust2

annotate piecrust/environment.py @ 460:55fc8918cb75

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Use batched jobs in the worker pool.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 11 Jul 2015 00:45:35 -0700
parents cb3446be44b7
children 81d9c3a3a0b5
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
1 import time
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import logging
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
3 import contextlib
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 412
diff changeset
4 from piecrust.cache import MemCache
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 logger = logging.getLogger(__name__)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 427
diff changeset
10 class AbortedSourceUseError(Exception):
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 427
diff changeset
11 pass
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 427
diff changeset
12
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 427
diff changeset
13
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
14 class ExecutionInfo(object):
96
0445a2232de7 Improvements and fixes to incremental baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 67
diff changeset
15 def __init__(self, page, render_ctx):
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
16 self.page = page
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
17 self.render_ctx = render_ctx
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
18 self.was_cache_valid = False
416
ff6cc43fb40c internal: Move `MemCache` to the `cache` module, remove threading locks.
Ludovic Chabant <ludovic@chabant.com>
parents: 412
diff changeset
19 self.start_time = time.perf_counter()
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
20
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
21
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
22 class ExecutionInfoStack(object):
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
23 def __init__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
24 self._page_stack = []
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
25
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
26 @property
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
27 def current_page_info(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
28 if len(self._page_stack) == 0:
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
29 return None
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
30 return self._page_stack[-1]
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
31
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
32 @property
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
33 def is_main_page(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
34 return len(self._page_stack) == 1
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
35
12
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
36 def hasPage(self, page):
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
37 for ei in self._page_stack:
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
38 if ei.page == page:
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
39 return True
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
40 return False
30a42341cfa8 Define page slugs properly, avoid recursions with debug data.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
41
96
0445a2232de7 Improvements and fixes to incremental baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 67
diff changeset
42 def pushPage(self, page, render_ctx):
0445a2232de7 Improvements and fixes to incremental baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 67
diff changeset
43 if len(self._page_stack) > 0:
0445a2232de7 Improvements and fixes to incremental baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 67
diff changeset
44 top = self._page_stack[-1]
0445a2232de7 Improvements and fixes to incremental baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 67
diff changeset
45 assert top.page is not page
0445a2232de7 Improvements and fixes to incremental baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 67
diff changeset
46 self._page_stack.append(ExecutionInfo(page, render_ctx))
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
47
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
48 def popPage(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
49 del self._page_stack[-1]
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
51 def clear(self):
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
52 self._page_stack = []
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
53
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 class Environment(object):
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 def __init__(self):
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
57 self.app = None
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
58 self.start_time = None
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
59 self.exec_info_stack = ExecutionInfoStack()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
60 self.was_cache_cleaned = False
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
61 self.base_asset_url_format = '%uri%'
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
62 self.page_repository = MemCache()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
63 self.rendered_segments_repository = MemCache()
111
208c652551a3 Quick fix for making the server correctly update referenced pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 96
diff changeset
64 self.fs_caches = {
208c652551a3 Quick fix for making the server correctly update referenced pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 96
diff changeset
65 'renders': self.rendered_segments_repository}
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
66 self.fs_cache_only_for_main_page = False
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 427
diff changeset
67 self.abort_source_use = False
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
68 self._default_layout_extensions = None
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
69 self._timers = {}
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
70
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
71 @property
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
72 def default_layout_extensions(self):
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
73 if self._default_layout_extensions is not None:
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
74 return self._default_layout_extensions
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
75
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
76 if self.app is None:
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
77 raise Exception("This environment has not been initialized yet.")
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
78
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
79 from piecrust.rendering import get_template_engine
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
80 dte = get_template_engine(self.app, None)
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
81 self._default_layout_extensions = ['.' + e.lstrip('.')
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
82 for e in dte.EXTENSIONS]
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
83 return self._default_layout_extensions
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
84
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
85 def initialize(self, app):
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
86 self.app = app
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
87 self.start_time = time.perf_counter()
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
88 self.exec_info_stack.clear()
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
89 self.was_cache_cleaned = False
32
43091c9837bf Fix problems with asset URLs.
Ludovic Chabant <ludovic@chabant.com>
parents: 12
diff changeset
90 self.base_asset_url_format = '%uri%'
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
91
371
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 336
diff changeset
92 self._onSubCacheDirChanged(app)
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93
412
a1567766c83c internal: Allow re-registering performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
94 def registerTimer(self, category, *, raise_if_registered=True):
a1567766c83c internal: Allow re-registering performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
95 if raise_if_registered and category in self._timers:
a1567766c83c internal: Allow re-registering performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
96 raise Exception("Timer '%s' has already been registered." %
a1567766c83c internal: Allow re-registering performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
97 category)
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
98 self._timers[category] = 0
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
99
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
100 @contextlib.contextmanager
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
101 def timerScope(self, category):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
102 start = time.perf_counter()
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
103 yield
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
104 self._timers[category] += time.perf_counter() - start
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
105
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
106 def stepTimer(self, category, value):
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
107 self._timers[category] += value
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
108
417
eef887cec776 internal: Add utility function for incrementing performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 416
diff changeset
109 def stepTimerSince(self, category, since):
eef887cec776 internal: Add utility function for incrementing performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 416
diff changeset
110 self.stepTimer(category, time.perf_counter() - since)
eef887cec776 internal: Add utility function for incrementing performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 416
diff changeset
111
371
c2ca72fb7f0b caching: Use separate caches for config variants and other contexts.
Ludovic Chabant <ludovic@chabant.com>
parents: 336
diff changeset
112 def _onSubCacheDirChanged(self, app):
111
208c652551a3 Quick fix for making the server correctly update referenced pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 96
diff changeset
113 for name, repo in self.fs_caches.items():
208c652551a3 Quick fix for making the server correctly update referenced pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 96
diff changeset
114 cache = app.cache.getCache(name)
208c652551a3 Quick fix for making the server correctly update referenced pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 96
diff changeset
115 repo.fs_cache = cache
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118 class StandardEnvironment(Environment):
1
aaa8fb7c8918 Re-arranged modules to reduce dependencies to builtin stuff.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
119 def __init__(self):
aaa8fb7c8918 Re-arranged modules to reduce dependencies to builtin stuff.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
120 super(StandardEnvironment, self).__init__()
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121