Mercurial > piecrust2

annotate piecrust/environment.py @ 1195:ae9387338db1 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
admin: add option to publish immediately
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 30 Dec 2022 16:48:04 -0800
parents a9a592f655e3
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
1 import time
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import logging
1153
a9a592f655e3 config: Add setting for enabling baking or serving posts in the future.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
3 import datetime
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
4 import contextlib
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 logger = logging.getLogger(__name__)
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
10 class ExecutionStats:
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
11 def __init__(self):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
12 self.timers = {}
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
13 self.counters = {}
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
14 self.manifests = {}
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
15
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
16 def registerTimer(self, category, *,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
17 raise_if_registered=True, time=0):
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
18 if raise_if_registered and category in self.timers:
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
19 raise Exception("Timer '%s' has already been registered." %
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
20 category)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
21 self.timers[category] = time
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
22
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
23 @contextlib.contextmanager
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
24 def timerScope(self, category):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
25 start = time.perf_counter()
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
26 yield
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
27 self.timers[category] += time.perf_counter() - start
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
28
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
29 def stepTimer(self, category, value):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
30 self.timers[category] += value
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
31
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
32 def stepTimerSince(self, category, since):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
33 self.stepTimer(category, time.perf_counter() - since)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
34
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
35 def registerCounter(self, category, *, raise_if_registered=True):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
36 if raise_if_registered and category in self.counters:
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
37 raise Exception("Counter '%s' has already been registered." %
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
38 category)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
39 self.counters[category] = 0
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
40
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
41 def stepCounter(self, category, inc=1):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
42 self.counters[category] += inc
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
43
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
44 def registerManifest(self, name, *, raise_if_registered=True):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
45 if raise_if_registered and name in self.manifests:
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
46 raise Exception("Manifest '%s' has already been registered." %
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
47 name)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
48 self.manifests[name] = []
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
49
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
50 def addManifestEntry(self, name, entry):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
51 self.manifests[name].append(entry)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
52
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
53 def mergeStats(self, other):
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
54 for oc, ov in other.timers.items():
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
55 v = self.timers.setdefault(oc, 0)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
56 self.timers[oc] = v + ov
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
57 for oc, ov in other.counters.items():
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
58 v = self.counters.setdefault(oc, 0)
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
59 self.counters[oc] = v + ov
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
60 for oc, ov in other.manifests.items():
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
61 v = self.manifests.setdefault(oc, [])
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
62 self.manifests[oc] = v + ov
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
63
991
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
64 def toData(self):
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
65 return {
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
66 'timers': self.timers.copy(),
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
67 'counters': self.counters.copy(),
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
68 'manifests': self.manifests.copy()}
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
69
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
70 def fromData(self, data):
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
71 self.timers = data['timers']
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
72 self.counters = data['counters']
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
73 self.manifests = data['manifests']
1857dbd4580f bake: Fix bugs introduced by bake optimizations, of course.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
74
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
75
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
76 class Environment:
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77 def __init__(self):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
78 from piecrust.cache import MemCache
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
79 from piecrust.rendering import RenderingContextStack
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
80
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
81 self.app = None
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
82 self.start_time = None
1153
a9a592f655e3 config: Add setting for enabling baking or serving posts in the future.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
83 self.start_datetime = None
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
84 self.was_cache_cleaned = False
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
85 self.page_repository = MemCache()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 1
diff changeset
86 self.rendered_segments_repository = MemCache()
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
87 self.render_ctx_stack = RenderingContextStack()
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
88 self.fs_cache_only_for_main_page = False
455
cb3446be44b7 bake: Abort "render first" jobs if we start using other pages.
Ludovic Chabant <ludovic@chabant.com>
parents: 427
diff changeset
89 self.abort_source_use = False
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
90 self._stats = ExecutionStats()
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
91
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
92 @property
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
93 def stats(self):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
94 return self._stats
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
95
114
371a6c879ab9 When possible, try and batch-load pages so we only lock once.
Ludovic Chabant <ludovic@chabant.com>
parents: 111
diff changeset
96 def initialize(self, app):
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
97 self.app = app
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
98 self.start_time = time.perf_counter()
1153
a9a592f655e3 config: Add setting for enabling baking or serving posts in the future.
Ludovic Chabant <ludovic@chabant.com>
parents: 991
diff changeset
99 self.start_datetime = datetime.datetime.now()
427
3b658190c02b performance: Compute default layout extensions only once.
Ludovic Chabant <ludovic@chabant.com>
parents: 417
diff changeset
100
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
101 self.rendered_segments_repository.fs_cache = \
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
102 app.cache.getCache('renders')
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 371
diff changeset
103
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
104 def _mergeCacheStats(self):
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
105 repos = [
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
106 ('RenderedSegmentsRepo', self.rendered_segments_repository),
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
107 ('PagesRepo', self.page_repository)]
687
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
108 for name, repo in repos:
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
109 self._stats.counters['%s_hit' % name] = repo._hits
61d606fbc313 bake: Change `show-timers` to `show-stats`, add stats.
Ludovic Chabant <ludovic@chabant.com>
parents: 666
diff changeset
110 self._stats.counters['%s_miss' % name] = repo._misses
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
111 self._stats.manifests['%s_missedKeys' % name] = \
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 687
diff changeset
112 list(repo._missed_keys)
417
eef887cec776 internal: Add utility function for incrementing performance timers.
Ludovic Chabant <ludovic@chabant.com>
parents: 416
diff changeset
113
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115 class StandardEnvironment(Environment):
1
aaa8fb7c8918 Re-arranged modules to reduce dependencies to builtin stuff.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
116 def __init__(self):
aaa8fb7c8918 Re-arranged modules to reduce dependencies to builtin stuff.
Ludovic Chabant <ludovic@chabant.com>
parents: 0
diff changeset
117 super(StandardEnvironment, self).__init__()
0
a212a3f2e3ee Initial commit.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118