Mercurial > piecrust2
annotate piecrust/baking/scheduler.py @ 415:0e9a94b7fdfa
bake: Improve bake record information.
* Store things in the bake record that require less interaction between the
master process and the workers. For instance, don't store the paginator
object in the render pass info -- instead, just store whether pagination
was used, and whether it had more items.
* Simplify information passing between workers and bake passes by saving the
rendering info to the JSON cache. This means the "render first sub" job
doesn't have to return anything except errors now.
* Add more performance counter info.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 20 Jun 2015 19:23:16 -0700 |
parents | 938be93215cb |
children |
rev | line source |
---|---|
150
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
1 import logging |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
2 import threading |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
3 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
4 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
5 logger = logging.getLogger(__name__) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
6 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
7 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
8 class BakeScheduler(object): |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
9 _EMPTY = object() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
10 _WAIT = object() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
11 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
12 def __init__(self, record, jobs=None): |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
13 self.record = record |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
14 self.jobs = list(jobs) if jobs is not None else [] |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
15 self._active_jobs = [] |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
16 self._lock = threading.Lock() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
17 self._added_event = threading.Event() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
18 self._done_event = threading.Event() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
19 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
20 def addJob(self, job): |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
21 logger.debug("Queuing job '%s:%s'." % ( |
158
1187739e5a19
Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents:
150
diff
changeset
|
22 job.factory.source.name, job.factory.rel_path)) |
150
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
23 with self._lock: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
24 self.jobs.append(job) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
25 self._added_event.set() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
26 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
27 def onJobFinished(self, job): |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
28 logger.debug("Removing job '%s:%s'." % ( |
158
1187739e5a19
Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents:
150
diff
changeset
|
29 job.factory.source.name, job.factory.rel_path)) |
150
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
30 with self._lock: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
31 self._active_jobs.remove(job) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
32 self._done_event.set() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
33 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
34 def getNextJob(self, wait_timeout=None, empty_timeout=None): |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
35 self._added_event.clear() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
36 self._done_event.clear() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
37 job = self._doGetNextJob() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
38 while job in (self._EMPTY, self._WAIT): |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
39 if job == self._EMPTY: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
40 if empty_timeout is None: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
41 return None |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
42 logger.debug("Waiting for a new job to be added...") |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
43 res = self._added_event.wait(empty_timeout) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
44 elif job == self._WAIT: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
45 if wait_timeout is None: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
46 return None |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
47 logger.debug("Waiting for a job to be finished...") |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
48 res = self._done_event.wait(wait_timeout) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
49 if not res: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
50 logger.debug("Timed-out. No job found.") |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
51 return None |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
52 job = self._doGetNextJob() |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
53 return job |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
54 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
55 def _doGetNextJob(self): |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
56 with self._lock: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
57 if len(self.jobs) == 0: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
58 return self._EMPTY |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
59 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
60 job = self.jobs.pop(0) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
61 first_job = job |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
62 while True: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
63 ready, wait_on_src = self._isJobReady(job) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
64 if ready: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
65 break |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
66 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
67 logger.debug("Job '%s:%s' isn't ready yet: waiting on pages " |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
68 "from source '%s' to finish baking." % |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
69 (job.factory.source.name, |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
70 job.factory.rel_path, wait_on_src)) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
71 self.jobs.append(job) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
72 job = self.jobs.pop(0) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
73 if job == first_job: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
74 # None of the jobs are ready... we need to wait. |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
75 self.jobs.append(job) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
76 return self._WAIT |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
77 |
158
1187739e5a19
Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents:
150
diff
changeset
|
78 logger.debug( |
1187739e5a19
Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents:
150
diff
changeset
|
79 "Job '%s:%s' is ready to go, moving to active queue." % |
1187739e5a19
Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents:
150
diff
changeset
|
80 (job.factory.source.name, job.factory.rel_path)) |
150
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
81 self._active_jobs.append(job) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
82 return job |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
83 |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
84 def _isJobReady(self, job): |
158
1187739e5a19
Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents:
150
diff
changeset
|
85 e = self.record.getPreviousEntry( |
1187739e5a19
Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents:
150
diff
changeset
|
86 job.factory.source.name, |
338
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
158
diff
changeset
|
87 job.factory.rel_path, |
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
158
diff
changeset
|
88 taxonomy_info=job.record_entry.taxonomy_info) |
150
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
89 if not e: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
90 return (True, None) |
338
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
158
diff
changeset
|
91 used_source_names = e.getAllUsedSourceNames() |
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
158
diff
changeset
|
92 for sn in used_source_names: |
150
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
93 if sn == job.factory.source.name: |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
94 continue |
338
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
158
diff
changeset
|
95 if any(filter(lambda j: j.factory.source.name == sn, |
938be93215cb
bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents:
158
diff
changeset
|
96 self.jobs)): |
150
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
97 return (False, sn) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
98 if any(filter(lambda j: j.factory.source.name == sn, |
158
1187739e5a19
Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents:
150
diff
changeset
|
99 self._active_jobs)): |
150
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
100 return (False, sn) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
101 return (True, None) |
91dcbb5fe1e8
Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff
changeset
|
102 |