Mercurial > piecrust2

annotate piecrust/baking/scheduler.py @ 264:74bea91c9630

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Don't store internal config values in the bake record. We sometimes store actual objects in the page config (like for instance page linkers) and we don't want that to be pickled.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 24 Feb 2015 23:18:23 -0800
parents 1187739e5a19
children 938be93215cb
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import logging
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import threading
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 logger = logging.getLogger(__name__)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 class BakeScheduler(object):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 _EMPTY = object()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 _WAIT = object()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 def __init__(self, record, jobs=None):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 self.record = record
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 self.jobs = list(jobs) if jobs is not None else []
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 self._active_jobs = []
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 self._lock = threading.Lock()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 self._added_event = threading.Event()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 self._done_event = threading.Event()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 def addJob(self, job):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 logger.debug("Queuing job '%s:%s'." % (
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
22 job.factory.source.name, job.factory.rel_path))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 with self._lock:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 self.jobs.append(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 self._added_event.set()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 def onJobFinished(self, job):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 logger.debug("Removing job '%s:%s'." % (
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
29 job.factory.source.name, job.factory.rel_path))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 with self._lock:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 self._active_jobs.remove(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 self._done_event.set()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 def getNextJob(self, wait_timeout=None, empty_timeout=None):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 self._added_event.clear()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self._done_event.clear()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 job = self._doGetNextJob()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 while job in (self._EMPTY, self._WAIT):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 if job == self._EMPTY:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 if empty_timeout is None:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 return None
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 logger.debug("Waiting for a new job to be added...")
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 res = self._added_event.wait(empty_timeout)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 elif job == self._WAIT:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 if wait_timeout is None:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 return None
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 logger.debug("Waiting for a job to be finished...")
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 res = self._done_event.wait(wait_timeout)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 if not res:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 logger.debug("Timed-out. No job found.")
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 return None
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 job = self._doGetNextJob()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 return job
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 def _doGetNextJob(self):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 with self._lock:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 if len(self.jobs) == 0:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 return self._EMPTY
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 job = self.jobs.pop(0)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 first_job = job
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 while True:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 ready, wait_on_src = self._isJobReady(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 if ready:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 break
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 logger.debug("Job '%s:%s' isn't ready yet: waiting on pages "
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 "from source '%s' to finish baking." %
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 (job.factory.source.name,
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 job.factory.rel_path, wait_on_src))
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 self.jobs.append(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 job = self.jobs.pop(0)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 if job == first_job:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 # None of the jobs are ready... we need to wait.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 self.jobs.append(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 return self._WAIT
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
78 logger.debug(
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
79 "Job '%s:%s' is ready to go, moving to active queue." %
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
80 (job.factory.source.name, job.factory.rel_path))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 self._active_jobs.append(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 return job
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 def _isJobReady(self, job):
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
85 e = self.record.getPreviousEntry(
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
86 job.factory.source.name,
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 job.factory.rel_path)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88 if not e:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 return (True, None)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 for sn, rp in e.used_source_names:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 if sn == job.factory.source.name:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
92 continue
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 if any(filter(lambda j: j.factory.source.name == sn, self.jobs)):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 return (False, sn)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
95 if any(filter(lambda j: j.factory.source.name == sn,
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
96 self._active_jobs)):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 return (False, sn)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 return (True, None)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99