Mercurial > piecrust2

annotate piecrust/baking/scheduler.py @ 380:f33712c4cfab

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
routing: Fix bugs with matching URLs with correct route but missing metadata. When matching a route like `/foo/%slug%` against an URL like `/foo`, the route will (correctly) return a match, but it will be completely missing the `slug` metadata, resulting in problems elsewhere. This change makes it so that any missing route metadata will be filled in with an empty string. And because this means generated URLs may differ from the incoming URL when using trailing slashes (`/foo/` _vs._ `/foo`), we make the assert in the chef server handle those discrepancies.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 10 May 2015 00:34:21 -0700
parents 938be93215cb
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import logging
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import threading
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 logger = logging.getLogger(__name__)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 class BakeScheduler(object):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 _EMPTY = object()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 _WAIT = object()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 def __init__(self, record, jobs=None):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 self.record = record
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 self.jobs = list(jobs) if jobs is not None else []
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 self._active_jobs = []
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 self._lock = threading.Lock()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 self._added_event = threading.Event()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 self._done_event = threading.Event()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 def addJob(self, job):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 logger.debug("Queuing job '%s:%s'." % (
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
22 job.factory.source.name, job.factory.rel_path))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 with self._lock:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 self.jobs.append(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 self._added_event.set()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 def onJobFinished(self, job):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 logger.debug("Removing job '%s:%s'." % (
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
29 job.factory.source.name, job.factory.rel_path))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30 with self._lock:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 self._active_jobs.remove(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 self._done_event.set()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 def getNextJob(self, wait_timeout=None, empty_timeout=None):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 self._added_event.clear()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 self._done_event.clear()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 job = self._doGetNextJob()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 while job in (self._EMPTY, self._WAIT):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 if job == self._EMPTY:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 if empty_timeout is None:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 return None
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 logger.debug("Waiting for a new job to be added...")
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 res = self._added_event.wait(empty_timeout)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 elif job == self._WAIT:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 if wait_timeout is None:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 return None
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 logger.debug("Waiting for a job to be finished...")
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 res = self._done_event.wait(wait_timeout)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 if not res:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 logger.debug("Timed-out. No job found.")
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 return None
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 job = self._doGetNextJob()
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 return job
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 def _doGetNextJob(self):
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 with self._lock:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 if len(self.jobs) == 0:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 return self._EMPTY
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 job = self.jobs.pop(0)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 first_job = job
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 while True:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 ready, wait_on_src = self._isJobReady(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 if ready:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 break
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 logger.debug("Job '%s:%s' isn't ready yet: waiting on pages "
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 "from source '%s' to finish baking." %
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 (job.factory.source.name,
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 job.factory.rel_path, wait_on_src))
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 self.jobs.append(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 job = self.jobs.pop(0)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 if job == first_job:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 # None of the jobs are ready... we need to wait.
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75 self.jobs.append(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76 return self._WAIT
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
78 logger.debug(
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
79 "Job '%s:%s' is ready to go, moving to active queue." %
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
80 (job.factory.source.name, job.factory.rel_path))
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 self._active_jobs.append(job)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 return job
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 def _isJobReady(self, job):
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
85 e = self.record.getPreviousEntry(
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
86 job.factory.source.name,
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 158
diff changeset
87 job.factory.rel_path,
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 158
diff changeset
88 taxonomy_info=job.record_entry.taxonomy_info)
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 if not e:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 return (True, None)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 158
diff changeset
91 used_source_names = e.getAllUsedSourceNames()
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 158
diff changeset
92 for sn in used_source_names:
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
93 if sn == job.factory.source.name:
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
94 continue
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 158
diff changeset
95 if any(filter(lambda j: j.factory.source.name == sn,
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 158
diff changeset
96 self.jobs)):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 return (False, sn)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
98 if any(filter(lambda j: j.factory.source.name == sn,
158
1187739e5a19 Fix some indentation and line lengths.
Ludovic Chabant <ludovic@chabant.com>
parents: 150
diff changeset
99 self._active_jobs)):
150
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 return (False, sn)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
101 return (True, None)
91dcbb5fe1e8 Split baking code in smaller files.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102