piecrust2: piecrust/baking/baker.py comparison

comparison piecrust/baking/baker.py @ 91:e88e330eb8dc

Improvements to incremental baking and cache invalidating.

author	Ludovic Chabant <ludovic@chabant.com>
date	Fri, 05 Sep 2014 00:42:13 -0700
parents	3471ffa059b2
children	0445a2232de7

comparison

equal deleted inserted replaced

-:e293f08d954e
+:e88e330eb8dc
 import shutil
 import hashlib
 import logging
 import threading
 import urllib.request, urllib.error, urllib.parse
-from piecrust.baking.records import TransitionalBakeRecord, BakeRecordPageEntry
+from piecrust.baking.records import (TransitionalBakeRecord,
+BakeRecordPageEntry,
+FLAG_OVERRIDEN, FLAG_SOURCE_MODIFIED)
 from piecrust.chefutil import format_timed, log_friendly_exception
 from piecrust.data.filters import (PaginationFilter, HasFilterClause,
 IsFilterClause, AndBooleanClause)
 from piecrust.processing.base import ProcessorPipeline
 from piecrust.rendering import PageRenderingContext, render_page
 else:
 bake_path.append(decoded_uri + '.html')
 return os.path.normpath(os.path.join(*bake_path))
-def bake(self, factory, route, taxonomy_name=None, taxonomy_term=None):
+def bake(self, factory, route, record_entry,
+taxonomy_name=None, taxonomy_term=None):
 pagination_filter = None
 custom_data = None
 if taxonomy_name and taxonomy_term:
 # Must bake a taxonomy listing page... we'll have to add a
 # pagination filter for only get matching posts, and the output
 "'%s:%s'." % (factory.ref_spec, uri,
 override.source_name, override.rel_path))
 logger.debug("'%s' [%s] is overriden by '%s:%s'. Skipping" %
 (factory.ref_spec, uri, override.source_name,
 override.rel_path))
-entry = BakeRecordPageEntry()
+record_entry.flags |= FLAG_OVERRIDEN
-entry.path = factory.path
+return
-entry.rel_path = factory.rel_path
-entry.source_name = factory.source.name
-entry.was_overriden = True
-if self.record:
-self.record.addEntry(entry)
-return entry
 cur_sub = 1
 has_more_subs = True
+force_this = self.force
 page = factory.buildPage()
-cur_record_entry = BakeRecordPageEntry(page)
+record_entry.config = page.config.get().copy()
-cur_record_entry.taxonomy_name = taxonomy_name
-cur_record_entry.taxonomy_term = taxonomy_term
 prev_record_entry = self.record.getPreviousEntry(
 factory.source.name, factory.rel_path,
 taxonomy_name, taxonomy_term)
 logger.debug("Baking '%s'..." % uri)
+# If the current page is known to use pages from other sources,
+# see if any of those got baked, or are going to be baked for some
+# reason. If so, we need to bake this one too.
+# (this happens for instance with the main page of a blog).
+if prev_record_entry:
+any_used_src_baked = False
+used_src_names = list(prev_record_entry.used_source_names)
+for src_name in used_src_names:
+entries = self.record.getCurrentEntries(src_name)
+for e in entries:
+if e.was_baked or e.flags & FLAG_SOURCE_MODIFIED:
+any_used_src_baked = True
+break
+if any_used_src_baked:
+break
+if any_used_src_baked:
+logger.debug("'%s' is known to use sources %s, at least one "
+"of which got baked. Will force bake this page. "
+% (uri, used_src_names))
+force_this = True
 while has_more_subs:
 sub_uri = self.getOutputUri(uri, cur_sub)
 out_path = self.getOutputPath(sub_uri)
 # Check for up-to-date outputs.
 do_bake = True
-if not self.force and prev_record_entry:
+if not force_this and prev_record_entry:
 try:
-in_path_time = page.path_mtime
+in_path_time = record_entry.path_mtime
 out_path_time = os.path.getmtime(out_path)
-if out_path_time > in_path_time:
+if out_path_time > in_path_time and not any_used_src_baked:
 do_bake = False
 except OSError:
 # File doesn't exist, we'll need to bake.
 pass
 dest_ap = os.path.join(out_assets_dir, os.path.basename(ap))
 logger.debug("  %s -> %s" % (ap, dest_ap))
 shutil.copy(ap, dest_ap)
 # Record what we did and figure out if we have more work.
-cur_record_entry.out_uris.append(sub_uri)
+record_entry.out_uris.append(sub_uri)
-cur_record_entry.out_paths.append(out_path)
+record_entry.out_paths.append(out_path)
-cur_record_entry.used_source_names |= ctx.used_source_names
+record_entry.used_source_names |= ctx.used_source_names
-cur_record_entry.used_taxonomy_terms |= ctx.used_taxonomy_terms
+record_entry.used_taxonomy_terms |= ctx.used_taxonomy_terms
 has_more_subs = False
 if ctx.used_pagination is not None:
-cur_record_entry.addUsedSource(ctx.used_pagination._source)
+record_entry.addUsedSource(ctx.used_pagination._source)
 if ctx.used_pagination.has_more:
 cur_sub += 1
 has_more_subs = True
-if self.record:
-self.record.addEntry(cur_record_entry)
-return cur_record_entry
 def _bakeSingle(self, page, sub_uri, num, out_path,
 pagination_filter=None, custom_data=None):
 ctx = PageRenderingContext(page, sub_uri)
 ctx.page_num = num
 self._bakeAssets(record)
 # Save the bake record.
 t = time.clock()
 record.current.bake_time = time.time()
+record.current.out_dir = self.out_dir
 record.collapseRecords()
 record.saveCurrent(record_cache.getCachePath(record_name))
 logger.debug(format_timed(t, 'saved bake record', colored=False))
 # All done.
 reason = "ordered to"
 elif not self.app.config.get('__cache_valid'):
 # The configuration file was changed, or we're running a new
 # version of the app.
 reason = "not valid anymore"
-elif not record.previous.bake_time:
+elif (not record.previous.bake_time or
+not record.previous.hasLatestVersion()):
 # We have no valid previous bake record.
 reason = "need bake record regeneration"
 else:
 # Check if any template has changed since the last bake. Since
 # there could be some advanced conditional logic going on, we'd
 cache_dir = self.app.cache.getCacheDir('baker')
 if os.path.isdir(cache_dir):
 logger.debug("Cleaning baker cache: %s" % cache_dir)
 shutil.rmtree(cache_dir)
 self.force = True
+record.incremental_count = 0
 logger.info(format_timed(start_time,
 "cleaned cache (reason: %s)" % reason))
 else:
+record.incremental_count += 1
 logger.debug(format_timed(start_time, "cache is assumed valid",
 colored=False))
 def _bakeRealm(self, record, realm, srclist):
 # Gather all page factories from the sources and queue them
 route = self.app.getRoute(source.name, fac.metadata)
 if route is None:
 logger.error("Can't get route for page: %s" % fac.ref_spec)
 continue
-logger.debug("Queuing: %s" % fac.ref_spec)
+entry = BakeRecordPageEntry(fac)
-queue.addJob(BakeWorkerJob(fac, route))
+record.addEntry(entry)
+queue.addJob(BakeWorkerJob(fac, route, entry))
 self._waitOnWorkerPool(pool, abort)
 def _bakeTaxonomies(self, record):
 logger.debug("Baking taxonomies")
 for term in terms:
 fac = PageFactory(tax_page_source, tax_page_rel_path,
 {tax.term_name: term})
 logger.debug("Queuing: %s [%s, %s]" %
 (fac.ref_spec, tax_name, term))
+entry = BakeRecordPageEntry(fac, tax_name, term)
+record.addEntry(entry)
 queue.addJob(
-BakeWorkerJob(fac, route, tax_name, term))
+BakeWorkerJob(fac, route, entry, tax_name, term))
 self._waitOnWorkerPool(pool, abort)
 def _bakeAssets(self, record):
 mounts = self.app.assets_dirs
 self._lock = threading.Lock()
 self._added_event = threading.Event()
 self._done_event = threading.Event()
 def addJob(self, job):
-logger.debug("Adding job '%s:%s' to scheduler." % (
+logger.debug("Queuing job '%s:%s'." % (
 job.factory.source.name, job.factory.rel_path))
 with self._lock:
 self.jobs.append(job)
 self._added_event.set()
 def onJobFinished(self, job):
-logger.debug("Removing job '%s:%s' from scheduler." % (
+logger.debug("Removing job '%s:%s'." % (
 job.factory.source.name, job.factory.rel_path))
 with self._lock:
 self._active_jobs.remove(job)
 self._done_event.set()
-def getNextJob(self, timeout=None):
+def getNextJob(self, wait_timeout=None, empty_timeout=None):
 self._added_event.clear()
 self._done_event.clear()
 job = self._doGetNextJob()
 while job in (self._EMPTY, self._WAIT):
-if timeout is None:
-return None
 if job == self._EMPTY:
+if empty_timeout is None:
+return None
 logger.debug("Waiting for a new job to be added...")
-res = self._added_event.wait(timeout)
+res = self._added_event.wait(empty_timeout)
 elif job == self._WAIT:
+if wait_timeout is None:
+return None
 logger.debug("Waiting for a job to be finished...")
-res = self._done_event.wait(timeout)
+res = self._done_event.wait(wait_timeout)
 if not res:
 logger.debug("Timed-out. No job found.")
 return None
 job = self._doGetNextJob()
 return job
 job.factory.source.name, job.factory.rel_path))
 self.jobs.append(job)
 job = self.jobs.pop(0)
 if job == first_job:
 # None of the jobs are ready... we need to wait.
+self.jobs.append(job)
 return self._WAIT
 logger.debug("Job '%s:%s' is ready to go, moving to active "
 "queue." % (job.factory.source.name, job.factory.rel_path))
 self._active_jobs.append(job)
 e = self.record.getPreviousEntry(job.factory.source.name,
 job.factory.rel_path)
 if not e:
 return True
 for sn in e.used_source_names:
+if sn == job.factory.source.name:
+continue
 if any(filter(lambda j: j.factory.source.name == sn, self.jobs)):
 return False
 if any(filter(lambda j: j.factory.source.name == sn,
 self._active_jobs)):
 return False
 self.work_queue = work_queue
 self.abort_event = abort_event
 class BakeWorkerJob(object):
-def __init__(self, factory, route, taxonomy_name=None, taxonomy_term=None):
+def __init__(self, factory, route, record_entry,
+taxonomy_name=None, taxonomy_term=None):
 self.factory = factory
 self.route = route
+self.record_entry = record_entry
 self.taxonomy_name = taxonomy_name
 self.taxonomy_term = taxonomy_term
 @property
 def source(self):
 self._page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force,
 ctx.record)
 def run(self):
 while(not self.ctx.abort_event.is_set()):
-job = self.ctx.work_queue.getNextJob()
+job = self.ctx.work_queue.getNextJob(wait_timeout=1)
 if job is None:
 logger.debug("[%d] No more work... shutting down." % self.wid)
 break
 try:
 break
 def _unsafeRun(self, job):
 start_time = time.clock()
-bake_res = self._page_baker.bake(job.factory, job.route,
+entry = job.record_entry
+self._page_baker.bake(job.factory, job.route, entry,
 taxonomy_name=job.taxonomy_name,
 taxonomy_term=job.taxonomy_term)
-if bake_res.was_baked:
+if entry.was_baked:
-uri = bake_res.out_uris[0]
+uri = entry.out_uris[0]
 friendly_uri = uri if uri != '' else '[main page]'
 friendly_count = ''
-if bake_res.num_subs > 1:
+if entry.num_subs > 1:
-friendly_count = ' (%d pages)' % bake_res.num_subs
+friendly_count = ' (%d pages)' % entry.num_subs
 logger.info(format_timed(start_time, '[%d] %s%s' %
 (self.wid, friendly_uri, friendly_count)))

Mercurial > piecrust2

comparison piecrust/baking/baker.py @ 91:e88e330eb8dc