changeset 1014:071f30aa04bb

bake: Do template caching in a background job if possible.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 28 Nov 2017 21:28:15 -0800
parents 6370ab74b2d5
children fa489c5e829e
files piecrust/baking/baker.py piecrust/baking/worker.py piecrust/workerpool.py
diffstat 3 files changed, 38 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
--- a/piecrust/baking/baker.py	Tue Nov 28 21:27:57 2017 -0800
+++ b/piecrust/baking/baker.py	Tue Nov 28 21:28:15 2017 -0800
@@ -98,8 +98,11 @@
         logger.info(format_timed(start_time, "setup baker"))
 
         # Load all sources, pre-cache templates.
+        load_start_time = time.perf_counter()
+        self._startPopulateTemplateCaches(pool)
         self._loadSources(ppmngr)
-        self._populateTemplateCaches()
+        self._endPopulateTemplateCache(pool)
+        logger.info(format_timed(load_start_time, "loaded site content"))
 
         # Bake the realms.
         self._bakeRealms(pool, ppmngr, record_histories)
@@ -206,8 +209,6 @@
         return ppmngr
 
     def _loadSources(self, ppmngr):
-        start_time = time.perf_counter()
-
         for ppinfo in ppmngr.getPipelineInfos():
             rec = ppinfo.record_history.current
             rec_entries = ppinfo.pipeline.loadAllContents()
@@ -215,19 +216,29 @@
                 for e in rec_entries:
                     rec.addEntry(e)
 
-        stats = self.app.env.stats
-        stats.stepTimer('LoadSourceContents', time.perf_counter() - start_time)
-        logger.info(format_timed(start_time, "loaded site content"))
-
-    def _populateTemplateCaches(self):
-        start_time = time.perf_counter()
+    def _startPopulateTemplateCaches(self, pool):
+        # If we can, cache templates in a worker process, so we can load
+        # the sources' pages in the main process in the meantime.
+        # But if we don't have any workers, well, we'll have to make do
+        # in the `_endPopulateTemplateCache` method.
+        if pool.pool_size == 0:
+            return
 
-        for eng in self.app.plugin_loader.getTemplateEngines():
-            eng.populateCache()
+        pool._callback = None
+        pool._error_callback = None
+        job = {'job_spec': ('__special__', 'populate_template_cache')}
+        pool.queueJobs([job])
 
-        stats = self.app.env.stats
-        stats.stepTimer('CacheTemplates', time.perf_counter() - start_time)
-        logger.info(format_timed(start_time, "cache templates"))
+    def _endPopulateTemplateCache(self, pool):
+        if pool.pool_size == 0:
+            # No workers... load the templates synchronously.
+            for eng in self.app.plugin_loader.getTemplateEngines():
+                eng.populateCache()
+        else:
+            # Wait for the job to finish.
+            pool.wait()
+            pool._callback = self._handleWorkerResult
+            pool._error_callback = self._handleWorkerError
 
     def _bakeRealms(self, pool, ppmngr, record_histories):
         # Bake the realms -- user first, theme second, so that a user item
--- a/piecrust/baking/worker.py	Tue Nov 28 21:27:57 2017 -0800
+++ b/piecrust/baking/worker.py	Tue Nov 28 21:28:15 2017 -0800
@@ -80,6 +80,15 @@
         source_name, item_spec = job['job_spec']
         logger.debug("Received job: %s@%s" % (source_name, item_spec))
 
+        # Check for special jobs.
+        if source_name == '__special__':
+            if item_spec == 'populate_template_cache':
+                for eng in self.app.plugin_loader.getTemplateEngines():
+                    eng.populateCache()
+            else:
+                raise Exception("Unknown special job: %s" % item_spec)
+            return {}
+
         # Run the job!
         job_start = time.perf_counter()
         pp = self.ppmngr.getPipeline(source_name)
--- a/piecrust/workerpool.py	Tue Nov 28 21:27:57 2017 -0800
+++ b/piecrust/workerpool.py	Tue Nov 28 21:28:15 2017 -0800
@@ -310,6 +310,10 @@
 
         stats.stepTimerSince('MasterInit', init_start_time)
 
+    @property
+    def pool_size(self):
+        return len(self._pool)
+
     def queueJobs(self, jobs):
         if self._closed:
             if self._error_on_join: