Mercurial > piecrust2
changeset 1014:071f30aa04bb
bake: Do template caching in a background job if possible.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Tue, 28 Nov 2017 21:28:15 -0800 |
parents | 6370ab74b2d5 |
children | fa489c5e829e |
files | piecrust/baking/baker.py piecrust/baking/worker.py piecrust/workerpool.py |
diffstat | 3 files changed, 38 insertions(+), 14 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/baking/baker.py Tue Nov 28 21:27:57 2017 -0800 +++ b/piecrust/baking/baker.py Tue Nov 28 21:28:15 2017 -0800 @@ -98,8 +98,11 @@ logger.info(format_timed(start_time, "setup baker")) # Load all sources, pre-cache templates. + load_start_time = time.perf_counter() + self._startPopulateTemplateCaches(pool) self._loadSources(ppmngr) - self._populateTemplateCaches() + self._endPopulateTemplateCache(pool) + logger.info(format_timed(load_start_time, "loaded site content")) # Bake the realms. self._bakeRealms(pool, ppmngr, record_histories) @@ -206,8 +209,6 @@ return ppmngr def _loadSources(self, ppmngr): - start_time = time.perf_counter() - for ppinfo in ppmngr.getPipelineInfos(): rec = ppinfo.record_history.current rec_entries = ppinfo.pipeline.loadAllContents() @@ -215,19 +216,29 @@ for e in rec_entries: rec.addEntry(e) - stats = self.app.env.stats - stats.stepTimer('LoadSourceContents', time.perf_counter() - start_time) - logger.info(format_timed(start_time, "loaded site content")) - - def _populateTemplateCaches(self): - start_time = time.perf_counter() + def _startPopulateTemplateCaches(self, pool): + # If we can, cache templates in a worker process, so we can load + # the sources' pages in the main process in the meantime. + # But if we don't have any workers, well, we'll have to make do + # in the `_endPopulateTemplateCache` method. + if pool.pool_size == 0: + return - for eng in self.app.plugin_loader.getTemplateEngines(): - eng.populateCache() + pool._callback = None + pool._error_callback = None + job = {'job_spec': ('__special__', 'populate_template_cache')} + pool.queueJobs([job]) - stats = self.app.env.stats - stats.stepTimer('CacheTemplates', time.perf_counter() - start_time) - logger.info(format_timed(start_time, "cache templates")) + def _endPopulateTemplateCache(self, pool): + if pool.pool_size == 0: + # No workers... load the templates synchronously. + for eng in self.app.plugin_loader.getTemplateEngines(): + eng.populateCache() + else: + # Wait for the job to finish. + pool.wait() + pool._callback = self._handleWorkerResult + pool._error_callback = self._handleWorkerError def _bakeRealms(self, pool, ppmngr, record_histories): # Bake the realms -- user first, theme second, so that a user item
--- a/piecrust/baking/worker.py Tue Nov 28 21:27:57 2017 -0800 +++ b/piecrust/baking/worker.py Tue Nov 28 21:28:15 2017 -0800 @@ -80,6 +80,15 @@ source_name, item_spec = job['job_spec'] logger.debug("Received job: %s@%s" % (source_name, item_spec)) + # Check for special jobs. + if source_name == '__special__': + if item_spec == 'populate_template_cache': + for eng in self.app.plugin_loader.getTemplateEngines(): + eng.populateCache() + else: + raise Exception("Unknown special job: %s" % item_spec) + return {} + # Run the job! job_start = time.perf_counter() pp = self.ppmngr.getPipeline(source_name)
--- a/piecrust/workerpool.py Tue Nov 28 21:27:57 2017 -0800 +++ b/piecrust/workerpool.py Tue Nov 28 21:28:15 2017 -0800 @@ -310,6 +310,10 @@ stats.stepTimerSince('MasterInit', init_start_time) + @property + def pool_size(self): + return len(self._pool) + def queueJobs(self, jobs): if self._closed: if self._error_on_join: