Mercurial > piecrust2
changeset 702:c62d83e17abf
bake: Some more optimizations.
Write the output files in a background thread to get away from the GIL.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Sat, 16 Apr 2016 22:50:07 -0700 |
| parents | 066d6156525c |
| children | dab26ab3d533 |
| files | piecrust/baking/single.py piecrust/baking/worker.py piecrust/workerpool.py |
| diffstat | 3 files changed, 46 insertions(+), 5 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/baking/single.py Sat Apr 16 22:48:57 2016 -0700 +++ b/piecrust/baking/single.py Sat Apr 16 22:50:07 2016 -0700 @@ -1,6 +1,8 @@ import os.path +import queue import shutil import logging +import threading import urllib.parse from piecrust import ASSET_DIR_SUFFIX from piecrust.baking.records import SubPageBakeInfo @@ -17,6 +19,24 @@ pass +def _text_writer(q): + while True: + item = q.get() + if item is not None: + out_path, txt = item + out_dir = os.path.dirname(out_path) + _ensure_dir_exists(out_dir) + + with open(out_path, 'w', encoding='utf8') as fp: + fp.write(txt) + + q.task_done() + else: + # Sentinel object, terminate the thread. + q.task_done() + break + + class PageBaker(object): def __init__(self, app, out_dir, force=False, copy_assets=True): self.app = app @@ -25,6 +45,16 @@ self.copy_assets = copy_assets self.site_root = app.config.get('site/root') self.pretty_urls = app.config.get('site/pretty_urls') + self._writer_queue = queue.Queue() + self._writer = threading.Thread( + name='PageSerializer', + target=_text_writer, + args=(self._writer_queue,)) + self._writer.start() + + def shutdown(self): + self._writer_queue.put_nowait(None) + self._writer.join() def getOutputPath(self, uri): uri_root, uri_path = split_uri(self.app, uri) @@ -162,11 +192,7 @@ rp = render_page(ctx) with self.app.env.timerScope("PageSerialize"): - out_dir = os.path.dirname(out_path) - _ensure_dir_exists(out_dir) - - with open(out_path, 'w', encoding='utf8') as fp: - fp.write(rp.content) + self._writer_queue.put_nowait((out_path, rp.content)) return rp
--- a/piecrust/baking/worker.py Sat Apr 16 22:48:57 2016 -0700 +++ b/piecrust/baking/worker.py Sat Apr 16 22:50:07 2016 -0700 @@ -81,6 +81,10 @@ 'type': 'stats', 'data': data} + def shutdown(self): + for jh in self.job_handlers.values(): + jh.shutdown() + JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE = range(0, 3) @@ -96,6 +100,9 @@ def handleJob(self, job): raise NotImplementedError() + def shutdown(self): + pass + def _get_errors(ex): errors = [] @@ -183,6 +190,9 @@ super(BakeJobHandler, self).__init__(ctx) self.page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force) + def shutdown(self): + self.page_baker.shutdown() + def handleJob(self, job): # Actually bake the page and all its sub-pages to the output folder. fac = load_factory(self.app, job['factory_info'])
--- a/piecrust/workerpool.py Sat Apr 16 22:48:57 2016 -0700 +++ b/piecrust/workerpool.py Sat Apr 16 22:50:07 2016 -0700 @@ -26,6 +26,9 @@ def getReport(self, pool_reports): return None + def shutdown(self): + pass + TASK_JOB = 0 TASK_BATCH = 1 @@ -149,6 +152,8 @@ reader_thread.join() writer_thread.join() + w.shutdown() + logger.debug("Worker %d completed %d tasks." % (wid, completed))