Mercurial > piecrust2
changeset 462:04abc97dd3b6
bake: Add CLI argument to specify job batch size.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Sat, 11 Jul 2015 18:49:50 -0700 |
| parents | b015e38d4ee1 |
| children | aefd2714d205 |
| files | piecrust/baking/baker.py piecrust/commands/builtin/baking.py piecrust/workerpool.py |
| diffstat | 3 files changed, 13 insertions(+), 1 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/baking/baker.py Sat Jul 11 17:51:56 2015 -0700 +++ b/piecrust/baking/baker.py Sat Jul 11 18:49:50 2015 -0700 @@ -541,6 +541,7 @@ from piecrust.baking.worker import BakeWorkerContext, BakeWorker worker_count = self.app.config.get('baker/workers') + batch_size = self.app.config.get('baker/batch_size') ctx = BakeWorkerContext( self.app.root_dir, self.app.cache.base_dir, self.out_dir, @@ -548,6 +549,7 @@ force=self.force, debug=self.app.debug) pool = WorkerPool( worker_count=worker_count, + batch_size=batch_size, worker_class=BakeWorker, initargs=(ctx,)) return pool
--- a/piecrust/commands/builtin/baking.py Sat Jul 11 17:51:56 2015 -0700 +++ b/piecrust/commands/builtin/baking.py Sat Jul 11 18:49:50 2015 -0700 @@ -42,6 +42,10 @@ help="The number of worker processes to spawn.", type=int, default=-1) parser.add_argument( + '--batch-size', + help="The number of jobs per batch.", + type=int, default=-1) + parser.add_argument( '--assets-only', help="Only bake the assets (don't bake the web pages).", action='store_true') @@ -90,6 +94,8 @@ def _bakeSources(self, ctx, out_dir): if ctx.args.workers > 0: ctx.app.config.set('baker/workers', ctx.args.workers) + if ctx.args.batch_size > 0: + ctx.app.config.set('baker/batch_size', ctx.args.batch_size) baker = Baker( ctx.app, out_dir, force=ctx.args.force)
--- a/piecrust/workerpool.py Sat Jul 11 17:51:56 2015 -0700 +++ b/piecrust/workerpool.py Sat Jul 11 18:49:50 2015 -0700 @@ -115,7 +115,8 @@ class WorkerPool(object): - def __init__(self, worker_class, worker_count=None, initargs=(), + def __init__(self, worker_class, initargs=(), + worker_count=None, batch_size=None, wrap_exception=False): worker_count = worker_count or os.cpu_count() or 1 @@ -131,6 +132,7 @@ self._quick_put = self._task_queue._writer.send self._quick_get = self._result_queue._reader.recv + self._batch_size = batch_size self._callback = None self._error_callback = None self._listener = None @@ -189,6 +191,8 @@ self._listener = res if chunk_size is None: + chunk_size = self._batch_size + if chunk_size is None: chunk_size = max(1, job_count // 50) logger.debug("Using chunk size of %d" % chunk_size)