diff piecrust/workerpool.py @ 462:04abc97dd3b6

bake: Add CLI argument to specify job batch size.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 11 Jul 2015 18:49:50 -0700
parents b015e38d4ee1
children 22a230d99621
line wrap: on
line diff
--- a/piecrust/workerpool.py	Sat Jul 11 17:51:56 2015 -0700
+++ b/piecrust/workerpool.py	Sat Jul 11 18:49:50 2015 -0700
@@ -115,7 +115,8 @@
 
 
 class WorkerPool(object):
-    def __init__(self, worker_class, worker_count=None, initargs=(),
+    def __init__(self, worker_class, initargs=(),
+                 worker_count=None, batch_size=None,
                  wrap_exception=False):
         worker_count = worker_count or os.cpu_count() or 1
 
@@ -131,6 +132,7 @@
             self._quick_put = self._task_queue._writer.send
             self._quick_get = self._result_queue._reader.recv
 
+        self._batch_size = batch_size
         self._callback = None
         self._error_callback = None
         self._listener = None
@@ -189,6 +191,8 @@
         self._listener = res
 
         if chunk_size is None:
+            chunk_size = self._batch_size
+        if chunk_size is None:
             chunk_size = max(1, job_count // 50)
             logger.debug("Using chunk size of %d" % chunk_size)