diff piecrust/workerpool.py @ 979:45ad976712ec

tests: Big push to get the tests to pass again. - Lots of fixes everywhere in the code. - Try to handle debug logging in the multiprocessing worker pool when running in pytest. Not perfect, but usable for now. - Replace all `.md` test files with `.html` since now a auto-format extension always sets the format. - Replace `out` with `outfiles` in most places since now blog archives are added to the bake output and I don't want to add expected outputs for blog archives everywhere.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 29 Oct 2017 22:51:57 -0700
parents 5485a11591ec
children 8adc27285d93
line wrap: on
line diff
--- a/piecrust/workerpool.py	Sun Oct 29 22:46:41 2017 -0700
+++ b/piecrust/workerpool.py	Sun Oct 29 22:51:57 2017 -0700
@@ -88,6 +88,30 @@
             _CRITICAL_WORKER_ERROR, None, False, params.wid, msg))
 
 
+def _pre_parse_pytest_args():
+    # If we are unit-testing, we need to translate our test logging
+    # arguments into something Chef can understand.
+    import argparse
+    parser = argparse.ArgumentParser()
+    # This is adapted from our `conftest.py`.
+    parser.add_argument('--log-debug', action='store_true')
+    parser.add_argument('--log-file')
+    res, _ = parser.parse_known_args(sys.argv[1:])
+
+    chef_args = []
+    if res.log_debug:
+        chef_args.append('--debug')
+    if res.log_file:
+        chef_args += ['--log', res.log_file]
+
+    root_logger = logging.getLogger()
+    while len(root_logger.handlers) > 0:
+        root_logger.removeHandler(root_logger.handlers[0])
+
+    from piecrust.main import _pre_parse_chef_args
+    _pre_parse_chef_args(chef_args)
+
+
 def _real_worker_func_unsafe(params):
     wid = params.wid
 
@@ -95,33 +119,18 @@
     stats.registerTimer('WorkerInit')
     init_start_time = time.perf_counter()
 
-    # If we are unit-testing, we didn't setup all the logging environment
-    # yet, since the executable is `py.test`. We need to translate our
-    # test logging arguments into something Chef can understand.
-    if params.is_unit_testing:
-        import argparse
-        parser = argparse.ArgumentParser()
-        # This is adapted from our `conftest.py`.
-        parser.add_argument('--log-debug', action='store_true')
-        parser.add_argument('--log-file')
-        res, _ = parser.parse_known_args(sys.argv[1:])
-
-        chef_args = []
-        if res.log_debug:
-            chef_args.append('--debug')
-        if res.log_file:
-            chef_args += ['--log', res.log_file]
-
-        from piecrust.main import _pre_parse_chef_args
-        _pre_parse_chef_args(chef_args)
-
     # In a context where `multiprocessing` is using the `spawn` forking model,
     # the new process doesn't inherit anything, so we lost all our logging
     # configuration here. Let's set it up again.
-    elif (hasattr(multiprocessing, 'get_start_method') and
+    if (hasattr(multiprocessing, 'get_start_method') and
             multiprocessing.get_start_method() == 'spawn'):
-        from piecrust.main import _pre_parse_chef_args
-        _pre_parse_chef_args(sys.argv[1:])
+        if not params.is_unit_testing:
+            from piecrust.main import _pre_parse_chef_args
+            _pre_parse_chef_args(sys.argv[1:])
+        else:
+            _pre_parse_pytest_args()
+    elif params.is_unit_testing:
+        _pre_parse_pytest_args()
 
     from piecrust.main import ColoredFormatter
     root_logger = logging.getLogger()
@@ -295,6 +304,11 @@
             self._event.clear()
             for job in jobs:
                 self._quick_put((TASK_JOB, job))
+        else:
+            with self._lock_jobs_left:
+                done = (self._jobs_left == 0)
+            if done:
+                self._event.set()
 
     def wait(self, timeout=None):
         if self._closed:
@@ -308,8 +322,10 @@
     def close(self):
         if self._closed:
             raise Exception("This worker pool has been closed.")
-        if self._jobs_left > 0 or not self._event.is_set():
+        if self._jobs_left > 0:
             raise Exception("A previous job queue has not finished yet.")
+        if not self._event.is_set():
+            raise Exception("A previous job queue hasn't been cleared.")
 
         logger.debug("Closing worker pool...")
         live_workers = list(filter(lambda w: w is not None, self._pool))