[PATCH] D31620: Always use the multiprocess module

[Rafael Ávila de Espíndola via Phabricator via llvm-commits]

rafael created this revision.
Herald added a reviewer: modocache.

This seems to work on freebsd these days. If someone could test on openbsd that would be awesome.


https://reviews.llvm.org/D31620

Files:
  llvm/utils/lit/lit/main.py
  llvm/utils/lit/lit/run.py


Index: llvm/utils/lit/lit/run.py
===================================================================
--- llvm/utils/lit/lit/run.py
+++ llvm/utils/lit/lit/run.py
@@ -12,11 +12,7 @@
 except ImportError:
     win32api = None
 
-try:
-    import multiprocessing
-except ImportError:
-    multiprocessing = None
-
+import multiprocessing
 import lit.Test
 
 ###
@@ -211,8 +207,7 @@
 
         test.setResult(result)
 
-    def execute_tests(self, display, jobs, max_time=None,
-                      use_processes=False):
+    def execute_tests(self, display, jobs, max_time=None):
         """
         execute_tests(display, jobs, [max_time])
 
@@ -234,27 +229,11 @@
         be given an UNRESOLVED result.
         """
 
-        # Choose the appropriate parallel execution implementation.
-        consumer = None
-        if jobs != 1 and use_processes and multiprocessing:
-            try:
-                task_impl = multiprocessing.Process
-                queue_impl = multiprocessing.Queue
-                sem_impl = multiprocessing.Semaphore
-                canceled_flag =  multiprocessing.Value('i', 0)
-                consumer = MultiprocessResultsConsumer(self, display, jobs)
-            except:
-                # multiprocessing fails to initialize with certain OpenBSD and
-                # FreeBSD Python versions: http://bugs.python.org/issue3770
-                # Unfortunately the error raised also varies by platform.
-                self.lit_config.note('failed to initialize multiprocessing')
-                consumer = None
-        if not consumer:
-            task_impl = threading.Thread
-            queue_impl = queue.Queue
-            sem_impl = threading.Semaphore
-            canceled_flag = LockedValue(0)
-            consumer = ThreadResultsConsumer(display)
+        task_impl = multiprocessing.Process
+        queue_impl = multiprocessing.Queue
+        sem_impl = multiprocessing.Semaphore
+        canceled_flag =  multiprocessing.Value('i', 0)
+        consumer = MultiprocessResultsConsumer(self, display, jobs)
 
         self.parallelism_semaphores = {k: sem_impl(v)
             for k, v in self.lit_config.parallelism_groups.items()}
Index: llvm/utils/lit/lit/main.py
===================================================================
--- llvm/utils/lit/lit/main.py
+++ llvm/utils/lit/lit/main.py
@@ -278,12 +278,6 @@
     debug_group.add_argument("--show-tests", dest="showTests",
                       help="Show all discovered tests",
                       action="store_true", default=False)
-    debug_group.add_argument("--use-processes", dest="useProcesses",
-                      help="Run tests in parallel with processes (not threads)",
-                      action="store_true", default=True)
-    debug_group.add_argument("--use-threads", dest="useProcesses",
-                      help="Run tests in parallel with threads (not processes)",
-                      action="store_false", default=True)
 
     opts = parser.parse_args()
     args = opts.test_paths
@@ -480,8 +474,7 @@
     startTime = time.time()
     display = TestingProgressDisplay(opts, len(run.tests), progressBar)
     try:
-        run.execute_tests(display, opts.numThreads, opts.maxTime,
-                          opts.useProcesses)
+        run.execute_tests(display, opts.numThreads, opts.maxTime)
     except KeyboardInterrupt:
         sys.exit(2)
     display.finish()


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D31620.93924.patch
Type: text/x-patch
Size: 3423 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20170403/12cd84a9/attachment-0001.bin>