[PATCH] D17609: [lit] Enqueue tests on a separate thread to not hit limits on parallel queues

[Filipe Cabecinhas via llvm-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rL263731: [lit] Enqueue tests on a separate thread to not hit limits on parallel queues (authored by filcab).

Changed prior to commit:
  http://reviews.llvm.org/D17609?vs=50639&id=50963#toc

Repository:
  rL LLVM

http://reviews.llvm.org/D17609

Files:
  llvm/trunk/utils/lit/lit/run.py

Index: llvm/trunk/utils/lit/lit/run.py
===================================================================
--- llvm/trunk/utils/lit/lit/run.py
+++ llvm/trunk/utils/lit/lit/run.py
@@ -44,11 +44,13 @@
     value = property(_get_value, _set_value)
 
 class TestProvider(object):
-    def __init__(self, tests, num_jobs, queue_impl, canceled_flag):
+    def __init__(self, queue_impl, canceled_flag):
         self.canceled_flag = canceled_flag
 
         # Create a shared queue to provide the test indices.
         self.queue = queue_impl()
+
+    def queue_tests(self, tests, num_jobs):
         for i in range(len(tests)):
             self.queue.put(i)
         for i in range(num_jobs):
@@ -229,7 +231,15 @@
             consumer = ThreadResultsConsumer(display)
 
         # Create the test provider.
-        provider = TestProvider(self.tests, jobs, queue_impl, canceled_flag)
+        provider = TestProvider(queue_impl, canceled_flag)
+
+        # Queue the tests outside the main thread because we can't guarantee
+        # that we can put() all the tests without blocking:
+        # https://docs.python.org/2/library/multiprocessing.html
+        # e.g: On Mac OS X, we will hang if we put 2^15 elements in the queue
+        # without taking any out.
+        queuer = task_impl(target=provider.queue_tests, args=(self.tests, jobs))
+        queuer.start()
 
         # Install a console-control signal handler on Windows.
         if win32api is not None:
@@ -252,6 +262,8 @@
             # Otherwise, execute the tests in parallel
             self._execute_tests_in_parallel(task_impl, provider, consumer, jobs)
 
+        queuer.join()
+
         # Cancel the timeout handler.
         if max_time is not None:
             timeout_timer.cancel()


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D17609.50963.patch
Type: text/x-patch
Size: 1764 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20160317/8cfdff47/attachment.bin>