[llvm] r375474 - [lit] Move increase_process_limit to ParallelRun

[Julian Lettner via llvm-commits]

Author: yln
Date: Mon Oct 21 18:13:30 2019
New Revision: 375474

URL: http://llvm.org/viewvc/llvm-project?rev=375474&view=rev
Log:
[lit] Move increase_process_limit to ParallelRun

Increasing the process limit only makes sense when we use multiple
processes.

Modified:
    llvm/trunk/utils/lit/lit/main.py
    llvm/trunk/utils/lit/lit/run.py

Modified: llvm/trunk/utils/lit/lit/main.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/main.py?rev=375474&r1=375473&r2=375474&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/main.py (original)
+++ llvm/trunk/utils/lit/lit/main.py Mon Oct 21 18:13:30 2019
@@ -182,32 +182,7 @@ def update_incremental_cache(test):
     fname = test.getFilePath()
     os.utime(fname, None)
 
-def increase_process_limit(litConfig, opts):
-    # Because some tests use threads internally, and at least on Linux each
-    # of these threads counts toward the current process limit, try to
-    # raise the (soft) process limit so that tests don't fail due to
-    # resource exhaustion.
-    try:
-        cpus = lit.util.detectCPUs()
-        desired_limit = opts.numWorkers * cpus * 2 # the 2 is a safety factor
-
-        # Import the resource module here inside this try block because it
-        # will likely fail on Windows.
-        import resource
-
-        max_procs_soft, max_procs_hard = resource.getrlimit(resource.RLIMIT_NPROC)
-        desired_limit = min(desired_limit, max_procs_hard)
-
-        if max_procs_soft < desired_limit:
-            resource.setrlimit(resource.RLIMIT_NPROC, (desired_limit, max_procs_hard))
-            litConfig.note('raised the process limit from %d to %d' % \
-                               (max_procs_soft, desired_limit))
-    except:
-        pass
-
 def run_tests(tests, litConfig, opts, numTotalTests):
-    increase_process_limit(litConfig, opts)
-
     display = lit.display.create_display(opts, len(tests), numTotalTests,
                                          opts.numWorkers)
     def progress_callback(test):

Modified: llvm/trunk/utils/lit/lit/run.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/run.py?rev=375474&r1=375473&r2=375474&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/run.py (original)
+++ llvm/trunk/utils/lit/lit/run.py Mon Oct 21 18:13:30 2019
@@ -109,6 +109,8 @@ class ParallelRun(Run):
             multiprocessing.BoundedSemaphore(v) for k, v in
             self.lit_config.parallelism_groups.items()}
 
+        self._increase_process_limit()
+
         # Start a process pool. Copy over the data shared between all test runs.
         # FIXME: Find a way to capture the worker process stderr. If the user
         # interrupts the workers before we make it into our task callback, they
@@ -144,3 +146,25 @@ class ParallelRun(Run):
             if self.hit_max_failures:
                 pool.terminate()
                 break
+
+    # Some tests use threads internally, and at least on Linux each of these
+    # threads counts toward the current process limit. Try to raise the (soft)
+    # process limit so that tests don't fail due to resource exhaustion.
+    def _increase_process_limit(self):
+        ncpus = lit.util.detectCPUs()
+        desired_limit = self.workers * ncpus * 2 # the 2 is a safety factor
+
+        # Importing the resource module will likely fail on Windows.
+        try:
+            import resource
+            NPROC = resource.RLIMIT_NPROC
+
+            soft_limit, hard_limit = resource.getrlimit(NPROC)
+            desired_limit = min(desired_limit, hard_limit)
+
+            if soft_limit < desired_limit:
+                resource.setrlimit(NPROC, (desired_limit, hard_limit))
+                self.lit_config.note('Raised process limit from %d to %d' % \
+                                        (soft_limit, desired_limit))
+        except Exception as ex:
+            self.lit_config.warning('Failed to raise process limit: %s' % ex)