[llvm] c3ef971 - [lit] Improve handling of timeouts and max failures

[Julian Lettner via llvm-commits]

Author: Julian Lettner
Date: 2020-04-02T01:24:02-07:00
New Revision: c3ef971d36b7f7c37a0f5cedb5c64fef7420bfea

URL: https://github.com/llvm/llvm-project/commit/c3ef971d36b7f7c37a0f5cedb5c64fef7420bfea
DIFF: https://github.com/llvm/llvm-project/commit/c3ef971d36b7f7c37a0f5cedb5c64fef7420bfea.diff

LOG: [lit] Improve handling of timeouts and max failures

This work prepares us for the overall goal of clean shutdown on user
keyboard interrupt [Ctrl+C].

Added: 
    

Modified: 
    llvm/utils/lit/lit/main.py
    llvm/utils/lit/lit/run.py

Removed: 
    


################################################################################
diff  --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 8c675c0e5ba4..a329115fafd0 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -189,8 +189,8 @@ def filter_by_shard(tests, run, shards, lit_config):
     return selected_tests
 
 
-def run_tests(tests, lit_config, opts, numTotalTests):
-    display = lit.display.create_display(opts, len(tests), numTotalTests,
+def run_tests(tests, lit_config, opts, discovered_tests):
+    display = lit.display.create_display(opts, len(tests), discovered_tests,
                                          opts.workers)
     def progress_callback(test):
         display.update(test)
@@ -201,12 +201,22 @@ def progress_callback(test):
                       opts.max_failures, opts.timeout)
 
     display.print_header()
+
+    interrupted = False
+    error = None
     try:
         execute_in_tmp_dir(run, lit_config)
-        display.clear(interrupted=False)
     except KeyboardInterrupt:
-        display.clear(interrupted=True)
-        print(' [interrupted by user]')
+        interrupted = True
+        error = '  interrupted by user'
+    except lit.run.MaxFailuresError:
+        error = 'warning: reached maximum number of test failures'
+    except lit.run.TimeoutError:
+        error = 'warning: reached timeout'
+
+    display.clear(interrupted)
+    if error:
+        sys.stderr.write('%s, skipping remaining tests\n' % error)
 
 
 def execute_in_tmp_dir(run, lit_config):

diff  --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py
index d69121195c63..4b91294ea8c2 100644
--- a/llvm/utils/lit/lit/run.py
+++ b/llvm/utils/lit/lit/run.py
@@ -14,6 +14,12 @@ def acquire(self): pass
     def release(self): pass
 
 
+class MaxFailuresError(Exception):
+    pass
+class TimeoutError(Exception):
+    pass
+
+
 class Run(object):
     """A concrete, configured testing run."""
 
@@ -45,8 +51,7 @@ def execute(self):
         computed. Tests which were not actually executed (for any reason) will
         be given an UNRESOLVED result.
         """
-        self.failure_count = 0
-        self.hit_max_failures = False
+        self.failures = 0
 
         # Larger timeouts (one year, positive infinity) don't work on Windows.
         one_week = 7 * 24 * 60 * 60  # days * hours * minutes * seconds
@@ -80,40 +85,37 @@ def _execute(self, deadline):
 
         async_results = [
             pool.apply_async(lit.worker.execute, args=[test],
-                             callback=lambda t, i=idx: self._process_completed(t, i))
-            for idx, test in enumerate(self.tests)]
+                             callback=self._process_completed)
+            for test in self.tests]
         pool.close()
 
-        for ar in async_results:
-            timeout = deadline - time.time()
+        try:
+            self._wait_for(async_results, deadline)
+        except:
+            pool.terminate()
+            raise
+        finally:
+            pool.join()
+
+    def _wait_for(self, async_results, deadline):
+        timeout = deadline - time.time()
+        for idx, ar in enumerate(async_results):
             try:
-                ar.get(timeout)
+                test = ar.get(timeout)
             except multiprocessing.TimeoutError:
-                # TODO(yln): print timeout error
-                pool.terminate()
-                break
-            if self.hit_max_failures:
-                pool.terminate()
-                break
-        pool.join()
-
-    # TODO(yln): as the comment says.. this is racing with the main thread waiting
-    # for results
-    def _process_completed(self, test, idx):
-        # Don't add any more test results after we've hit the maximum failure
-        # count.  Otherwise we're racing with the main thread, which is going
-        # to terminate the process pool soon.
-        if self.hit_max_failures:
+                raise TimeoutError()
+            else:
+                self.tests[idx] = test
+                if test.isFailure():
+                    self.failures += 1
+                    if self.failures == self.max_failures:
+                        raise MaxFailuresError()
+
+    def _process_completed(self, test):
+        # Avoid racing with the main thread, which is going to terminate the
+        # process pool soon.
+        if self.failures == self.max_failures:
             return
-
-        self.tests[idx] = test
-
-        # Use test.isFailure() for correct XFAIL and XPASS handling
-        if test.isFailure():
-            self.failure_count += 1
-            if self.failure_count == self.max_failures:
-                self.hit_max_failures = True
-
         self.progress_callback(test)
 
     # TODO(yln): interferes with progress bar