[llvm] r282452 - [lit] Add a --max-failures option.

[Daniel Dunbar via llvm-commits]

Author: ddunbar
Date: Mon Sep 26 18:38:23 2016
New Revision: 282452

URL: http://llvm.org/viewvc/llvm-project?rev=282452&view=rev
Log:
[lit] Add a --max-failures option.

 - This is primarily useful as a "fail fast" mode for lit, where it will stop
   running tests after the first failure.

 - Patch by Max Moiseev.

Added:
    llvm/trunk/utils/lit/tests/max-failures.py
Modified:
    llvm/trunk/utils/lit/lit/LitConfig.py
    llvm/trunk/utils/lit/lit/main.py
    llvm/trunk/utils/lit/lit/run.py

Modified: llvm/trunk/utils/lit/lit/LitConfig.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/LitConfig.py?rev=282452&r1=282451&r2=282452&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/LitConfig.py (original)
+++ llvm/trunk/utils/lit/lit/LitConfig.py Mon Sep 26 18:38:23 2016
@@ -23,7 +23,8 @@ class LitConfig(object):
                  useValgrind, valgrindLeakCheck, valgrindArgs,
                  noExecute, debug, isWindows,
                  params, config_prefix = None,
-                 maxIndividualTestTime = 0):
+                 maxIndividualTestTime = 0,
+                 maxFailures = None):
         # The name of the test runner.
         self.progname = progname
         # The items to add to the PATH environment variable.
@@ -60,6 +61,7 @@ class LitConfig(object):
             self.valgrindArgs.extend(self.valgrindUserArgs)
 
         self.maxIndividualTestTime = maxIndividualTestTime
+        self.maxFailures = maxFailures
 
     @property
     def maxIndividualTestTime(self):

Modified: llvm/trunk/utils/lit/lit/main.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/main.py?rev=282452&r1=282451&r2=282452&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/main.py (original)
+++ llvm/trunk/utils/lit/lit/main.py Mon Sep 26 18:38:23 2016
@@ -232,6 +232,9 @@ def main_with_tmp(builtinParameters):
                      help="Maximum time to spend running a single test (in seconds)."
                      "0 means no time limit. [Default: 0]",
                     type=int, default=None)
+    group.add_option("", "--max-failures", dest="maxFailures",
+                     help="Stop execution after the given number of failures.",
+                     action="store", type=int, default=None)
     parser.add_option_group(group)
 
     group = OptionGroup(parser, "Test Selection")
@@ -291,6 +294,9 @@ def main_with_tmp(builtinParameters):
        else:
                opts.numThreads = 1
 
+    if opts.maxFailures == 0:
+        parser.error("Setting --max-failures to 0 does not have any effect.")
+
     inputs = args
 
     # Create the user defined parameters.
@@ -324,7 +330,8 @@ def main_with_tmp(builtinParameters):
         isWindows = isWindows,
         params = userParams,
         config_prefix = opts.configPrefix,
-        maxIndividualTestTime = maxIndividualTestTime)
+        maxIndividualTestTime = maxIndividualTestTime,
+        maxFailures = opts.maxFailures)
 
     # Perform test discovery.
     run = lit.run.Run(litConfig,
@@ -477,7 +484,8 @@ def main_with_tmp(builtinParameters):
                        ('Expected Failing Tests', lit.Test.XFAIL),
                        ('Timed Out Tests', lit.Test.TIMEOUT)):
         if (lit.Test.XFAIL == code and not opts.show_xfail) or \
-           (lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
+           (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
+           (lit.Test.UNRESOLVED == code and (opts.maxFailures is not None)):
             continue
         elts = byCode.get(code)
         if not elts:

Modified: llvm/trunk/utils/lit/lit/run.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/run.py?rev=282452&r1=282451&r2=282452&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/run.py (original)
+++ llvm/trunk/utils/lit/lit/run.py Mon Sep 26 18:38:23 2016
@@ -152,6 +152,19 @@ def run_one_tester(run, provider, displa
 
 ###
 
+def handleFailures(provider, consumer, maxFailures):
+    class _Display(object):
+        def __init__(self, display):
+            self.display = display
+            self.maxFailures = maxFailures or object()
+            self.failedCount = 0
+        def update(self, test):
+            self.display.update(test)
+            self.failedCount += (test.result.code == lit.Test.FAIL)
+            if self.failedCount == self.maxFailures:
+                provider.cancel()
+    consumer.display = _Display(consumer.display)
+
 class Run(object):
     """
     This class represents a concrete, configured testing run.
@@ -232,6 +245,7 @@ class Run(object):
 
         # Create the test provider.
         provider = TestProvider(queue_impl, canceled_flag)
+        handleFailures(provider, consumer, self.lit_config.maxFailures)
 
         # Queue the tests outside the main thread because we can't guarantee
         # that we can put() all the tests without blocking:

Added: llvm/trunk/utils/lit/tests/max-failures.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/tests/max-failures.py?rev=282452&view=auto
==============================================================================
--- llvm/trunk/utils/lit/tests/max-failures.py (added)
+++ llvm/trunk/utils/lit/tests/max-failures.py Mon Sep 26 18:38:23 2016
@@ -0,0 +1,14 @@
+# Check the behavior of --max-failures option.
+#
+# RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out
+# RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/shtest-shell >> %t.out
+# RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/shtest-shell >> %t.out
+# RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/shtest-shell 2>> %t.out
+# RUN: FileCheck < %t.out %s
+#
+# END.
+
+# CHECK: Failing Tests (3)
+# CHECK: Failing Tests (1)
+# CHECK: Failing Tests (2)
+# CHECK: error: Setting --max-failures to 0 does not have any effect.