[llvm] r282452 - [lit] Add a --max-failures option.
Daniel Dunbar via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 26 16:38:23 PDT 2016
Author: ddunbar
Date: Mon Sep 26 18:38:23 2016
New Revision: 282452
URL: http://llvm.org/viewvc/llvm-project?rev=282452&view=rev
Log:
[lit] Add a --max-failures option.
- This is primarily useful as a "fail fast" mode for lit, where it will stop
running tests after the first failure.
- Patch by Max Moiseev.
Added:
llvm/trunk/utils/lit/tests/max-failures.py
Modified:
llvm/trunk/utils/lit/lit/LitConfig.py
llvm/trunk/utils/lit/lit/main.py
llvm/trunk/utils/lit/lit/run.py
Modified: llvm/trunk/utils/lit/lit/LitConfig.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/LitConfig.py?rev=282452&r1=282451&r2=282452&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/LitConfig.py (original)
+++ llvm/trunk/utils/lit/lit/LitConfig.py Mon Sep 26 18:38:23 2016
@@ -23,7 +23,8 @@ class LitConfig(object):
useValgrind, valgrindLeakCheck, valgrindArgs,
noExecute, debug, isWindows,
params, config_prefix = None,
- maxIndividualTestTime = 0):
+ maxIndividualTestTime = 0,
+ maxFailures = None):
# The name of the test runner.
self.progname = progname
# The items to add to the PATH environment variable.
@@ -60,6 +61,7 @@ class LitConfig(object):
self.valgrindArgs.extend(self.valgrindUserArgs)
self.maxIndividualTestTime = maxIndividualTestTime
+ self.maxFailures = maxFailures
@property
def maxIndividualTestTime(self):
Modified: llvm/trunk/utils/lit/lit/main.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/main.py?rev=282452&r1=282451&r2=282452&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/main.py (original)
+++ llvm/trunk/utils/lit/lit/main.py Mon Sep 26 18:38:23 2016
@@ -232,6 +232,9 @@ def main_with_tmp(builtinParameters):
help="Maximum time to spend running a single test (in seconds)."
"0 means no time limit. [Default: 0]",
type=int, default=None)
+ group.add_option("", "--max-failures", dest="maxFailures",
+ help="Stop execution after the given number of failures.",
+ action="store", type=int, default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
@@ -291,6 +294,9 @@ def main_with_tmp(builtinParameters):
else:
opts.numThreads = 1
+ if opts.maxFailures == 0:
+ parser.error("Setting --max-failures to 0 does not have any effect.")
+
inputs = args
# Create the user defined parameters.
@@ -324,7 +330,8 @@ def main_with_tmp(builtinParameters):
isWindows = isWindows,
params = userParams,
config_prefix = opts.configPrefix,
- maxIndividualTestTime = maxIndividualTestTime)
+ maxIndividualTestTime = maxIndividualTestTime,
+ maxFailures = opts.maxFailures)
# Perform test discovery.
run = lit.run.Run(litConfig,
@@ -477,7 +484,8 @@ def main_with_tmp(builtinParameters):
('Expected Failing Tests', lit.Test.XFAIL),
('Timed Out Tests', lit.Test.TIMEOUT)):
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
- (lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
+ (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
+ (lit.Test.UNRESOLVED == code and (opts.maxFailures is not None)):
continue
elts = byCode.get(code)
if not elts:
Modified: llvm/trunk/utils/lit/lit/run.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/run.py?rev=282452&r1=282451&r2=282452&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/run.py (original)
+++ llvm/trunk/utils/lit/lit/run.py Mon Sep 26 18:38:23 2016
@@ -152,6 +152,19 @@ def run_one_tester(run, provider, displa
###
+def handleFailures(provider, consumer, maxFailures):
+ class _Display(object):
+ def __init__(self, display):
+ self.display = display
+ self.maxFailures = maxFailures or object()
+ self.failedCount = 0
+ def update(self, test):
+ self.display.update(test)
+ self.failedCount += (test.result.code == lit.Test.FAIL)
+ if self.failedCount == self.maxFailures:
+ provider.cancel()
+ consumer.display = _Display(consumer.display)
+
class Run(object):
"""
This class represents a concrete, configured testing run.
@@ -232,6 +245,7 @@ class Run(object):
# Create the test provider.
provider = TestProvider(queue_impl, canceled_flag)
+ handleFailures(provider, consumer, self.lit_config.maxFailures)
# Queue the tests outside the main thread because we can't guarantee
# that we can put() all the tests without blocking:
Added: llvm/trunk/utils/lit/tests/max-failures.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/tests/max-failures.py?rev=282452&view=auto
==============================================================================
--- llvm/trunk/utils/lit/tests/max-failures.py (added)
+++ llvm/trunk/utils/lit/tests/max-failures.py Mon Sep 26 18:38:23 2016
@@ -0,0 +1,14 @@
+# Check the behavior of --max-failures option.
+#
+# RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out
+# RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/shtest-shell >> %t.out
+# RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/shtest-shell >> %t.out
+# RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/shtest-shell 2>> %t.out
+# RUN: FileCheck < %t.out %s
+#
+# END.
+
+# CHECK: Failing Tests (3)
+# CHECK: Failing Tests (1)
+# CHECK: Failing Tests (2)
+# CHECK: error: Setting --max-failures to 0 does not have any effect.
More information about the llvm-commits
mailing list