[llvm] cbe42a9 - [lit] Add SKIPPED test result category
Julian Lettner via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 10 15:15:06 PDT 2020
Author: Julian Lettner
Date: 2020-04-10T15:13:30-07:00
New Revision: cbe42a9d5fa76eb56cbe615bd64cff2e6a8a9d65
URL: https://github.com/llvm/llvm-project/commit/cbe42a9d5fa76eb56cbe615bd64cff2e6a8a9d65
DIFF: https://github.com/llvm/llvm-project/commit/cbe42a9d5fa76eb56cbe615bd64cff2e6a8a9d65.diff
LOG: [lit] Add SKIPPED test result category
Track and print the number of skipped tests. Skipped tests are tests
that should have been executed but weren't due to:
* user interrupt [Ctrl+C]
* --max-time (overall lit timeout)
* --max-failures
This is part of a larger effort to ensure that all discovered tests are
properly accounted for.
Add test for overall lit timeout feature (`--max-time` option) to
observe skipped tests. Extend test for `--max-failures` option.
Reviewed By: jdenny
Differential Revision: https://reviews.llvm.org/D77819
Added:
llvm/utils/lit/tests/Inputs/max-time/fast.txt
llvm/utils/lit/tests/Inputs/max-time/lit.cfg
llvm/utils/lit/tests/Inputs/max-time/slow.txt
llvm/utils/lit/tests/max-time.py
Modified:
llvm/utils/lit/lit/Test.py
llvm/utils/lit/lit/main.py
llvm/utils/lit/lit/run.py
llvm/utils/lit/tests/max-failures.py
Removed:
################################################################################
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index c809783dae7e..d4ae528d99d4 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -36,6 +36,7 @@ def __repr__(self):
UNRESOLVED = ResultCode('UNRESOLVED', True)
UNSUPPORTED = ResultCode('UNSUPPORTED', False)
TIMEOUT = ResultCode('TIMEOUT', True)
+SKIPPED = ResultCode('SKIPPED', False)
# Test metric values.
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 73e33f530acd..d0ee1fa8a754 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -89,12 +89,14 @@ def main(builtin_params={}):
run_tests(filtered_tests, lit_config, opts, len(discovered_tests))
elapsed = time.time() - start
- executed_tests = [t for t in filtered_tests if t.result]
+ # TODO(yln): eventually, all functions below should act on discovered_tests
+ executed_tests = [
+ t for t in filtered_tests if t.result.code != lit.Test.SKIPPED]
if opts.time_tests:
print_histogram(executed_tests)
- print_results(executed_tests, elapsed, opts)
+ print_results(filtered_tests, elapsed, opts)
if opts.output_path:
#TODO(yln): pass in discovered_tests
@@ -256,6 +258,7 @@ def print_histogram(tests):
]
all_codes = [
+ (lit.Test.SKIPPED, 'Skipped Tests', 'Skipped'),
(lit.Test.UNSUPPORTED, 'Unsupported Tests', 'Unsupported'),
(lit.Test.PASS, 'Expected Passes', ''),
(lit.Test.FLAKYPASS, 'Passes With Retry', ''),
@@ -277,11 +280,11 @@ def print_results(tests, elapsed, opts):
def print_group(code, label, tests, opts):
if not tests:
return
- if code == lit.Test.PASS:
+ # TODO(yln): FLAKYPASS? Make this more consistent!
+ if code in {lit.Test.SKIPPED, lit.Test.PASS}:
return
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
- (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
- (lit.Test.UNRESOLVED == code and (opts.max_failures is not None)):
+ (lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
return
print('*' * 20)
print('%s Tests (%d):' % (label, len(tests)))
diff --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py
index 212e909dcea1..9f2d712b107d 100644
--- a/llvm/utils/lit/lit/run.py
+++ b/llvm/utils/lit/lit/run.py
@@ -42,7 +42,7 @@ def execute(self):
Upon completion, each test in the run will have its result
computed. Tests which were not actually executed (for any reason) will
- be given an UNRESOLVED result.
+ be marked SKIPPED.
"""
self.failures = 0
@@ -51,12 +51,13 @@ def execute(self):
timeout = self.timeout or one_week
deadline = time.time() + timeout
- self._execute(deadline)
-
- # Mark any tests that weren't run as UNRESOLVED.
- for test in self.tests:
- if test.result is None:
- test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
+ try:
+ self._execute(deadline)
+ finally:
+ skipped = lit.Test.Result(lit.Test.SKIPPED)
+ for test in self.tests:
+ if test.result is None:
+ test.setResult(skipped)
def _execute(self, deadline):
self._increase_process_limit()
diff --git a/llvm/utils/lit/tests/Inputs/max-time/fast.txt b/llvm/utils/lit/tests/Inputs/max-time/fast.txt
new file mode 100644
index 000000000000..18efe9e49e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/max-time/fast.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/max-time/lit.cfg b/llvm/utils/lit/tests/Inputs/max-time/lit.cfg
new file mode 100644
index 000000000000..724adfe998e1
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/max-time/lit.cfg
@@ -0,0 +1,6 @@
+import lit.formats
+config.name = 'lit-time'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/max-time/slow.txt b/llvm/utils/lit/tests/Inputs/max-time/slow.txt
new file mode 100644
index 000000000000..61275048d1e3
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/max-time/slow.txt
@@ -0,0 +1 @@
+RUN: sleep 60
diff --git a/llvm/utils/lit/tests/max-failures.py b/llvm/utils/lit/tests/max-failures.py
index 267d7eec324d..cdfe546c1ec2 100644
--- a/llvm/utils/lit/tests/max-failures.py
+++ b/llvm/utils/lit/tests/max-failures.py
@@ -1,14 +1,23 @@
# Check the behavior of --max-failures option.
#
-# RUN: not %{lit} -j 1 -v %{inputs}/max-failures > %t.out
-# RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/max-failures >> %t.out
-# RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/max-failures >> %t.out
-# RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/max-failures 2>> %t.out
+# RUN: not %{lit} -j 1 %{inputs}/max-failures > %t.out 2>&1
+# RUN: not %{lit} --max-failures=1 -j 1 %{inputs}/max-failures >> %t.out 2>&1
+# RUN: not %{lit} --max-failures=2 -j 1 %{inputs}/max-failures >> %t.out 2>&1
+# RUN: not %{lit} --max-failures=0 -j 1 %{inputs}/max-failures 2>> %t.out
# RUN: FileCheck < %t.out %s
#
# END.
-# CHECK: Failing Tests (35)
-# CHECK: Failing Tests (1)
-# CHECK: Failing Tests (2)
+# CHECK-NOT: reached maximum number of test failures
+# CHECK-NOT: Skipped Tests
+# CHECK: Unexpected Failures: 35
+
+# CHECK: reached maximum number of test failures, skipping remaining tests
+# CHECK: Skipped Tests : 41
+# CHECK: Unexpected Failures: 1
+
+# CHECK: reached maximum number of test failures, skipping remaining tests
+# CHECK: Skipped Tests : 40
+# CHECK: Unexpected Failures: 2
+
# CHECK: error: argument --max-failures: requires positive integer, but found '0'
diff --git a/llvm/utils/lit/tests/max-time.py b/llvm/utils/lit/tests/max-time.py
new file mode 100644
index 000000000000..a4cb0139336e
--- /dev/null
+++ b/llvm/utils/lit/tests/max-time.py
@@ -0,0 +1,7 @@
+# Test overall lit timeout (--max-time).
+#
+# RUN: %{lit} %{inputs}/max-time --max-time=1 2>&1 | FileCheck %s
+
+# CHECK: reached timeout, skipping remaining tests
+# CHECK: Skipped Tests : 1
+# CHECK: Expected Passes: 1
More information about the llvm-commits
mailing list