[llvm] 04ce013 - Reapply "[llvm][lit] Add option to run only the failed tests" (#171588)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 12 02:29:11 PST 2025
Author: Michael Buch
Date: 2025-12-12T10:29:07Z
New Revision: 04ce013d7c87c361b14878b0038d41d37290151f
URL: https://github.com/llvm/llvm-project/commit/04ce013d7c87c361b14878b0038d41d37290151f
DIFF: https://github.com/llvm/llvm-project/commit/04ce013d7c87c361b14878b0038d41d37290151f.diff
LOG: Reapply "[llvm][lit] Add option to run only the failed tests" (#171588)
This reverts commit 3847648e84d2ff5194f605a8a9a5c0a5e5174939.
Relands https://github.com/llvm/llvm-project/pull/158043 which got
auto-merged on a revision which wasn't approved.
The only addition to the approved version was that we adjust how we set
the time for failed tests. We used to just assign it the negative value
of the elapsed time. But if the test failed with `0` seconds (which some
of the new tests do), we would mark it `-0`. But the check for whether
something failed checks for `time < 0`. That messed with the new
`--filter-failed` option of this PR. This was only an issue on Windows
CI, but presumably can happen on any platform. Happy to do this in a
separate PR.
---- Original PR
This patch adds a new --filter-failed option to llvm-lit, which when
set, will only run the tests that have previously failed.
Added:
llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
llvm/utils/lit/tests/filter-failed-delete.py
llvm/utils/lit/tests/filter-failed-rerun.py
llvm/utils/lit/tests/filter-failed.py
Modified:
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/TestTimes.py
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/main.py
Removed:
################################################################################
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index bbc149740494a..bbeafd2b9234c 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -314,6 +314,11 @@ The timing data is stored in the `test_exec_root` in a file named
place of this option, which is especially useful in environments where the
call to ``lit`` is issued indirectly.
+.. option:: --filter-failed
+
+ Run only those tests that previously failed. Tests that have been newly added
+ but not yet run are not included.
+
.. option:: --xfail LIST
Treat those tests whose name is in the semicolon separated list ``LIST`` as
diff --git a/llvm/utils/lit/lit/TestTimes.py b/llvm/utils/lit/lit/TestTimes.py
index a2c0e0527b84b..7ec1a1b97b717 100644
--- a/llvm/utils/lit/lit/TestTimes.py
+++ b/llvm/utils/lit/lit/TestTimes.py
@@ -22,7 +22,12 @@ def record_test_times(tests, lit_config):
continue
if not t.suite.exec_root in times_by_suite:
times_by_suite[t.suite.exec_root] = read_test_times(t.suite)
- time = -t.result.elapsed if t.isFailure() else t.result.elapsed
+
+ # Mark the elapsed time for failed tests as negative so LIT can distingiush failed from
+ # successful test runs just based on the time value. For this heuristic to work for tests
+ # whose elapsed time is '0', we set it to a small negative constant.
+ time = min(-t.result.elapsed, -1.0e-6) if t.isFailure() else t.result.elapsed
+
# The "path" here is only used as a key into a dictionary. It is never
# used as an actual path to a filesystem API, therefore we use '/' as
# the canonical separator so that Unix and Windows machines can share
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 5c2ff4e70a3aa..ee05847e2c765 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -423,6 +423,12 @@ def parse_args():
help="Filter out tests with paths matching the given regular expression",
default=os.environ.get("LIT_FILTER_OUT", "^$"),
)
+ selection_group.add_argument(
+ "--filter-failed",
+ dest="filterFailed",
+ help="Only run tests which failed in the previous run",
+ action="store_true",
+ )
selection_group.add_argument(
"--xfail",
metavar="LIST",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 07e809b168dc2..77b23bf560c6e 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -90,6 +90,9 @@ def main(builtin_params={}):
and not opts.filter_out.search(t.getFullName())
]
+ if opts.filterFailed:
+ selected_tests = [t for t in selected_tests if t.previous_failure]
+
if not selected_tests:
sys.stderr.write(
"error: filter did not match any tests "
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
new file mode 100644
index 0000000000000..15eb81a5f5e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
@@ -0,0 +1 @@
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
new file mode 100644
index 0000000000000..5aee4eb132d6e
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt b/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
new file mode 100644
index 0000000000000..6814cda401483
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
@@ -0,0 +1,2 @@
+RUN: false
+XFAIL: *
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
new file mode 100644
index 0000000000000..66b8a6a5a187c
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
@@ -0,0 +1,2 @@
+RUN: true
+XFAIL: *
diff --git a/llvm/utils/lit/tests/filter-failed-delete.py b/llvm/utils/lit/tests/filter-failed-delete.py
new file mode 100644
index 0000000000000..68bb840425ea3
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed-delete.py
@@ -0,0 +1,16 @@
+# Shows behaviour when a previously failed test was deleted
+# before running with --filter-failed.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t
+#
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-FIRST
+#
+# RUN: rm %t%{fs-sep}fail.txt
+# RUN: not %{lit} --filter-failed %t | FileCheck %s --check-prefix=CHECK-SECOND
+
+# CHECK-FIRST: Testing: 5 tests
+# CHECK-FIRST: FAIL: filter-failed :: fail.txt
+
+# CHECK-SECOND: Testing: 2 of 4 tests
+# CHECK-SECOND-NOT: filter-failed :: fail.txt
diff --git a/llvm/utils/lit/tests/filter-failed-rerun.py b/llvm/utils/lit/tests/filter-failed-rerun.py
new file mode 100644
index 0000000000000..7db12c26a02ff
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed-rerun.py
@@ -0,0 +1,18 @@
+# Checks that --filter-failed won't re-run tests that have passed
+# since the last time --filter-failed was run.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t
+#
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-FIRST
+#
+# RUN: cp %t%{fs-sep}pass.txt %t%{fs-sep}fail.txt
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-SECOND
+# RUN: not %{lit} --filter-failed %t | FileCheck %s --check-prefix=CHECK-THIRD
+
+# CHECK-FIRST: FAIL: filter-failed :: fail.txt
+
+# CHECK-SECOND: PASS: filter-failed :: fail.txt
+
+# CHECK-THIRD: Testing: 2 of 5 tests
+# CHECK-THIRD-NOT: filter-failed :: fail.txt
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
new file mode 100644
index 0000000000000..3c1c6fe010077
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -0,0 +1,23 @@
+# Checks that --filter-failed only runs tests that previously failed.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t
+#
+# RUN: not %{lit} %t
+#
+# RUN: echo "RUN: false" > %t%{fs-sep}new-fail.txt
+# RUN: echo "RUN: true" > %t%{fs-sep}new-pass.txt
+#
+# RUN: not %{lit} --filter-failed %t | FileCheck %s
+
+# CHECK: Testing: 3 of 7 tests
+# CHECK-DAG: FAIL: filter-failed :: fail.txt
+# CHECK-DAG: UNRESOLVED: filter-failed :: unresolved.txt
+# CHECK-DAG: XPASS: filter-failed :: xpass.txt
+
+# CHECK: Testing Time:
+# CHECK: Total Discovered Tests:
+# CHECK-NEXT: Excluded : 4 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
More information about the llvm-commits
mailing list