[llvm] Reapply "[llvm][lit] Add option to run only the failed tests" (PR #171588)
Michael Buch via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 10 02:09:49 PST 2025
https://github.com/Michael137 created https://github.com/llvm/llvm-project/pull/171588
This reverts commit 3847648e84d2ff5194f605a8a9a5c0a5e5174939.
Relands https://github.com/llvm/llvm-project/pull/158043 which got auto-merged on a revision which wasn't approved.
The only addition to the approved version was that we adjust how we set the time for failed tests. We used to just assign it the negative value of the elapsed time. But if the test failed with `0` seconds (which some of the new tests do), we would mark it `-0`. But the check for whether something failed checks for `time < 0`. That messed with the new `--filter-failed` option of this PR. This was only an issue on Windows CI, but presumably can happen on any platform. Happy to do this in a separate PR.
>From d3b07c40c2ebd91b632418354be97ecc5a7c8c6e Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Wed, 10 Dec 2025 10:05:33 +0000
Subject: [PATCH] Reapply "[llvm][lit] Add option to run only the failed tests"
(#171582)
This reverts commit 3847648e84d2ff5194f605a8a9a5c0a5e5174939.
Relands https://github.com/llvm/llvm-project/pull/158043 which got auto-merged on a revision which wasn't approved.
The only addition to the approved version was that we adjust how we set the time for failed tests. We used to just assign it the negative value of the elapsed time. But if the test failed with `0` seconds (which some of the new tests do), we would mark it `-0`. But the check for whether something failed checks for `time < 0`. That messed with the new `--filter-failed` option of this PR. This was only an issue on Windows CI, but presumably can happen on any platform. Happy to do this in a separate PR.
---
llvm/docs/CommandGuide/lit.rst | 5 ++++
llvm/utils/lit/lit/TestTimes.py | 2 +-
llvm/utils/lit/lit/cl_arguments.py | 6 +++++
llvm/utils/lit/lit/main.py | 3 +++
.../lit/tests/Inputs/filter-failed/fail.txt | 1 +
.../lit/tests/Inputs/filter-failed/lit.cfg | 7 ++++++
.../lit/tests/Inputs/filter-failed/pass.txt | 1 +
.../tests/Inputs/filter-failed/unresolved.txt | 0
.../lit/tests/Inputs/filter-failed/xfail.txt | 2 ++
.../lit/tests/Inputs/filter-failed/xpass.txt | 2 ++
llvm/utils/lit/tests/filter-failed-delete.py | 16 +++++++++++++
llvm/utils/lit/tests/filter-failed-rerun.py | 18 +++++++++++++++
llvm/utils/lit/tests/filter-failed.py | 23 +++++++++++++++++++
13 files changed, 85 insertions(+), 1 deletion(-)
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
create mode 100644 llvm/utils/lit/tests/filter-failed-delete.py
create mode 100644 llvm/utils/lit/tests/filter-failed-rerun.py
create mode 100644 llvm/utils/lit/tests/filter-failed.py
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index bbc149740494a..bbeafd2b9234c 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -314,6 +314,11 @@ The timing data is stored in the `test_exec_root` in a file named
place of this option, which is especially useful in environments where the
call to ``lit`` is issued indirectly.
+.. option:: --filter-failed
+
+ Run only those tests that previously failed. Tests that have been newly added
+ but not yet run are not included.
+
.. option:: --xfail LIST
Treat those tests whose name is in the semicolon separated list ``LIST`` as
diff --git a/llvm/utils/lit/lit/TestTimes.py b/llvm/utils/lit/lit/TestTimes.py
index a2c0e0527b84b..0c18e3e015828 100644
--- a/llvm/utils/lit/lit/TestTimes.py
+++ b/llvm/utils/lit/lit/TestTimes.py
@@ -22,7 +22,7 @@ def record_test_times(tests, lit_config):
continue
if not t.suite.exec_root in times_by_suite:
times_by_suite[t.suite.exec_root] = read_test_times(t.suite)
- time = -t.result.elapsed if t.isFailure() else t.result.elapsed
+ time = min(-t.result.elapsed, -1.0e-6) if t.isFailure() else t.result.elapsed
# The "path" here is only used as a key into a dictionary. It is never
# used as an actual path to a filesystem API, therefore we use '/' as
# the canonical separator so that Unix and Windows machines can share
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 5c2ff4e70a3aa..ee05847e2c765 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -423,6 +423,12 @@ def parse_args():
help="Filter out tests with paths matching the given regular expression",
default=os.environ.get("LIT_FILTER_OUT", "^$"),
)
+ selection_group.add_argument(
+ "--filter-failed",
+ dest="filterFailed",
+ help="Only run tests which failed in the previous run",
+ action="store_true",
+ )
selection_group.add_argument(
"--xfail",
metavar="LIST",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 07e809b168dc2..77b23bf560c6e 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -90,6 +90,9 @@ def main(builtin_params={}):
and not opts.filter_out.search(t.getFullName())
]
+ if opts.filterFailed:
+ selected_tests = [t for t in selected_tests if t.previous_failure]
+
if not selected_tests:
sys.stderr.write(
"error: filter did not match any tests "
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
new file mode 100644
index 0000000000000..15eb81a5f5e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
@@ -0,0 +1 @@
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
new file mode 100644
index 0000000000000..5aee4eb132d6e
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt b/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
new file mode 100644
index 0000000000000..6814cda401483
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
@@ -0,0 +1,2 @@
+RUN: false
+XFAIL: *
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
new file mode 100644
index 0000000000000..66b8a6a5a187c
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
@@ -0,0 +1,2 @@
+RUN: true
+XFAIL: *
diff --git a/llvm/utils/lit/tests/filter-failed-delete.py b/llvm/utils/lit/tests/filter-failed-delete.py
new file mode 100644
index 0000000000000..68bb840425ea3
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed-delete.py
@@ -0,0 +1,16 @@
+# Shows behaviour when a previously failed test was deleted
+# before running with --filter-failed.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t
+#
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-FIRST
+#
+# RUN: rm %t%{fs-sep}fail.txt
+# RUN: not %{lit} --filter-failed %t | FileCheck %s --check-prefix=CHECK-SECOND
+
+# CHECK-FIRST: Testing: 5 tests
+# CHECK-FIRST: FAIL: filter-failed :: fail.txt
+
+# CHECK-SECOND: Testing: 2 of 4 tests
+# CHECK-SECOND-NOT: filter-failed :: fail.txt
diff --git a/llvm/utils/lit/tests/filter-failed-rerun.py b/llvm/utils/lit/tests/filter-failed-rerun.py
new file mode 100644
index 0000000000000..7db12c26a02ff
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed-rerun.py
@@ -0,0 +1,18 @@
+# Checks that --filter-failed won't re-run tests that have passed
+# since the last time --filter-failed was run.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t
+#
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-FIRST
+#
+# RUN: cp %t%{fs-sep}pass.txt %t%{fs-sep}fail.txt
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-SECOND
+# RUN: not %{lit} --filter-failed %t | FileCheck %s --check-prefix=CHECK-THIRD
+
+# CHECK-FIRST: FAIL: filter-failed :: fail.txt
+
+# CHECK-SECOND: PASS: filter-failed :: fail.txt
+
+# CHECK-THIRD: Testing: 2 of 5 tests
+# CHECK-THIRD-NOT: filter-failed :: fail.txt
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
new file mode 100644
index 0000000000000..3c1c6fe010077
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -0,0 +1,23 @@
+# Checks that --filter-failed only runs tests that previously failed.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t
+#
+# RUN: not %{lit} %t
+#
+# RUN: echo "RUN: false" > %t%{fs-sep}new-fail.txt
+# RUN: echo "RUN: true" > %t%{fs-sep}new-pass.txt
+#
+# RUN: not %{lit} --filter-failed %t | FileCheck %s
+
+# CHECK: Testing: 3 of 7 tests
+# CHECK-DAG: FAIL: filter-failed :: fail.txt
+# CHECK-DAG: UNRESOLVED: filter-failed :: unresolved.txt
+# CHECK-DAG: XPASS: filter-failed :: xpass.txt
+
+# CHECK: Testing Time:
+# CHECK: Total Discovered Tests:
+# CHECK-NEXT: Excluded : 4 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
More information about the llvm-commits
mailing list