[llvm] [llvm][lit] Add option to run only the failed tests (PR #158043)
Michael Buch via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 12 01:51:43 PDT 2025
https://github.com/Michael137 updated https://github.com/llvm/llvm-project/pull/158043
>From d0775919cc03a2d1dbee739b926996e4f0ad78c5 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Thu, 11 Sep 2025 12:22:45 +0100
Subject: [PATCH 1/7] [llvm][lit] Add option to run only the failed tests
This patch adds a new `--filter-failed` option to `llvm-lit`, which when
set, will only run the tests that have previously failed.
---
llvm/utils/lit/lit/cl_arguments.py | 6 ++++++
llvm/utils/lit/lit/main.py | 3 +++
.../lit/tests/Inputs/ignore-fail/pass.txt | 1 +
llvm/utils/lit/tests/filter-failed.py | 18 ++++++++++++++++++
llvm/utils/lit/tests/ignore-fail.py | 2 ++
5 files changed, 30 insertions(+)
create mode 100644 llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
create mode 100644 llvm/utils/lit/tests/filter-failed.py
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 8238bc42395af..a87715f16df2b 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -302,6 +302,12 @@ def parse_args():
help="Filter out tests with paths matching the given regular expression",
default=os.environ.get("LIT_FILTER_OUT", "^$"),
)
+ selection_group.add_argument(
+ "--filter-failed",
+ dest="filterFailed",
+ help="Only run tests which failed in the previous run.",
+ action="store_true",
+ )
selection_group.add_argument(
"--xfail",
metavar="LIST",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index a585cc0abdd48..6c650724bb33d 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -90,6 +90,9 @@ def main(builtin_params={}):
and not opts.filter_out.search(t.getFullName())
]
+ if opts.filterFailed:
+ selected_tests = [t for t in selected_tests if t.previous_failure]
+
if not selected_tests:
sys.stderr.write(
"error: filter did not match any tests "
diff --git a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
new file mode 100644
index 0000000000000..074f14cf7fc34
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -0,0 +1,18 @@
+# Checks that --filter-failed only runs tests that previously failed.
+
+# RUN: not %{lit} %{inputs}/ignore-fail
+# RUN: not %{lit} --filter-failed %{inputs}/ignore-fail | FileCheck %s
+
+# END.
+
+# CHECK: Testing: 3 of 5 tests
+# CHECK-DAG: FAIL: ignore-fail :: fail.txt
+# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
+# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+
+# CHECK: Testing Time:
+# CHECK: Total Discovered Tests:
+# CHECK-NEXT: Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 494c6e092c906..51196fbae9e5e 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -10,9 +10,11 @@
# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
# CHECK-DAG: XFAIL: ignore-fail :: xfail.txt
# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+# CHECK-DAG: PASS: ignore-fail :: pass.txt
# CHECK: Testing Time:
# CHECK: Total Discovered Tests:
+# CHECK-NEXT: Passed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
>From bdfbb404f4b05fca0575a52f7c8e1ba747c98b86 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Thu, 11 Sep 2025 17:13:25 +0100
Subject: [PATCH 2/7] fixup! docs
---
llvm/docs/CommandGuide/lit.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 15c249d8e6d31..d2ec64cb354da 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -314,6 +314,10 @@ The timing data is stored in the `test_exec_root` in a file named
place of this option, which is especially useful in environments where the
call to ``lit`` is issued indirectly.
+.. option:: --filter-failed
+
+ Run only those tests that previously failed.
+
.. option:: --xfail LIST
Treat those tests whose name is in the semicolon separated list ``LIST`` as
>From 90092411a846889e9abb388a8dbd3a7d22af29cd Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:30:14 +0100
Subject: [PATCH 3/7] fixup! omit terminating '.' in help text
---
llvm/utils/lit/lit/cl_arguments.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index a87715f16df2b..231d180b63cbd 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -305,7 +305,7 @@ def parse_args():
selection_group.add_argument(
"--filter-failed",
dest="filterFailed",
- help="Only run tests which failed in the previous run.",
+ help="Only run tests which failed in the previous run",
action="store_true",
)
selection_group.add_argument(
>From 83b5ba8bbd15b4456b14cc32230f2ecbff6ece88 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:30:30 +0100
Subject: [PATCH 4/7] fixup! expand docs
---
llvm/docs/CommandGuide/lit.rst | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index d2ec64cb354da..0d1019824c3cb 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -316,7 +316,8 @@ The timing data is stored in the `test_exec_root` in a file named
.. option:: --filter-failed
- Run only those tests that previously failed.
+ Run only those tests that previously failed. Tests that have been newly added
+ but not yet run are not included.
.. option:: --xfail LIST
>From 0e59dfca5b658debd424c83e6ecb39a772158176 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:31:07 +0100
Subject: [PATCH 5/7] fixup! fix spacing in test
---
llvm/utils/lit/tests/filter-failed.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
index 074f14cf7fc34..444509dc02b42 100644
--- a/llvm/utils/lit/tests/filter-failed.py
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -10,7 +10,7 @@
# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
-# CHECK: Testing Time:
+# CHECK: Testing Time:
# CHECK: Total Discovered Tests:
# CHECK-NEXT: Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
>From 63d3a548e774fcdbf767a404c305dc0edb96c175 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:48:58 +0100
Subject: [PATCH 6/7] fixup! add 'newly added tests' test case
---
.../lit/tests/Inputs/filter-failed/fail.txt | 1 +
.../lit/tests/Inputs/filter-failed/lit.cfg | 7 +++++++
.../lit/tests/Inputs/filter-failed/pass.txt | 1 +
.../tests/Inputs/filter-failed/unresolved.txt | 0
.../lit/tests/Inputs/filter-failed/xfail.txt | 2 ++
.../lit/tests/Inputs/filter-failed/xpass.txt | 2 ++
llvm/utils/lit/tests/filter-failed.py | 21 ++++++++++++-------
7 files changed, 27 insertions(+), 7 deletions(-)
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
new file mode 100644
index 0000000000000..15eb81a5f5e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
@@ -0,0 +1 @@
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
new file mode 100644
index 0000000000000..5aee4eb132d6e
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt b/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
new file mode 100644
index 0000000000000..6814cda401483
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
@@ -0,0 +1,2 @@
+RUN: false
+XFAIL: *
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
new file mode 100644
index 0000000000000..66b8a6a5a187c
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
@@ -0,0 +1,2 @@
+RUN: true
+XFAIL: *
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
index 444509dc02b42..f0199f57bf5ae 100644
--- a/llvm/utils/lit/tests/filter-failed.py
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -1,18 +1,25 @@
# Checks that --filter-failed only runs tests that previously failed.
-# RUN: not %{lit} %{inputs}/ignore-fail
-# RUN: not %{lit} --filter-failed %{inputs}/ignore-fail | FileCheck %s
+# RUN: not %{lit} %{inputs}/filter-failed
+#
+# RUN: rm -f %{inputs}/filter-failed/new-fail.txt
+# RUN: echo "RUN: false" > %{inputs}/filter-failed/new-fail.txt
+#
+# RUN: rm -f %{inputs}/filter-failed/new-pass.txt
+# RUN: echo "RUN: true" > %{inputs}/filter-failed/new-pass.txt
+#
+# RUN: not %{lit} --filter-failed %{inputs}/filter-failed | FileCheck %s
# END.
-# CHECK: Testing: 3 of 5 tests
-# CHECK-DAG: FAIL: ignore-fail :: fail.txt
-# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
-# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+# CHECK: Testing: 3 of 7 tests
+# CHECK-DAG: FAIL: filter-failed :: fail.txt
+# CHECK-DAG: UNRESOLVED: filter-failed :: unresolved.txt
+# CHECK-DAG: XPASS: filter-failed :: xpass.txt
# CHECK: Testing Time:
# CHECK: Total Discovered Tests:
-# CHECK-NEXT: Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Excluded : 4 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
>From d6d8f2847e7ed524644431feb5acf504588d25a6 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:51:29 +0100
Subject: [PATCH 7/7] fixup! revert changes to ignore-fail
---
llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt | 1 -
llvm/utils/lit/tests/ignore-fail.py | 2 --
2 files changed, 3 deletions(-)
delete mode 100644 llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
diff --git a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
deleted file mode 100644
index 18efe9e49e95b..0000000000000
--- a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
+++ /dev/null
@@ -1 +0,0 @@
-RUN: true
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 51196fbae9e5e..494c6e092c906 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -10,11 +10,9 @@
# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
# CHECK-DAG: XFAIL: ignore-fail :: xfail.txt
# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
-# CHECK-DAG: PASS: ignore-fail :: pass.txt
# CHECK: Testing Time:
# CHECK: Total Discovered Tests:
-# CHECK-NEXT: Passed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
More information about the llvm-commits
mailing list