[llvm] [llvm][lit] Add option to run only the failed tests (PR #158043)

[Michael Buch via llvm-commits]

https://github.com/Michael137 updated https://github.com/llvm/llvm-project/pull/158043

>From d0775919cc03a2d1dbee739b926996e4f0ad78c5 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Thu, 11 Sep 2025 12:22:45 +0100
Subject: [PATCH 1/2] [llvm][lit] Add option to run only the failed tests

This patch adds a new `--filter-failed` option to `llvm-lit`, which when
set, will only run the tests that have previously failed.
---
 llvm/utils/lit/lit/cl_arguments.py             |  6 ++++++
 llvm/utils/lit/lit/main.py                     |  3 +++
 .../lit/tests/Inputs/ignore-fail/pass.txt      |  1 +
 llvm/utils/lit/tests/filter-failed.py          | 18 ++++++++++++++++++
 llvm/utils/lit/tests/ignore-fail.py            |  2 ++
 5 files changed, 30 insertions(+)
 create mode 100644 llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
 create mode 100644 llvm/utils/lit/tests/filter-failed.py

diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 8238bc42395af..a87715f16df2b 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -302,6 +302,12 @@ def parse_args():
         help="Filter out tests with paths matching the given regular expression",
         default=os.environ.get("LIT_FILTER_OUT", "^$"),
     )
+    selection_group.add_argument(
+        "--filter-failed",
+        dest="filterFailed",
+        help="Only run tests which failed in the previous run.",
+        action="store_true",
+    )
     selection_group.add_argument(
         "--xfail",
         metavar="LIST",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index a585cc0abdd48..6c650724bb33d 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -90,6 +90,9 @@ def main(builtin_params={}):
         and not opts.filter_out.search(t.getFullName())
     ]
 
+    if opts.filterFailed:
+        selected_tests = [t for t in selected_tests if t.previous_failure]
+
     if not selected_tests:
         sys.stderr.write(
             "error: filter did not match any tests "
diff --git a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
new file mode 100644
index 0000000000000..074f14cf7fc34
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -0,0 +1,18 @@
+# Checks that --filter-failed only runs tests that previously failed.
+
+# RUN: not %{lit} %{inputs}/ignore-fail
+# RUN: not %{lit} --filter-failed %{inputs}/ignore-fail | FileCheck %s
+
+# END.
+
+# CHECK: Testing: 3 of 5 tests
+# CHECK-DAG: FAIL: ignore-fail :: fail.txt
+# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
+# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+
+#      CHECK: Testing Time:
+# CHECK: Total Discovered Tests:
+# CHECK-NEXT:   Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 494c6e092c906..51196fbae9e5e 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -10,9 +10,11 @@
 # CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
 # CHECK-DAG: XFAIL: ignore-fail :: xfail.txt
 # CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+# CHECK-DAG: PASS: ignore-fail :: pass.txt
 
 #      CHECK: Testing Time:
 # CHECK: Total Discovered Tests:
+# CHECK-NEXT:   Passed : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Failed : 1 {{\([0-9]*\.[0-9]*%\)}}

>From bdfbb404f4b05fca0575a52f7c8e1ba747c98b86 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Thu, 11 Sep 2025 17:13:25 +0100
Subject: [PATCH 2/2] fixup! docs

---
 llvm/docs/CommandGuide/lit.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 15c249d8e6d31..d2ec64cb354da 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -314,6 +314,10 @@ The timing data is stored in the `test_exec_root` in a file named
   place of this option, which is especially useful in environments where the
   call to ``lit`` is issued indirectly.
 
+.. option:: --filter-failed
+
+  Run only those tests that previously failed.
+
 .. option:: --xfail LIST
 
   Treat those tests whose name is in the semicolon separated list ``LIST`` as