[llvm] [llvm][lit] Add option to run only the failed tests (PR #158043)

Michael Buch via llvm-commits llvm-commits at lists.llvm.org
Fri Sep 12 01:51:43 PDT 2025


https://github.com/Michael137 updated https://github.com/llvm/llvm-project/pull/158043

>From d0775919cc03a2d1dbee739b926996e4f0ad78c5 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Thu, 11 Sep 2025 12:22:45 +0100
Subject: [PATCH 1/7] [llvm][lit] Add option to run only the failed tests

This patch adds a new `--filter-failed` option to `llvm-lit`, which when
set, will only run the tests that have previously failed.
---
 llvm/utils/lit/lit/cl_arguments.py             |  6 ++++++
 llvm/utils/lit/lit/main.py                     |  3 +++
 .../lit/tests/Inputs/ignore-fail/pass.txt      |  1 +
 llvm/utils/lit/tests/filter-failed.py          | 18 ++++++++++++++++++
 llvm/utils/lit/tests/ignore-fail.py            |  2 ++
 5 files changed, 30 insertions(+)
 create mode 100644 llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
 create mode 100644 llvm/utils/lit/tests/filter-failed.py

diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 8238bc42395af..a87715f16df2b 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -302,6 +302,12 @@ def parse_args():
         help="Filter out tests with paths matching the given regular expression",
         default=os.environ.get("LIT_FILTER_OUT", "^$"),
     )
+    selection_group.add_argument(
+        "--filter-failed",
+        dest="filterFailed",
+        help="Only run tests which failed in the previous run.",
+        action="store_true",
+    )
     selection_group.add_argument(
         "--xfail",
         metavar="LIST",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index a585cc0abdd48..6c650724bb33d 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -90,6 +90,9 @@ def main(builtin_params={}):
         and not opts.filter_out.search(t.getFullName())
     ]
 
+    if opts.filterFailed:
+        selected_tests = [t for t in selected_tests if t.previous_failure]
+
     if not selected_tests:
         sys.stderr.write(
             "error: filter did not match any tests "
diff --git a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
new file mode 100644
index 0000000000000..074f14cf7fc34
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -0,0 +1,18 @@
+# Checks that --filter-failed only runs tests that previously failed.
+
+# RUN: not %{lit} %{inputs}/ignore-fail
+# RUN: not %{lit} --filter-failed %{inputs}/ignore-fail | FileCheck %s
+
+# END.
+
+# CHECK: Testing: 3 of 5 tests
+# CHECK-DAG: FAIL: ignore-fail :: fail.txt
+# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
+# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+
+#      CHECK: Testing Time:
+# CHECK: Total Discovered Tests:
+# CHECK-NEXT:   Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 494c6e092c906..51196fbae9e5e 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -10,9 +10,11 @@
 # CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
 # CHECK-DAG: XFAIL: ignore-fail :: xfail.txt
 # CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+# CHECK-DAG: PASS: ignore-fail :: pass.txt
 
 #      CHECK: Testing Time:
 # CHECK: Total Discovered Tests:
+# CHECK-NEXT:   Passed : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Failed : 1 {{\([0-9]*\.[0-9]*%\)}}

>From bdfbb404f4b05fca0575a52f7c8e1ba747c98b86 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Thu, 11 Sep 2025 17:13:25 +0100
Subject: [PATCH 2/7] fixup! docs

---
 llvm/docs/CommandGuide/lit.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 15c249d8e6d31..d2ec64cb354da 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -314,6 +314,10 @@ The timing data is stored in the `test_exec_root` in a file named
   place of this option, which is especially useful in environments where the
   call to ``lit`` is issued indirectly.
 
+.. option:: --filter-failed
+
+  Run only those tests that previously failed.
+
 .. option:: --xfail LIST
 
   Treat those tests whose name is in the semicolon separated list ``LIST`` as

>From 90092411a846889e9abb388a8dbd3a7d22af29cd Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:30:14 +0100
Subject: [PATCH 3/7] fixup! omit terminating '.' in help text

---
 llvm/utils/lit/lit/cl_arguments.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index a87715f16df2b..231d180b63cbd 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -305,7 +305,7 @@ def parse_args():
     selection_group.add_argument(
         "--filter-failed",
         dest="filterFailed",
-        help="Only run tests which failed in the previous run.",
+        help="Only run tests which failed in the previous run",
         action="store_true",
     )
     selection_group.add_argument(

>From 83b5ba8bbd15b4456b14cc32230f2ecbff6ece88 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:30:30 +0100
Subject: [PATCH 4/7] fixup! expand docs

---
 llvm/docs/CommandGuide/lit.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index d2ec64cb354da..0d1019824c3cb 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -316,7 +316,8 @@ The timing data is stored in the `test_exec_root` in a file named
 
 .. option:: --filter-failed
 
-  Run only those tests that previously failed.
+  Run only those tests that previously failed. Tests that have been newly added
+  but not yet run are not included.
 
 .. option:: --xfail LIST
 

>From 0e59dfca5b658debd424c83e6ecb39a772158176 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:31:07 +0100
Subject: [PATCH 5/7] fixup! fix spacing in test

---
 llvm/utils/lit/tests/filter-failed.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
index 074f14cf7fc34..444509dc02b42 100644
--- a/llvm/utils/lit/tests/filter-failed.py
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -10,7 +10,7 @@
 # CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
 # CHECK-DAG: XPASS: ignore-fail :: xpass.txt
 
-#      CHECK: Testing Time:
+# CHECK: Testing Time:
 # CHECK: Total Discovered Tests:
 # CHECK-NEXT:   Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}

>From 63d3a548e774fcdbf767a404c305dc0edb96c175 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:48:58 +0100
Subject: [PATCH 6/7] fixup! add 'newly added tests' test case

---
 .../lit/tests/Inputs/filter-failed/fail.txt   |  1 +
 .../lit/tests/Inputs/filter-failed/lit.cfg    |  7 +++++++
 .../lit/tests/Inputs/filter-failed/pass.txt   |  1 +
 .../tests/Inputs/filter-failed/unresolved.txt |  0
 .../lit/tests/Inputs/filter-failed/xfail.txt  |  2 ++
 .../lit/tests/Inputs/filter-failed/xpass.txt  |  2 ++
 llvm/utils/lit/tests/filter-failed.py         | 21 ++++++++++++-------
 7 files changed, 27 insertions(+), 7 deletions(-)
 create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
 create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
 create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
 create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
 create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
 create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt

diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
new file mode 100644
index 0000000000000..15eb81a5f5e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
@@ -0,0 +1 @@
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
new file mode 100644
index 0000000000000..5aee4eb132d6e
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt b/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
new file mode 100644
index 0000000000000..6814cda401483
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
@@ -0,0 +1,2 @@
+RUN: false
+XFAIL: *
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
new file mode 100644
index 0000000000000..66b8a6a5a187c
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
@@ -0,0 +1,2 @@
+RUN: true
+XFAIL: *
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
index 444509dc02b42..f0199f57bf5ae 100644
--- a/llvm/utils/lit/tests/filter-failed.py
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -1,18 +1,25 @@
 # Checks that --filter-failed only runs tests that previously failed.
 
-# RUN: not %{lit} %{inputs}/ignore-fail
-# RUN: not %{lit} --filter-failed %{inputs}/ignore-fail | FileCheck %s
+# RUN: not %{lit} %{inputs}/filter-failed
+#
+# RUN: rm -f %{inputs}/filter-failed/new-fail.txt
+# RUN: echo "RUN: false" > %{inputs}/filter-failed/new-fail.txt
+#
+# RUN: rm -f %{inputs}/filter-failed/new-pass.txt
+# RUN: echo "RUN: true" > %{inputs}/filter-failed/new-pass.txt
+#
+# RUN: not %{lit} --filter-failed %{inputs}/filter-failed | FileCheck %s
 
 # END.
 
-# CHECK: Testing: 3 of 5 tests
-# CHECK-DAG: FAIL: ignore-fail :: fail.txt
-# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
-# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+# CHECK: Testing: 3 of 7 tests
+# CHECK-DAG: FAIL: filter-failed :: fail.txt
+# CHECK-DAG: UNRESOLVED: filter-failed :: unresolved.txt
+# CHECK-DAG: XPASS: filter-failed :: xpass.txt
 
 # CHECK: Testing Time:
 # CHECK: Total Discovered Tests:
-# CHECK-NEXT:   Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Excluded : 4 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}

>From d6d8f2847e7ed524644431feb5acf504588d25a6 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:51:29 +0100
Subject: [PATCH 7/7] fixup! revert changes to ignore-fail

---
 llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt | 1 -
 llvm/utils/lit/tests/ignore-fail.py              | 2 --
 2 files changed, 3 deletions(-)
 delete mode 100644 llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt

diff --git a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
deleted file mode 100644
index 18efe9e49e95b..0000000000000
--- a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
+++ /dev/null
@@ -1 +0,0 @@
-RUN: true
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 51196fbae9e5e..494c6e092c906 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -10,11 +10,9 @@
 # CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
 # CHECK-DAG: XFAIL: ignore-fail :: xfail.txt
 # CHECK-DAG: XPASS: ignore-fail :: xpass.txt
-# CHECK-DAG: PASS: ignore-fail :: pass.txt
 
 #      CHECK: Testing Time:
 # CHECK: Total Discovered Tests:
-# CHECK-NEXT:   Passed : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NEXT:   Failed : 1 {{\([0-9]*\.[0-9]*%\)}}



More information about the llvm-commits mailing list