[llvm] [llvm][lit] Add option to run only the failed tests (PR #158043)
Michael Buch via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 15 10:06:01 PDT 2025
https://github.com/Michael137 updated https://github.com/llvm/llvm-project/pull/158043
>From d0775919cc03a2d1dbee739b926996e4f0ad78c5 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Thu, 11 Sep 2025 12:22:45 +0100
Subject: [PATCH 1/8] [llvm][lit] Add option to run only the failed tests
This patch adds a new `--filter-failed` option to `llvm-lit`, which when
set, will only run the tests that have previously failed.
---
llvm/utils/lit/lit/cl_arguments.py | 6 ++++++
llvm/utils/lit/lit/main.py | 3 +++
.../lit/tests/Inputs/ignore-fail/pass.txt | 1 +
llvm/utils/lit/tests/filter-failed.py | 18 ++++++++++++++++++
llvm/utils/lit/tests/ignore-fail.py | 2 ++
5 files changed, 30 insertions(+)
create mode 100644 llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
create mode 100644 llvm/utils/lit/tests/filter-failed.py
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 8238bc42395af..a87715f16df2b 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -302,6 +302,12 @@ def parse_args():
help="Filter out tests with paths matching the given regular expression",
default=os.environ.get("LIT_FILTER_OUT", "^$"),
)
+ selection_group.add_argument(
+ "--filter-failed",
+ dest="filterFailed",
+ help="Only run tests which failed in the previous run.",
+ action="store_true",
+ )
selection_group.add_argument(
"--xfail",
metavar="LIST",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index a585cc0abdd48..6c650724bb33d 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -90,6 +90,9 @@ def main(builtin_params={}):
and not opts.filter_out.search(t.getFullName())
]
+ if opts.filterFailed:
+ selected_tests = [t for t in selected_tests if t.previous_failure]
+
if not selected_tests:
sys.stderr.write(
"error: filter did not match any tests "
diff --git a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
new file mode 100644
index 0000000000000..074f14cf7fc34
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -0,0 +1,18 @@
+# Checks that --filter-failed only runs tests that previously failed.
+
+# RUN: not %{lit} %{inputs}/ignore-fail
+# RUN: not %{lit} --filter-failed %{inputs}/ignore-fail | FileCheck %s
+
+# END.
+
+# CHECK: Testing: 3 of 5 tests
+# CHECK-DAG: FAIL: ignore-fail :: fail.txt
+# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
+# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+
+# CHECK: Testing Time:
+# CHECK: Total Discovered Tests:
+# CHECK-NEXT: Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 494c6e092c906..51196fbae9e5e 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -10,9 +10,11 @@
# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
# CHECK-DAG: XFAIL: ignore-fail :: xfail.txt
# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+# CHECK-DAG: PASS: ignore-fail :: pass.txt
# CHECK: Testing Time:
# CHECK: Total Discovered Tests:
+# CHECK-NEXT: Passed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
>From bdfbb404f4b05fca0575a52f7c8e1ba747c98b86 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Thu, 11 Sep 2025 17:13:25 +0100
Subject: [PATCH 2/8] fixup! docs
---
llvm/docs/CommandGuide/lit.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 15c249d8e6d31..d2ec64cb354da 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -314,6 +314,10 @@ The timing data is stored in the `test_exec_root` in a file named
place of this option, which is especially useful in environments where the
call to ``lit`` is issued indirectly.
+.. option:: --filter-failed
+
+ Run only those tests that previously failed.
+
.. option:: --xfail LIST
Treat those tests whose name is in the semicolon separated list ``LIST`` as
>From 90092411a846889e9abb388a8dbd3a7d22af29cd Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:30:14 +0100
Subject: [PATCH 3/8] fixup! omit terminating '.' in help text
---
llvm/utils/lit/lit/cl_arguments.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index a87715f16df2b..231d180b63cbd 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -305,7 +305,7 @@ def parse_args():
selection_group.add_argument(
"--filter-failed",
dest="filterFailed",
- help="Only run tests which failed in the previous run.",
+ help="Only run tests which failed in the previous run",
action="store_true",
)
selection_group.add_argument(
>From 83b5ba8bbd15b4456b14cc32230f2ecbff6ece88 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:30:30 +0100
Subject: [PATCH 4/8] fixup! expand docs
---
llvm/docs/CommandGuide/lit.rst | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index d2ec64cb354da..0d1019824c3cb 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -316,7 +316,8 @@ The timing data is stored in the `test_exec_root` in a file named
.. option:: --filter-failed
- Run only those tests that previously failed.
+ Run only those tests that previously failed. Tests that have been newly added
+ but not yet run are not included.
.. option:: --xfail LIST
>From 0e59dfca5b658debd424c83e6ecb39a772158176 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:31:07 +0100
Subject: [PATCH 5/8] fixup! fix spacing in test
---
llvm/utils/lit/tests/filter-failed.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
index 074f14cf7fc34..444509dc02b42 100644
--- a/llvm/utils/lit/tests/filter-failed.py
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -10,7 +10,7 @@
# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
-# CHECK: Testing Time:
+# CHECK: Testing Time:
# CHECK: Total Discovered Tests:
# CHECK-NEXT: Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
>From 63d3a548e774fcdbf767a404c305dc0edb96c175 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:48:58 +0100
Subject: [PATCH 6/8] fixup! add 'newly added tests' test case
---
.../lit/tests/Inputs/filter-failed/fail.txt | 1 +
.../lit/tests/Inputs/filter-failed/lit.cfg | 7 +++++++
.../lit/tests/Inputs/filter-failed/pass.txt | 1 +
.../tests/Inputs/filter-failed/unresolved.txt | 0
.../lit/tests/Inputs/filter-failed/xfail.txt | 2 ++
.../lit/tests/Inputs/filter-failed/xpass.txt | 2 ++
llvm/utils/lit/tests/filter-failed.py | 21 ++++++++++++-------
7 files changed, 27 insertions(+), 7 deletions(-)
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
new file mode 100644
index 0000000000000..15eb81a5f5e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt
@@ -0,0 +1 @@
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
new file mode 100644
index 0000000000000..5aee4eb132d6e
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt b/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
new file mode 100644
index 0000000000000..6814cda401483
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt
@@ -0,0 +1,2 @@
+RUN: false
+XFAIL: *
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
new file mode 100644
index 0000000000000..66b8a6a5a187c
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt
@@ -0,0 +1,2 @@
+RUN: true
+XFAIL: *
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
index 444509dc02b42..f0199f57bf5ae 100644
--- a/llvm/utils/lit/tests/filter-failed.py
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -1,18 +1,25 @@
# Checks that --filter-failed only runs tests that previously failed.
-# RUN: not %{lit} %{inputs}/ignore-fail
-# RUN: not %{lit} --filter-failed %{inputs}/ignore-fail | FileCheck %s
+# RUN: not %{lit} %{inputs}/filter-failed
+#
+# RUN: rm -f %{inputs}/filter-failed/new-fail.txt
+# RUN: echo "RUN: false" > %{inputs}/filter-failed/new-fail.txt
+#
+# RUN: rm -f %{inputs}/filter-failed/new-pass.txt
+# RUN: echo "RUN: true" > %{inputs}/filter-failed/new-pass.txt
+#
+# RUN: not %{lit} --filter-failed %{inputs}/filter-failed | FileCheck %s
# END.
-# CHECK: Testing: 3 of 5 tests
-# CHECK-DAG: FAIL: ignore-fail :: fail.txt
-# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
-# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
+# CHECK: Testing: 3 of 7 tests
+# CHECK-DAG: FAIL: filter-failed :: fail.txt
+# CHECK-DAG: UNRESOLVED: filter-failed :: unresolved.txt
+# CHECK-DAG: XPASS: filter-failed :: xpass.txt
# CHECK: Testing Time:
# CHECK: Total Discovered Tests:
-# CHECK-NEXT: Excluded : 2 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT: Excluded : 4 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
>From d6d8f2847e7ed524644431feb5acf504588d25a6 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Fri, 12 Sep 2025 09:51:29 +0100
Subject: [PATCH 7/8] fixup! revert changes to ignore-fail
---
llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt | 1 -
llvm/utils/lit/tests/ignore-fail.py | 2 --
2 files changed, 3 deletions(-)
delete mode 100644 llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
diff --git a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt b/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
deleted file mode 100644
index 18efe9e49e95b..0000000000000
--- a/llvm/utils/lit/tests/Inputs/ignore-fail/pass.txt
+++ /dev/null
@@ -1 +0,0 @@
-RUN: true
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 51196fbae9e5e..494c6e092c906 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -10,11 +10,9 @@
# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
# CHECK-DAG: XFAIL: ignore-fail :: xfail.txt
# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
-# CHECK-DAG: PASS: ignore-fail :: pass.txt
# CHECK: Testing Time:
# CHECK: Total Discovered Tests:
-# CHECK-NEXT: Passed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
>From cab413c9c04fb68b41db98ad84131d0b476a511f Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12 at gmail.com>
Date: Mon, 15 Sep 2025 18:05:45 +0100
Subject: [PATCH 8/8] fixup! add more tests
---
.../Inputs/filter-failed-delete/fail.txt | 1 +
.../tests/Inputs/filter-failed-delete/lit.cfg | 7 +++++++
.../Inputs/filter-failed-delete/pass.txt | 1 +
.../filter-failed-delete/unresolved.txt | 0
.../tests/Inputs/filter-failed-rerun/fail.txt | 1 +
.../tests/Inputs/filter-failed-rerun/lit.cfg | 7 +++++++
.../tests/Inputs/filter-failed-rerun/pass.txt | 1 +
.../Inputs/filter-failed-rerun/xpass.txt | 2 ++
llvm/utils/lit/tests/filter-failed-delete.py | 14 +++++++++++++
llvm/utils/lit/tests/filter-failed-rerun.py | 20 +++++++++++++++++++
llvm/utils/lit/tests/filter-failed.py | 2 --
11 files changed, 54 insertions(+), 2 deletions(-)
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed-delete/fail.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed-delete/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed-delete/pass.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed-delete/unresolved.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed-rerun/fail.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed-rerun/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed-rerun/pass.txt
create mode 100644 llvm/utils/lit/tests/Inputs/filter-failed-rerun/xpass.txt
create mode 100644 llvm/utils/lit/tests/filter-failed-delete.py
create mode 100644 llvm/utils/lit/tests/filter-failed-rerun.py
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed-delete/fail.txt b/llvm/utils/lit/tests/Inputs/filter-failed-delete/fail.txt
new file mode 100644
index 0000000000000..15eb81a5f5e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed-delete/fail.txt
@@ -0,0 +1 @@
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed-delete/lit.cfg b/llvm/utils/lit/tests/Inputs/filter-failed-delete/lit.cfg
new file mode 100644
index 0000000000000..6c6dfa8e184c3
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed-delete/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed-delete"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed-delete/pass.txt b/llvm/utils/lit/tests/Inputs/filter-failed-delete/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed-delete/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed-delete/unresolved.txt b/llvm/utils/lit/tests/Inputs/filter-failed-delete/unresolved.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed-rerun/fail.txt b/llvm/utils/lit/tests/Inputs/filter-failed-rerun/fail.txt
new file mode 100644
index 0000000000000..15eb81a5f5e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed-rerun/fail.txt
@@ -0,0 +1 @@
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed-rerun/lit.cfg b/llvm/utils/lit/tests/Inputs/filter-failed-rerun/lit.cfg
new file mode 100644
index 0000000000000..25b79c4a368f0
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed-rerun/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed-rerun"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed-rerun/pass.txt b/llvm/utils/lit/tests/Inputs/filter-failed-rerun/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed-rerun/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/filter-failed-rerun/xpass.txt b/llvm/utils/lit/tests/Inputs/filter-failed-rerun/xpass.txt
new file mode 100644
index 0000000000000..66b8a6a5a187c
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/filter-failed-rerun/xpass.txt
@@ -0,0 +1,2 @@
+RUN: true
+XFAIL: *
diff --git a/llvm/utils/lit/tests/filter-failed-delete.py b/llvm/utils/lit/tests/filter-failed-delete.py
new file mode 100644
index 0000000000000..2432db193414f
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed-delete.py
@@ -0,0 +1,14 @@
+# Shows behaviour when a previously failed test was deleted
+# before running with --filter-failed.
+
+# RUN: not %{lit} %{inputs}/filter-failed-delete | FileCheck %s --check-prefix=CHECK-FIRST
+#
+# RUN: mv %{inputs}/filter-failed-delete/fail.txt %{inputs}/filter-failed-delete/fail.txt.bk
+# RUN: not %{lit} --filter-failed %{inputs}/filter-failed-delete > %s.rerun.log
+# RUN: mv %{inputs}/filter-failed-delete/fail.txt.bk %{inputs}/filter-failed-delete/fail.txt
+#
+# RUN: cat %s.rerun.log | FileCheck %s --check-prefix=CHECK-RERUN
+
+# CHECK-FIRST: FAIL: filter-failed-delete :: fail.txt
+
+# CHECK-RERUN-NOT: filter-failed-delete :: fail.txt
diff --git a/llvm/utils/lit/tests/filter-failed-rerun.py b/llvm/utils/lit/tests/filter-failed-rerun.py
new file mode 100644
index 0000000000000..d43c592dc7273
--- /dev/null
+++ b/llvm/utils/lit/tests/filter-failed-rerun.py
@@ -0,0 +1,20 @@
+# Checks that --filter-failed won't re-run tests that have passed
+# since the last time --filter-failed has run.
+
+# RUN: not %{lit} %{inputs}/filter-failed-rerun | FileCheck %s --check-prefix=CHECK-FIRST
+#
+# RUN: mv %{inputs}/filter-failed-rerun/fail.txt %{inputs}/filter-failed-rerun/fail.txt.bk
+# RUN: cp %{inputs}/filter-failed-rerun/pass.txt %{inputs}/filter-failed-rerun/fail.txt
+# RUN: not %{lit} %{inputs}/filter-failed-rerun > %s.rerun-1.log
+# RUN: not %{lit} --filter-failed %{inputs}/filter-failed-rerun > %s.rerun-2.log
+# RUN: mv %{inputs}/filter-failed-rerun/fail.txt.bk %{inputs}/filter-failed-rerun/fail.txt
+#
+# RUN: cat %s.rerun-1.log | FileCheck %s --check-prefix=CHECK-RERUN1
+# RUN: cat %s.rerun-2.log | FileCheck %s --check-prefix=CHECK-RERUN2
+
+# CHECK-FIRST: FAIL: filter-failed-rerun :: fail.txt
+
+# CHECK-RERUN1: PASS: filter-failed-rerun :: fail.txt
+
+# CHECK-RERUN2: Testing: 1 of 3 tests
+# CHECK-RERUN2-NOT: filter-failed-rerun :: fail.txt
diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py
index f0199f57bf5ae..76c529f5b54ef 100644
--- a/llvm/utils/lit/tests/filter-failed.py
+++ b/llvm/utils/lit/tests/filter-failed.py
@@ -10,8 +10,6 @@
#
# RUN: not %{lit} --filter-failed %{inputs}/filter-failed | FileCheck %s
-# END.
-
# CHECK: Testing: 3 of 7 tests
# CHECK-DAG: FAIL: filter-failed :: fail.txt
# CHECK-DAG: UNRESOLVED: filter-failed :: unresolved.txt
More information about the llvm-commits
mailing list