[llvm] b383efc - [lit] Optionally exclude xfail tests (#151191)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 31 12:50:41 PDT 2025
Author: Mircea Trofin
Date: 2025-07-31T21:50:38+02:00
New Revision: b383efc3e3e18b57e1a13bd59aef4b5964410321
URL: https://github.com/llvm/llvm-project/commit/b383efc3e3e18b57e1a13bd59aef4b5964410321
DIFF: https://github.com/llvm/llvm-project/commit/b383efc3e3e18b57e1a13bd59aef4b5964410321.diff
LOG: [lit] Optionally exclude xfail tests (#151191)
See the related issue. We want to set up a build bot where `opt` runs with `-enable-profcheck`, which inserts `MD_prof` before running the rest of the pipeline requested from `opt`, and then validates resulting profile information (more info in the RFC linked by the aforementioned issue)
In that setup, we will also ignore `FileCheck`: while the profile info inserted is, currently, equivalent to the profile info a pass would observe via `BranchProbabilityInfo`/`BlockFrequencyInfo`, (1) we may want to change that, and (2) some tests are quite sensitive to the output IR, and break if, for instance, extra metadata is present (which it would be due to `-enable-profcheck`). Since we're just interested in profile consistency on the upcoming bot, ignoring `FileCheck` is simpler and sufficient. However, this has the effect of passing XFAIL tests. Rather than listing them all, the alternative is to just exclude XFAIL tests.
This PR adds support for that by introducing a `--exclude-xfail` option to `llvm-lit`.
Issue #147390
Added:
llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail-conditionally.txt
Modified:
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/Test.py
llvm/utils/lit/lit/TestRunner.py
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/main.py
llvm/utils/lit/tests/xfail-cl.py
Removed:
################################################################################
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 938b7f9f8b6f3..eb90e950a3770 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -356,6 +356,11 @@ The timing data is stored in the `test_exec_root` in a file named
primary purpose is to suppress an ``XPASS`` result without modifying a test
case that uses the ``XFAIL`` directive.
+.. option:: --exclude-xfail
+
+ ``XFAIL`` tests won't be run, unless they are listed in the ``--xfail-not``
+ (or ``LIT_XFAIL_NOT``) lists.
+
.. option:: --num-shards M
Divide the set of selected tests into ``M`` equal-sized subsets or
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 1bd5ba8dc2070..7290977724170 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -247,6 +247,9 @@ def __init__(
# and will be honored when the test result is supplied.
self.xfails = []
+ # Exclude this test if it's xfail.
+ self.exclude_xfail = False
+
# If true, ignore all items in self.xfails.
self.xfail_not = False
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 73db67aedb739..e7cd70766a3dd 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -2175,6 +2175,8 @@ def parseIntegratedTestScript(test, additional_parsers=[], require_script=True):
assert parsed["DEFINE:"] == script
assert parsed["REDEFINE:"] == script
test.xfails += parsed["XFAIL:"] or []
+ if test.exclude_xfail and test.isExpectedToFail():
+ return lit.Test.Result(Test.EXCLUDED, "excluding XFAIL tests")
test.requires += parsed["REQUIRES:"] or []
test.unsupported += parsed["UNSUPPORTED:"] or []
if parsed["ALLOW_RETRIES:"]:
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 3292554ab5ff7..e88951520e660 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -303,6 +303,16 @@ def parse_args():
help="do not XFAIL tests with paths in the semicolon separated list",
default=os.environ.get("LIT_XFAIL_NOT", ""),
)
+ selection_group.add_argument(
+ "--exclude-xfail",
+ help="exclude XFAIL tests (unless they are in the --xfail-not list). "
+ "Note: This option is implemented in "
+ "lit.TestRunner.parseIntegratedTestScript and so will have no effect on "
+ "test formats that do not call that and do not implement the option "
+ "separately.",
+ default=False,
+ action="store_true",
+ )
selection_group.add_argument(
"--num-shards",
dest="numShards",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 0939838b78ceb..9650a0e901173 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -240,6 +240,8 @@ def mark_xfail(selected_tests, opts):
t.xfails += "*"
if test_file in opts.xfail_not or test_full_name in opts.xfail_not:
t.xfail_not = True
+ if opts.exclude_xfail:
+ t.exclude_xfail = True
def mark_excluded(discovered_tests, selected_tests):
diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail-conditionally.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail-conditionally.txt
new file mode 100644
index 0000000000000..6fdecd6233835
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail-conditionally.txt
@@ -0,0 +1,2 @@
+# XFAIL: this-does-not-exist
+# RUN: true
\ No newline at end of file
diff --git a/llvm/utils/lit/tests/xfail-cl.py b/llvm/utils/lit/tests/xfail-cl.py
index ef1bb0414cfea..f1e0e335e396f 100644
--- a/llvm/utils/lit/tests/xfail-cl.py
+++ b/llvm/utils/lit/tests/xfail-cl.py
@@ -5,6 +5,18 @@
# RUN: %{inputs}/xfail-cl \
# RUN: | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: %{lit} --xfail 'false.txt;false2.txt;top-level-suite :: b :: test.txt' \
+# RUN: --exclude-xfail \
+# RUN: %{inputs}/xfail-cl \
+# RUN: | FileCheck --check-prefixes=CHECK-EXCLUDED,CHECK-EXCLUDED-NOOVERRIDE %s
+
+# RUN: %{lit} --xfail 'false.txt;false2.txt;top-level-suite :: b :: test.txt' \
+# RUN: --xfail-not 'true-xfail.txt' \
+# RUN: --exclude-xfail \
+# RUN: %{inputs}/xfail-cl \
+# RUN: | FileCheck --check-prefixes=CHECK-EXCLUDED,CHECK-EXCLUDED-OVERRIDE %s
+
+
# RUN: env LIT_XFAIL='false.txt;false2.txt;top-level-suite :: b :: test.txt' \
# RUN: LIT_XFAIL_NOT='true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \
# RUN: %{lit} %{inputs}/xfail-cl \
@@ -23,7 +35,7 @@
# END.
-# CHECK-FILTER: Testing: 10 tests, {{[0-9]*}} workers
+# CHECK-FILTER: Testing: 11 tests, {{[0-9]*}} workers
# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test.txt
# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test.txt
# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: a :: false.txt
@@ -37,3 +49,17 @@
# CHECK-OVERRIDE: Testing: 1 tests, {{[0-9]*}} workers
# CHECK-OVERRIDE: {{^}}PASS: top-level-suite :: true-xfail.txt
+
+# CHECK-EXCLUDED: Testing: 11 tests, {{[0-9]*}} workers
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: a :: false.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: a :: test-xfail.txt
+# CHECK-EXCLUDED-DAG: {{^}}PASS: top-level-suite :: a :: test.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: b :: false.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: b :: test-xfail.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: b :: test.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: false.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: false2.txt
+# CHECK-EXCLUDED-DAG: {{^}}PASS: top-level-suite :: true-xfail-conditionally.txt
+# CHECK-EXCLUDED-NOOVERRIDE-DAG: {{^}}EXCLUDED: top-level-suite :: true-xfail.txt
+# CHECK-EXCLUDED-OVERRIDE-DAG: {{^}}PASS: top-level-suite :: true-xfail.txt
+# CHECK-EXCLUDED-DAG: {{^}}PASS: top-level-suite :: true.txt
More information about the llvm-commits
mailing list