[llvm] b8355b7 - [lit] Add --xfail-not/LIT_XFAIL_NOT
Joel E. Denny via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 16 16:14:38 PDT 2021
Author: Joel E. Denny
Date: 2021-07-16T19:13:34-04:00
New Revision: b8355b71260e762af4e125a2241817cffe40bf4e
URL: https://github.com/llvm/llvm-project/commit/b8355b71260e762af4e125a2241817cffe40bf4e
DIFF: https://github.com/llvm/llvm-project/commit/b8355b71260e762af4e125a2241817cffe40bf4e.diff
LOG: [lit] Add --xfail-not/LIT_XFAIL_NOT
For example, I need this lately in my CI config:
LIT_XFAIL_NOT='libomptarget :: nvptx64-nvidia-cuda :: unified_shared_memory/api.c'
That test specifies an XFAIL directive, but I get an XPASS result.
Reviewed By: jhenderson
Differential Revision: https://reviews.llvm.org/D106022
Added:
llvm/utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt
llvm/utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt
llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail.txt
Modified:
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/Test.py
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/main.py
llvm/utils/lit/tests/xfail-cl.py
Removed:
################################################################################
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index dc2b57cda8a4e..9c48d3abb8df7 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -263,6 +263,15 @@ The timing data is stored in the `test_exec_root` in a file named
LIT_XFAIL="affinity/kmp-hw-subset.c;libomptarget :: x86_64-pc-linux-gnu :: offloading/memory_manager.cpp"
+.. option:: --xfail-not=LIST
+
+ Do not treat the specified tests as ``XFAIL``. The environment variable
+ ``LIT_XFAIL_NOT`` can also be used in place of this option. The syntax is the
+ same as for :option:`--xfail` and ``LIT_XFAIL``. :option:`--xfail-not` and
+ ``LIT_XFAIL_NOT`` always override all other ``XFAIL`` specifications,
+ including an :option:`--xfail` appearing later on the command line. The
+ primary purpose is to suppress an ``XPASS`` result without modifying a test
+ case that uses the ``XFAIL`` directive.
ADDITIONAL OPTIONS
------------------
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 45ab12a85bd5e..77b9c235e40d4 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -231,6 +231,9 @@ def __init__(self, suite, path_in_suite, config, file_path = None):
# handlers, and will be honored when the test result is supplied.
self.xfails = []
+ # If true, ignore all items in self.xfails.
+ self.xfail_not = False
+
# A list of conditions that must be satisfied before running the test.
# Each condition is a boolean expression of features. All of them
# must be True for the test to run.
@@ -309,6 +312,9 @@ def isExpectedToFail(self):
Throws ValueError if an XFAIL line has a syntax error.
"""
+ if self.xfail_not:
+ return False
+
features = self.config.available_features
triple = getattr(self.suite.config, 'target_triple', "")
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 3eb1870bf16d9..70e0c8d6a17ec 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -171,6 +171,11 @@ def parse_args():
type=_semicolon_list,
help="XFAIL tests with paths in the semicolon separated list",
default=os.environ.get("LIT_XFAIL", ""))
+ selection_group.add_argument("--xfail-not",
+ metavar="LIST",
+ type=_semicolon_list,
+ help="do not XFAIL tests with paths in the semicolon separated list",
+ default=os.environ.get("LIT_XFAIL_NOT", ""))
selection_group.add_argument("--num-shards",
dest="numShards",
metavar="M",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index b60c30c68457c..6924aee3d5148 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -197,6 +197,8 @@ def mark_xfail(selected_tests, opts):
test_full_name = t.getFullName()
if test_file in opts.xfail or test_full_name in opts.xfail:
t.xfails += '*'
+ if test_file in opts.xfail_not or test_full_name in opts.xfail_not:
+ t.xfail_not = True
def mark_excluded(discovered_tests, selected_tests):
excluded_tests = set(discovered_tests) - set(selected_tests)
diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt
new file mode 100644
index 0000000000000..b0d8552333611
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt
@@ -0,0 +1,2 @@
+# XFAIL: *
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt
new file mode 100644
index 0000000000000..f0ae3fecf2da8
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt
@@ -0,0 +1,2 @@
+# XFAIL: *
+# RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail.txt
new file mode 100644
index 0000000000000..b0d8552333611
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail.txt
@@ -0,0 +1,2 @@
+# XFAIL: *
+# RUN: true
diff --git a/llvm/utils/lit/tests/xfail-cl.py b/llvm/utils/lit/tests/xfail-cl.py
index 1d5fdb079c2c0..ef1bb0414cfea 100644
--- a/llvm/utils/lit/tests/xfail-cl.py
+++ b/llvm/utils/lit/tests/xfail-cl.py
@@ -1,16 +1,29 @@
# Check that XFAILing works via command line or env var.
# RUN: %{lit} --xfail 'false.txt;false2.txt;top-level-suite :: b :: test.txt' \
-# RUN: %{inputs}/xfail-cl \
+# RUN: --xfail-not 'true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \
+# RUN: %{inputs}/xfail-cl \
# RUN: | FileCheck --check-prefix=CHECK-FILTER %s
# RUN: env LIT_XFAIL='false.txt;false2.txt;top-level-suite :: b :: test.txt' \
+# RUN: LIT_XFAIL_NOT='true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \
# RUN: %{lit} %{inputs}/xfail-cl \
# RUN: | FileCheck --check-prefix=CHECK-FILTER %s
+# Check that --xfail-not and LIT_XFAIL_NOT always have precedence.
+
+# RUN: env LIT_XFAIL=true-xfail.txt \
+# RUN: %{lit} --xfail true-xfail.txt --xfail-not true-xfail.txt \
+# RUN: --xfail true-xfail.txt %{inputs}/xfail-cl/true-xfail.txt \
+# RUN: | FileCheck --check-prefix=CHECK-OVERRIDE %s
+
+# RUN: env LIT_XFAIL_NOT=true-xfail.txt LIT_XFAIL=true-xfail.txt \
+# RUN: %{lit} --xfail true-xfail.txt %{inputs}/xfail-cl/true-xfail.txt \
+# RUN: | FileCheck --check-prefix=CHECK-OVERRIDE %s
+
# END.
-# CHECK-FILTER: Testing: 7 tests, {{[1-7]}} workers
+# CHECK-FILTER: Testing: 10 tests, {{[0-9]*}} workers
# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test.txt
# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test.txt
# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: a :: false.txt
@@ -18,3 +31,9 @@
# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: false.txt
# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: false2.txt
# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: true.txt
+# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: true-xfail.txt
+# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test-xfail.txt
+# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test-xfail.txt
+
+# CHECK-OVERRIDE: Testing: 1 tests, {{[0-9]*}} workers
+# CHECK-OVERRIDE: {{^}}PASS: top-level-suite :: true-xfail.txt
More information about the llvm-commits
mailing list