[llvm] 45d058e - [lit] Add --xfail and --filter-out (inverse of --filter)
David Zarzycki via llvm-commits
llvm-commits at lists.llvm.org
Sat Feb 20 02:43:58 PST 2021
Author: David Zarzycki
Date: 2021-02-20T05:43:29-05:00
New Revision: 45d058e56d43e5d1b322d290764fc883c1b2e0dc
URL: https://github.com/llvm/llvm-project/commit/45d058e56d43e5d1b322d290764fc883c1b2e0dc
DIFF: https://github.com/llvm/llvm-project/commit/45d058e56d43e5d1b322d290764fc883c1b2e0dc.diff
LOG: [lit] Add --xfail and --filter-out (inverse of --filter)
In semi-automated environments, XFAILing or filtering out known regressions without actually committing changes or temporarily modifying the test suite can be quite useful.
Reviewed By: yln
Differential Revision: https://reviews.llvm.org/D96662
Added:
llvm/utils/lit/tests/Inputs/xfail-cl/false.txt
llvm/utils/lit/tests/Inputs/xfail-cl/false2.txt
llvm/utils/lit/tests/Inputs/xfail-cl/lit.cfg
llvm/utils/lit/tests/Inputs/xfail-cl/true.txt
llvm/utils/lit/tests/xfail-cl.py
Modified:
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/main.py
llvm/utils/lit/tests/selecting.py
Removed:
################################################################################
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 9a8135359967..c071a3dbc4d0 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -214,6 +214,21 @@ SELECTION OPTIONS
of this option, which is especially useful in environments where the call
to ``lit`` is issued indirectly.
+.. option:: --filter-out=REGEXP
+
+ Filter out those tests whose name matches the regular expression specified in
+ ``REGEXP``. The environment variable ``LIT_FILTER_OUT`` can be also used in
+ place of this option, which is especially useful in environments where the
+ call to ``lit`` is issued indirectly.
+
+.. option:: --xfail=LIST
+
+ Treat those tests whose name is in the semicolon separated list ``LIST`` as
+ ``XFAIL``. This can be helpful when one does not want to modify the test
+ suite. The environment variable ``LIT_XFAIL`` can be also used in place of
+ this option, which is especially useful in environments where the call to
+ ``lit`` is issued indirectly.
+
ADDITIONAL OPTIONS
------------------
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 8111ff725495..62c46705867e 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -158,6 +158,16 @@ def parse_args():
type=_case_insensitive_regex,
help="Only run tests with paths matching the given regular expression",
default=os.environ.get("LIT_FILTER", ".*"))
+ selection_group.add_argument("--filter-out",
+ metavar="REGEX",
+ type=_case_insensitive_regex,
+ help="Filter out tests with paths matching the given regular expression",
+ default=os.environ.get("LIT_FILTER_OUT", "^$"))
+ selection_group.add_argument("--xfail",
+ metavar="LIST",
+ type=_semicolon_list,
+ help="XFAIL tests with paths in the semicolon separated list",
+ default=os.environ.get("LIT_XFAIL", ""))
selection_group.add_argument("--num-shards",
dest="numShards",
metavar="M",
@@ -242,6 +252,10 @@ def _case_insensitive_regex(arg):
raise _error("invalid regular expression: '{}', {}", arg, reason)
+def _semicolon_list(arg):
+ return arg.split(';')
+
+
def _error(desc, *args):
msg = desc.format(*args)
return argparse.ArgumentTypeError(msg)
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 801687e44a69..33ce1109b9ee 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -68,7 +68,9 @@ def main(builtin_params={}):
determine_order(discovered_tests, opts.order)
selected_tests = [t for t in discovered_tests if
- opts.filter.search(t.getFullName())]
+ opts.filter.search(t.getFullName()) and not
+ opts.filter_out.search(t.getFullName())]
+
if not selected_tests:
sys.stderr.write('error: filter did not match any tests '
'(of %d discovered). ' % len(discovered_tests))
@@ -95,6 +97,8 @@ def main(builtin_params={}):
selected_tests = selected_tests[:opts.max_tests]
+ mark_xfail(discovered_tests, opts)
+
mark_excluded(discovered_tests, selected_tests)
start = time.time()
@@ -189,6 +193,11 @@ def filter_by_shard(tests, run, shards, lit_config):
return selected_tests
+def mark_xfail(selected_tests, opts):
+ for t in selected_tests:
+ if os.sep.join(t.path_in_suite) in opts.xfail:
+ t.xfails += '*'
+
def mark_excluded(discovered_tests, selected_tests):
excluded_tests = set(discovered_tests) - set(selected_tests)
result = lit.Test.Result(lit.Test.EXCLUDED)
diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/false.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/false.txt
new file mode 100644
index 000000000000..49932c3006e1
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xfail-cl/false.txt
@@ -0,0 +1 @@
+# RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/false2.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/false2.txt
new file mode 100644
index 000000000000..49932c3006e1
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xfail-cl/false2.txt
@@ -0,0 +1 @@
+# RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/lit.cfg b/llvm/utils/lit/tests/Inputs/xfail-cl/lit.cfg
new file mode 100644
index 000000000000..4568d61bfbcf
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xfail-cl/lit.cfg
@@ -0,0 +1,4 @@
+import lit.formats
+config.name = 'top-level-suite'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/true.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/true.txt
new file mode 100644
index 000000000000..b80b60b7a279
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/xfail-cl/true.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/selecting.py b/llvm/utils/lit/tests/selecting.py
index 286a917261dc..b02219019404 100644
--- a/llvm/utils/lit/tests/selecting.py
+++ b/llvm/utils/lit/tests/selecting.py
@@ -9,9 +9,12 @@
# CHECK-BAD-PATH: error: did not discover any tests for provided path(s)
# Check that we exit with an error if we filter out all tests, but allow it with --allow-empty-runs.
+# Check that we exit with an error if we skip all tests, but allow it with --allow-empty-runs.
#
# RUN: not %{lit} --filter 'nonexistent' %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s
# RUN: %{lit} --filter 'nonexistent' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s
+# RUN: not %{lit} --filter-out '.*' %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s
+# RUN: %{lit} --filter-out '.*' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s
# CHECK-BAD-FILTER: error: filter did not match any tests (of 5 discovered).
# CHECK-BAD-FILTER-ERROR: Use '--allow-empty-runs' to suppress this error.
# CHECK-BAD-FILTER-ALLOW: Suppressing error because '--allow-empty-runs' was specified.
@@ -21,6 +24,9 @@
# RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
# RUN: %{lit} --filter 'O[A-Z]E' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
# RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: %{lit} --filter-out 'test-t[a-z]' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: %{lit} --filter-out 'test-t[A-Z]' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: env LIT_FILTER_OUT='test-t[a-z]' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
# CHECK-FILTER: Testing: 2 of 5 tests
# CHECK-FILTER: Excluded: 3
diff --git a/llvm/utils/lit/tests/xfail-cl.py b/llvm/utils/lit/tests/xfail-cl.py
new file mode 100644
index 000000000000..c55e07408af8
--- /dev/null
+++ b/llvm/utils/lit/tests/xfail-cl.py
@@ -0,0 +1,9 @@
+# Check that regex-XFAILing works and can be configured via env var.
+#
+# RUN: %{lit} --xfail 'false.txt;false2.txt' %{inputs}/xfail-cl | FileCheck --check-prefix=CHECK-FILTER %s
+# RUN: env LIT_XFAIL='false.txt;false2.txt' %{lit} %{inputs}/xfail-cl | FileCheck --check-prefix=CHECK-FILTER %s
+# END:
+# CHECK-FILTER: Testing: 3 tests, 3 workers
+# CHECK-FILTER-DAG: {{XFAIL}}: top-level-suite :: false.txt
+# CHECK-FILTER-DAG: {{XFAIL}}: top-level-suite :: false2.txt
+# CHECK-FILTER-DAG: PASS: top-level-suite :: true.txt
More information about the llvm-commits
mailing list