[llvm] [lit] Add --show-tests-json to list tests and their requirements (PR #227773)
Yasser Khan via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 09:45:54 PDT 2026
https://github.com/yassermkhan updated https://github.com/llvm/llvm-project/pull/227773
>From 1c88b11d0108080abbf02d8c46660a6dae3780a9 Mon Sep 17 00:00:00 2001
From: yakha <yakha at microsoft.com>
Date: Wed, 30 Sep 2026 11:31:05 -0400
Subject: [PATCH 1/2] [lit] Add --show-tests-json
List discovered tests as JSON, grouped by suite, with relative paths
and normalized REQUIRES expressions.
Reuse existing discovery and parsing without running tests.
Add documentation and tests.
---
llvm/docs/CommandGuide/lit.md | 8 +
llvm/utils/lit/lit/BooleanExpression.py | 71 +++
llvm/utils/lit/lit/TestRunner.py | 2 +
llvm/utils/lit/lit/cl_arguments.py | 5 +
llvm/utils/lit/lit/formats/base.py | 4 +
llvm/utils/lit/lit/formats/shtest.py | 7 +
llvm/utils/lit/lit/main.py | 65 +++
.../bad-continuation.test | 2 +
.../show-tests-json-invalid/bad-regex.test | 1 +
.../Inputs/show-tests-json-invalid/bad.test | 1 +
.../Inputs/show-tests-json-invalid/lit.cfg | 5 +
.../tests/Inputs/show-tests-json/empty.test | 1 +
.../tests/Inputs/show-tests-json/known.test | 8 +
.../lit/tests/Inputs/show-tests-json/lit.cfg | 16 +
.../tests/Inputs/show-tests-json/options.rsp | 2 +
.../Inputs/show-tests-json/same-a/lit.cfg | 5 +
.../Inputs/show-tests-json/same-a/test.test | 1 +
.../Inputs/show-tests-json/same-b/lit.cfg | 6 +
.../Inputs/show-tests-json/same-b/test.test | 1 +
.../show-tests-json/unknown/binary.test | 1 +
.../show-tests-json/unknown/lit.local.cfg | 3 +
llvm/utils/lit/tests/show-tests-json.py | 445 ++++++++++++++++++
22 files changed, 660 insertions(+)
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad-continuation.test
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad-regex.test
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad.test
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json-invalid/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/empty.test
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/known.test
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/options.rsp
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/same-a/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/same-a/test.test
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/same-b/lit.cfg
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/same-b/test.test
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/unknown/binary.test
create mode 100644 llvm/utils/lit/tests/Inputs/show-tests-json/unknown/lit.local.cfg
create mode 100644 llvm/utils/lit/tests/show-tests-json.py
diff --git a/llvm/docs/CommandGuide/lit.md b/llvm/docs/CommandGuide/lit.md
index 43d664942d532..c3594b7b122b2 100644
--- a/llvm/docs/CommandGuide/lit.md
+++ b/llvm/docs/CommandGuide/lit.md
@@ -112,6 +112,14 @@ Enable -v, but for all tests not just failed tests.
Write test results to the provided path.
:::
+:::{option} --show-tests-json
+List all discovered tests as JSON and exit without running tests.
+Group tests by suite, with relative logical paths and normalized requirements.
+`requires` is `{}` for no requirements, `null` for unknown metadata,
+or a feature-expression string or logical-expression object otherwise.
+Suite configuration still runs.
+:::
+
:::{option} --no-progress-bar
Do not use curses based progress bar.
:::
diff --git a/llvm/utils/lit/lit/BooleanExpression.py b/llvm/utils/lit/lit/BooleanExpression.py
index 9b9573d2f3f14..15384e72cfb97 100644
--- a/llvm/utils/lit/lit/BooleanExpression.py
+++ b/llvm/utils/lit/lit/BooleanExpression.py
@@ -153,6 +153,77 @@ def parseAll(self):
self.expect(BooleanExpression.END)
return self.value
+ @staticmethod
+ def normalize(string):
+ """Parse an expression into a normalized tree with string feature leaves."""
+ try:
+ return _ExpressionTree(string, set()).parseAll()
+ except ValueError as e:
+ raise ValueError(str(e) + ("\nin expression: %r" % string))
+
+ @staticmethod
+ def combine(op, operands):
+ """Flatten, deduplicate, and sort operands without expanding expressions."""
+ identity = op == "and"
+ flattened = []
+ for operand in operands:
+ if isinstance(operand, bool):
+ if operand != identity:
+ return operand
+ continue
+ if isinstance(operand, dict) and operand["op"] == op:
+ flattened.extend(operand["operands"])
+ else:
+ flattened.append(operand)
+ if not flattened:
+ return identity
+ unique = {_tree_key(operand): operand for operand in flattened}
+ ordered = [unique[key] for key in sorted(unique)]
+ if len(ordered) == 1:
+ return ordered[0]
+ return {"op": op, "operands": ordered}
+
+
+def _tree_key(tree):
+ if isinstance(tree, bool):
+ return (0, tree)
+ if isinstance(tree, str):
+ return (1, tree)
+ op = tree["op"]
+ if op == "not":
+ return (2, op, _tree_key(tree["operand"]))
+ return (2, op, tuple(_tree_key(child) for child in tree["operands"]))
+
+
+class _ExpressionTree(BooleanExpression):
+ def parseMATCH(self):
+ token = self.token
+ super().parseMATCH()
+ self.value = token
+
+ def parseNOT(self):
+ if self.accept("!"):
+ self.parseNOT()
+ self.value = {"op": "not", "operand": self.value}
+ else:
+ super().parseNOT()
+
+ def parseAND(self):
+ self.parseNOT()
+ operands = [self.value]
+ while self.accept("&&"):
+ self.parseNOT()
+ operands.append(self.value)
+ self.value = self.combine("and", operands)
+
+ def parseOR(self):
+ self.parseAND()
+ operands = [self.value]
+ while self.accept("||"):
+ self.parseAND()
+ operands.append(self.value)
+ self.value = self.combine("or", operands)
+
#######
# Tests
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 425255125db9c..714c1a05ac44a 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1990,6 +1990,8 @@ def _handleBooleanExpr(line_number, line, output):
"""A parser for BOOLEAN_EXPR type keywords"""
parts = [s.strip() for s in line.split(",") if s.strip() != ""]
if output and output[-1][-1] == "\\":
+ if not parts:
+ raise ValueError("Empty continuation in boolean expression")
output[-1] = output[-1][:-1] + parts[0]
del parts[0]
if output is None:
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index be0f154a10552..b8c24a75aa9af 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -529,6 +529,11 @@ def parse_args():
debug_group.add_argument(
"--show-tests", help="Show all discovered tests and exit", action="store_true"
)
+ debug_group.add_argument(
+ "--show-tests-json",
+ help="Show all discovered tests and feature annotations as JSON on stdout and exit",
+ action="store_true",
+ )
debug_group.add_argument(
"--show-used-features",
help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
diff --git a/llvm/utils/lit/lit/formats/base.py b/llvm/utils/lit/lit/formats/base.py
index 6e53613e7d1cb..47328768198d4 100644
--- a/llvm/utils/lit/lit/formats/base.py
+++ b/llvm/utils/lit/lit/formats/base.py
@@ -5,6 +5,10 @@
class TestFormat:
+ def getTestRequirements(self, test):
+ """Return requirement expressions, or None if they cannot be exposed."""
+ return None
+
def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
"""
Given the path to a test in the test suite, generates the Lit tests associated
diff --git a/llvm/utils/lit/lit/formats/shtest.py b/llvm/utils/lit/lit/formats/shtest.py
index 80490c1f3227a..d9d7e67e3b09d 100644
--- a/llvm/utils/lit/lit/formats/shtest.py
+++ b/llvm/utils/lit/lit/formats/shtest.py
@@ -42,3 +42,10 @@ def execute(self, test, litConfig):
self.extra_substitutions,
self.preamble_commands,
)
+
+ def getTestRequirements(self, test):
+ """Include preexisting and parsed REQUIRES without changing the test."""
+ parsed = lit.TestRunner._parseKeywords(
+ test.getSourcePath(), require_script=False
+ )
+ return list(test.requires) + (parsed["REQUIRES:"] or [])
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 0e6bee691d400..79140cd6008aa 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -5,8 +5,10 @@
"""
import itertools
+import json
import os
import platform
+import re
import sys
import time
@@ -18,6 +20,7 @@
import lit.run
import lit.Test
import lit.util
+from lit.BooleanExpression import BooleanExpression
from lit.formats.googletest import GoogleTest
from lit.TestTimes import record_test_times
@@ -55,6 +58,16 @@ def main(builtin_params={}):
sys.stderr.write("error: did not discover any tests for provided path(s)\n")
sys.exit(2)
+ if opts.show_tests_json:
+ try:
+ inventory = build_test_inventory(discovered_tests)
+ except (ValueError, re.error, OSError) as e:
+ sys.stderr.write("error: unable to export test metadata: %s\n" % e)
+ sys.exit(2)
+ json.dump(inventory, sys.stdout, indent=2, sort_keys=True)
+ sys.stdout.write("\n")
+ sys.exit(0)
+
if opts.show_suites or opts.show_tests:
print_discovered(discovered_tests, opts.show_suites, opts.show_tests)
sys.exit(0)
@@ -375,3 +388,55 @@ def print_summary(total_tests, tests_by_code, quiet, elapsed):
label = label.ljust(max_label_len)
count = str(count).rjust(max_count_len)
print(" %s: %s (%.2f%%)" % (label, count, float(count) / total_tests * 100))
+
+
+def build_test_inventory(tests):
+ def sort_key(test):
+ suite = test.suite
+ return (
+ suite.config.name,
+ os.path.normcase(os.path.abspath(suite.source_root)),
+ os.path.normcase(os.path.abspath(suite.exec_root)),
+ suite.source_root,
+ suite.exec_root,
+ test.path_in_suite,
+ )
+
+ suites = []
+ suite_entries = {}
+ for test in sorted(tests, key=sort_key):
+ suite = test.suite
+ suite_key = (suite.config.name, suite.source_root, suite.exec_root)
+ if suite_key not in suite_entries:
+ suite_entries[suite_key] = {
+ "name": suite.config.name,
+ "source_root": suite.source_root,
+ "exec_root": suite.exec_root,
+ "tests": [],
+ }
+ suites.append(suite_entries[suite_key])
+ requirements_hook = getattr(test.config.test_format, "getTestRequirements", None)
+ try:
+ raw = requirements_hook(test) if requirements_hook else None
+ if raw is None:
+ requires = None
+ elif not raw:
+ requires = {}
+ else:
+ requires = BooleanExpression.combine(
+ "and", [BooleanExpression.normalize(expr) for expr in raw]
+ )
+ except (ValueError, re.error, OSError, RecursionError) as e:
+ raise ValueError(
+ "%s (%s): %s" % (test.getFullName(), test.getSourcePath(), e)
+ ) from e
+
+ suite_entries[suite_key]["tests"].append(
+ {
+ "path_in_suite": (
+ os.path.join(*test.path_in_suite) if test.path_in_suite else ""
+ ),
+ "requires": requires,
+ }
+ )
+ return {"schema_version": 1, "suites": suites}
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad-continuation.test b/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad-continuation.test
new file mode 100644
index 0000000000000..a6d97af10b363
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad-continuation.test
@@ -0,0 +1,2 @@
+# REQUIRES: a && \
+# REQUIRES:
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad-regex.test b/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad-regex.test
new file mode 100644
index 0000000000000..cd9983e03f37b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad-regex.test
@@ -0,0 +1 @@
+# UNSUPPORTED: feature={{[}}
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad.test b/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad.test
new file mode 100644
index 0000000000000..09560578bb500
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/bad.test
@@ -0,0 +1 @@
+# REQUIRES: a &&
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/lit.cfg b/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/lit.cfg
new file mode 100644
index 0000000000000..c6feafe8df8e9
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json-invalid/lit.cfg
@@ -0,0 +1,5 @@
+import lit.formats
+
+config.name = "invalid-annotations"
+config.suffixes = [".test"]
+config.test_format = lit.formats.ShTest()
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/empty.test b/llvm/utils/lit/tests/Inputs/show-tests-json/empty.test
new file mode 100644
index 0000000000000..ded0e3f1de4f9
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/empty.test
@@ -0,0 +1 @@
+# RUN: exit 99
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/known.test b/llvm/utils/lit/tests/Inputs/show-tests-json/known.test
new file mode 100644
index 0000000000000..6a2f4acb568e5
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/known.test
@@ -0,0 +1,8 @@
+# REQUIRES: z && (a && z), feature={{[a-z]+}} || b
+# REQUIRES: !disabled && \
+# REQUIRES: preexisting
+# UNSUPPORTED: unsupported-{{[a-z]+}} && !enabled
+# XFAIL: *, bug || failure
+# END.
+# REQUIRES: ignored-after-end
+# RUN: exit 99
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/lit.cfg b/llvm/utils/lit/tests/Inputs/show-tests-json/lit.cfg
new file mode 100644
index 0000000000000..bf15c06d95b7e
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/lit.cfg
@@ -0,0 +1,16 @@
+import lit.formats
+
+
+class AnnotatedShTest(lit.formats.ShTest):
+ def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
+ for test in super().getTestsForPath(testSuite, path_in_suite, litConfig, localConfig):
+ if path_in_suite[-1] == "known.test":
+ test.requires.append("preexisting")
+ test.unsupported.append("disabled")
+ test.xfails.append("old-xfail")
+ yield test
+
+
+config.name = "json-inventory"
+config.suffixes = [".test"]
+config.test_format = AnnotatedShTest()
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/options.rsp b/llvm/utils/lit/tests/Inputs/show-tests-json/options.rsp
new file mode 100644
index 0000000000000..7c343d13698a2
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/options.rsp
@@ -0,0 +1,2 @@
+--show-tests-json
+--filter=does-not-match
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/same-a/lit.cfg b/llvm/utils/lit/tests/Inputs/show-tests-json/same-a/lit.cfg
new file mode 100644
index 0000000000000..69b376b48d4e7
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/same-a/lit.cfg
@@ -0,0 +1,5 @@
+import lit.formats
+
+config.name = "same-name"
+config.suffixes = [".test"]
+config.test_format = lit.formats.ShTest()
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/same-a/test.test b/llvm/utils/lit/tests/Inputs/show-tests-json/same-a/test.test
new file mode 100644
index 0000000000000..896c5a3f1f449
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/same-a/test.test
@@ -0,0 +1 @@
+# REQUIRES: a
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/same-b/lit.cfg b/llvm/utils/lit/tests/Inputs/show-tests-json/same-b/lit.cfg
new file mode 100644
index 0000000000000..d311c8bfe89bb
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/same-b/lit.cfg
@@ -0,0 +1,6 @@
+import lit.formats
+
+config.name = "same-name"
+config.suffixes = [".test"]
+config.test_format = lit.formats.ShTest()
+config.unsupported = True
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/same-b/test.test b/llvm/utils/lit/tests/Inputs/show-tests-json/same-b/test.test
new file mode 100644
index 0000000000000..b65b507320d20
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/same-b/test.test
@@ -0,0 +1 @@
+# REQUIRES: b
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/unknown/binary.test b/llvm/utils/lit/tests/Inputs/show-tests-json/unknown/binary.test
new file mode 100644
index 0000000000000..2c0a5472f2db7
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/unknown/binary.test
@@ -0,0 +1 @@
+No integrated-test directives are available for this format.
diff --git a/llvm/utils/lit/tests/Inputs/show-tests-json/unknown/lit.local.cfg b/llvm/utils/lit/tests/Inputs/show-tests-json/unknown/lit.local.cfg
new file mode 100644
index 0000000000000..7583d78eba515
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-tests-json/unknown/lit.local.cfg
@@ -0,0 +1,3 @@
+import lit.formats
+
+config.test_format = lit.formats.ExecutableTest()
diff --git a/llvm/utils/lit/tests/show-tests-json.py b/llvm/utils/lit/tests/show-tests-json.py
new file mode 100644
index 0000000000000..00ed7c96a16e1
--- /dev/null
+++ b/llvm/utils/lit/tests/show-tests-json.py
@@ -0,0 +1,445 @@
+# RUN: %{python} %s
+# END.
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from types import SimpleNamespace
+
+from lit.BooleanExpression import BooleanExpression
+from lit.formats.shtest import ShTest
+from lit.Test import Test, TestSuite
+from lit.main import build_test_inventory
+
+
+INPUTS = os.path.join(os.path.dirname(__file__), "Inputs")
+LIT = os.path.join(os.path.dirname(__file__), "..", "lit.py")
+SUITE = os.path.join(INPUTS, "show-tests-json")
+
+
+def run_lit(*args):
+ env = os.environ.copy()
+ for variable in ("LIT_OPTS", "LIT_XFAIL", "LIT_XFAIL_NOT", "LIT_UNSUPPORTED"):
+ env.pop(variable, None)
+ return subprocess.run(
+ [sys.executable, LIT, *args],
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ )
+
+
+def evaluate_tree(tree, features):
+ if isinstance(tree, bool):
+ return tree
+ if isinstance(tree, str):
+ return BooleanExpression.evaluate(tree, features)
+ op = tree["op"]
+ if op == "not":
+ return not evaluate_tree(tree["operand"], features)
+ if op == "and":
+ return all(evaluate_tree(child, features) for child in tree["operands"])
+ if op == "or":
+ return any(evaluate_tree(child, features) for child in tree["operands"])
+ raise AssertionError("unexpected expression operator: " + op)
+
+
+class ShowTestsJsonTest(unittest.TestCase):
+ def inventory(self, *args):
+ result = run_lit("--show-tests-json", SUITE, *args)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ return json.loads(result.stdout)
+
+ def test_discovery_and_metadata(self):
+ result = self.inventory("--filter=does-not-match", "--max-tests=1")
+ self.assertEqual(result["schema_version"], 1)
+ self.assertEqual(set(result), {"schema_version", "suites"})
+ suites = result["suites"]
+ self.assertEqual(len(suites), 3)
+ self.assertEqual(
+ [suite["name"] for suite in suites],
+ ["json-inventory", "same-name", "same-name"],
+ )
+ self.assertEqual(
+ [suite["source_root"] for suite in suites],
+ [
+ os.path.abspath(SUITE),
+ os.path.abspath(os.path.join(SUITE, "same-a")),
+ os.path.abspath(os.path.join(SUITE, "same-b")),
+ ],
+ )
+ self.assertEqual(
+ [suite["exec_root"] for suite in suites],
+ [suite["source_root"] for suite in suites],
+ )
+ self.assertTrue(
+ all(
+ set(suite) == {"name", "source_root", "exec_root", "tests"}
+ for suite in suites
+ )
+ )
+ tests = suites[0]["tests"]
+ self.assertEqual(len(tests), 3)
+ self.assertEqual(
+ [test["path_in_suite"] for test in tests],
+ [
+ "empty.test",
+ "known.test",
+ os.path.join("unknown", "binary.test"),
+ ],
+ )
+ known = tests[1]
+ self.assertEqual(set(known), {"path_in_suite", "requires"})
+ self.assertEqual(
+ known["requires"],
+ {
+ "op": "and",
+ "operands": [
+ "a",
+ "preexisting",
+ "z",
+ {"op": "not", "operand": "disabled"},
+ {
+ "op": "or",
+ "operands": ["b", "feature={{[a-z]+}}"],
+ },
+ ],
+ },
+ )
+ self.assertEqual(tests[0]["requires"], {})
+ self.assertIsNone(tests[2]["requires"])
+ self.assertEqual(
+ [suite["tests"] for suite in suites[1:]],
+ [
+ [{"path_in_suite": "test.test", "requires": "a"}],
+ [{"path_in_suite": "test.test", "requires": "b"}],
+ ],
+ )
+
+ def test_exported_conditions_preserve_feature_semantics(self):
+ requires = self.inventory("--filter=does-not-match")["suites"][0]["tests"][1][
+ "requires"
+ ]
+ expressions = [
+ "preexisting",
+ "z && (a && z)",
+ "feature={{[a-z]+}} || b",
+ "!disabled && preexisting",
+ ]
+ for features in (
+ (),
+ ("a", "z", "preexisting", "feature=foo"),
+ ("a", "z", "preexisting", "b", "disabled"),
+ ("a", "z", "preexisting", "feature=123"),
+ ("a", "z", "preexisting", "b"),
+ ):
+ with self.subTest(features=features):
+ self.assertEqual(
+ evaluate_tree(requires, features),
+ all(
+ BooleanExpression.evaluate(expr, features)
+ for expr in expressions
+ ),
+ )
+ self.assertEqual(
+ evaluate_tree(requires, features),
+ set(("a", "z", "preexisting")).issubset(features)
+ and "disabled" not in features
+ and ("b" in features or "feature=foo" in features),
+ )
+
+ def test_separate_exec_root_and_source_paths(self):
+ suite = os.path.join(INPUTS, "exec-discovery")
+ result = run_lit("--show-tests-json", suite)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ record = next(
+ suite for suite in json.loads(result.stdout)["suites"]
+ if suite["name"] == "top-level-suite"
+ )
+ self.assertIn(
+ os.path.join("subdir", "test-three.py"),
+ [test["path_in_suite"] for test in record["tests"]],
+ )
+ test = next(
+ test for test in record["tests"]
+ if test["path_in_suite"] == "test-one.txt"
+ )
+ self.assertEqual(test["requires"], {})
+ self.assertEqual(
+ record["source_root"],
+ os.path.abspath(os.path.join(INPUTS, "discovery")),
+ )
+ self.assertEqual(record["exec_root"], os.path.abspath(suite))
+
+ def test_deterministic_output_and_response_file(self):
+ first = run_lit("@" + os.path.join(SUITE, "options.rsp"), SUITE)
+ second = run_lit("@" + os.path.join(SUITE, "options.rsp"), SUITE)
+ reordered = run_lit(
+ "--show-tests-json",
+ *(
+ os.path.join(SUITE, subdir, filename)
+ for subdir, filename in (
+ ("same-b", "test.test"),
+ ("unknown", "binary.test"),
+ ("", "known.test"),
+ ("", "empty.test"),
+ ("same-a", "test.test"),
+ )
+ ),
+ )
+ self.assertEqual(first.returncode, 0, first.stderr)
+ self.assertEqual(reordered.returncode, 0, reordered.stderr)
+ self.assertEqual(first.stdout, second.stdout)
+ self.assertEqual(first.stdout, reordered.stdout)
+ self.assertEqual(
+ first.stdout,
+ json.dumps(json.loads(first.stdout), indent=2, sort_keys=True) + "\n",
+ )
+
+ def test_deprecated_order_option_preserves_warning(self):
+ result = run_lit("--show-tests-json", "--incremental", SUITE)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ warning, _, inventory = result.stdout.partition("\n")
+ self.assertEqual(
+ warning,
+ "WARNING: --incremental is deprecated. Failing tests now always run first.",
+ )
+ self.assertEqual(json.loads(inventory)["schema_version"], 1)
+
+ def test_invalid_metadata_has_diagnostic_without_partial_json(self):
+ result = run_lit(
+ "--show-tests-json",
+ os.path.join(INPUTS, "show-tests-json-invalid", "bad.test"),
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(result.stdout, "")
+ self.assertIn("invalid-annotations :: bad.test", result.stderr)
+ self.assertIn("a &&", result.stderr)
+
+ def test_invalid_regex_has_diagnostic(self):
+ result = run_lit(
+ "--show-tests-json",
+ os.path.join(INPUTS, "show-tests-json-invalid", "bad-regex.test"),
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(result.stdout, "")
+ self.assertIn("invalid-annotations :: bad-regex.test", result.stderr)
+ self.assertIn("unterminated character set", result.stderr)
+
+ def test_empty_boolean_continuation_has_test_specific_diagnostic(self):
+ result = run_lit(
+ "--show-tests-json",
+ os.path.join(INPUTS, "show-tests-json-invalid", "bad-continuation.test"),
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(result.stdout, "")
+ self.assertIn("invalid-annotations :: bad-continuation.test", result.stderr)
+ self.assertIn("Empty continuation in boolean expression", result.stderr)
+ self.assertNotIn("Traceback", result.stderr)
+
+ def test_deep_boolean_expression_has_test_specific_diagnostic(self):
+ config = SimpleNamespace(name="json-inventory", test_format=ShTest())
+ suite = TestSuite(config.name, SUITE, SUITE, config)
+ test = Test(suite, ["known.test"], config)
+ test.requires.append("!" * 2000 + "a")
+ with self.assertRaisesRegex(
+ ValueError, "json-inventory :: known.test"
+ ) as error:
+ build_test_inventory([test])
+ self.assertIn(
+ os.path.abspath(os.path.join(SUITE, "known.test")), str(error.exception)
+ )
+ self.assertIsInstance(error.exception.__cause__, RecursionError)
+
+ def test_duck_typed_format_without_metadata_is_unknown(self):
+ config = SimpleNamespace(name="duck", test_format=object())
+ suite = TestSuite("other-name", SUITE, SUITE, config)
+ test = Test(suite, ["virtual", "case"], config, file_path="elsewhere")
+ record = build_test_inventory([test])["suites"][0]
+ self.assertEqual(
+ record,
+ {
+ "name": "duck",
+ "source_root": SUITE,
+ "exec_root": SUITE,
+ "tests": [
+ {
+ "path_in_suite": os.path.join("virtual", "case"),
+ "requires": None,
+ }
+ ],
+ },
+ )
+
+ def test_empty_logical_path_and_known_empty_requirements(self):
+ config = SimpleNamespace(name="virtual", test_format=ShTest())
+ suite = TestSuite(config.name, SUITE, SUITE, config)
+ # Use a duck format for the empty virtual path: ShTest needs a source file.
+ empty = Test(suite, (), SimpleNamespace(test_format=object()))
+ ordinary = Test(suite, ("empty.test",), config)
+ tests = build_test_inventory([ordinary, empty])["suites"][0]["tests"]
+ self.assertEqual(
+ tests,
+ [
+ {"path_in_suite": "", "requires": None},
+ {"path_in_suite": "empty.test", "requires": {}},
+ ],
+ )
+
+ def test_literal_true_and_boolean_identities(self):
+ config = SimpleNamespace(name="bool", test_format=ShTest())
+ suite = TestSuite(config.name, SUITE, SUITE, config)
+ test = Test(suite, ("empty.test",), config)
+ test.requires.append("true")
+ requires = build_test_inventory([test])["suites"][0]["tests"][0]["requires"]
+ self.assertEqual(requires, "true")
+ self.assertNotIsInstance(requires, bool)
+ self.assertTrue(evaluate_tree(requires, ()))
+ self.assertEqual(BooleanExpression.combine("and", []), True)
+ self.assertEqual(BooleanExpression.combine("or", []), False)
+ self.assertEqual(BooleanExpression.combine("and", ["a", True, "a"]), "a")
+ self.assertEqual(BooleanExpression.combine("or", ["a", False, "a"]), "a")
+ self.assertEqual(BooleanExpression.combine("and", ["a", False]), False)
+ self.assertEqual(BooleanExpression.combine("or", ["a", True]), True)
+ self.assertEqual(
+ BooleanExpression.combine(
+ "and", ["true", True, {"op": "not", "operand": "a"}]
+ ),
+ {"op": "and", "operands": ["true", {"op": "not", "operand": "a"}]},
+ )
+
+ def test_unsatisfiable_requirements_are_not_empty(self):
+ config = SimpleNamespace(name="contradiction", test_format=ShTest())
+ suite = TestSuite(config.name, SUITE, SUITE, config)
+ test = Test(suite, ("empty.test",), config)
+ test.requires.append("a && !a")
+ requires = build_test_inventory([test])["suites"][0]["tests"][0]["requires"]
+ self.assertEqual(
+ requires,
+ {"op": "and", "operands": ["a", {"op": "not", "operand": "a"}]},
+ )
+ self.assertFalse(evaluate_tree(requires, ()))
+ self.assertFalse(evaluate_tree(requires, ("a",)))
+
+ def test_json_takes_precedence_over_inspection_and_result_options(self):
+ expected = self.inventory()
+ for option in (
+ "--show-tests",
+ "--show-suites",
+ "--show-used-features",
+ "-o",
+ "--xunit-xml-output",
+ "--resultdb-output",
+ "--time-trace-output",
+ "--wtt-output",
+ ):
+ with self.subTest(option=option), tempfile.TemporaryDirectory() as directory:
+ report_path = os.path.join(directory, "unused-report.json")
+ args = [option]
+ if option in (
+ "-o",
+ "--xunit-xml-output",
+ "--resultdb-output",
+ "--time-trace-output",
+ "--wtt-output",
+ ):
+ args.append(report_path)
+ result = run_lit("--show-tests-json", *args, SUITE)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(json.loads(result.stdout), expected)
+ self.assertFalse(os.path.exists(report_path))
+
+ def test_text_inspection_unchanged(self):
+ result = run_lit("--show-tests", SUITE)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("-- Available Tests --", result.stdout)
+ self.assertNotIn('"schema_version"', result.stdout)
+
+ def test_parsing_does_not_mutate_prepopulated_annotations(self):
+ test = SimpleNamespace(
+ getSourcePath=lambda: os.path.join(SUITE, "known.test"),
+ requires=["preexisting"],
+ unsupported=["disabled"],
+ xfails=["old-xfail"],
+ )
+ requirements = ShTest().getTestRequirements(test)
+ self.assertEqual(
+ requirements,
+ [
+ "preexisting",
+ "z && (a && z)",
+ "feature={{[a-z]+}} || b",
+ "!disabled && preexisting",
+ ],
+ )
+ self.assertEqual(test.requires, ["preexisting"])
+ self.assertEqual(test.unsupported, ["disabled"])
+ self.assertEqual(test.xfails, ["old-xfail"])
+
+
+class ExpressionTreeTest(unittest.TestCase):
+ def test_structural_normalization(self):
+ left = BooleanExpression.normalize("(b && a) && (a && b)")
+ self.assertEqual(
+ left,
+ {
+ "op": "and",
+ "operands": ["a", "b"],
+ },
+ )
+ self.assertEqual(
+ BooleanExpression.normalize("b || (a || b)"),
+ {
+ "op": "or",
+ "operands": ["a", "b"],
+ },
+ )
+ self.assertEqual(
+ BooleanExpression.normalize("!a && (b || feature={{[a-z]+}})"),
+ {
+ "op": "and",
+ "operands": [
+ {"op": "not", "operand": "a"},
+ {
+ "op": "or",
+ "operands": ["b", "feature={{[a-z]+}}"],
+ },
+ ],
+ },
+ )
+ self.assertEqual(BooleanExpression.normalize("true"), "true")
+ self.assertTrue(evaluate_tree(BooleanExpression.normalize("true"), ()))
+ self.assertFalse(evaluate_tree(BooleanExpression.normalize("true-ish"), ()))
+ self.assertNotEqual(
+ BooleanExpression.normalize("a && (b || c)"),
+ BooleanExpression.normalize("(a && b) || (a && c)"),
+ )
+
+ def test_combination_preserves_order_independent_normal_form(self):
+ expressions = ["b", "a && b", "!c", "a"]
+ forward = BooleanExpression.combine(
+ "and", [BooleanExpression.normalize(expr) for expr in expressions]
+ )
+ backward = BooleanExpression.combine(
+ "and", [BooleanExpression.normalize(expr) for expr in reversed(expressions)]
+ )
+ self.assertEqual(forward, backward)
+ self.assertEqual(
+ forward,
+ {
+ "op": "and",
+ "operands": ["a", "b", {"op": "not", "operand": "c"}],
+ },
+ )
+
+ def test_syntax_error(self):
+ with self.assertRaisesRegex(ValueError, "in expression: 'a &&'"):
+ BooleanExpression.normalize("a &&")
+
+
+if __name__ == "__main__":
+ unittest.main()
>From 4019ac6ba1cd0e054135e3cbf502281af13ceb6a Mon Sep 17 00:00:00 2001
From: yakha <yakha at microsoft.com>
Date: Wed, 30 Sep 2026 12:45:37 -0400
Subject: [PATCH 2/2] [lit] Fix formatting in --show-tests-json code and tests
---
llvm/utils/lit/lit/main.py | 4 +++-
llvm/utils/lit/tests/show-tests-json.py | 10 ++++++----
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 79140cd6008aa..a9c9ce408efea 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -415,7 +415,9 @@ def sort_key(test):
"tests": [],
}
suites.append(suite_entries[suite_key])
- requirements_hook = getattr(test.config.test_format, "getTestRequirements", None)
+ requirements_hook = getattr(
+ test.config.test_format, "getTestRequirements", None
+ )
try:
raw = requirements_hook(test) if requirements_hook else None
if raw is None:
diff --git a/llvm/utils/lit/tests/show-tests-json.py b/llvm/utils/lit/tests/show-tests-json.py
index 00ed7c96a16e1..3afc6d164fb73 100644
--- a/llvm/utils/lit/tests/show-tests-json.py
+++ b/llvm/utils/lit/tests/show-tests-json.py
@@ -157,7 +157,8 @@ def test_separate_exec_root_and_source_paths(self):
result = run_lit("--show-tests-json", suite)
self.assertEqual(result.returncode, 0, result.stderr)
record = next(
- suite for suite in json.loads(result.stdout)["suites"]
+ suite
+ for suite in json.loads(result.stdout)["suites"]
if suite["name"] == "top-level-suite"
)
self.assertIn(
@@ -165,8 +166,7 @@ def test_separate_exec_root_and_source_paths(self):
[test["path_in_suite"] for test in record["tests"]],
)
test = next(
- test for test in record["tests"]
- if test["path_in_suite"] == "test-one.txt"
+ test for test in record["tests"] if test["path_in_suite"] == "test-one.txt"
)
self.assertEqual(test["requires"], {})
self.assertEqual(
@@ -337,7 +337,9 @@ def test_json_takes_precedence_over_inspection_and_result_options(self):
"--time-trace-output",
"--wtt-output",
):
- with self.subTest(option=option), tempfile.TemporaryDirectory() as directory:
+ with self.subTest(
+ option=option
+ ), tempfile.TemporaryDirectory() as directory:
report_path = os.path.join(directory, "unused-report.json")
args = [option]
if option in (
More information about the llvm-commits
mailing list