[llvm] 5b55eb1 - [lit] Add a method to lit.TestFormat to get the list of tests associated to a path
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 27 13:27:59 PDT 2023
Author: Louis Dionne
Date: 2023-06-27T16:27:08-04:00
New Revision: 5b55eb1e888430abfbcde8a84c99ca35fb3c5374
URL: https://github.com/llvm/llvm-project/commit/5b55eb1e888430abfbcde8a84c99ca35fb3c5374
DIFF: https://github.com/llvm/llvm-project/commit/5b55eb1e888430abfbcde8a84c99ca35fb3c5374.diff
LOG: [lit] Add a method to lit.TestFormat to get the list of tests associated to a path
Lit TestFormat classes already needed to implement the getTestsInDirectory
method. However, some test formats may want to expand a single test path
to multiple Lit tests, for example in the case of a test that actually
generates other Lit tests.
To accommodate that, this commit adds the getTestsForPath method to
TestFormat. This method can be used to turn a single path in a Lit
test suite into a list of tests associated to that path.
Differential Revision: https://reviews.llvm.org/D151664
Added:
llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py
llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg
llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/x.test
Modified:
libcxx/utils/libcxx/test/format.py
llvm/utils/lit/examples/many-tests/ManyTests.py
llvm/utils/lit/lit/discovery.py
llvm/utils/lit/lit/formats/base.py
llvm/utils/lit/tests/discovery.py
Removed:
################################################################################
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index 8a96dbceef4e9..13e40738d30ae 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -182,7 +182,7 @@ def parseScript(test, preamble):
return script
-class CxxStandardLibraryTest(lit.formats.TestFormat):
+class CxxStandardLibraryTest(lit.formats.FileBasedTest):
"""
Lit test format for the C++ Standard Library conformance test suite.
@@ -278,7 +278,7 @@ class CxxStandardLibraryTest(lit.formats.TestFormat):
in conjunction with the %{build} substitution.
"""
- def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
+ def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig):
SUPPORTED_SUFFIXES = [
"[.]pass[.]cpp$",
"[.]pass[.]mm$",
@@ -293,22 +293,22 @@ def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
"[.]verify[.]cpp$",
"[.]fail[.]cpp$",
]
+
sourcePath = testSuite.getSourcePath(pathInSuite)
- for filename in os.listdir(sourcePath):
- # Ignore dot files and excluded tests.
- if filename.startswith(".") or filename in localConfig.excludes:
- continue
-
- filepath = os.path.join(sourcePath, filename)
- if not os.path.isdir(filepath):
- if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]):
- # If this is a generated test, run the generation step and add
- # as many Lit tests as necessary.
- if re.search('[.]gen[.][^.]+$', filename):
- for test in self._generateGenTest(testSuite, pathInSuite + (filename,), litConfig, localConfig):
- yield test
- else:
- yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig)
+ filename = os.path.basename(sourcePath)
+
+ # Ignore dot files, excluded tests and tests with an unsupported suffix
+ hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES])
+ if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename):
+ return
+
+ # If this is a generated test, run the generation step and add
+ # as many Lit tests as necessary.
+ if re.search('[.]gen[.][^.]+$', filename):
+ for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig):
+ yield test
+ else:
+ yield lit.Test.Test(testSuite, pathInSuite, localConfig)
def execute(self, test, litConfig):
VERIFY_FLAGS = (
diff --git a/llvm/utils/lit/examples/many-tests/ManyTests.py b/llvm/utils/lit/examples/many-tests/ManyTests.py
index df3edb55b85e9..89e818a037c39 100644
--- a/llvm/utils/lit/examples/many-tests/ManyTests.py
+++ b/llvm/utils/lit/examples/many-tests/ManyTests.py
@@ -1,7 +1,7 @@
-from lit import Test
+from lit import Test, TestFormat
-class ManyTests(object):
+class ManyTests(TestFormat):
def __init__(self, N=10000):
self.N = N
diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py
index 5bfe1eb5acd71..e421a87b55664 100644
--- a/llvm/utils/lit/lit/discovery.py
+++ b/llvm/utils/lit/lit/discovery.py
@@ -163,35 +163,43 @@ def getTestsInSuite(
if not os.path.isdir(source_path):
test_dir_in_suite = path_in_suite[:-1]
lc = getLocalConfig(ts, test_dir_in_suite, litConfig, localConfigCache)
- test = Test.Test(ts, path_in_suite, lc)
-
- # Issue a error if the specified test would not be run if
- # the user had specified the containing directory instead of
- # of naming the test directly. This helps to avoid writing
- # tests which are not executed. The check adds some performance
- # overhead which might be important if a large number of tests
- # are being run directly.
- # This check can be disabled by using --no-indirectly-run-check or
- # setting the standalone_tests variable in the suite's configuration.
- if (
- indirectlyRunCheck
- and lc.test_format is not None
- and not lc.standalone_tests
- ):
- found = False
- for res in lc.test_format.getTestsInDirectory(
- ts, test_dir_in_suite, litConfig, lc
+
+ # TODO: Stop checking for indirectlyRunCheck and lc.standalone_tests here
+ # once we remove --no-indirectly-run-check, which is not needed anymore
+ # now that we error out when trying to run a test that wouldn't be
+ # discovered in the directory.
+ fallbackOnSingleTest = lc.test_format is None or not indirectlyRunCheck or lc.standalone_tests
+ tests = [Test.Test(ts, path_in_suite, lc)] if fallbackOnSingleTest else \
+ lc.test_format.getTestsForPath(ts, path_in_suite, litConfig, lc)
+
+ for test in tests:
+ # Issue a error if the specified test would not be run if
+ # the user had specified the containing directory instead of
+ # of naming the test directly. This helps to avoid writing
+ # tests which are not executed. The check adds some performance
+ # overhead which might be important if a large number of tests
+ # are being run directly.
+ # This check can be disabled by using --no-indirectly-run-check or
+ # setting the standalone_tests variable in the suite's configuration.
+ if (
+ indirectlyRunCheck
+ and lc.test_format is not None
+ and not lc.standalone_tests
):
- if test.getFullName() == res.getFullName():
- found = True
- break
- if not found:
- litConfig.error(
- "%r would not be run indirectly: change name or LIT config"
- "(e.g. suffixes or standalone_tests variables)" % test.getFullName()
- )
-
- yield test
+ found = False
+ for res in lc.test_format.getTestsInDirectory(
+ ts, test_dir_in_suite, litConfig, lc
+ ):
+ if test.getFullName() == res.getFullName():
+ found = True
+ break
+ if not found:
+ litConfig.error(
+ "%r would not be run indirectly: change name or LIT config"
+ "(e.g. suffixes or standalone_tests variables)" % test.getFullName()
+ )
+
+ yield test
return
# Otherwise we have a directory to search for tests, start by getting the
diff --git a/llvm/utils/lit/lit/formats/base.py b/llvm/utils/lit/lit/formats/base.py
index 0f8e984b2ab48..8430f6fe9ed58 100644
--- a/llvm/utils/lit/lit/formats/base.py
+++ b/llvm/utils/lit/lit/formats/base.py
@@ -6,27 +6,42 @@
class TestFormat(object):
- pass
-
+ def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
+ """
+ Given the path to a test in the test suite, generates the Lit tests associated
+ to that path. There can be zero, one or more tests. For example, some testing
+ formats allow expanding a single path in the test suite into multiple Lit tests
+ (e.g. they are generated on the fly).
+ """
+ yield lit.Test.Test(testSuite, path_in_suite, localConfig)
###
class FileBasedTest(TestFormat):
+ def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
+ """
+ Expand each path in a test suite to a Lit test using that path and assuming
+ it is a file containing the test. File extensions excluded by the configuration
+ or not contained in the allowed extensions are ignored.
+ """
+ filename = path_in_suite[-1]
+
+ # Ignore dot files and excluded tests.
+ if filename.startswith(".") or filename in localConfig.excludes:
+ return
+
+ base, ext = os.path.splitext(filename)
+ if ext in localConfig.suffixes:
+ yield lit.Test.Test(testSuite, path_in_suite, localConfig)
+
def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
source_path = testSuite.getSourcePath(path_in_suite)
for filename in os.listdir(source_path):
- # Ignore dot files and excluded tests.
- if filename.startswith(".") or filename in localConfig.excludes:
- continue
-
filepath = os.path.join(source_path, filename)
if not os.path.isdir(filepath):
- base, ext = os.path.splitext(filename)
- if ext in localConfig.suffixes:
- yield lit.Test.Test(
- testSuite, path_in_suite + (filename,), localConfig
- )
+ for t in self.getTestsForPath(testSuite, path_in_suite + (filename,), litConfig, localConfig):
+ yield t
###
diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py
new file mode 100644
index 0000000000000..c6f469fc81d53
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py
@@ -0,0 +1,12 @@
+import os
+import lit.formats
+
+class CustomFormat(lit.formats.ShTest):
+ def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
+ for sub in ['one.test', 'two.test']:
+ basePath = os.path.dirname(testSuite.getExecPath(path_in_suite))
+ os.makedirs(basePath, exist_ok=True)
+ generatedFile = os.path.join(basePath, sub)
+ with open(generatedFile, 'w') as f:
+ f.write('RUN: true')
+ yield lit.Test.Test(testSuite, (generatedFile, ), localConfig)
diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg
new file mode 100644
index 0000000000000..10192a6cbf137
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg
@@ -0,0 +1,9 @@
+import sys, os
+sys.path.append(os.path.dirname(__file__))
+from custom_format import CustomFormat
+
+config.name = "discovery-getTestsForPath-suite"
+config.suffixes = [".test"]
+config.test_format = CustomFormat()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/x.test b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/x.test
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/lit/tests/discovery.py b/llvm/utils/lit/tests/discovery.py
index 221a634ce4354..5bf25574e15b3 100644
--- a/llvm/utils/lit/tests/discovery.py
+++ b/llvm/utils/lit/tests/discovery.py
@@ -134,14 +134,15 @@
# CHECK-ASEXEC-DIRECT-TEST: -- Available Tests --
# CHECK-ASEXEC-DIRECT-TEST: top-level-suite :: subdir/test-three
-# Check an error is emitted when the directly named test would not be run
-# indirectly (e.g. when the directory containing the test is specified).
+# Check that an error is emitted when the directly named test does not satisfy
+# the test config's requirements.
#
# RUN: not %{lit} \
# RUN: %{inputs}/discovery/test.not-txt 2>%t.err
-# RUN: FileCheck --check-prefix=CHECK-ERROR-INDIRECT-RUN-CHECK < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-ERROR-INPUT-CONTAINED-NO-TESTS < %t.err %s
#
-# CHECK-ERROR-INDIRECT-RUN-CHECK: error: 'top-level-suite :: test.not-txt' would not be run indirectly
+# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: warning: input 'Inputs/discovery/test.not-txt' contained no tests
+# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: error: did not discover any tests for provided path(s)
# Check that no error is emitted with --no-indirectly-run-check.
#
@@ -178,6 +179,15 @@
#
# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)
+# Check that a single file path can result in multiple tests being discovered if
+# the test format implements those semantics.
+#
+# RUN: %{lit} %{inputs}/discovery-getTestsForPath/x.test > %t.out
+# RUN: FileCheck --check-prefix=CHECK-getTestsForPath < %t.out %s
+#
+# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}one.test
+# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}two.test
+
# Check that we don't recurse infinitely when loading an site specific test
# suite located inside the test source root.
#
More information about the llvm-commits
mailing list