[libcxx] [libcxxabi] [llvm] [libc++][WIP] Move the libc++ test format to Lit (PR #90803)
via llvm-commits
llvm-commits at lists.llvm.org
Wed May 1 17:12:56 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-libcxxabi
Author: Louis Dionne (ldionne)
<details>
<summary>Changes</summary>
This allows the generally-useful parts of the test format to be shipped alongside Lit, which would make it usable for third-party developers as well. This came up at C++Now in the context of the Beman project where we'd like to set up a non-LLVM project that can use the same testing framework as libc++.
With the test format moved to Lit, the format would be available after installing Lit with pip, which is really convenient.
---
Patch is 37.44 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/90803.diff
6 Files Affected:
- (modified) libcxx/test/configs/cmake-bridge.cfg.in (+1-1)
- (modified) libcxx/utils/libcxx/test/dsl.py (+3-3)
- (modified) libcxx/utils/libcxx/test/format.py (+53-363)
- (modified) libcxxabi/test/configs/cmake-bridge.cfg.in (+1-1)
- (modified) llvm/utils/lit/lit/formats/__init__.py (+1)
- (added) llvm/utils/lit/lit/formats/standardlibrarytest.py (+355)
``````````diff
diff --git a/libcxx/test/configs/cmake-bridge.cfg.in b/libcxx/test/configs/cmake-bridge.cfg.in
index 84b3270a8940ac..b220e5ebcafb17 100644
--- a/libcxx/test/configs/cmake-bridge.cfg.in
+++ b/libcxx/test/configs/cmake-bridge.cfg.in
@@ -18,7 +18,7 @@ import libcxx.test.format
# Basic configuration of the test suite
config.name = os.path.basename('@LIBCXX_TEST_CONFIG@')
config.test_source_root = os.path.join('@LIBCXX_SOURCE_DIR@', 'test')
-config.test_format = libcxx.test.format.CxxStandardLibraryTest()
+config.test_format = libcxx.test.format.LibcxxTest()
config.recursiveExpansionLimit = 10
config.test_exec_root = os.path.join('@CMAKE_BINARY_DIR@', 'test')
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 387862ae6f496d..6177dc9dccd327 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -99,7 +99,7 @@ def _executeWithFakeConfig(test, commands):
order="smart",
params={},
)
- return libcxx.test.format._executeScriptInternal(test, litConfig, commands)
+ return lit.formats.standardlibrarytest._executeScriptInternal(test, litConfig, commands)
def _makeConfigTest(config):
@@ -121,12 +121,12 @@ def _makeConfigTest(config):
class TestWrapper(lit.Test.Test):
def __enter__(self):
- testDir, _ = libcxx.test.format._getTempPaths(self)
+ testDir, _ = lit.formats.standardlibrarytest._getTempPaths(self)
os.makedirs(testDir)
return self
def __exit__(self, *args):
- testDir, _ = libcxx.test.format._getTempPaths(self)
+ testDir, _ = lit.formats.standardlibrarytest._getTempPaths(self)
shutil.rmtree(testDir)
os.remove(tmp.name)
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index 7e5281c0b74064..e74e4838f7dfe6 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -7,50 +7,8 @@
# ===----------------------------------------------------------------------===##
import lit
-import libcxx.test.config as config
import lit.formats
import os
-import re
-
-
-def _getTempPaths(test):
- """
- Return the values to use for the %T and %t substitutions, respectively.
-
- The difference between this and Lit's default behavior is that we guarantee
- that %T is a path unique to the test being run.
- """
- tmpDir, _ = lit.TestRunner.getTempPaths(test)
- _, testName = os.path.split(test.getExecPath())
- tmpDir = os.path.join(tmpDir, testName + ".dir")
- tmpBase = os.path.join(tmpDir, "t")
- return tmpDir, tmpBase
-
-
-def _checkBaseSubstitutions(substitutions):
- substitutions = [s for (s, _) in substitutions]
- for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
- assert s in substitutions, "Required substitution {} was not provided".format(s)
-
-def _executeScriptInternal(test, litConfig, commands):
- """
- Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands)
-
- TODO: This really should be easier to access from Lit itself
- """
- parsedCommands = parseScript(test, preamble=commands)
-
- _, tmpBase = _getTempPaths(test)
- execDir = os.path.dirname(test.getExecPath())
- try:
- res = lit.TestRunner.executeScriptInternal(
- test, litConfig, tmpBase, parsedCommands, execDir, debug=False
- )
- except lit.TestRunner.ScriptFatal as e:
- res = ("", str(e), 127, None)
- (out, err, exitCode, timeoutInfo) = res
-
- return (out, err, exitCode, timeoutInfo, parsedCommands)
def _validateModuleDependencies(modules):
@@ -61,334 +19,66 @@ def _validateModuleDependencies(modules):
)
-def parseScript(test, preamble):
- """
- Extract the script from a test, with substitutions applied.
-
- Returns a list of commands ready to be executed.
-
- - test
- The lit.Test to parse.
-
- - preamble
- A list of commands to perform before any command in the test.
- These commands can contain unexpanded substitutions, but they
- must not be of the form 'RUN:' -- they must be proper commands
- once substituted.
- """
- # Get the default substitutions
- tmpDir, tmpBase = _getTempPaths(test)
+def _buildModule(test, litConfig, command):
+ tmpDir, tmpBase = lit.formats.standardlibrarytest._getTempPaths(test)
+ execDir = os.path.dirname(test.getExecPath())
substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase)
- # Check base substitutions and add the %{build}, %{verify} and %{run} convenience substitutions
- #
- # Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as
- # errors, which doesn't make sense for clang-verify tests because we may want to check
- # for specific warning diagnostics.
- _checkBaseSubstitutions(substitutions)
- substitutions.append(
- ("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe")
+ substituted = lit.TestRunner.applySubstitutions(
+ [command], substitutions, recursion_limit=test.config.recursiveExpansionLimit
)
- substitutions.append(
- (
- "%{verify}",
- "%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0",
- )
- )
- substitutions.append(("%{run}", "%{exec} %t.exe"))
-
- # Parse the test file, including custom directives
- additionalCompileFlags = []
- fileDependencies = []
- modules = [] # The enabled modules
- moduleCompileFlags = [] # The compilation flags to use modules
- parsers = [
- lit.TestRunner.IntegratedTestKeywordParser(
- "FILE_DEPENDENCIES:",
- lit.TestRunner.ParserKind.LIST,
- initial_value=fileDependencies,
- ),
- lit.TestRunner.IntegratedTestKeywordParser(
- "ADDITIONAL_COMPILE_FLAGS:",
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=additionalCompileFlags,
- ),
- lit.TestRunner.IntegratedTestKeywordParser(
- "MODULE_DEPENDENCIES:",
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=modules,
- ),
- ]
-
- # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first
- # class support for conditional keywords in Lit, which would allow evaluating arbitrary
- # Lit boolean expressions instead.
- for feature in test.config.available_features:
- parser = lit.TestRunner.IntegratedTestKeywordParser(
- "ADDITIONAL_COMPILE_FLAGS({}):".format(feature),
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=additionalCompileFlags,
- )
- parsers.append(parser)
-
- scriptInTest = lit.TestRunner.parseIntegratedTestScript(
- test, additional_parsers=parsers, require_script=not preamble
- )
- if isinstance(scriptInTest, lit.Test.Result):
- return scriptInTest
-
- script = []
-
- # For each file dependency in FILE_DEPENDENCIES, inject a command to copy
- # that file to the execution directory. Execute the copy from %S to allow
- # relative paths from the test directory.
- for dep in fileDependencies:
- script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)]
- script += preamble
- script += scriptInTest
-
- # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS.
- # Modules need to be built with the same compilation flags as the
- # test. So add these flags before adding the modules.
- substitutions = config._appendToSubstitution(
- substitutions, "%{compile_flags}", " ".join(additionalCompileFlags)
- )
-
- if modules:
- _validateModuleDependencies(modules)
-
- # The moduleCompileFlags are added to the %{compile_flags}, but
- # the modules need to be built without these flags. So expand the
- # %{compile_flags} eagerly and hardcode them in the build script.
- compileFlags = config._getSubstitution("%{compile_flags}", test.config)
-
- # Building the modules needs to happen before the other script
- # commands are executed. Therefore the commands are added to the
- # front of the list.
- if "std.compat" in modules:
- script.insert(
- 0,
- "%dbg(MODULE std.compat) %{cxx} %{flags} "
- f"{compileFlags} "
- "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
- "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std.
- "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm",
- )
- moduleCompileFlags.extend(
- ["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"]
- )
-
- # Make sure the std module is built before std.compat. Libc++'s
- # std.compat module depends on the std module. It is not
- # known whether the compiler expects the modules in the order of
- # their dependencies. However it's trivial to provide them in
- # that order.
- script.insert(
- 0,
- "%dbg(MODULE std) %{cxx} %{flags} "
- f"{compileFlags} "
- "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
- "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm",
- )
- moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"])
-
- # Add compile flags required for the modules.
- substitutions = config._appendToSubstitution(
- substitutions, "%{compile_flags}", " ".join(moduleCompileFlags)
+ (out, err, exitCode, _) = lit.TestRunner.executeScriptInternal(test, litConfig, tmpBase, substituted, execDir)
+ if exitCode != 0:
+ return lit.Test.Result(
+ lit.Test.UNRESOLVED, "Failed to build module std for '{}':\n{}\n{}".format(test.getFilePath(), out, err)
)
- # Perform substitutions in the script itself.
- script = lit.TestRunner.applySubstitutions(
- script, substitutions, recursion_limit=test.config.recursiveExpansionLimit
- )
-
- return script
-
-
-class CxxStandardLibraryTest(lit.formats.FileBasedTest):
- """
- Lit test format for the C++ Standard Library conformance test suite.
-
- Lit tests are contained in files that follow a certain pattern, which determines the semantics of the test.
- Under the hood, we basically generate a builtin Lit shell test that follows the ShTest format, and perform
- the appropriate operations (compile/link/run). See
- https://libcxx.llvm.org/TestingLibcxx.html#test-names
- for a complete description of those semantics.
-
- Substitution requirements
- ===============================
- The test format operates by assuming that each test's configuration provides
- the following substitutions, which it will reuse in the shell scripts it
- constructs:
- %{cxx} - A command that can be used to invoke the compiler
- %{compile_flags} - Flags to use when compiling a test case
- %{link_flags} - Flags to use when linking a test case
- %{flags} - Flags to use either when compiling or linking a test case
- %{exec} - A command to prefix the execution of executables
-
- Note that when building an executable (as opposed to only compiling a source
- file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used
- in the same command line. In other words, the test format doesn't perform
- separate compilation and linking steps in this case.
-
- Additional provided substitutions and features
- ==============================================
- The test format will define the following substitutions for use inside tests:
-
- %{build}
- Expands to a command-line that builds the current source
- file with the %{flags}, %{compile_flags} and %{link_flags}
- substitutions, and that produces an executable named %t.exe.
-
- %{verify}
- Expands to a command-line that builds the current source
- file with the %{flags} and %{compile_flags} substitutions
- and enables clang-verify. This can be used to write .sh.cpp
- tests that use clang-verify. Note that this substitution can
- only be used when the 'verify-support' feature is available.
-
- %{run}
- Equivalent to `%{exec} %t.exe`. This is intended to be used
- in conjunction with the %{build} substitution.
- """
-
- def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig):
- SUPPORTED_SUFFIXES = [
- "[.]pass[.]cpp$",
- "[.]pass[.]mm$",
- "[.]compile[.]pass[.]cpp$",
- "[.]compile[.]pass[.]mm$",
- "[.]compile[.]fail[.]cpp$",
- "[.]link[.]pass[.]cpp$",
- "[.]link[.]pass[.]mm$",
- "[.]link[.]fail[.]cpp$",
- "[.]sh[.][^.]+$",
- "[.]gen[.][^.]+$",
- "[.]verify[.]cpp$",
- ]
-
- sourcePath = testSuite.getSourcePath(pathInSuite)
- filename = os.path.basename(sourcePath)
-
- # Ignore dot files, excluded tests and tests with an unsupported suffix
- hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES])
- if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename):
- return
-
- # If this is a generated test, run the generation step and add
- # as many Lit tests as necessary.
- if re.search('[.]gen[.][^.]+$', filename):
- for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig):
- yield test
- else:
- yield lit.Test.Test(testSuite, pathInSuite, localConfig)
+class LibcxxTest(lit.formats.StandardLibraryTest):
def execute(self, test, litConfig):
- supportsVerify = "verify-support" in test.config.available_features
- filename = test.path_in_suite[-1]
-
- if re.search("[.]sh[.][^.]+$", filename):
- steps = [] # The steps are already in the script
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".compile.pass.cpp") or filename.endswith(
- ".compile.pass.mm"
- ):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".compile.fail.cpp"):
- steps = [
- "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".link.fail.cpp"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o",
- "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe",
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".verify.cpp"):
- if not supportsVerify:
- return lit.Test.Result(
- lit.Test.UNSUPPORTED,
- "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(
- test.getFullName()
- ),
- )
- steps = ["%dbg(COMPILED WITH) %{verify}"]
- return self._executeShTest(test, litConfig, steps)
- # Make sure to check these ones last, since they will match other
- # suffixes above too.
- elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe",
- "%dbg(EXECUTED AS) %{exec} %t.exe",
- ]
- return self._executeShTest(test, litConfig, steps)
- else:
- return lit.Test.Result(
- lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)
+ # Parse any MODULE_DEPENDENCIES in the test file.
+ modules = []
+ parsers = [
+ lit.TestRunner.IntegratedTestKeywordParser(
+ "MODULE_DEPENDENCIES:",
+ lit.TestRunner.ParserKind.SPACE_LIST,
+ initial_value=modules,
)
-
- def _executeShTest(self, test, litConfig, steps):
- if test.config.unsupported:
- return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported")
-
- script = parseScript(test, steps)
- if isinstance(script, lit.Test.Result):
- return script
-
- if litConfig.noExecute:
- return lit.Test.Result(
- lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS
- )
- else:
- _, tmpBase = _getTempPaths(test)
- useExternalSh = False
- return lit.TestRunner._runShTest(
- test, litConfig, useExternalSh, script, tmpBase
+ ]
+ lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, require_script=False)
+
+ # Build the modules if needed and tweak the compiler flags of the rest of the test so
+ # it knows about the just-built modules.
+ moduleCompileFlags = []
+ if modules:
+ _validateModuleDependencies(modules)
+
+ # Make sure the std module is built before std.compat. Libc++'s
+ # std.compat module depends on the std module. It is not
+ # known whether the compiler expects the modules in the order of
+ # their dependencies. However it's trivial to provide them in
+ # that order.
+ command = ("%dbg(MODULE std) %{cxx} %{flags} %{compile_flags} "
+ "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
+ "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm")
+ res = _buildModule(test, litConfig, command)
+ if isinstance(res, lit.Test.Result):
+ return res
+ moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"])
+
+ if "std.compat" in modules:
+ command = ("%dbg(MODULE std.compat) %{cxx} %{flags} %{compile_flags} "
+ "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
+ "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std.
+ "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm")
+ res = _buildModule(test, litConfig, command)
+ if isinstance(res, lit.Test.Result):
+ return res
+ moduleCompileFlags.extend(["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"])
+
+ # Add compile flags required for the test to use the just-built modules
+ test.config.substitutions = lit.formats.standardlibrarytest._appendToSubstitution(
+ test.config.substitutions, "%{compile_flags}", " ".join(moduleCompileFlags)
)
- def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
- generator = lit.Test.Test(testSuite, pathInSuite, localConfig)
-
- # Make sure we have a directory to execute the generator test in
- generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
- os.makedirs(generatorExecDir, exist_ok=True)
-
- # Run the generator test
- steps = [] # Steps must already be in the script
- (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
- if exitCode != 0:
- raise RuntimeErr...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/90803
More information about the llvm-commits
mailing list