[libcxx-commits] [libcxx] f039cae - [libc++] Add support for generated tests in the libc++ test format
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Thu May 25 14:17:56 PDT 2023
Author: Louis Dionne
Date: 2023-05-25T14:17:45-07:00
New Revision: f039caec338300b794f0edf0ef10df49b297ec37
URL: https://github.com/llvm/llvm-project/commit/f039caec338300b794f0edf0ef10df49b297ec37
DIFF: https://github.com/llvm/llvm-project/commit/f039caec338300b794f0edf0ef10df49b297ec37.diff
LOG: [libc++] Add support for generated tests in the libc++ test format
A recurring problem recently has been that libc++ has several generated
tests which all need to be re-generated before committing a change. This
creates noise during code reviews and friction for contributors.
Furthermore, the way we generated most of these tests resulted in
extremely bad compilation times when using modules, because we defined
a macro before compiling each file.
This commit introduces a new kind of test called a '.gen' test. These
tests are normal shell tests, however the Lit test format will run the
test to discover the actual Lit tests it should run. This basically
allows generating a Lit test suite on the fly using arbitrary code,
which can be used in the future to generate tests like our __verbose_abort
tests and several others.
Differential Revision: https://reviews.llvm.org/D151258
Added:
libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp
libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp
libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp
Modified:
libcxx/utils/libcxx/test/dsl.py
libcxx/utils/libcxx/test/format.py
Removed:
################################################################################
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp b/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp
new file mode 100644
index 0000000000000..1915bede471f5
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp
@@ -0,0 +1,11 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate no tests at all
+
+// RUN: true
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp b/libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp
new file mode 100644
index 0000000000000..077baa11d89b6
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp
@@ -0,0 +1,11 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate one test
+
+// RUN: echo "//--- test1.compile.pass.cpp"
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp b/libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp
new file mode 100644
index 0000000000000..91ddf5208dcfd
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp
@@ -0,0 +1,12 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate two tests
+
+// RUN: echo "//--- test1.compile.pass.cpp"
+// RUN: echo "//--- test2.compile.pass.cpp"
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 66cf8176e0459..8a753f59e8178 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -179,9 +179,7 @@ def programOutput(config, program, args=None):
"Failed to run program, cmd:\n{}\nstderr is:\n{}".format(runcmd, err)
)
- actualOut = re.search("# command output:\n(.+)\n$", out, flags=re.DOTALL)
- actualOut = actualOut.group(1) if actualOut else ""
- return actualOut
+ return libcxx.test.format._parseLitOutput(out)
@_memoizeExpensiveOperation(
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index 6827b1b74b740..8a96dbceef4e9 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -6,6 +6,8 @@
#
# ===----------------------------------------------------------------------===##
+import contextlib
+import io
import lit
import lit.formats
import os
@@ -33,6 +35,38 @@ def _checkBaseSubstitutions(substitutions):
for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
assert s in substitutions, "Required substitution {} was not provided".format(s)
+def _parseLitOutput(fullOutput):
+ """
+ Parse output of a Lit ShTest to extract the actual output of the contained commands.
+
+ This takes output of the form
+
+ $ ":" "RUN: at line 11"
+ $ "echo" "OUTPUT1"
+ # command output:
+ OUTPUT1
+
+ $ ":" "RUN: at line 12"
+ $ "echo" "OUTPUT2"
+ # command output:
+ OUTPUT2
+
+ and returns a string containing
+
+ OUTPUT1
+ OUTPUT2
+
+ as-if the commands had been run directly. This is a workaround for the fact
+ that Lit doesn't let us execute ShTest and retrieve the raw output without
+ injecting additional Lit output around it.
+ """
+ parsed = ''
+ for output in re.split('[$]\s*":"\s*"RUN: at line \d+"', fullOutput):
+ if output: # skip blank lines
+ commandOutput = re.search("# command output:\n(.+)\n$", output, flags=re.DOTALL)
+ if commandOutput:
+ parsed += commandOutput.group(1)
+ return parsed
def _executeScriptInternal(test, litConfig, commands):
"""
@@ -170,6 +204,16 @@ class CxxStandardLibraryTest(lit.formats.TestFormat):
FOO.sh.<anything> - A builtin Lit Shell test
+ FOO.gen.<anything> - A .sh test that generates one or more Lit tests on the
+ fly. Executing this test must generate one or more files
+ as expected by LLVM split-file, and each generated file
+ leads to a separate Lit test that runs that file as
+ defined by the test format. This can be used to generate
+ multiple Lit tests from a single source file, which is
+ useful for testing repetitive properties in the library.
+ Be careful not to abuse this since this is not a replacement
+ for usual code reuse techniques.
+
FOO.verify.cpp - Compiles with clang-verify. This type of test is
automatically marked as UNSUPPORTED if the compiler
does not support Clang-verify.
@@ -245,6 +289,7 @@ def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
"[.]link[.]pass[.]mm$",
"[.]link[.]fail[.]cpp$",
"[.]sh[.][^.]+$",
+ "[.]gen[.][^.]+$",
"[.]verify[.]cpp$",
"[.]fail[.]cpp$",
]
@@ -257,9 +302,13 @@ def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
filepath = os.path.join(sourcePath, filename)
if not os.path.isdir(filepath):
if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]):
- yield lit.Test.Test(
- testSuite, pathInSuite + (filename,), localConfig
- )
+ # If this is a generated test, run the generation step and add
+ # as many Lit tests as necessary.
+ if re.search('[.]gen[.][^.]+$', filename):
+ for test in self._generateGenTest(testSuite, pathInSuite + (filename,), litConfig, localConfig):
+ yield test
+ else:
+ yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig)
def execute(self, test, litConfig):
VERIFY_FLAGS = (
@@ -356,3 +405,42 @@ def _executeShTest(self, test, litConfig, steps):
return lit.TestRunner._runShTest(
test, litConfig, useExternalSh, script, tmpBase
)
+
+ def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
+ generator = lit.Test.Test(testSuite, pathInSuite, localConfig)
+
+ # Make sure we have a directory to execute the generator test in
+ generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
+ os.makedirs(generatorExecDir, exist_ok=True)
+
+ # Run the generator test
+ steps = [] # Steps must already be in the script
+ (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
+ if exitCode != 0:
+ raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}")
+
+ # Split the generated output into multiple files and generate one test for each file
+ parsed = _parseLitOutput(out)
+ for (subfile, content) in self._splitFile(parsed):
+ generatedFile = testSuite.getExecPath(pathInSuite + (subfile, ))
+ os.makedirs(os.path.dirname(generatedFile), exist_ok=True)
+ with open(generatedFile, 'w') as f:
+ f.write(content)
+ yield lit.Test.Test(testSuite, (generatedFile,), localConfig)
+
+ def _splitFile(self, input):
+ DELIM = r'^(//|#)---(.+)'
+ lines = input.splitlines()
+ currentFile = None
+ thisFileContent = []
+ for line in lines:
+ match = re.match(DELIM, line)
+ if match:
+ if currentFile is not None:
+ yield (currentFile, '\n'.join(thisFileContent))
+ currentFile = match.group(2).strip()
+ thisFileContent = []
+ assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}"
+ thisFileContent.append(line)
+ if currentFile is not None:
+ yield (currentFile, '\n'.join(thisFileContent))
More information about the libcxx-commits
mailing list