[libcxx] [llvm] [libc++] Allow specifying a configuration in test-at-commit (PR #224381)
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 17 11:47:10 PDT 2026
https://github.com/ldionne created https://github.com/llvm/llvm-project/pull/224381
test-at-commit always generated its own Lit configuration pointing at the libc++ installation given with --libcxx-installation, which meant it could only ever test libc++. This patch generalizes this to allow support passing a custom Lit config, which allows using the script for a wider array of use cases.
>From 1105c41ee8bfe12349d7a0edb39e1a21de85ed0f Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Wed, 16 Sep 2026 23:00:09 -0700
Subject: [PATCH] [libc++] Allow specifying a configuration in test-at-commit
test-at-commit always generated its own Lit configuration pointing at the libc++
installation given with --libcxx-installation, which meant it could only ever
test libc++. This patch generalizes this to allow support passing a custom Lit
config, which allows using the script for a wider array of use cases.
---
.github/workflows/libcxx-pr-benchmark.yml | 4 +-
libcxx/test/configs/installed-libc++.cfg.in | 46 ++++++++++++++++++++
libcxx/utils/ci/lnt/run-benchmarks | 3 +-
libcxx/utils/ci/run-buildbot | 4 +-
libcxx/utils/test-at-commit | 47 +++++----------------
5 files changed, 63 insertions(+), 41 deletions(-)
create mode 100644 libcxx/test/configs/installed-libc++.cfg.in
diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml
index bad08ce8afa72..30274e242d2a0 100644
--- a/.github/workflows/libcxx-pr-benchmark.yml
+++ b/.github/workflows/libcxx-pr-benchmark.yml
@@ -166,9 +166,9 @@ jobs:
# Run 5 times so we can pick the median, and interleave baseline and candidate to mitigate the impact of
# environmental noise
for _ in $(seq 1 5); do
- ./libcxx/utils/test-at-commit --libcxx-installation install/baseline -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS"
+ ./libcxx/utils/test-at-commit --test-config "${PWD}/libcxx/test/configs/installed-libc++.cfg.in" -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/baseline" "$BENCHMARKS"
./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee -a baseline.lnt
- ./libcxx/utils/test-at-commit --libcxx-installation install/candidate -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS"
+ ./libcxx/utils/test-at-commit --test-config "${PWD}/libcxx/test/configs/installed-libc++.cfg.in" -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/candidate" "$BENCHMARKS"
./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee -a candidate.lnt
done
diff --git a/libcxx/test/configs/installed-libc++.cfg.in b/libcxx/test/configs/installed-libc++.cfg.in
new file mode 100644
index 0000000000000..44563c4637f46
--- /dev/null
+++ b/libcxx/test/configs/installed-libc++.cfg.in
@@ -0,0 +1,46 @@
+#
+# This testing configuration runs the test suite against a libc++ that is already installed at
+# the given path.
+#
+# The additional '--param libcxx_installation=<PATH>' Lit parameter must be provided when invoking
+# Lit. It must point to the root of the installation, i.e. the directory containing 'include/c++/v1'
+# and 'lib'.
+#
+
+import os, site
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+import libcxx.test.params, libcxx.test.config, libcxx.test.dsl
+
+# Additional parameters for testing an installed libc++
+LIBCXX_INSTALLATION_PARAMETERS = [
+ libcxx.test.dsl.Parameter(name='libcxx_installation', type=str,
+ actions=lambda path: [libcxx.test.dsl.AddSubstitution('%{libcxx-installation}', os.path.abspath(path))],
+ help="""
+ The root of the libc++ installation to run the tests against. This is used to find the
+ headers and to link against the library.
+ """),
+]
+
+# Configure the compiler and flags
+config.substitutions.append(('%{flags}',
+ '-pthread' + (' -isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else '')
+))
+config.substitutions.append(('%{compile_flags}',
+ '-nostdinc++ -I %{libcxx-installation}/include/c++/v1 -I %{libcxx-dir}/test/support'
+))
+config.substitutions.append(('%{link_flags}',
+ '-nostdlib++ -L %{libcxx-installation}/lib -Wl,-rpath,%{libcxx-installation}/lib -lc++'
+))
+config.substitutions.append(('%{exec}',
+ '%{executor} --execdir %{temp} -- '
+))
+
+libcxx.test.config.configure(
+ # The installation parameters must be applied before the default ones: the default parameters
+ # probe the compiler (e.g. to select the default -std=), which is not usable until the substitutions
+ # above have been resolved.
+ LIBCXX_INSTALLATION_PARAMETERS + libcxx.test.params.DEFAULT_PARAMETERS,
+ libcxx.test.features.DEFAULT_FEATURES,
+ config,
+ lit_config
+)
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index 8dfb3a449c8c0..9c617a251baf8 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -172,11 +172,12 @@ def main(argv):
'--git-repo', args.git_repo,
'--build-dir', artifacts / 'benchmarks',
'--test-suite-commit', args.test_suite_commit,
- '--libcxx-installation', artifacts / 'libcxx-install',
+ '--test-config', args.git_repo / 'libcxx/test/configs/installed-libc++.cfg.in',
'--compiler', args.compiler,
'--',
'-j1', '--time-tests', '--test-output=failed',
'--param', f'compiler={args.compiler}',
+ '--param', f'libcxx_installation={artifacts / "libcxx-install"}',
'--param', 'optimization=speed',
'--param', 'std=c++26',
'--param', 'enable_werror=False', # older versions of the library trigger new warnings, don't fail
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 3ff6aabf07cfe..f2c3d1f67916e 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -306,10 +306,10 @@ test-tools)
step "Running a benchmark through the benchmark tooling pipeline"
"${MONOREPO_ROOT}/libcxx/utils/test-at-commit" \
--git-repo "${MONOREPO_ROOT}" \
- --libcxx-installation "${INSTALL_DIR}" \
+ --test-config "${MONOREPO_ROOT}/libcxx/test/configs/installed-libc++.cfg.in" \
--build-dir "${BUILD_DIR}/benchmarks" \
--compiler "${CXX}" \
- -- -sv --param optimization=speed \
+ -- -sv --param optimization=speed --param "libcxx_installation=${INSTALL_DIR}" \
"${MONOREPO_ROOT}/libcxx/test/benchmarks/hash.bench.cpp"
step "Consolidate benchmark results"
diff --git a/libcxx/utils/test-at-commit b/libcxx/utils/test-at-commit
index b02c571ca9b1b..5ff9607112433 100755
--- a/libcxx/utils/test-at-commit
+++ b/libcxx/utils/test-at-commit
@@ -9,30 +9,6 @@ import tempfile
PARENT_DIR = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
-LIT_CONFIG_FILE = """
-#
-# This testing configuration handles running the test suite against a version
-# of libc++ installed at the given path.
-#
-
-config.substitutions.append(('%{{flags}}',
- '-pthread' + (' -isysroot {{}}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else '')
-))
-config.substitutions.append(('%{{compile_flags}}', '-nostdinc++ -I {INSTALL_ROOT}/include/c++/v1 -I %{{libcxx-dir}}/test/support'))
-config.substitutions.append(('%{{link_flags}}', '-nostdlib++ -L {INSTALL_ROOT}/lib -Wl,-rpath,{INSTALL_ROOT}/lib -lc++'))
-config.substitutions.append(('%{{exec}}', '%{{executor}} --execdir %{{temp}} -- '))
-
-import os, site
-site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
-import libcxx.test.params, libcxx.test.config
-libcxx.test.config.configure(
- libcxx.test.params.DEFAULT_PARAMETERS,
- libcxx.test.features.DEFAULT_FEATURES,
- config,
- lit_config
-)
-"""
-
# Unofficial list of directories required to build libc++. This is a best guess that should work
# when checking out the monorepo at most commits, but it's technically not guaranteed to work
# (especially for much older commits).
@@ -77,14 +53,18 @@ def directory_path(string):
def main(argv):
parser = argparse.ArgumentParser(
prog='test-at-commit',
- description='Test the provided libc++ installation against the test suite at the specified commit (or '
- 'the currently checked-out sources by default). This makes it easier to perform historical '
- 'analyses of libc++ behavior, gather historical performance data, bisect issues, and so on.')
+ description='Test a Standard Library against the test suite at the specified commit (or the currently '
+ 'checked-out sources by default). This makes it easier to perform historical analyses of '
+ 'libc++ behavior, gather historical performance data, bisect issues, and so on.')
parser.add_argument('--build-dir', '-B', type=pathlib.Path, required=True,
help='Path to create the build directory for running the test suite at. The results of the tests '
'are located in that directory after the run.')
- parser.add_argument('--libcxx-installation', type=pathlib.Path, required=True,
- help='Path to the directory where a copy of libc++ to run tests on is installed.')
+ parser.add_argument('--test-config', type=str, required=True,
+ help='The testing configuration to use, which determines the Standard Library being tested. This has '
+ 'the same meaning as the LIBCXX_TEST_CONFIG CMake option: either an absolute path to a Lit '
+ 'configuration file, or a path relative to the libcxx/test/configs directory of the test suite '
+ 'being used. Note that when --test-suite-commit is used, a relative path is resolved against '
+ 'that older version of the test suite, so an absolute path is usually what you want.')
parser.add_argument('--compiler', type=pathlib.Path, required=True,
help='Path to the compiler to use to run the tests.')
parser.add_argument('--test-suite-commit', type=str, required=False,
@@ -99,7 +79,6 @@ def main(argv):
args = parser.parse_args(argv)
args.build_dir = args.build_dir.resolve()
- args.libcxx_installation = args.libcxx_installation.resolve()
# Gather lit options
lit_options = []
@@ -120,17 +99,13 @@ def main(argv):
checkout_dirs = [d for d in LIBCXX_REQUIRED_DIRECTORIES if exists_in_commit(args.git_repo, args.test_suite_commit, d)]
checkout_subdirectories(args.git_repo, args.test_suite_commit, checkout_dirs, test_suite_sources)
- # Configure the test suite in the specified build directory
+ # Configure the test suite in the specified build directory.
args.build_dir.mkdir(parents=True, exist_ok=True)
- lit_cfg = (args.build_dir / 'temp_lit_cfg.cfg.in').absolute()
- with open(lit_cfg, 'w') as f:
- f.write(LIT_CONFIG_FILE.format(INSTALL_ROOT=args.libcxx_installation))
-
test_suite_cmd = ['cmake', '-B', args.build_dir, '-S', test_suite_sources / 'runtimes', '-G', 'Ninja']
test_suite_cmd += ['-D', f'CMAKE_CXX_COMPILER={args.compiler}']
test_suite_cmd += ['-D', 'LLVM_ENABLE_RUNTIMES=libcxx;libcxxabi']
test_suite_cmd += ['-D', 'LIBCXXABI_USE_LLVM_UNWINDER=OFF']
- test_suite_cmd += ['-D', f'LIBCXX_TEST_CONFIG={lit_cfg}']
+ test_suite_cmd += ['-D', f'LIBCXX_TEST_CONFIG={args.test_config}']
subprocess.check_call(test_suite_cmd)
# Run the specified tests against the built library
More information about the llvm-commits
mailing list