[libcxx] [llvm] [libc++] Add a benchmark machine for libstdc++ (PR #226346)
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 07:37:51 PDT 2026
https://github.com/ldionne updated https://github.com/llvm/llvm-project/pull/226346
>From 16f2aee0c114693e10e398c68619debaca81c66a Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Fri, 25 Sep 2026 10:24:28 -0400
Subject: [PATCH] [libc++] Add a benchmark machine for libstdc++
Add a machine that benchmarks the libstdc++ shipped on macOS instead of
a libc++ build. To enable that, generalize the scripts and workflows to
make more aspects of machines.json optional (e.g. support machines that
don't need to build libc++).
---
.github/workflows/libcxx-benchmark-commit.yml | 34 +++++++++++++------
.github/workflows/libcxx-benchmark-cron.yml | 5 ++-
.github/workflows/libcxx-pr-benchmark.yml | 15 ++++----
libcxx/utils/ci/lnt/README.md | 26 +++++++++++---
libcxx/utils/ci/lnt/machines.json | 25 ++++++++++++--
libcxx/utils/ci/lnt/run-benchmarks | 20 ++++++-----
libcxx/utils/ci/run-buildbot | 4 +--
7 files changed, 93 insertions(+), 36 deletions(-)
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
index 0aafda54561306..4c62087f2eeded 100644
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ b/.github/workflows/libcxx-benchmark-commit.yml
@@ -107,8 +107,8 @@ jobs:
runs-on: ${{ matrix.runner }}
env:
COMPILER: ${{ matrix.cxx }}
- # Where we install the library. This lives inside the workspace so that actions/checkout
- # cleans it up between runs on self-hosted runners.
+ # Where the library is installed when we build it. This lives inside the workspace
+ # so that actions/checkout cleans it up between runs on self-hosted runners.
INSTALL_DIR: ${{ github.workspace }}/install
steps:
- name: Checkout the LLVM monorepo
@@ -131,9 +131,13 @@ jobs:
- name: Install dependencies via Homebrew
if: runner.os == 'macOS'
+ env:
+ # Extra packages needed by the test configuration
+ EXTRA_BREW_PACKAGES: ${{ join(matrix.brew-packages, ' ') }}
run: |
+ read -ra extra_packages <<< "${EXTRA_BREW_PACKAGES}"
brew update
- brew install ninja cmake python at 3.14
+ brew install ninja cmake python at 3.14 "${extra_packages[@]}"
echo "$(brew --prefix python at 3.14)/bin" >> "$GITHUB_PATH"
- name: Diagnose tools in use
@@ -149,11 +153,14 @@ jobs:
source .venv/bin/activate
pip install -r libcxx/utils/ci/lnt/requirements.txt
- # Build the library.
+ # Build the library. This is only done when the machine configuration provides a 'build' key,
+ # otherwise we assume that the configuration does not require building anything (e.g. a system
+ # standard library).
#
# A build failure is tolerated on purpose, since the library doesn't build at every historical commit.
# We still submit an empty run for such commits.
- name: Build libc++ at ${{ inputs.commit }}
+ if: ${{ matrix.build != '' }}
continue-on-error: true
uses: ./.github/workflows/libcxx/build-at-commit
with:
@@ -161,15 +168,17 @@ jobs:
commit: ${{ inputs.commit }}
install-dir: ${{ env.INSTALL_DIR }}
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.cmake-cache }}
+ cmake-cache: ${{ matrix.build.cmake-cache }}
- name: Run the benchmarks
env:
BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
+ BUILT_LIBCXX: ${{ matrix.build != '' }}
COMMIT: ${{ inputs.commit }}
FILTER: ${{ inputs.filter }}
LIT_PARAMS: ${{ join(matrix.lit-params, ' ') }}
LNT_MACHINE: ${{ matrix.lnt-machine }}
+ TEST_CONFIG: ${{ matrix.test-config }}
run: |
source .venv/bin/activate
filter_arg=()
@@ -183,17 +192,22 @@ jobs:
lit_params+=(--param "${param}")
done
- # If the library build failed, still create an install directory so we
- # run the benchmarks. The benchmarks will fail, and we'll submit an empty
- # LNT report.
- mkdir -p "${INSTALL_DIR}"
+ # When the library is built in this workflow, point the testing configuration at the installation.
+ # Everything else needed by the configuration comes from machines.json.
+ if [ "${BUILT_LIBCXX}" = "true" ]; then
+ # If the library build failed, still create an install directory so we
+ # run the benchmarks. The benchmarks will fail, and we'll submit an empty
+ # LNT report.
+ mkdir -p "${INSTALL_DIR}"
+ lit_params+=(--param "libcxx_installation=${INSTALL_DIR}")
+ fi
libcxx/utils/ci/lnt/run-benchmarks \
--test-suite-commit "${BENCHMARK_SUITE_VERSION}" \
--machine "${LNT_MACHINE}" \
--compiler "${COMPILER}" \
--benchmark-commit "${COMMIT}" \
- --libcxx-installation "${INSTALL_DIR}" \
+ --test-config "${TEST_CONFIG}" \
"${filter_arg[@]}" \
--output "${COMMIT}.json" \
-- "${lit_params[@]}"
diff --git a/.github/workflows/libcxx-benchmark-cron.yml b/.github/workflows/libcxx-benchmark-cron.yml
index d25328c89fd38f..5bad2531e30770 100644
--- a/.github/workflows/libcxx-benchmark-cron.yml
+++ b/.github/workflows/libcxx-benchmark-cron.yml
@@ -58,7 +58,10 @@ jobs:
with:
script: |
const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
- core.setOutput('matrix', JSON.stringify(config.map(cfg => ({
+ // Only machines with a `coverage` key are tracked by this workflow. Other machines may exist but we
+ // don't generate historical data for them.
+ const tracked = config.filter(cfg => cfg.coverage);
+ core.setOutput('matrix', JSON.stringify(tracked.map(cfg => ({
machine: cfg['lnt-machine'],
...cfg.coverage,
}))));
diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml
index 39cab7b72c5cce..e5de26cc1f0f25 100644
--- a/.github/workflows/libcxx-pr-benchmark.yml
+++ b/.github/workflows/libcxx-pr-benchmark.yml
@@ -71,9 +71,11 @@ jobs:
const match = context.payload.comment.body.match(/\/libcxx-bot benchmark (.+)/);
core.setOutput('benchmarks', match ? match[1] : '');
- // Benchmark on the same configurations that we track performance on.
+ // Benchmark the configurations defined in machines.json. We only consider configurations that
+ // build something, since there's nothing to do an A/B comparison with for other configurations.
const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
- core.setOutput('matrix', JSON.stringify(config.map(({coverage, ...machine}) => machine)));
+ const buildable = config.filter(cfg => cfg.build);
+ core.setOutput('matrix', JSON.stringify(buildable.map(({coverage, ...machine}) => machine)));
- name: Update comment with link to the run
uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
@@ -115,6 +117,7 @@ jobs:
LNT_MACHINE: ${{ matrix.lnt-machine }}
PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}
PR_BASE: ${{ needs.extract-info.outputs.pr_base }}
+ TEST_CONFIG: ${{ matrix.test-config }}
TOOLING: ${{ github.workspace }}/tooling
steps:
- name: Checkout the PR
@@ -179,7 +182,7 @@ jobs:
commit: ${{ steps.baseline.outputs.commit }}
install-dir: ${{ github.workspace }}/install/baseline
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.cmake-cache }}
+ cmake-cache: ${{ matrix.build.cmake-cache }}
- name: Build the candidate
uses: ./tooling/.github/workflows/libcxx/build-at-commit
@@ -188,7 +191,7 @@ jobs:
commit: ${{ needs.extract-info.outputs.pr_head }}
install-dir: ${{ github.workspace }}/install/candidate
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.cmake-cache }}
+ cmake-cache: ${{ matrix.build.cmake-cache }}
- name: Run baseline and candidate interleaved
run: |
@@ -203,9 +206,9 @@ jobs:
# Run 5 times so we can pick the median, and interleave baseline and candidate to mitigate the impact of
# environmental noise
for _ in $(seq 1 5); do
- "${TOOLING}/libcxx/utils/test-at-commit" --test-config installed-libc++.cfg.in -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/baseline" "$BENCHMARKS"
+ "${TOOLING}/libcxx/utils/test-at-commit" --test-config "${TEST_CONFIG}" -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/baseline" "$BENCHMARKS"
"${TOOLING}/libcxx/utils/consolidate-benchmarks" benchmarks/baseline | tee -a baseline.lnt
- "${TOOLING}/libcxx/utils/test-at-commit" --test-config installed-libc++.cfg.in -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/candidate" "$BENCHMARKS"
+ "${TOOLING}/libcxx/utils/test-at-commit" --test-config "${TEST_CONFIG}" -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/candidate" "$BENCHMARKS"
"${TOOLING}/libcxx/utils/consolidate-benchmarks" benchmarks/candidate | tee -a candidate.lnt
done
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index ed1ecae79857c2..ff5dcc0cce9a90 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -67,6 +67,17 @@ single source of truth for all workflows that run benchmarks (PR benchmarking, r
historical benchmarks, etc). Each entry contains variables used by the various workflows
and the LNT machine name that the results will be reported under.
+The `test-config` key selects the Lit testing configuration to benchmark. This is used to e.g.
+select which Standard Library is being measured. The `lit-params` key provides additional lit
+parameters to pass when running the benchmarks.
+
+The `build` key allows providing the CMake cache to use when building the library before running
+the benchmarks. If `build` is not present, building libc++ is skipped for that configuration.
+
+The `coverage` key establishes how far back and at which frequency performance should be measured
+for that configuration. A machine without a `coverage` entry can be defined, but it won't result
+in historical data.
+
## Running benchmarks locally
On GitHub, the `libcxx-benchmark-commit.yml` workflow is used to run benchmarks and report
@@ -74,17 +85,22 @@ results to a LNT instance. This workflow wraps the `libcxx/utils/ci/lnt/run-benc
which can be used to benchmark locally:
```
-run-benchmarks --test-suite-commit <SHA1> --machine <MACHINE> \
- --compiler clang++ --benchmark-commit <SHA2> \
- --libcxx-installation <PATH> \
- --output result.json \
- -- --param std=c++26 --param optimization=speed
+run-benchmarks --test-suite-commit <SHA1> --machine <MACHINE> \
+ --compiler clang++ --benchmark-commit <SHA2> \
+ --test-config installed-libc++.cfg.in \
+ --output result.json \
+ -- --param std=c++26 --param optimization=speed \
+ --param libcxx_installation=<PATH>
```
This will run the benchmarks (using the test suite at the specified `SHA1`) against the installation
of libc++ at `PATH` (which is assumed to be libc++ as-of `SHA2`), and produce a LNT-ready JSON report.
The results can then be submitted to a LNT instance if desired.
+Note that `run-benchmarks` does not build anything: the library being benchmarked must have been built
+or installed beforehand. How to pick up that library is determined by `--test-config` and any Lit parameters
+passed.
+
## Setting up a local LNT instance
```
diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json
index fce5eea24085bf..187cfec16d7554 100644
--- a/libcxx/utils/ci/lnt/machines.json
+++ b/libcxx/utils/ci/lnt/machines.json
@@ -5,8 +5,11 @@
"cxx": "clang++",
"xcode-version": "26.6",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
+ "build": {
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake"
+ },
"lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in",
"coverage": {
"since": "2023-01-01",
"every": "week",
@@ -21,8 +24,11 @@
"cxx": "clang++",
"xcode-version": "26.6",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake",
+ "build": {
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake"
+ },
"lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in",
"coverage": {
"since": "2023-12-01",
"every": "week",
@@ -36,8 +42,11 @@
"runner": "llvm-premerge-libcxx-runners",
"cxx": "clang++-22",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
+ "build": {
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake"
+ },
"lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in",
"coverage": {
"since": "2023-01-01",
"every": "week",
@@ -45,5 +54,15 @@
"max-in-flight": 8,
"lnt-url": "https://lnt.llvm.org"
}
+ },
+ {
+ "lnt-machine": "macos-26.6.2-arm64-libstdcxx16-20260925",
+ "runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
+ "cxx": "clang++",
+ "xcode-version": "26.6",
+ "benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
+ "brew-packages": ["gcc at 16"],
+ "lit-params": ["std=c++26", "optimization=speed", "libstdcxx_compiler=g++-16"],
+ "test-config": "stdlib-libstdc++.cfg.in"
}
]
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index f95a2b431dc392..ca6fbbd4141fae 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -107,12 +107,17 @@ def dict_to_params(d):
def main(argv):
parser = argparse.ArgumentParser(
prog='run-benchmarks',
- description='Benchmark libc++ at the given commit and produce a LNT JSON report.',
+ description='Run the libc++ benchmark suite against a Standard Library and produce a LNT JSON report. '
+ 'The library being benchmarked is selected by the testing configuration passed to --test-config.',
epilog='This script depends on the modules listed in `libcxx/utils/ci/lnt/requirements.txt`.')
- parser.add_argument('--libcxx-installation', type=pathlib.Path, required=True,
- help='Directory where libc++ is installed.')
+ parser.add_argument('--test-config', type=str, required=True,
+ help='The Lit testing configuration to use. This has the same meaning as for test-at-commit: either an '
+ 'absolute path to a Lit configuration file, or a path relative to the libcxx/test/configs directory '
+ 'of the test suite being used.')
parser.add_argument('--benchmark-commit', type=str, required=True,
- help='The SHA representing the version of the library to benchmark.')
+ help='The SHA that the results are attributed to in LNT. When benchmarking libc++, this is the version '
+ 'of the library being benchmarked. When benchmarking another Standard Library, this is merely a label '
+ 'placing the results on the LNT time axis.')
parser.add_argument('--test-suite-commit', type=str, required=True,
help='The SHA representing the version of the test suite to use for benchmarking.')
parser.add_argument('--compiler', type=str, required=True,
@@ -183,8 +188,6 @@ def main(argv):
# check dependencies.
if args.output.exists():
sys.exit(f'error: output report {args.output} already exists; not overwriting it')
- if not args.libcxx_installation.exists():
- sys.exit(f'error: libc++ installation directory {args.libcxx_installation} does not exist')
if args.artifacts_dir is not None and args.artifacts_dir.exists():
sys.exit(f'error: artifacts directory {args.artifacts_dir} already exists; not overwriting it')
if shutil.which('lnt') is None:
@@ -200,10 +203,9 @@ def main(argv):
artifacts.mkdir(parents=True, exist_ok=True)
logging.info(f'Storing artifacts in {artifacts}')
- logging.info(f'Running benchmarks from {args.test_suite_commit} against libc++ at {args.libcxx_installation}')
+ logging.info(f'Running benchmarks from {args.test_suite_commit} against {args.test_config}')
# Always add some lit parameters that are required or make sense when running benchmarks.
lit_args = ['--param', f'compiler={args.compiler}',
- '--param', f'libcxx_installation={args.libcxx_installation}',
'--param', 'enable_werror=False'] # older versions of the library trigger new warnings, don't fail
if args.spec_dir is not None:
lit_args += ['--param', f'spec_dir={args.spec_dir}']
@@ -213,7 +215,7 @@ def main(argv):
'--build-dir', artifacts / 'benchmarks-build',
'--test-suite-commit', args.test_suite_commit,
'--tmp-src-dir', artifacts / 'benchmarks-src',
- '--test-config', 'installed-libc++.cfg.in',
+ '--test-config', args.test_config,
'--compiler', args.compiler,
'--',
'-j1', '--time-tests', '--test-output=failed',
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 572f326c4fb0a2..3bfb5bb5fb02c3 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -337,14 +337,14 @@ EOF
"${MONOREPO_ROOT}/libcxx/utils/ci/lnt/run-benchmarks" \
--dry-run -vv \
--git-repo "${MONOREPO_ROOT}" \
- --libcxx-installation "${INSTALL_DIR}" \
+ --test-config installed-libc++.cfg.in \
--benchmark-commit HEAD \
--test-suite-commit HEAD \
--machine test-tools \
--compiler "${CXX}" \
--filter hash.bench.cpp \
--output "${BUILD_DIR}/report.json" \
- -- --param optimization=speed
+ -- --param optimization=speed --param "libcxx_installation=${INSTALL_DIR}"
step "Submit a LNT report (dry-run)"
"${MONOREPO_ROOT}/libcxx/utils/ci/lnt/submit-benchmarks" \
More information about the llvm-commits
mailing list