[libcxx] [llvm] [libc++] Add a benchmark machine for libstdc++ (PR #226346)
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 21:57:20 PDT 2026
https://github.com/ldionne updated https://github.com/llvm/llvm-project/pull/226346
>From cd21a72fc7f30d74a9169913be1fd933a607ab43 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 24 Sep 2026 22:06:28 -0400
Subject: [PATCH 1/2] [libc++] Add a benchmark machine for libstdc++
Add a machine that benchmarks the libstdc++ shipped on macOS instead of
a libc++ build. To enable that, generalize the scripts and workflows to
make more aspects of machines.json optional (e.g. support machines that
don't need to build libc++).
---
.github/workflows/libcxx-benchmark-commit.yml | 28 +++++++++++++------
.github/workflows/libcxx-benchmark-cron.yml | 5 +++-
.github/workflows/libcxx-pr-benchmark.yml | 15 ++++++----
libcxx/utils/ci/lnt/README.md | 26 +++++++++++++----
libcxx/utils/ci/lnt/machines.json | 23 +++++++++++++--
libcxx/utils/ci/lnt/run-benchmarks | 20 +++++++------
libcxx/utils/ci/run-buildbot | 4 +--
7 files changed, 86 insertions(+), 35 deletions(-)
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
index 0aafda5456130..0e12a0470f284 100644
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ b/.github/workflows/libcxx-benchmark-commit.yml
@@ -107,8 +107,8 @@ jobs:
runs-on: ${{ matrix.runner }}
env:
COMPILER: ${{ matrix.cxx }}
- # Where we install the library. This lives inside the workspace so that actions/checkout
- # cleans it up between runs on self-hosted runners.
+ # Where the library is installed when we build it. This lives inside the workspace
+ # so that actions/checkout cleans it up between runs on self-hosted runners.
INSTALL_DIR: ${{ github.workspace }}/install
steps:
- name: Checkout the LLVM monorepo
@@ -149,11 +149,14 @@ jobs:
source .venv/bin/activate
pip install -r libcxx/utils/ci/lnt/requirements.txt
- # Build the library.
+ # Build the library. This is only done when the machine configuration provides a 'build' key,
+ # otherwise we assume that the configuration does not require building anything (e.g. a system
+ # standard library).
#
# A build failure is tolerated on purpose, since the library doesn't build at every historical commit.
# We still submit an empty run for such commits.
- name: Build libc++ at ${{ inputs.commit }}
+ if: ${{ matrix.build != '' }}
continue-on-error: true
uses: ./.github/workflows/libcxx/build-at-commit
with:
@@ -161,15 +164,17 @@ jobs:
commit: ${{ inputs.commit }}
install-dir: ${{ env.INSTALL_DIR }}
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.cmake-cache }}
+ cmake-cache: ${{ matrix.build.cmake-cache }}
- name: Run the benchmarks
env:
BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
+ BUILT_LIBCXX: ${{ matrix.build != '' }}
COMMIT: ${{ inputs.commit }}
FILTER: ${{ inputs.filter }}
LIT_PARAMS: ${{ join(matrix.lit-params, ' ') }}
LNT_MACHINE: ${{ matrix.lnt-machine }}
+ TEST_CONFIG: ${{ matrix.test-config }}
run: |
source .venv/bin/activate
filter_arg=()
@@ -183,17 +188,22 @@ jobs:
lit_params+=(--param "${param}")
done
- # If the library build failed, still create an install directory so we
- # run the benchmarks. The benchmarks will fail, and we'll submit an empty
- # LNT report.
- mkdir -p "${INSTALL_DIR}"
+ # When the library is built in this workflow, point the testing configuration at the installation.
+ # Everything else needed by the configuration comes from machines.json.
+ if [ "${BUILT_LIBCXX}" = "true" ]; then
+ # If the library build failed, still create an install directory so we
+ # run the benchmarks. The benchmarks will fail, and we'll submit an empty
+ # LNT report.
+ mkdir -p "${INSTALL_DIR}"
+ lit_params+=(--param "libcxx_installation=${INSTALL_DIR}")
+ fi
libcxx/utils/ci/lnt/run-benchmarks \
--test-suite-commit "${BENCHMARK_SUITE_VERSION}" \
--machine "${LNT_MACHINE}" \
--compiler "${COMPILER}" \
--benchmark-commit "${COMMIT}" \
- --libcxx-installation "${INSTALL_DIR}" \
+ --test-config "${TEST_CONFIG}" \
"${filter_arg[@]}" \
--output "${COMMIT}.json" \
-- "${lit_params[@]}"
diff --git a/.github/workflows/libcxx-benchmark-cron.yml b/.github/workflows/libcxx-benchmark-cron.yml
index d25328c89fd38..5bad2531e3077 100644
--- a/.github/workflows/libcxx-benchmark-cron.yml
+++ b/.github/workflows/libcxx-benchmark-cron.yml
@@ -58,7 +58,10 @@ jobs:
with:
script: |
const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
- core.setOutput('matrix', JSON.stringify(config.map(cfg => ({
+ // Only machines with a `coverage` key are tracked by this workflow. Other machines may exist but we
+ // don't generate historical data for them.
+ const tracked = config.filter(cfg => cfg.coverage);
+ core.setOutput('matrix', JSON.stringify(tracked.map(cfg => ({
machine: cfg['lnt-machine'],
...cfg.coverage,
}))));
diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml
index 39cab7b72c5cc..e5de26cc1f0f2 100644
--- a/.github/workflows/libcxx-pr-benchmark.yml
+++ b/.github/workflows/libcxx-pr-benchmark.yml
@@ -71,9 +71,11 @@ jobs:
const match = context.payload.comment.body.match(/\/libcxx-bot benchmark (.+)/);
core.setOutput('benchmarks', match ? match[1] : '');
- // Benchmark on the same configurations that we track performance on.
+ // Benchmark the configurations defined in machines.json. We only consider configurations that
+ // build something, since there's nothing to do an A/B comparison with for other configurations.
const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
- core.setOutput('matrix', JSON.stringify(config.map(({coverage, ...machine}) => machine)));
+ const buildable = config.filter(cfg => cfg.build);
+ core.setOutput('matrix', JSON.stringify(buildable.map(({coverage, ...machine}) => machine)));
- name: Update comment with link to the run
uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
@@ -115,6 +117,7 @@ jobs:
LNT_MACHINE: ${{ matrix.lnt-machine }}
PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}
PR_BASE: ${{ needs.extract-info.outputs.pr_base }}
+ TEST_CONFIG: ${{ matrix.test-config }}
TOOLING: ${{ github.workspace }}/tooling
steps:
- name: Checkout the PR
@@ -179,7 +182,7 @@ jobs:
commit: ${{ steps.baseline.outputs.commit }}
install-dir: ${{ github.workspace }}/install/baseline
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.cmake-cache }}
+ cmake-cache: ${{ matrix.build.cmake-cache }}
- name: Build the candidate
uses: ./tooling/.github/workflows/libcxx/build-at-commit
@@ -188,7 +191,7 @@ jobs:
commit: ${{ needs.extract-info.outputs.pr_head }}
install-dir: ${{ github.workspace }}/install/candidate
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.cmake-cache }}
+ cmake-cache: ${{ matrix.build.cmake-cache }}
- name: Run baseline and candidate interleaved
run: |
@@ -203,9 +206,9 @@ jobs:
# Run 5 times so we can pick the median, and interleave baseline and candidate to mitigate the impact of
# environmental noise
for _ in $(seq 1 5); do
- "${TOOLING}/libcxx/utils/test-at-commit" --test-config installed-libc++.cfg.in -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/baseline" "$BENCHMARKS"
+ "${TOOLING}/libcxx/utils/test-at-commit" --test-config "${TEST_CONFIG}" -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/baseline" "$BENCHMARKS"
"${TOOLING}/libcxx/utils/consolidate-benchmarks" benchmarks/baseline | tee -a baseline.lnt
- "${TOOLING}/libcxx/utils/test-at-commit" --test-config installed-libc++.cfg.in -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/candidate" "$BENCHMARKS"
+ "${TOOLING}/libcxx/utils/test-at-commit" --test-config "${TEST_CONFIG}" -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/candidate" "$BENCHMARKS"
"${TOOLING}/libcxx/utils/consolidate-benchmarks" benchmarks/candidate | tee -a candidate.lnt
done
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index ed1ecae79857c..ff5dcc0cce9a9 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -67,6 +67,17 @@ single source of truth for all workflows that run benchmarks (PR benchmarking, r
historical benchmarks, etc). Each entry contains variables used by the various workflows
and the LNT machine name that the results will be reported under.
+The `test-config` key selects the Lit testing configuration to benchmark. This is used to e.g.
+select which Standard Library is being measured. The `lit-params` key provides additional lit
+parameters to pass when running the benchmarks.
+
+The `build` key allows providing the CMake cache to use when building the library before running
+the benchmarks. If `build` is not present, building libc++ is skipped for that configuration.
+
+The `coverage` key establishes how far back and at which frequency performance should be measured
+for that configuration. A machine without a `coverage` entry can be defined, but it won't result
+in historical data.
+
## Running benchmarks locally
On GitHub, the `libcxx-benchmark-commit.yml` workflow is used to run benchmarks and report
@@ -74,17 +85,22 @@ results to a LNT instance. This workflow wraps the `libcxx/utils/ci/lnt/run-benc
which can be used to benchmark locally:
```
-run-benchmarks --test-suite-commit <SHA1> --machine <MACHINE> \
- --compiler clang++ --benchmark-commit <SHA2> \
- --libcxx-installation <PATH> \
- --output result.json \
- -- --param std=c++26 --param optimization=speed
+run-benchmarks --test-suite-commit <SHA1> --machine <MACHINE> \
+ --compiler clang++ --benchmark-commit <SHA2> \
+ --test-config installed-libc++.cfg.in \
+ --output result.json \
+ -- --param std=c++26 --param optimization=speed \
+ --param libcxx_installation=<PATH>
```
This will run the benchmarks (using the test suite at the specified `SHA1`) against the installation
of libc++ at `PATH` (which is assumed to be libc++ as-of `SHA2`), and produce a LNT-ready JSON report.
The results can then be submitted to a LNT instance if desired.
+Note that `run-benchmarks` does not build anything: the library being benchmarked must have been built
+or installed beforehand. How to pick up that library is determined by `--test-config` and any Lit parameters
+passed.
+
## Setting up a local LNT instance
```
diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json
index fce5eea24085b..2099daa41d62a 100644
--- a/libcxx/utils/ci/lnt/machines.json
+++ b/libcxx/utils/ci/lnt/machines.json
@@ -5,8 +5,11 @@
"cxx": "clang++",
"xcode-version": "26.6",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
+ "build": {
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake"
+ },
"lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in",
"coverage": {
"since": "2023-01-01",
"every": "week",
@@ -21,8 +24,11 @@
"cxx": "clang++",
"xcode-version": "26.6",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake",
+ "build": {
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake"
+ },
"lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in",
"coverage": {
"since": "2023-12-01",
"every": "week",
@@ -36,8 +42,11 @@
"runner": "llvm-premerge-libcxx-runners",
"cxx": "clang++-22",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
+ "build": {
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake"
+ },
"lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in",
"coverage": {
"since": "2023-01-01",
"every": "week",
@@ -45,5 +54,13 @@
"max-in-flight": 8,
"lnt-url": "https://lnt.llvm.org"
}
+ },
+ {
+ "lnt-machine": "linux-x86_64-libstdcxx-20260924",
+ "runner": "llvm-premerge-libcxx-runners",
+ "cxx": "clang++-22",
+ "benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
+ "lit-params": ["std=c++26", "optimization=speed", "libstdcxx_compiler=g++-16"],
+ "test-config": "stdlib-libstdc++.cfg.in"
}
]
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index f95a2b431dc39..ca6fbbd4141fa 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -107,12 +107,17 @@ def dict_to_params(d):
def main(argv):
parser = argparse.ArgumentParser(
prog='run-benchmarks',
- description='Benchmark libc++ at the given commit and produce a LNT JSON report.',
+ description='Run the libc++ benchmark suite against a Standard Library and produce a LNT JSON report. '
+ 'The library being benchmarked is selected by the testing configuration passed to --test-config.',
epilog='This script depends on the modules listed in `libcxx/utils/ci/lnt/requirements.txt`.')
- parser.add_argument('--libcxx-installation', type=pathlib.Path, required=True,
- help='Directory where libc++ is installed.')
+ parser.add_argument('--test-config', type=str, required=True,
+ help='The Lit testing configuration to use. This has the same meaning as for test-at-commit: either an '
+ 'absolute path to a Lit configuration file, or a path relative to the libcxx/test/configs directory '
+ 'of the test suite being used.')
parser.add_argument('--benchmark-commit', type=str, required=True,
- help='The SHA representing the version of the library to benchmark.')
+ help='The SHA that the results are attributed to in LNT. When benchmarking libc++, this is the version '
+ 'of the library being benchmarked. When benchmarking another Standard Library, this is merely a label '
+ 'placing the results on the LNT time axis.')
parser.add_argument('--test-suite-commit', type=str, required=True,
help='The SHA representing the version of the test suite to use for benchmarking.')
parser.add_argument('--compiler', type=str, required=True,
@@ -183,8 +188,6 @@ def main(argv):
# check dependencies.
if args.output.exists():
sys.exit(f'error: output report {args.output} already exists; not overwriting it')
- if not args.libcxx_installation.exists():
- sys.exit(f'error: libc++ installation directory {args.libcxx_installation} does not exist')
if args.artifacts_dir is not None and args.artifacts_dir.exists():
sys.exit(f'error: artifacts directory {args.artifacts_dir} already exists; not overwriting it')
if shutil.which('lnt') is None:
@@ -200,10 +203,9 @@ def main(argv):
artifacts.mkdir(parents=True, exist_ok=True)
logging.info(f'Storing artifacts in {artifacts}')
- logging.info(f'Running benchmarks from {args.test_suite_commit} against libc++ at {args.libcxx_installation}')
+ logging.info(f'Running benchmarks from {args.test_suite_commit} against {args.test_config}')
# Always add some lit parameters that are required or make sense when running benchmarks.
lit_args = ['--param', f'compiler={args.compiler}',
- '--param', f'libcxx_installation={args.libcxx_installation}',
'--param', 'enable_werror=False'] # older versions of the library trigger new warnings, don't fail
if args.spec_dir is not None:
lit_args += ['--param', f'spec_dir={args.spec_dir}']
@@ -213,7 +215,7 @@ def main(argv):
'--build-dir', artifacts / 'benchmarks-build',
'--test-suite-commit', args.test_suite_commit,
'--tmp-src-dir', artifacts / 'benchmarks-src',
- '--test-config', 'installed-libc++.cfg.in',
+ '--test-config', args.test_config,
'--compiler', args.compiler,
'--',
'-j1', '--time-tests', '--test-output=failed',
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 572f326c4fb0a..3bfb5bb5fb02c 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -337,14 +337,14 @@ EOF
"${MONOREPO_ROOT}/libcxx/utils/ci/lnt/run-benchmarks" \
--dry-run -vv \
--git-repo "${MONOREPO_ROOT}" \
- --libcxx-installation "${INSTALL_DIR}" \
+ --test-config installed-libc++.cfg.in \
--benchmark-commit HEAD \
--test-suite-commit HEAD \
--machine test-tools \
--compiler "${CXX}" \
--filter hash.bench.cpp \
--output "${BUILD_DIR}/report.json" \
- -- --param optimization=speed
+ -- --param optimization=speed --param "libcxx_installation=${INSTALL_DIR}"
step "Submit a LNT report (dry-run)"
"${MONOREPO_ROOT}/libcxx/utils/ci/lnt/submit-benchmarks" \
>From aa38106aa4112b563ef4eb757060514bc133e309 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 24 Sep 2026 23:14:57 -0400
Subject: [PATCH 2/2] [TEMPORARY] Exercise the libstdc++ benchmark machine on
the pull request
DO NOT MERGE -- this commit is meant to be dropped before landing.
None of the benchmarking workflows run on a pull request, so a change to
machines.json or to run-benchmarks currently lands without ever having been
executed. This adds a workflow that runs on the pull request itself and covers
the two things that are otherwise unverified: that `matrix.build != ''` selects
exactly the machines with a `build` key, and that a machine which builds nothing
can be benchmarked end-to-end and produces a report that isn't empty.
---
...x-TEMPORARY-exercise-libstdcxx-machine.yml | 311 ++++++++++++++++++
1 file changed, 311 insertions(+)
create mode 100644 .github/workflows/libcxx-TEMPORARY-exercise-libstdcxx-machine.yml
diff --git a/.github/workflows/libcxx-TEMPORARY-exercise-libstdcxx-machine.yml b/.github/workflows/libcxx-TEMPORARY-exercise-libstdcxx-machine.yml
new file mode 100644
index 0000000000000..140ef3cd9b16f
--- /dev/null
+++ b/.github/workflows/libcxx-TEMPORARY-exercise-libstdcxx-machine.yml
@@ -0,0 +1,311 @@
+# ###########################################################################
+# TEMPORARY -- DO NOT MERGE. Delete this file before landing the PR.
+# ###########################################################################
+#
+# The benchmarking workflows are only reachable via workflow_dispatch, an issue
+# comment or a schedule, so none of them run on a pull request. That means a
+# change to machines.json or to run-benchmarks gets no real coverage before it
+# lands. This workflow runs on the pull request itself so that the libstdc++
+# machine added by this PR is actually exercised.
+#
+# It checks the two things that are otherwise unverified:
+#
+# 1. That `matrix.build != ''` selects exactly the machines that have a `build`
+# key. This relies on GitHub coercing an object to NaN when comparing against
+# a string, which is documented but subtle enough to be worth confirming
+# against the real machines.json.
+#
+# 2. That a machine which builds nothing can actually be benchmarked end-to-end,
+# and that it produces a report with results in it. A configuration that fails
+# to compile still produces a valid (but empty) LNT report, so the number of
+# results is what distinguishes "worked" from "silently measured nothing".
+#
+# 3. Whether the same thing is feasible on macOS at all. That job is exploratory
+# and is expected to fail: locally, Apple Clang cannot consume the libstdc++
+# headers shipped by Homebrew's GCC on arm64, since GCC's own stddef.h refers
+# to __float128, which Clang does not support on aarch64-apple-darwin. It is
+# marked continue-on-error so that it reports the answer without hiding the
+# result of the job above.
+
+name: "[libc++] TEMPORARY: exercise the libstdc++ benchmark machine"
+
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ paths:
+ - '.github/workflows/libcxx-TEMPORARY-exercise-libstdcxx-machine.yml'
+ - '.github/workflows/libcxx-benchmark-commit.yml'
+ - '.github/workflows/libcxx-benchmark-cron.yml'
+ - '.github/workflows/libcxx-pr-benchmark.yml'
+ - '.github/workflows/libcxx/**'
+ - 'libcxx/utils/ci/lnt/**'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ select-machines:
+ if: github.repository_owner == 'llvm'
+ runs-on: ubuntu-26.04
+ outputs:
+ all: ${{ steps.select.outputs.all }}
+ nothing-to-build: ${{ steps.select.outputs.nothing-to-build }}
+ steps:
+ - name: Checkout the machine definitions
+ uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ persist-credentials: false
+ sparse-checkout: libcxx/utils/ci/lnt/machines.json
+ sparse-checkout-cone-mode: false
+
+ - name: Select the machines
+ id: select
+ uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
+
+ // `expected-build` is the ground truth, computed here in Javascript. The job below
+ // compares it against what the Github expression evaluates to.
+ core.setOutput('all', JSON.stringify(config.map(cfg => ({
+ 'lnt-machine': cfg['lnt-machine'],
+ build: cfg.build,
+ 'expected-build': Boolean(cfg.build),
+ }))));
+
+ // Only the machines that don't build anything are benchmarked here: the ones that do
+ // are already covered by the existing flow, and they take much longer to run.
+ core.setOutput('nothing-to-build', JSON.stringify(config.filter(cfg => !cfg.build)));
+
+ # Confirm that `matrix.build != ''` means "this machine has a build key", for every machine
+ # actually defined in machines.json.
+ check-build-predicate:
+ needs: select-machines
+ runs-on: ubuntu-26.04
+ strategy:
+ matrix:
+ include: ${{ fromJSON(needs.select-machines.outputs.all) }}
+ fail-fast: false
+ name: "Check build predicate for ${{ matrix.lnt-machine }}"
+ steps:
+ - name: Compare the expression against the ground truth
+ env:
+ MACHINE: ${{ matrix.lnt-machine }}
+ EXPECTED: ${{ matrix.expected-build }}
+ # The spelling used by the workflows.
+ ACTUAL: ${{ matrix.build != '' }}
+ # Two alternative spellings, reported for information only.
+ VIA_MEMBER: ${{ matrix.build.cmake-cache != '' }}
+ VIA_TRUTHINESS: ${{ matrix.build && 'true' || 'false' }}
+ run: |
+ echo "machine : ${MACHINE}"
+ echo "has a build key (ground truth) : ${EXPECTED}"
+ echo "matrix.build != '' : ${ACTUAL}"
+ echo "matrix.build.cmake-cache != '' : ${VIA_MEMBER}"
+ echo "truthiness of matrix.build : ${VIA_TRUTHINESS}"
+
+ if [ "${ACTUAL}" != "${EXPECTED}" ]; then
+ echo "::error::matrix.build != '' evaluated to '${ACTUAL}' for ${MACHINE}, expected '${EXPECTED}'"
+ exit 1
+ fi
+
+ # Benchmark a machine that builds nothing, end to end, the same way libcxx-benchmark-commit.yml
+ # does it. Results are never submitted to LNT.
+ benchmark:
+ needs: select-machines
+ strategy:
+ matrix:
+ include: ${{ fromJSON(needs.select-machines.outputs.nothing-to-build) }}
+ fail-fast: false
+ name: "Benchmark ${{ matrix.lnt-machine }}"
+ # Deliberately not matrix.runner. The pool the machine actually targets
+ # (llvm-premerge-libcxx-runners) had a multi-hour backlog, and what this job is meant to
+ # exercise is the configuration, not the runner. This is the pool and image that normal
+ # libc++ pull request testing uses, and the image provides both compilers the configuration
+ # needs: clang++-22 to build the tests, and g++-16 for the libstdc++ we benchmark against.
+ runs-on: llvm-premerge-linux-32-runners
+ container:
+ image: ghcr.io/llvm/libcxx-linux-builder:d79f222aea61926e7c81fa8908b12f52ea70111f
+ timeout-minutes: 120
+ env:
+ BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
+ COMPILER: ${{ matrix.cxx }}
+ LIT_PARAMS: ${{ join(matrix.lit-params, ' ') }}
+ LNT_MACHINE: ${{ matrix.lnt-machine }}
+ TEST_CONFIG: ${{ matrix.test-config }}
+ # Benchmarking the whole suite would take far too long for a pull request.
+ FILTER: 'hash.bench.cpp'
+ steps:
+ - name: Checkout the LLVM monorepo
+ uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ persist-credentials: false
+ # run-benchmarks resolves --benchmark-commit and checks out the test suite at
+ # --test-suite-commit, both of which need the full history.
+ fetch-depth: 0
+ fetch-tags: true
+
+ - name: Diagnose tools in use
+ run: |
+ cmake --version
+ ninja --version
+ "${COMPILER}" --version
+ python3 --version
+
+ - name: Setup virtual environment
+ run: |
+ # --system-site-packages to match what the conformance tests do in this same image.
+ python3 -m venv --system-site-packages .venv
+ source .venv/bin/activate
+ pip install -r libcxx/utils/ci/lnt/requirements.txt
+
+ - name: Run the benchmarks
+ env:
+ BENCHMARK_COMMIT: ${{ github.event.pull_request.head.sha }}
+ run: |
+ source .venv/bin/activate
+
+ read -ra configured_params <<< "${LIT_PARAMS}"
+ lit_params=()
+ for param in "${configured_params[@]}"; do
+ lit_params+=(--param "${param}")
+ done
+
+ # -vv so the Lit invocation and its output show up in the log, which is the whole
+ # point of running this on the pull request.
+ libcxx/utils/ci/lnt/run-benchmarks -vv \
+ --test-suite-commit "${BENCHMARK_SUITE_VERSION}" \
+ --machine "${LNT_MACHINE}" \
+ --compiler "${COMPILER}" \
+ --benchmark-commit "${BENCHMARK_COMMIT}" \
+ --test-config "${TEST_CONFIG}" \
+ --filter "${FILTER}" \
+ --output report.json \
+ -- "${lit_params[@]}"
+
+ - name: Check that the report actually contains results
+ run: |
+ python3 - <<'PY'
+ import json, sys
+
+ with open('report.json') as f:
+ report = json.load(f)
+
+ machine = report.get('machine', {})
+ tests = report.get('tests', [])
+ print('machine name :', machine.get('name'))
+ print('machine info :', json.dumps(machine.get('info', {}), indent=2, sort_keys=True))
+ print('results :', len(tests))
+ for test in tests[:10]:
+ print(' ', test.get('name'))
+
+ # A configuration that fails to build still produces a valid but empty report, which
+ # would be submitted to LNT as a run with no results. That is what we're guarding against.
+ if not tests:
+ sys.exit('error: the report contains no results, so the benchmarks did not build or run')
+ PY
+
+ # Same thing, on macOS. This is exploratory rather than a gate: Apple Clang is not generally able
+ # to consume the libstdc++ headers that Homebrew's GCC ships, so this is expected to fail. The
+ # point is to find out on the real runner rather than by assumption, which is also why the
+ # configuration is spelled out here instead of being added to machines.json.
+ benchmark-macos:
+ if: github.repository_owner == 'llvm'
+ name: "Benchmark libstdc++ on macOS (exploratory)"
+ continue-on-error: true
+ runs-on: ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"]
+ timeout-minutes: 120
+ env:
+ # Kept in sync with the macOS entries of machines.json by hand, since this job is temporary.
+ BENCHMARK_SUITE_VERSION: '4ebb19efe952ded7f9d937731418baf3b28babd0'
+ COMPILER: clang++
+ LIT_PARAMS: 'std=c++26 optimization=speed'
+ LNT_MACHINE: macos-libstdcxx-exploratory
+ TEST_CONFIG: stdlib-libstdc++.cfg.in
+ XCODE_VERSION: '26.6'
+ FILTER: 'hash.bench.cpp'
+ steps:
+ - name: Checkout the LLVM monorepo
+ uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ persist-credentials: false
+ fetch-depth: 0
+ fetch-tags: true
+
+ - name: Select Xcode
+ run: echo "DEVELOPER_DIR=/Applications/Xcode_${XCODE_VERSION}.app/Contents/Developer" >> "${GITHUB_ENV}"
+
+ - name: Install dependencies via Homebrew
+ run: |
+ brew update
+ # GCC is what provides the libstdc++ we benchmark against; it is not otherwise needed
+ # on these runners, which is part of what makes this configuration exploratory.
+ brew install ninja cmake python at 3.14 gcc
+ echo "$(brew --prefix python at 3.14)/bin" >> "${GITHUB_PATH}"
+
+ - name: Locate the GCC whose libstdc++ we benchmark
+ run: |
+ gxx="$(ls "$(brew --prefix)"/bin/g++-[0-9]* 2>/dev/null | sort -V | tail -1)"
+ if [ -z "${gxx}" ]; then
+ echo "::error::no versioned g++ found under $(brew --prefix)/bin"
+ exit 1
+ fi
+ echo "Using ${gxx}"
+ "${gxx}" --version | head -1
+ echo "LIBSTDCXX_COMPILER=${gxx}" >> "${GITHUB_ENV}"
+
+ - name: Diagnose tools in use
+ run: |
+ cmake --version
+ ninja --version
+ "${COMPILER}" --version
+ python3 --version
+
+ - name: Setup virtual environment
+ run: |
+ python3 -m venv .venv
+ source .venv/bin/activate
+ pip install -r libcxx/utils/ci/lnt/requirements.txt
+
+ - name: Run the benchmarks
+ env:
+ BENCHMARK_COMMIT: ${{ github.event.pull_request.head.sha }}
+ run: |
+ source .venv/bin/activate
+
+ read -ra configured_params <<< "${LIT_PARAMS}"
+ lit_params=()
+ for param in "${configured_params[@]}"; do
+ lit_params+=(--param "${param}")
+ done
+ lit_params+=(--param "libstdcxx_compiler=${LIBSTDCXX_COMPILER}")
+
+ libcxx/utils/ci/lnt/run-benchmarks -vv \
+ --test-suite-commit "${BENCHMARK_SUITE_VERSION}" \
+ --machine "${LNT_MACHINE}" \
+ --compiler "${COMPILER}" \
+ --benchmark-commit "${BENCHMARK_COMMIT}" \
+ --test-config "${TEST_CONFIG}" \
+ --filter "${FILTER}" \
+ --output report.json \
+ -- "${lit_params[@]}"
+
+ - name: Check that the report actually contains results
+ run: |
+ python3 - <<'PY'
+ import json, sys
+
+ with open('report.json') as f:
+ report = json.load(f)
+
+ tests = report.get('tests', [])
+ print('results :', len(tests))
+ for test in tests[:10]:
+ print(' ', test.get('name'))
+
+ if not tests:
+ sys.exit('error: the report contains no results, so the benchmarks did not build or run')
+ PY
More information about the llvm-commits
mailing list