[libcxx] [llvm] [libc++] Remove unused generalization from the libc++ benchmarking pipeline (PR #227369)
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 08:46:15 PDT 2026
https://github.com/ldionne updated https://github.com/llvm/llvm-project/pull/227369
>From 4b4a63e8a71e288aa0c283a8801c47eb04e691cd Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Tue, 29 Sep 2026 11:35:59 -0400
Subject: [PATCH] [libc++] Remove unused generalization from the libc++
benchmarking pipeline
I initially tried reusing the same workflow file to run libc++ and libstdc++
benchmarks, but since then I have given up on that approach. As part of that,
I had generalized the workflow and machines.json to support not building
the library. Since that capability won't be useful, I am removing the
generalization to simplify (a bit) the workflow and machines.json.
Also introduce a `pr-benchmarks` key in machines.json to explicitly opt
into the PR benchmarking job: otherwise we'd have to implicitly key off
of the existence of `cmake-cache`, which is awkward.
---
.github/workflows/libcxx-benchmark-commit.yml | 27 +++++++------------
.github/workflows/libcxx-pr-benchmark.yml | 11 ++++----
libcxx/utils/ci/lnt/README.md | 13 ++++-----
libcxx/utils/ci/lnt/machines.json | 16 +++++------
libcxx/utils/ci/lnt/run-benchmarks | 4 +--
5 files changed, 29 insertions(+), 42 deletions(-)
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
index 7a333a1a0222c3..77204b4a27b5be 100644
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ b/.github/workflows/libcxx-benchmark-commit.yml
@@ -107,8 +107,8 @@ jobs:
runs-on: ${{ matrix.runner }}
env:
COMPILER: ${{ matrix.cxx }}
- # Where the library is installed when we build it. This lives inside the workspace
- # so that actions/checkout cleans it up between runs on self-hosted runners.
+ # Where we install the library. This lives inside the workspace so that actions/checkout
+ # cleans it up between runs on self-hosted runners.
INSTALL_DIR: ${{ github.workspace }}/install
steps:
- name: Checkout the LLVM monorepo
@@ -149,14 +149,11 @@ jobs:
source .venv/bin/activate
pip install -r libcxx/utils/ci/lnt/requirements.txt
- # Build the library. This is only done when the machine configuration provides a 'build' key,
- # otherwise we assume that the configuration does not require building anything (e.g. a system
- # standard library).
+ # Build the library.
#
# A build failure is tolerated on purpose, since the library doesn't build at every historical commit.
# We still submit an empty run for such commits.
- name: Build libc++ at ${{ inputs.commit }}
- if: ${{ matrix.build != '' }}
continue-on-error: true
uses: ./.github/workflows/libcxx/build-at-commit
with:
@@ -164,12 +161,11 @@ jobs:
commit: ${{ inputs.commit }}
install-dir: ${{ env.INSTALL_DIR }}
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.build.cmake-cache }}
+ cmake-cache: ${{ matrix.cmake-cache }}
- name: Run the benchmarks
env:
BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
- BUILT_LIBCXX: ${{ matrix.build != '' }}
COMMIT: ${{ inputs.commit }}
FILTER: ${{ inputs.filter }}
LIT_PARAMS: ${{ join(matrix.lit-params, ' ') }}
@@ -188,15 +184,10 @@ jobs:
lit_params+=(--param "${param}")
done
- # When the library is built in this workflow, point the testing configuration at the installation.
- # Everything else needed by the configuration comes from machines.json.
- if [ "${BUILT_LIBCXX}" = "true" ]; then
- # If the library build failed, still create an install directory so we
- # run the benchmarks. The benchmarks will fail, and we'll submit an empty
- # LNT report.
- mkdir -p "${INSTALL_DIR}"
- lit_params+=(--param "libcxx_installation=${INSTALL_DIR}")
- fi
+ # If the library build failed, still create an install directory so we
+ # run the benchmarks. The benchmarks will fail, and we'll submit an empty
+ # LNT report.
+ mkdir -p "${INSTALL_DIR}"
libcxx/utils/ci/lnt/run-benchmarks \
--test-suite-commit "${BENCHMARK_SUITE_VERSION}" \
@@ -206,7 +197,7 @@ jobs:
--test-config "${TEST_CONFIG}" \
"${filter_arg[@]}" \
--output "${COMMIT}.json" \
- -- "${lit_params[@]}"
+ -- "${lit_params[@]}" --param "libcxx_installation=${INSTALL_DIR}"
cat "${COMMIT}.json"
diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml
index 6d184317b806c8..efeca9be1e857b 100644
--- a/.github/workflows/libcxx-pr-benchmark.yml
+++ b/.github/workflows/libcxx-pr-benchmark.yml
@@ -71,11 +71,10 @@ jobs:
const match = context.payload.comment.body.match(/\/libcxx-bot benchmark (.+)/);
core.setOutput('benchmarks', match ? match[1] : '');
- // Benchmark the configurations defined in machines.json. We only consider configurations that
- // build something, since there's nothing to do an A/B comparison with for other configurations.
+ // Benchmark the configurations defined in machines.json that opt into PR benchmarking.
const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
- const buildable = config.filter(cfg => cfg.build);
- core.setOutput('matrix', JSON.stringify(buildable.map(({coverage, ...machine}) => machine)));
+ const selected = config.filter(cfg => cfg['pr-benchmarks']);
+ core.setOutput('matrix', JSON.stringify(selected.map(({coverage, 'pr-benchmarks': _, ...machine}) => machine)));
- name: Update comment with link to the run
uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
@@ -181,7 +180,7 @@ jobs:
commit: ${{ steps.baseline.outputs.commit }}
install-dir: ${{ github.workspace }}/install/baseline
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.build.cmake-cache }}
+ cmake-cache: ${{ matrix.cmake-cache }}
- name: Build the candidate
uses: ./tooling/.github/workflows/libcxx/build-at-commit
@@ -190,7 +189,7 @@ jobs:
commit: ${{ needs.extract-info.outputs.pr_head }}
install-dir: ${{ github.workspace }}/install/candidate
compiler: ${{ matrix.cxx }}
- cmake-cache: ${{ matrix.build.cmake-cache }}
+ cmake-cache: ${{ matrix.cmake-cache }}
- name: Run baseline and candidate interleaved
run: |
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index a60fef83f506d7..2aa6b4648a7b09 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -67,17 +67,18 @@ single source of truth for all workflows that run benchmarks (PR benchmarking, r
historical benchmarks, etc). Each entry contains variables used by the various workflows
and the LNT machine name that the results will be reported under.
-The `test-config` key selects the Lit testing configuration to benchmark. This is used to e.g.
-select which Standard Library is being measured. The `lit-params` key provides additional lit
-parameters to pass when running the benchmarks.
-
-The `build` key allows providing the CMake cache to use when building the library before running
-the benchmarks. If `build` is not present, building libc++ is skipped for that configuration.
+The `cmake-cache` key provides the CMake cache to use when building libc++ before running the
+benchmarks. The `test-config` key selects the Lit testing configuration used to run the benchmarks
+against that build, and the `lit-params` key provides additional Lit parameters to pass when running
+the benchmarks.
The `coverage` key establishes how far back and at which frequency performance should be measured
for that configuration. A machine without a `coverage` entry can be defined, but it won't result
in historical data.
+The `pr-benchmarks` key determines whether the machine is used when benchmarking pull requests
+with `/libcxx-bot benchmark`.
+
## Running benchmarks locally
On GitHub, the `libcxx-benchmark-commit.yml` workflow is used to run benchmarks and report
diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json
index 32a5ca0802e986..ae3296e6990946 100644
--- a/libcxx/utils/ci/lnt/machines.json
+++ b/libcxx/utils/ci/lnt/machines.json
@@ -2,12 +2,11 @@
{
"lnt-machine": "macos-26.6.2-arm64-20260924",
"runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
+ "pr-benchmarks": true,
"cxx": "clang++",
"xcode-version": "26.6",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "build": {
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake"
- },
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
"lit-params": ["std=c++26", "optimization=speed"],
"test-config": "installed-libc++.cfg.in",
"coverage": {
@@ -21,12 +20,11 @@
{
"lnt-machine": "macos-26.6.2-arm64-hardenedfast-20260924",
"runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
+ "pr-benchmarks": true,
"cxx": "clang++",
"xcode-version": "26.6",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "build": {
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake"
- },
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake",
"lit-params": ["std=c++26", "optimization=speed"],
"test-config": "installed-libc++.cfg.in",
"coverage": {
@@ -40,11 +38,10 @@
{
"lnt-machine": "linux-x86_64-20260924",
"runner": "llvm-premerge-libcxx-runners",
+ "pr-benchmarks": true,
"cxx": "clang++-22",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
- "build": {
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake"
- },
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
"lit-params": ["std=c++26", "optimization=speed"],
"test-config": "installed-libc++.cfg.in",
"coverage": {
@@ -58,6 +55,7 @@
{
"lnt-machine": "macos-26.6.2-arm64-libstdcxx-20260928",
"runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
+ "pr-benchmarks": false,
"cxx": "clang++",
"xcode-version": "26.6",
"benchmark-suite-version": "4ebb19efe952ded7f9d937731418baf3b28babd0",
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index ca6fbbd4141fae..afd8669d2e8165 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -115,9 +115,7 @@ def main(argv):
'absolute path to a Lit configuration file, or a path relative to the libcxx/test/configs directory '
'of the test suite being used.')
parser.add_argument('--benchmark-commit', type=str, required=True,
- help='The SHA that the results are attributed to in LNT. When benchmarking libc++, this is the version '
- 'of the library being benchmarked. When benchmarking another Standard Library, this is merely a label '
- 'placing the results on the LNT time axis.')
+ help='The SHA representing the version of the library to benchmark.')
parser.add_argument('--test-suite-commit', type=str, required=True,
help='The SHA representing the version of the test suite to use for benchmarking.')
parser.add_argument('--compiler', type=str, required=True,
More information about the llvm-commits
mailing list