[libcxx-commits] [libcxx] [llvm] [libc++] Extract LNT benchmark validation into a reusable workflow (PR #232155)
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Oct 14 18:53:06 PDT 2026
https://github.com/ldionne created https://github.com/llvm/llvm-project/pull/232155
This allows sharing the logic for validating the LNT machines definition between the libc++ and the libstdc++ benchmarking jobs. As part of that, I noticed that these jobs only used a subset of the fields in machines.json, so this PR regroups them into a sub-entry. There are now two aspects in machines.json: "when" we measure something, and "how" we measure something, and both are sub-objects.
>From 4748116fee77496a17b103a8952cf10989844d1f Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Wed, 14 Oct 2026 21:12:51 -0400
Subject: [PATCH] [libc++] Extract LNT benchmark validation into a reusable
workflow
This allows sharing the logic for validating the LNT machines definition
between the libc++ and the libstdc++ benchmarking jobs. As part of that,
I noticed that these jobs only used a subset of the fields in machines.json,
so this PR regroups them into a sub-entry. There are now two aspects in
machines.json: "when" we measure something, and "how" we measure something,
and both are sub-objects.
---
.github/workflows/libcxx-benchmark-commit.yml | 84 +++++--------------
.../workflows/libcxx-benchmark-libstdcxx.yml | 80 ++++--------------
.github/workflows/libcxx-lnt-validate.yml | 82 ++++++++++++++++++
.github/workflows/libcxx-pr-benchmark.yml | 2 +-
libcxx/utils/ci/lnt/README.md | 22 +++--
libcxx/utils/ci/lnt/machines.json | 60 +++++++------
6 files changed, 167 insertions(+), 163 deletions(-)
create mode 100644 .github/workflows/libcxx-lnt-validate.yml
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
index c579ad1b6eade6..593ffd5a3bfbb2 100644
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ b/.github/workflows/libcxx-benchmark-commit.yml
@@ -45,72 +45,31 @@ on:
default: https://lnt.llvm.org
jobs:
- # Determine which configuration to run based on the `lnt-machine` input.
- select-machine:
- runs-on: ubuntu-26.04
- outputs:
- matrix: ${{ steps.select.outputs.matrix }}
- steps:
- - name: Checkout the machine definitions
- uses: actions/checkout at 3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- with:
- persist-credentials: false
- # Disabling cone mode allows checking out exactly the single file we need.
- sparse-checkout: libcxx/utils/ci/lnt/machines.json
- sparse-checkout-cone-mode: false
-
- - name: Select the configuration to benchmark on
- id: select
- uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- LNT_MACHINE: ${{ inputs.lnt-machine }}
- BENCHMARK_SUITE_OVERRIDE: ${{ inputs.benchmark-suite-override }}
- SUBMIT_LNT: ${{ inputs.submit-lnt }}
- with:
- script: |
- const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
-
- const requested = process.env.LNT_MACHINE;
- const selected = config.filter(cfg => cfg['lnt-machine'] === requested);
- if (selected.length === 0) {
- const known = config.map(cfg => cfg['lnt-machine']).join(', ');
- core.setFailed(`Unknown LNT machine '${requested}' (known machines: ${known})`);
- return;
- }
-
- // Honor benchmark suite version override and make sure we don't submit if an incorrect
- // override is provided.
- const version_override = (process.env.BENCHMARK_SUITE_OVERRIDE || '').trim().toLowerCase();
- for (const cfg of selected) {
- const pinned = cfg['benchmark-suite-version'];
- if (process.env.SUBMIT_LNT === 'true' && version_override && version_override !== pinned) {
- core.setFailed(`Refusing to submit results for ${cfg['lnt-machine']}, since the benchmark suite was `
- + `overridden to version ${version_override}, which is different from the version `
- + `pinned in machines.json (${pinned}).`);
- return;
- }
-
- if (version_override) {
- cfg['benchmark-suite-version'] = version_override;
- }
- }
-
- core.setOutput('matrix', JSON.stringify(selected));
+ # Validate the inputs and select the configuration of the machine to benchmark on.
+ validate:
+ uses: $/.github/workflows/libcxx-lnt-validate.yml
+ with:
+ lnt-machine: ${{ inputs.lnt-machine }}
+ benchmark-suite-override: ${{ inputs.benchmark-suite-override }}
+ submit-lnt: ${{ inputs.submit-lnt }}
run-benchmarks:
+ name: Run benchmarks on ${{ inputs.lnt-machine }}
needs:
- - select-machine
- strategy:
- matrix:
- include: ${{ fromJSON(needs.select-machine.outputs.matrix) }}
- fail-fast: false
- runs-on: ${{ matrix.runner }}
- container: ${{ matrix.container || null }}
+ - validate
+ runs-on: ${{ fromJSON(needs.validate.outputs.setup).runner }}
+ container: ${{ fromJSON(needs.validate.outputs.setup).container || null }}
defaults:
run:
shell: bash
env:
- COMPILER: ${{ matrix.cxx }}
+ BENCHMARK_SUITE_VERSION: ${{ fromJSON(needs.validate.outputs.setup).benchmark-suite-version }}
+ CMAKE_CACHE: ${{ fromJSON(needs.validate.outputs.setup).cmake-cache }}
+ COMPILER: ${{ fromJSON(needs.validate.outputs.setup).cxx }}
+ LIT_PARAMS: ${{ join(fromJSON(needs.validate.outputs.setup).lit-params, ' ') }}
+ LNT_MACHINE: ${{ inputs.lnt-machine }}
+ TEST_CONFIG: ${{ fromJSON(needs.validate.outputs.setup).test-config }}
+ XCODE_VERSION: ${{ fromJSON(needs.validate.outputs.setup).xcode-version }}
# Where we install the library. This lives inside the workspace so that actions/checkout
# cleans it up between runs on self-hosted runners.
INSTALL_DIR: ${{ github.workspace }}/install
@@ -130,7 +89,7 @@ jobs:
- name: Select Xcode
if: runner.os == 'macOS'
- run: echo "DEVELOPER_DIR=/Applications/Xcode_${{ matrix.xcode-version }}.app/Contents/Developer" >> $GITHUB_ENV
+ run: echo "DEVELOPER_DIR=/Applications/Xcode_${XCODE_VERSION}.app/Contents/Developer" >> "${GITHUB_ENV}"
- name: Install dependencies via Homebrew
if: runner.os == 'macOS'
@@ -159,7 +118,6 @@ jobs:
- name: Build libc++ at ${{ inputs.commit }}
continue-on-error: true
env:
- CMAKE_CACHE: ${{ matrix.cmake-cache }}
COMMIT: ${{ inputs.commit }}
run: |
# Clean up any stale install directory
@@ -175,12 +133,8 @@ jobs:
- name: Run the benchmarks
env:
- BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
COMMIT: ${{ inputs.commit }}
FILTER: ${{ inputs.filter }}
- LIT_PARAMS: ${{ join(matrix.lit-params, ' ') }}
- LNT_MACHINE: ${{ matrix.lnt-machine }}
- TEST_CONFIG: ${{ matrix.test-config }}
run: |
source .venv/bin/activate
filter_arg=()
diff --git a/.github/workflows/libcxx-benchmark-libstdcxx.yml b/.github/workflows/libcxx-benchmark-libstdcxx.yml
index aa7fdcbd26cc0c..fa322fe6ce7201 100644
--- a/.github/workflows/libcxx-benchmark-libstdcxx.yml
+++ b/.github/workflows/libcxx-benchmark-libstdcxx.yml
@@ -41,73 +41,30 @@ on:
default: https://lnt.llvm.org
jobs:
- # Determine which configuration to run based on the `lnt-machine` input.
- select-machine:
- runs-on: ubuntu-26.04
- outputs:
- matrix: ${{ steps.select.outputs.matrix }}
- steps:
- - name: Checkout the machine definitions
- uses: actions/checkout at 3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- with:
- persist-credentials: false
- # Disabling cone mode allows checking out exactly the single file we need.
- sparse-checkout: libcxx/utils/ci/lnt/machines.json
- sparse-checkout-cone-mode: false
-
- - name: Select the configuration to benchmark on
- id: select
- uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- LNT_MACHINE: ${{ inputs.lnt-machine }}
- BENCHMARK_SUITE_OVERRIDE: ${{ inputs.benchmark-suite-override }}
- SUBMIT_LNT: ${{ inputs.submit-lnt }}
- with:
- script: |
- const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
-
- const requested = process.env.LNT_MACHINE;
- const selected = config.filter(cfg => cfg['lnt-machine'] === requested);
- if (selected.length === 0) {
- const known = config.map(cfg => cfg['lnt-machine']).join(', ');
- core.setFailed(`Unknown LNT machine '${requested}' (known machines: ${known})`);
- return;
- }
-
- // Honor benchmark suite version override and make sure we don't submit if an incorrect
- // override is provided.
- const version_override = (process.env.BENCHMARK_SUITE_OVERRIDE || '').trim().toLowerCase();
- for (const cfg of selected) {
- const pinned = cfg['benchmark-suite-version'];
- if (process.env.SUBMIT_LNT === 'true' && version_override && version_override !== pinned) {
- core.setFailed(`Refusing to submit results for ${cfg['lnt-machine']}, since the benchmark suite was `
- + `overridden to version ${version_override}, which is different from the version `
- + `pinned in machines.json (${pinned}).`);
- return;
- }
-
- if (version_override) {
- cfg['benchmark-suite-version'] = version_override;
- }
- }
-
- core.setOutput('matrix', JSON.stringify(selected));
+ # Validate the inputs and select the configuration of the machine to benchmark on.
+ validate:
+ uses: $/.github/workflows/libcxx-lnt-validate.yml
+ with:
+ lnt-machine: ${{ inputs.lnt-machine }}
+ benchmark-suite-override: ${{ inputs.benchmark-suite-override }}
+ submit-lnt: ${{ inputs.submit-lnt }}
run-benchmarks:
+ name: Run benchmarks on ${{ inputs.lnt-machine }}
needs:
- - select-machine
- strategy:
- matrix:
- include: ${{ fromJSON(needs.select-machine.outputs.matrix) }}
- fail-fast: false
- runs-on: ${{ matrix.runner }}
- container: ${{ matrix.container || null }}
+ - validate
+ runs-on: ${{ fromJSON(needs.validate.outputs.setup).runner }}
+ container: ${{ fromJSON(needs.validate.outputs.setup).container || null }}
defaults:
run:
shell: bash
env:
- COMPILER: ${{ matrix.cxx }}
+ BENCHMARK_SUITE_VERSION: ${{ fromJSON(needs.validate.outputs.setup).benchmark-suite-version }}
+ COMPILER: ${{ fromJSON(needs.validate.outputs.setup).cxx }}
LIBSTDCXX_VERSION: ${{ inputs.libstdcxx-version }}
+ LIT_PARAMS: ${{ join(fromJSON(needs.validate.outputs.setup).lit-params, ' ') }}
+ LNT_MACHINE: ${{ inputs.lnt-machine }}
+ XCODE_VERSION: ${{ fromJSON(needs.validate.outputs.setup).xcode-version }}
steps:
- name: Checkout the LLVM monorepo
uses: actions/checkout at 3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -125,7 +82,7 @@ jobs:
- name: Select Xcode
if: runner.os == 'macOS'
- run: echo "DEVELOPER_DIR=/Applications/Xcode_${{ matrix.xcode-version }}.app/Contents/Developer" >> $GITHUB_ENV
+ run: echo "DEVELOPER_DIR=/Applications/Xcode_${XCODE_VERSION}.app/Contents/Developer" >> "${GITHUB_ENV}"
- name: Install dependencies via Homebrew
if: runner.os == 'macOS'
@@ -149,10 +106,7 @@ jobs:
- name: Run the benchmarks
env:
- BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
FILTER: ${{ inputs.filter }}
- LIT_PARAMS: ${{ join(matrix.lit-params, ' ') }}
- LNT_MACHINE: ${{ matrix.lnt-machine }}
run: |
read -ra configured_params <<< "${LIT_PARAMS}"
lit_args=()
diff --git a/.github/workflows/libcxx-lnt-validate.yml b/.github/workflows/libcxx-lnt-validate.yml
new file mode 100644
index 00000000000000..bc4832a301070b
--- /dev/null
+++ b/.github/workflows/libcxx-lnt-validate.yml
@@ -0,0 +1,82 @@
+# This file defines a reusable workflow that validates the inputs of the benchmarking workflows and
+# outputs the `setup` of the selected LNT machine, as defined in machines.json. Any override of the
+# benchmark suite version is also applied on top.
+#
+# See libcxx/utils/ci/lnt/README.md for a description of machines.json.
+
+name: "[libc++] Validate benchmark inputs"
+
+permissions:
+ contents: read
+
+on:
+ workflow_call:
+ inputs:
+ lnt-machine:
+ description: 'The LNT machine configuration to use when benchmarking'
+ required: true
+ type: string
+ benchmark-suite-override:
+ description: 'Override the version of the benchmark suite to use (a LLVM monorepo SHA)'
+ required: false
+ type: string
+ submit-lnt:
+ description: 'Whether the results are going to be submitted to LNT'
+ required: true
+ type: boolean
+ outputs:
+ setup:
+ description: 'The setup of the selected machine, as a JSON object'
+ value: ${{ jobs.validate.outputs.setup }}
+
+jobs:
+ validate:
+ runs-on: ubuntu-26.04
+ outputs:
+ setup: ${{ steps.validate.outputs.setup }}
+ steps:
+ - name: Checkout the machine definitions
+ uses: actions/checkout at 3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ # Disabling cone mode allows checking out exactly the single file we need.
+ sparse-checkout: libcxx/utils/ci/lnt/machines.json
+ sparse-checkout-cone-mode: false
+
+ - name: Validate the inputs and select the machine setup
+ id: validate
+ uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ LNT_MACHINE: ${{ inputs.lnt-machine }}
+ BENCHMARK_SUITE_OVERRIDE: ${{ inputs.benchmark-suite-override }}
+ SUBMIT_LNT: ${{ inputs.submit-lnt }}
+ with:
+ script: |
+ const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
+
+ const requested = process.env.LNT_MACHINE;
+ const selected = config.filter(cfg => cfg['lnt-machine'] === requested);
+ if (selected.length !== 1) {
+ const known = config.map(cfg => cfg['lnt-machine']).join(', ');
+ core.setFailed(`Expected exactly one LNT machine named '${requested}', found ${selected.length} `
+ + `(known machines: ${known})`);
+ return;
+ }
+
+ // Honor any benchmark suite version override, but make sure we don't submit to LNT with a benchmark
+ // suite version that differs from what's specified in machines.json. This would fail at submission
+ // time anyway, only much later.
+ const setup = selected[0].setup;
+ const pinned = setup['benchmark-suite-version'];
+ const version_override = (process.env.BENCHMARK_SUITE_OVERRIDE || '').trim().toLowerCase();
+ if (process.env.SUBMIT_LNT === 'true' && version_override && version_override !== pinned) {
+ core.setFailed(`Refusing to submit results for ${requested}, since the benchmark suite was `
+ + `overridden to version ${version_override}, which is different from the version `
+ + `pinned in machines.json (${pinned}).`);
+ return;
+ }
+
+ if (version_override) {
+ setup['benchmark-suite-version'] = version_override;
+ }
+ core.setOutput('setup', JSON.stringify(setup));
diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml
index 7ea3ec3ebc1995..6608e479add41d 100644
--- a/.github/workflows/libcxx-pr-benchmark.yml
+++ b/.github/workflows/libcxx-pr-benchmark.yml
@@ -74,7 +74,7 @@ jobs:
// Benchmark the configurations defined in machines.json that opt into PR benchmarking.
const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
const selected = config.filter(cfg => cfg['pr-benchmarks']);
- core.setOutput('matrix', JSON.stringify(selected.map(({coverage, ...machine}) => machine)));
+ core.setOutput('matrix', JSON.stringify(selected.map(cfg => ({'lnt-machine': cfg['lnt-machine'], ...cfg.setup}))));
- name: Update comment with link to the run
uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index 37cc35dcdf807d..6bf5a181db899a 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -67,14 +67,20 @@ single source of truth for all workflows that run benchmarks (PR benchmarking, r
historical benchmarks, etc). Each entry contains variables used by the various workflows
and the LNT machine name that the results will be reported under.
-The `runner` key selects the GitHub Actions runner(s) to benchmark on. When the runner set
-requires jobs to run in a container, the `container` key provides the image to use. Otherwise,
-the benchmarks run directly on the host.
-
-The `cmake-cache` key provides the CMake cache to use when building libc++ before running the
-benchmarks. The `test-config` key selects the Lit testing configuration used to run the benchmarks
-against that build, and the `lit-params` key provides additional Lit parameters to pass when running
-the benchmarks.
+The `setup` key describes how benchmarks are run on that machine. It contains the following
+entries:
+
+- `runner`: the GitHub Actions runner(s) to benchmark on.
+- `container`: the image to use if running in a container. Otherwise, the benchmarks run directly
+ on the host.
+- `cxx`: the C++ compiler to use.
+- `xcode-version`: the version of Xcode to use (only relevant on macOS).
+- `benchmark-suite-version`: the version of the benchmark suite to use (a LLVM monorepo SHA).
+- `cmake-cache`: the CMake cache to use when building libc++ before running the benchmarks (only
+ relevant when benchmarking libc++).
+- `test-config`: the Lit testing configuration used to run the benchmarks against that build (only
+ relevant when benchmarking libc++).
+- `lit-params`: additional Lit parameters to pass when running the benchmarks.
The `coverage` key establishes how far back and at which frequency performance should be measured
for that configuration. A machine without a `coverage` entry can be defined, but it won't result
diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json
index 3b34b12084cafc..35ff7017909027 100644
--- a/libcxx/utils/ci/lnt/machines.json
+++ b/libcxx/utils/ci/lnt/machines.json
@@ -1,14 +1,16 @@
[
{
"lnt-machine": "macos-26.6.2-arm64-2026-09-29",
- "runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
"pr-benchmarks": true,
- "cxx": "clang++",
- "xcode-version": "26.6",
- "benchmark-suite-version": "a0e3ae753ad39cd8f2b1b9b3e2f1ad7da3c088d2",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
- "lit-params": ["std=c++26", "optimization=speed"],
- "test-config": "installed-libc++.cfg.in",
+ "setup": {
+ "runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
+ "cxx": "clang++",
+ "xcode-version": "26.6",
+ "benchmark-suite-version": "a0e3ae753ad39cd8f2b1b9b3e2f1ad7da3c088d2",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
+ "lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in"
+ },
"coverage": {
"since": "2023-01-01",
"every": "week",
@@ -19,14 +21,16 @@
},
{
"lnt-machine": "macos-26.6.2-arm64-hardenedfast-2026-09-29",
- "runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
"pr-benchmarks": true,
- "cxx": "clang++",
- "xcode-version": "26.6",
- "benchmark-suite-version": "a0e3ae753ad39cd8f2b1b9b3e2f1ad7da3c088d2",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake",
- "lit-params": ["std=c++26", "optimization=speed"],
- "test-config": "installed-libc++.cfg.in",
+ "setup": {
+ "runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
+ "cxx": "clang++",
+ "xcode-version": "26.6",
+ "benchmark-suite-version": "a0e3ae753ad39cd8f2b1b9b3e2f1ad7da3c088d2",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake",
+ "lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in"
+ },
"coverage": {
"since": "2023-12-01",
"every": "week",
@@ -37,14 +41,16 @@
},
{
"lnt-machine": "linux-x86_64-2026-09-29",
- "runner": "llvm-premerge-linux-32-runners",
- "container": "ghcr.io/llvm/libcxx-linux-builder:5e1edea790610e7ef8c5e06b6e5185fe5dd1c830",
"pr-benchmarks": true,
- "cxx": "clang++-22",
- "benchmark-suite-version": "a0e3ae753ad39cd8f2b1b9b3e2f1ad7da3c088d2",
- "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
- "lit-params": ["std=c++26", "optimization=speed"],
- "test-config": "installed-libc++.cfg.in",
+ "setup": {
+ "runner": "llvm-premerge-linux-32-runners",
+ "container": "ghcr.io/llvm/libcxx-linux-builder:5e1edea790610e7ef8c5e06b6e5185fe5dd1c830",
+ "cxx": "clang++-22",
+ "benchmark-suite-version": "a0e3ae753ad39cd8f2b1b9b3e2f1ad7da3c088d2",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
+ "lit-params": ["std=c++26", "optimization=speed"],
+ "test-config": "installed-libc++.cfg.in"
+ },
"coverage": {
"since": "2023-01-01",
"every": "week",
@@ -55,11 +61,13 @@
},
{
"lnt-machine": "macos-26.6.2-arm64-libstdcxx-2026-09-29",
- "runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
"pr-benchmarks": false,
- "cxx": "clang++",
- "xcode-version": "26.6",
- "benchmark-suite-version": "a0e3ae753ad39cd8f2b1b9b3e2f1ad7da3c088d2",
- "lit-params": ["std=c++26", "optimization=speed"]
+ "setup": {
+ "runner": ["self-hosted", "macOS", "26.6.2", "ARM64", "apple-runners"],
+ "cxx": "clang++",
+ "xcode-version": "26.6",
+ "benchmark-suite-version": "a0e3ae753ad39cd8f2b1b9b3e2f1ad7da3c088d2",
+ "lit-params": ["std=c++26", "optimization=speed"]
+ }
}
]
More information about the libcxx-commits
mailing list