[libcxx] [llvm] [libc++] Enforce the pinned benchmark suite version when submitting to LNT (PR #215570)
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 06:51:29 PDT 2026
https://github.com/ldionne updated https://github.com/llvm/llvm-project/pull/215570
>From 521febb017cf4e10e46fa9a2483605732dfb9ca8 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Tue, 11 Aug 2026 09:25:10 -0400
Subject: [PATCH 1/2] [libc++] Enforce the pinned benchmark suite version when
submitting to LNT
Instead of passing a benchmark suite version when dispatching jobs,
enforce that the version pinned in machines.json is used when submitting
to LNT. This will prevent bad data from making it into the LNT instance
by accident.
We still allow overriding the test suite when doing a workflow dispatch
of libcxx-benchmark-commit.yml since that is useful for e.g. dry-running
different versions of the test suite, but they shouldn't be submitted.
---
.github/workflows/libcxx-benchmark-commit.yml | 30 ++++++++++++++++---
.github/workflows/libcxx-benchmark-cron.yml | 5 ++--
libcxx/utils/ci/lnt/README.md | 3 +-
libcxx/utils/ci/lnt/common.py | 9 ------
libcxx/utils/ci/lnt/dispatch-benchmarks | 4 ---
libcxx/utils/ci/lnt/machines.json | 4 +--
6 files changed, 31 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
index f55c11b0d93c6..54987b83286f4 100644
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ b/.github/workflows/libcxx-benchmark-commit.yml
@@ -21,9 +21,13 @@ on:
description: 'The LNT machine to run the benchmarks on'
required: true
type: string
- benchmark-suite-version:
- description: 'The version of the benchmark suite to use (a LLVM monorepo SHA)'
- required: true
+ benchmark-suite-override:
+ description: |
+ Override the version of the benchmark suite to use (a LLVM monorepo SHA). By default, the version pinned
+ for this machine in machines.json is used. This override can be used to dry-run benchmarks with arbitrary
+ commits of the benchmark suite, but only the version pinned in machines.json can be used when actually
+ submitting to LNT.
+ required: false
type: string
filter:
description: 'An optional filter to determine which benchmarks to run'
@@ -60,6 +64,8 @@ jobs:
uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
LNT_MACHINE: ${{ inputs.lnt-machine }}
+ BENCHMARK_SUITE_OVERRIDE: ${{ inputs.benchmark-suite-override }}
+ SUBMIT_LNT: ${{ inputs.submit-lnt }}
with:
script: |
const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
@@ -72,6 +78,22 @@ jobs:
return;
}
+ // Make sure we don't submit if an incorrect override is provided.
+ const version_override = (process.env.BENCHMARK_SUITE_OVERRIDE || '').trim().toLowerCase();
+ for (const cfg of selected) {
+ const pinned = cfg['benchmark-suite-version'];
+ if (process.env.SUBMIT_LNT === 'true' && version_override && version_override !== pinned) {
+ core.setFailed(`Refusing to submit results for ${cfg['lnt-machine']}, since the benchmark suite was `
+ + `overridden to version ${version_override}, which is different from the version `
+ + `pinned in machines.json (${pinned}).`);
+ return;
+ }
+
+ if (version_override) {
+ cfg['benchmark-suite-version'] = version_override;
+ }
+ }
+
core.setOutput('matrix', JSON.stringify(selected));
run-benchmarks:
@@ -126,7 +148,7 @@ jobs:
- name: Run the benchmarks
env:
COMMIT: ${{ inputs.commit }}
- BENCHMARK_SUITE_VERSION: ${{ inputs.benchmark-suite-version }}
+ BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
FILTER: ${{ inputs.filter }}
LNT_MACHINE: ${{ matrix.lnt-machine }}
run: |
diff --git a/.github/workflows/libcxx-benchmark-cron.yml b/.github/workflows/libcxx-benchmark-cron.yml
index fcd63314cf743..1bc224eb7fc74 100644
--- a/.github/workflows/libcxx-benchmark-cron.yml
+++ b/.github/workflows/libcxx-benchmark-cron.yml
@@ -82,7 +82,6 @@ jobs:
EVERY: ${{ matrix.every }}
SAMPLES: ${{ matrix.samples }}
MAX_IN_FLIGHT: ${{ matrix.max-in-flight }}
- BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
LNT_URL: ${{ matrix.lnt-url }}
ALLOW_MISSING_MACHINE: ${{ inputs.allow-missing-machine || 'false' }}
@@ -133,6 +132,6 @@ jobs:
dry_run=(--dry-run)
fi
- libcxx/utils/ci/lnt/dispatch-benchmarks --work-items plan.jsonl --test-suite-commit "${BENCHMARK_SUITE_VERSION}" \
- --max-in-flight "${MAX_IN_FLIGHT}" --lnt-url "${LNT_URL}" \
+ libcxx/utils/ci/lnt/dispatch-benchmarks --work-items plan.jsonl --lnt-url "${LNT_URL}" \
+ --max-in-flight "${MAX_IN_FLIGHT}" \
--output dispatched.jsonl "${dry_run[@]}"
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index f3345d343d2d6..5f8af9b55ba87 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -42,8 +42,7 @@ plan-benchmarks --commit-list anchor-commits.txt
# Request the corresponding workflow runs, at most 4 at a time to be a good citizen.
export GITHUB_TOKEN=$(gh auth token)
-dispatch-benchmarks --work-items plan.jsonl --test-suite-commit <benchmark suite SHA> \
- --max-in-flight 4 --dry-run
+dispatch-benchmarks --work-items plan.jsonl --max-in-flight 4 --dry-run
```
In a nutshell, `select-anchor-commits` produces the list of anchor commits that we want
diff --git a/libcxx/utils/ci/lnt/common.py b/libcxx/utils/ci/lnt/common.py
index 0ee20eea825aa..7ff423271322f 100644
--- a/libcxx/utils/ci/lnt/common.py
+++ b/libcxx/utils/ci/lnt/common.py
@@ -15,15 +15,6 @@ def is_sha(string: str) -> bool:
return len(string) == 40 and all(c in '0123456789abcdef' for c in string.lower())
-def sha(string: str) -> str:
- """
- An argparse type for a full commit SHA, normalized to lowercase.
- """
- if not is_sha(string):
- raise argparse.ArgumentTypeError(f'expected a full 40-character SHA, got {string!r}')
- return string.lower()
-
-
def at_least(minimum: int) -> Callable[[str], int]:
"""
Return an argparse type that accepts integers no smaller than `minimum`.
diff --git a/libcxx/utils/ci/lnt/dispatch-benchmarks b/libcxx/utils/ci/lnt/dispatch-benchmarks
index 59600119238c9..45a343cafe64c 100755
--- a/libcxx/utils/ci/lnt/dispatch-benchmarks
+++ b/libcxx/utils/ci/lnt/dispatch-benchmarks
@@ -242,7 +242,6 @@ def workflow_inputs(args: argparse.Namespace, item: common.WorkItem) -> Dict[str
"""Return the inputs to dispatch the workflow with for a work item."""
inputs = {'commit': item.commit,
'lnt-machine': item.machine,
- 'benchmark-suite-version': args.test_suite_commit,
# Submission is not optional: a run whose results are not recorded
# anywhere would be requested again on every future invocation.
# During a dry run nothing runs, so nothing is submitted either.
@@ -312,9 +311,6 @@ def main(argv: List[str]) -> int:
parser.add_argument('--work-items', type=argparse.FileType('r'), default=sys.stdin,
help='A file containing the plan, one JSON object per line. By default, this is read from '
'standard input.')
- parser.add_argument('--test-suite-commit', type=common.sha, required=True,
- help='The version of the benchmark suite to use, as a monorepo SHA. Pinning this is what '
- 'makes results comparable across commits.')
parser.add_argument('--max-in-flight', type=common.at_least(1), default=4,
help='The largest number of workflow runs allowed to exist at once for a single LNT machine. '
'This counts runs that are queued but not yet started.')
diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json
index 4b9321a21330a..792ebc81cca5c 100644
--- a/libcxx/utils/ci/lnt/machines.json
+++ b/libcxx/utils/ci/lnt/machines.json
@@ -5,8 +5,8 @@
"cxx": "clang++",
"running-on": "macos",
"xcode-version": "26.5",
+ "benchmark-suite-version": "810062e9771d8b47c25a89311139c621c13bbf9f",
"coverage": {
- "benchmark-suite-version": "810062e9771d8b47c25a89311139c621c13bbf9f",
"since": "2025-01-01",
"every": "week",
"samples": 3,
@@ -19,8 +19,8 @@
"runner": "llvm-premerge-libcxx-runners",
"cxx": "clang++-22",
"running-on": "linux",
+ "benchmark-suite-version": "810062e9771d8b47c25a89311139c621c13bbf9f",
"coverage": {
- "benchmark-suite-version": "810062e9771d8b47c25a89311139c621c13bbf9f",
"since": "2025-01-01",
"every": "week",
"samples": 3,
>From 3b927106dcd77d8752beb63e788816d9de55d344 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Tue, 11 Aug 2026 09:51:16 -0400
Subject: [PATCH 2/2] Improve comment
---
.github/workflows/libcxx-benchmark-commit.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
index 54987b83286f4..c8c7f03104fc2 100644
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ b/.github/workflows/libcxx-benchmark-commit.yml
@@ -78,7 +78,8 @@ jobs:
return;
}
- // Make sure we don't submit if an incorrect override is provided.
+ // Honor benchmark suite version override and make sure we don't submit if an incorrect
+ // override is provided.
const version_override = (process.env.BENCHMARK_SUITE_OVERRIDE || '').trim().toLowerCase();
for (const cfg of selected) {
const pinned = cfg['benchmark-suite-version'];
More information about the llvm-commits
mailing list