[libcxx-commits] [libcxx] [llvm] [libc++] Submit the results of PR benchmarks to LNT and link to them (PR #237102)
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Thu Oct 29 22:25:00 PDT 2026
https://github.com/ldionne created https://github.com/llvm/llvm-project/pull/237102
Now that we produce LNT v5 JSON in the benchmarking jobs, we can submit it to LNT as a data point with an expiration date and include the link in the comment we post to the PR.
The expiration date should help avoid bloating the LNT instance if this kind of PR benchmarking becomes more mainstream.
>From e723d829f72d4ae3bca648241a29598e63b08e22 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 29 Oct 2026 21:34:54 -0700
Subject: [PATCH] [libc++] Submit the results of PR benchmarks to LNT and link
to them
Now that we produce LNT v5 JSON in the benchmarking jobs, we can
submit it to LNT as a data point with an expiration date and include
the link in the comment we post to the PR.
The expiration date should help avoid bloating the LNT instance if
this kind of PR benchmarking becomes more mainstream.
---
.github/workflows/libcxx-pr-benchmark.yml | 68 ++++++++++++++++++-
.../workflows/libcxx-run-ab-benchmarks.yml | 2 +
libcxx/utils/ci/lnt/common.py | 12 +++-
libcxx/utils/ci/lnt/create-lnt-report | 14 +++-
4 files changed, 91 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml
index f5a6863aef7e1..b630146e74961 100644
--- a/.github/workflows/libcxx-pr-benchmark.yml
+++ b/.github/workflows/libcxx-pr-benchmark.yml
@@ -131,10 +131,33 @@ jobs:
artifact-name: ab-results-${{ matrix.lnt-machine }}
lnt-artifact-name: lnt-reports-${{ matrix.lnt-machine }}
+ # Submit the results of each machine to LNT, which allows linking to LNT's comparison page in the report.
+ # Machines whose benchmarks didn't complete have no reports, so submitting them fails too.
+ submit:
+ name: Submit ${{ matrix.lnt-machine }}
+ needs:
+ - extract-info
+ - run-benchmarks
+ # Submit the results of the machines that succeeded even if others failed.
+ if: ${{ !cancelled() && needs.extract-info.result == 'success' }}
+ permissions:
+ contents: read
+ strategy:
+ matrix:
+ include: ${{ fromJSON(needs.extract-info.outputs.matrix) }}
+ fail-fast: false
+ uses: $/.github/workflows/lnt-submit.yml
+ with:
+ artifact-name: lnt-reports-${{ matrix.lnt-machine }}
+ test-suite: libcxx
+ secrets:
+ LNT_SUBMISSION_TOKEN: ${{ secrets.LNT_SUBMISSION_TOKEN }}
+
report:
needs:
- extract-info
- run-benchmarks
+ - submit
# Report results even if some of the benchmark runs failed.
if: ${{ always() && needs.extract-info.result == 'success' }}
runs-on: ubuntu-26.04
@@ -148,17 +171,60 @@ jobs:
path: ${{ runner.temp }}/results
merge-multiple: true
+ # Each artifact is downloaded in its own directory, since the reports of all machines have the same names.
+ - name: Download the LNT reports
+ uses: actions/download-artifact at 9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ pattern: lnt-reports-*
+ path: ${{ runner.temp }}/lnt-reports
+
- name: Update comment with results
uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
MATRIX: ${{ needs.extract-info.outputs.matrix }}
RESULTS_DIR: ${{ runner.temp }}/results
+ LNT_REPORTS_DIR: ${{ runner.temp }}/lnt-reports
+ LNT_URL: https://lnt.llvm.org
+ LNT_TEST_SUITE: libcxx
+ BASELINE: ${{ needs.extract-info.outputs.baseline }}
+ CANDIDATE: ${{ needs.extract-info.outputs.candidate }}
with:
script: |
const fs = require('fs');
const path = require('path');
const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+ // Return a link to the comparison of the baseline and the candidate on LNT, or nothing if there are no
+ // results for that machine.
+ function lnt_comparison(machine) {
+ const suite = process.env.LNT_TEST_SUITE;
+ const dir = path.join(process.env.LNT_REPORTS_DIR, `lnt-reports-${machine}`);
+ if (!fs.existsSync(dir)) {
+ return '';
+ }
+ const reports = fs.readdirSync(dir).filter(f => f.endsWith('.json'))
+ .map(f => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')));
+ if (reports.length === 0) {
+ return '';
+ }
+
+ // Select the runs of this job explicitly, since other runs may exist for the same commits (e.g.
+ // when several pull requests have the same baseline).
+ const url = new URL('/compare', process.env.LNT_URL);
+ for (const [side, commit] of [['a', process.env.BASELINE], ['b', process.env.CANDIDATE]]) {
+ const runs = reports.filter(r => r.commit.value === commit);
+ url.searchParams.append(`suite_${side}`, suite);
+ url.searchParams.append(`machine_${side}`, reports[0].machine.name);
+ url.searchParams.append(`commit_${side}`, commit);
+ for (const run of runs) {
+ url.searchParams.append(`runs_${side}`, run.uuid);
+ }
+ }
+ url.searchParams.append('metric', 'execution_time');
+ // This ends up in an HTML block of the comment, where Markdown links aren't rendered.
+ return ` (<a href="${url.href.replaceAll('&', '&')}">view on LNT</a>)`;
+ }
+
// Report the results of every machine we benchmarked on. Machines without results are those where
// the comparison failed.
const machines = JSON.parse(process.env.MATRIX).map(cfg => cfg['lnt-machine']);
@@ -170,7 +236,7 @@ jobs:
return [
'<details>',
'<summary>',
- `Benchmark results for ${machine}:`,
+ `Benchmark results for ${machine}${lnt_comparison(machine)}:`,
'</summary>',
'',
'```',
diff --git a/.github/workflows/libcxx-run-ab-benchmarks.yml b/.github/workflows/libcxx-run-ab-benchmarks.yml
index d672bc6e63c90..3a0b8b406a0bb 100644
--- a/.github/workflows/libcxx-run-ab-benchmarks.yml
+++ b/.github/workflows/libcxx-run-ab-benchmarks.yml
@@ -199,6 +199,8 @@ jobs:
--commit "${commit}" \
--compiler "${COMPILER}" \
--no-ordinal \
+ --untracked-machine \
+ --delete-after-inactive-days 180 \
--run-parameter "test_suite_commit=${BENCHMARK_SUITE_VERSION}" \
--run-parameter "ci_run=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
--lit-build-dir "benchmarks/${build}" \
diff --git a/libcxx/utils/ci/lnt/common.py b/libcxx/utils/ci/lnt/common.py
index 9b03de931e654..33f6dba107fa8 100644
--- a/libcxx/utils/ci/lnt/common.py
+++ b/libcxx/utils/ci/lnt/common.py
@@ -113,15 +113,21 @@ def benchmarked_commits(lnt_url: str, test_suite: str, machine: str,
def write_lnt_report(path: pathlib.Path, machine: str, machine_fields: dict, commit: dict, tests: List[dict],
- measured_at: datetime.datetime, run_parameters: Optional[dict] = None) -> None:
- """Write a LNT v5 run submission, ready to be submitted with `lnt v5 submit`."""
+ measured_at: datetime.datetime, run_parameters: Optional[dict] = None,
+ machine_settings: Optional[dict] = None) -> None:
+ """
+ Write a LNT v5 run submission, ready to be submitted with `lnt v5 submit`.
+
+ `machine_settings` holds the built-in attributes of the machine (e.g. `tracked`), as opposed to
+ its fields.
+ """
report = {
'format_version': '5',
# Pick the UUID ourselves so that we can safely retry submitting this run if needed without
# running the risk of storing the same run twice.
'uuid': str(uuid.uuid4()),
'measured_at': measured_at.astimezone(datetime.timezone.utc).strftime(LNT_TIMESTAMP_FORMAT),
- 'machine': {'name': machine, 'fields': machine_fields},
+ 'machine': {'name': machine, 'fields': machine_fields, **(machine_settings or {})},
'commit': commit,
'run_parameters': run_parameters or {},
'tests': tests,
diff --git a/libcxx/utils/ci/lnt/create-lnt-report b/libcxx/utils/ci/lnt/create-lnt-report
index b66ed46f7795d..83170b1549012 100755
--- a/libcxx/utils/ci/lnt/create-lnt-report
+++ b/libcxx/utils/ci/lnt/create-lnt-report
@@ -175,6 +175,11 @@ def main(argv: List[str]) -> int:
ordinal.add_argument('--no-ordinal', action='store_true',
help='Never give the commit an ordinal. This is useful for results that are not meant to be part of a '
'time series, and required when the repository does not have the full history (e.g. a shallow clone).')
+ parser.add_argument('--untracked-machine', action='store_true',
+ help='Mark the machine as not tracked, which keeps it out of the dashboards and the automatic machine '
+ 'selection of LNT. This is useful for machines holding throwaway results, like A/B comparisons.')
+ parser.add_argument('--delete-after-inactive-days', type=common.at_least(1), default=None, metavar='DAYS',
+ help='Ask LNT to delete the machine and the commit after they have been inactive for that many days.')
parser.add_argument('--measured-at', type=utc_timestamp, default=None, metavar='YYYY-MM-DDTHH:MM:SSZ',
help='The UTC time at which the benchmarks were run. Defaults to the current time.')
parser.add_argument('--run-parameter', type=common.key_value, action='append', default=[], metavar='KEY=VALUE',
@@ -219,10 +224,17 @@ def main(argv: List[str]) -> int:
if args.lit_build_dir is not None:
machine_fields.update(gather_lit_features(args.lit_build_dir, lit_options))
+ machine_settings = {}
+ if args.untracked_machine:
+ machine_settings['tracked'] = False
+ if args.delete_after_inactive_days is not None:
+ machine_settings['delete_after_inactive_days'] = args.delete_after_inactive_days
+ commit['delete_after_inactive_days'] = args.delete_after_inactive_days
+
measured_at = args.measured_at or datetime.datetime.now(datetime.timezone.utc)
args.output.parent.mkdir(parents=True, exist_ok=True)
common.write_lnt_report(args.output, args.machine, machine_fields, commit, tests, measured_at,
- run_parameters=dict(args.run_parameter))
+ run_parameters=dict(args.run_parameter), machine_settings=machine_settings)
logging.info(f'Report written to {args.output}')
return 0
More information about the libcxx-commits
mailing list