[libcxx-commits] [libcxx] [libc++] Update the LNT tooling to LNT v5 (PR #232866)
via libcxx-commits
libcxx-commits at lists.llvm.org
Fri Oct 16 10:49:59 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-libcxx
Author: Louis Dionne (ldionne)
<details>
<summary>Changes</summary>
lnt.llvm.org now runs LNT v5, which has a different submission format and REST API than the LNT v4 instance. This updates the scripts that talk to LNT accordingly.
---
Patch is 27.34 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/232866.diff
7 Files Affected:
- (modified) libcxx/utils/ci/lnt/README.md (+8-21)
- (modified) libcxx/utils/ci/lnt/common.py (+81-1)
- (modified) libcxx/utils/ci/lnt/plan-benchmarks (+27-29)
- (modified) libcxx/utils/ci/lnt/requirements.txt (-1)
- (modified) libcxx/utils/ci/lnt/run-benchmarks (+43-35)
- (modified) libcxx/utils/ci/lnt/run-libstdcxx-benchmarks (+20-29)
- (modified) libcxx/utils/ci/lnt/submit-benchmarks (+55-25)
``````````diff
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index f846631d79a54..8155a3ec94b31 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -106,7 +106,8 @@ run-benchmarks --test-suite-commit <SHA1> --machine <MACHINE> \
This will run the benchmarks (using the test suite at the specified `SHA1`) against the installation
of libc++ at `PATH` (which is assumed to be libc++ as-of `SHA2`), and produce a LNT-ready JSON report.
-The results can then be submitted to a LNT instance if desired.
+The results can then be submitted to a LNT instance with `submit-benchmarks` if desired (requires an
+API key).
Note that `run-benchmarks` does not build anything: the library being benchmarked must have been built
or installed beforehand. How to pick up that library is determined by `--test-config` and any Lit parameters
@@ -115,25 +116,11 @@ passed.
## Test suite schemas
`schemas/` holds the schemas of the `libcxx` and `libstdcxx` test suites as we submit them to LNT.
-This describes the various metrics and fields we submit to the instance. For the `libcxx` test suite,
-the ordinal (the ordering between commits) is obtained by `git rev-list --count`.
+This describes the various metrics and fields we submit to the instance.
-## Setting up a local LNT instance
+For the `libcxx` test suite, the commits are libc++ commits and their ordinal (the ordering between
+commits) is obtained by `git rev-list --count`. Commits not on the `main` branch (e.g. commits made
+on PR branches) don't have an ordinal.
-```
-# Create an instance and run a server
-lnt create my-instance
-echo "api_auth_token = 'example_token'" >> my-instance/lnt.cfg
-lnt runserver my-instance
-
-# In another terminal, create the libcxx test suite on the locally-running server
-cat <<EOF > lnt-admin-config.yaml
-lnt_url: "http://localhost:8000"
-database: default
-auth_token: example_token
-EOF
-lnt admin --config lnt-admin-config.yaml --testsuite libcxx test-suite add libcxx/utils/ci/lnt/schemas/libcxx.yaml
-
-# Then submit to the local instance
-submit-benchmarks --lnt-url http://localhost:8000 --test-suite libcxx result.json
-```
+For the `libstdcxx` test suite, the commits are versions of libstdc++ (e.g. `16.2.0`), and their
+ordinal is `major * 10000 + minor * 100 + patch`.
diff --git a/libcxx/utils/ci/lnt/common.py b/libcxx/utils/ci/lnt/common.py
index 7ff423271322f..fd202cb4745f5 100644
--- a/libcxx/utils/ci/lnt/common.py
+++ b/libcxx/utils/ci/lnt/common.py
@@ -6,15 +6,95 @@
#
# ===----------------------------------------------------------------------===##
-from typing import Callable, NamedTuple
+from typing import Callable, Dict, Iterable, List, NamedTuple, Optional
import argparse
+import datetime
import json
+import pathlib
+import urllib.parse
+import urllib.request
+import uuid
+
+
+# The User-Agent we send to LNT. This is set explicitly since some deployments
+# reject requests made with the default User-Agent of urllib.
+USER_AGENT = 'libcxx-lnt-tooling'
+
+# LNT instances can be slow: timeout after that many seconds.
+LNT_TIMEOUT_SECONDS = 120
+
+
+class LNTError(Exception):
+ pass
def is_sha(string: str) -> bool:
return len(string) == 40 and all(c in '0123456789abcdef' for c in string.lower())
+def lnt_suite_url(lnt_url: str, test_suite: str) -> str:
+ """Return the URL of a test suite in the REST API of a LNT instance."""
+ return f'{lnt_url.rstrip("/")}/api/suites/{urllib.parse.quote(test_suite, safe="")}'
+
+
+def lnt_request(url: str, body: Optional[bytes] = None, token: Optional[str] = None) -> dict:
+ """
+ Send a request to LNT and return its JSON response. If a body is provided, this issues
+ a POST request, otherwise a GET. Errors are raised as urllib raises them.
+ """
+ headers = {'User-Agent': USER_AGENT}
+ if token is not None:
+ headers['Authorization'] = f'Bearer {token}'
+ if body is not None:
+ headers['Content-Type'] = 'application/json'
+ request = urllib.request.Request(url, data=body, headers=headers)
+ with urllib.request.urlopen(request, timeout=LNT_TIMEOUT_SECONDS) as response:
+ return json.loads(response.read().decode())
+
+
+def write_lnt_report(path: pathlib.Path, machine: str, machine_fields: dict, commit: dict, tests: List[dict],
+ measured_at: datetime.datetime, run_parameters: Optional[dict] = None) -> None:
+ """Write a LNT v5 run submission, ready to be submitted with submit-benchmarks."""
+ report = {
+ 'format_version': '5',
+ # Pick the UUID ourselves so that we can safely retry submitting this run if needed without
+ # running the risk of storing the same run twice.
+ 'uuid': str(uuid.uuid4()),
+ 'measured_at': measured_at.astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
+ 'machine': {'name': machine, 'fields': machine_fields},
+ 'commit': commit,
+ 'run_parameters': run_parameters or {},
+ 'tests': tests,
+ }
+ with open(path, 'w') as f:
+ json.dump(report, f, indent=2, allow_nan=False)
+
+
+def parse_lnt_results(lines: Iterable[str]) -> List[dict]:
+ """
+ Turn `<test>.<metric> <value>` lines (as produced by consolidate-benchmarks) into the list
+ of tests expected in a LNT v5 run submission.
+
+ A test can have several values for the same metric, for example when it was repeated. The
+ n-th value of each metric is taken to belong to the n-th sample of that test.
+ """
+ tests: Dict[str, Dict[str, List[float]]] = {}
+ for line in lines:
+ if not line.strip():
+ continue
+ (key, value) = line.split()
+ (name, metric) = key.rsplit('.', 1)
+ tests.setdefault(name, {}).setdefault(metric, []).append(float(value))
+
+ result = []
+ for (name, metrics) in tests.items():
+ count = max(len(values) for values in metrics.values())
+ samples = [{metric: values[i] for (metric, values) in metrics.items() if i < len(values)}
+ for i in range(count)]
+ result.append({'name': name, 'samples': samples})
+ return result
+
+
def at_least(minimum: int) -> Callable[[str], int]:
"""
Return an argparse type that accepts integers no smaller than `minimum`.
diff --git a/libcxx/utils/ci/lnt/plan-benchmarks b/libcxx/utils/ci/lnt/plan-benchmarks
index eb97e229b4317..ff89a5ce9dd88 100755
--- a/libcxx/utils/ci/lnt/plan-benchmarks
+++ b/libcxx/utils/ci/lnt/plan-benchmarks
@@ -17,35 +17,30 @@ import sys
import tabulate
import urllib.error
import urllib.parse
-import urllib.request
-# LNT instances can be slow: timeout after that many seconds.
-TIMEOUT_SECONDS = 120
+# How many runs to ask for at once. This is the largest page size LNT allows.
+PAGE_SIZE = 10000
class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter,
argparse.RawDescriptionHelpFormatter):
"""Show defaults, but keep the paragraphs of the description intact."""
-class LNTError(Exception):
- pass
-
def get(url: str) -> Optional[dict]:
"""GET a JSON resource, returning None if the server answers 404."""
logging.debug(f'GET {url}')
try:
- with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response:
- return json.loads(response.read().decode())
+ return common.lnt_request(url)
except urllib.error.HTTPError as error:
if error.code == 404:
return None
- raise LNTError(f'GET {url} failed with HTTP {error.code}: {error.reason}')
+ raise common.LNTError(f'GET {url} failed with HTTP {error.code}: {error.reason}')
except urllib.error.URLError as error:
- raise LNTError(f'GET {url} failed: {error.reason}')
+ raise common.LNTError(f'GET {url} failed: {error.reason}')
except (json.JSONDecodeError, UnicodeDecodeError) as error:
- raise LNTError(f'GET {url} returned a malformed response: {error}')
+ raise common.LNTError(f'GET {url} returned a malformed response: {error}')
def benchmarked_commits(lnt_url: str, test_suite: str, machine: str) -> Optional[Dict[str, int]]:
@@ -53,25 +48,28 @@ def benchmarked_commits(lnt_url: str, test_suite: str, machine: str) -> Optional
Return {commit: number of runs} for a machine, or None if the instance has no
such machine.
"""
- base = f'{lnt_url.rstrip("/")}/api/db_default/v4/{test_suite}'
- result = get(f'{base}/machines/{urllib.parse.quote(machine)}')
- if result is None:
- listing = get(f'{base}/machines')
- if listing is None:
- raise LNTError(f'{lnt_url} has no test suite named {test_suite}')
- named = [m for m in listing.get('machines', []) if m.get('name') == machine]
- if len(named) > 1:
- ids = ', '.join(str(m.get('id')) for m in named)
- raise LNTError(f'ambiguous name: {lnt_url} has {len(named)} machines named {machine} (ids {ids})')
- return None
+ base = common.lnt_suite_url(lnt_url, test_suite)
+ # Listing runs answers 404 both for an unknown suite and for an unknown machine: tell them apart.
+ if get(base) is None:
+ raise common.LNTError(f'{lnt_url} has no test suite named {test_suite}')
counts: Dict[str, int] = {}
- for run in result.get('runs', []):
- if 'git_sha' not in run:
- raise LNTError(f'run {run.get("id")} has no git_sha field. Is {test_suite} the right test suite?')
- commit = str(run['git_sha']).strip().lower() # normalize commit SHAs
- counts[commit] = counts.get(commit, 0) + 1
- return counts
+ params = {'machine': machine, 'limit': PAGE_SIZE}
+ while True:
+ page = get(f'{base}/runs?{urllib.parse.urlencode(params)}')
+ if page is None:
+ if 'cursor' in params:
+ raise common.LNTError(f'machine {machine} disappeared from {lnt_url} while listing its runs')
+ return None
+ for run in page.get('items', []):
+ if 'commit' not in run:
+ raise common.LNTError(f'run {run.get("uuid")} has no commit')
+ commit = str(run['commit']).strip().lower() # normalize commit SHAs
+ counts[commit] = counts.get(commit, 0) + 1
+ cursor = (page.get('cursor') or {}).get('next')
+ if cursor is None:
+ return counts
+ params['cursor'] = cursor
def plan(commits: Sequence[str],
@@ -197,5 +195,5 @@ def main(argv: List[str]) -> int:
if __name__ == '__main__':
try:
sys.exit(main(sys.argv[1:]))
- except (ValueError, OSError, LNTError) as error:
+ except (ValueError, OSError, common.LNTError) as error:
sys.exit(f'error: {error}')
diff --git a/libcxx/utils/ci/lnt/requirements.txt b/libcxx/utils/ci/lnt/requirements.txt
index 3876354ca072c..673c81551e8ff 100644
--- a/libcxx/utils/ci/lnt/requirements.txt
+++ b/libcxx/utils/ci/lnt/requirements.txt
@@ -1,3 +1,2 @@
-llvm-lnt
PyGithub
tabulate
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index 07eb22958a46e..cf2e4373c9eee 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -8,14 +8,15 @@
# ===----------------------------------------------------------------------===##
import argparse
+import common
import contextlib
+import datetime
import json
import logging
import os
import pathlib
import platform
import re
-import shutil
import subprocess
import sys
import tempfile
@@ -84,31 +85,36 @@ def gather_lit_features(lit, lit_args, tests):
reported[REPORTED_FEATURES[name]] = value
return reported
-def gather_run_information(args):
+def gather_commit_information(args):
"""
- Gather the run information to upload to LNT as part of the submission.
+ Gather the information about the benchmarked commit to upload to LNT as part of the submission.
"""
- info = {}
- # TODO: For now, we don't include commit information. This happens to be a long string and
- # production instances of LNT don't accept those as part of submissions for the time being.
- # info['commit_info'] = subprocess.check_output(['git', '-C', args.git_repo, 'show', args.benchmark_commit, '--no-patch']).decode()
- info['git_sha'] = subprocess.check_output(['git', '-C', args.git_repo, 'rev-parse', args.benchmark_commit]).decode().strip()
- return info
+ def git(*command):
+ return subprocess.check_output(['git', '-C', args.git_repo, *command]).decode().strip()
-def dict_to_params(d):
- """
- Return a list of 'key=value' strings from a dictionary.
- """
- res = []
- for (k, v) in d.items():
- res.append(f'{k}={v}')
- return res
+ sha = git('rev-parse', '--verify', f'{args.benchmark_commit}^{{commit}}')
+ commit = {'value': sha, 'fields': {'short_sha': sha[:12]}}
+
+ # Only commits on the main branch get an ordinal so we can order them on time series. Other commits
+ # can exist, but they are not totally ordered with the rest.
+ ancestor = subprocess.run(['git', '-C', args.git_repo, 'merge-base', '--is-ancestor', sha, args.main_branch])
+ if ancestor.returncode == 0:
+ commit['ordinal'] = int(git('rev-list', '--count', sha))
+ elif ancestor.returncode == 1:
+ logging.info(f'{sha} is not on {args.main_branch}: not giving it an ordinal')
+ else:
+ raise RuntimeError(f'could not determine whether {sha} is on {args.main_branch} (does that branch exist?)')
+ return commit
+
+def key_value(string):
+ if '=' not in string:
+ raise argparse.ArgumentTypeError(f'expected KEY=VALUE, got {string!r}')
+ return tuple(string.split('=', 1))
def main(argv):
parser = argparse.ArgumentParser(
prog='run-benchmarks',
- description='Benchmark libc++ at the given commit and produce a LNT JSON report.',
- epilog='This script depends on the modules listed in `libcxx/utils/ci/lnt/requirements.txt`.')
+ description='Benchmark libc++ at the given commit and produce a LNT JSON report.')
parser.add_argument('--test-config', type=str, required=True,
help='The Lit testing configuration to use. This has the same meaning as for test-at-commit: either an '
'absolute path to a Lit configuration file, or a path relative to the libcxx/test/configs directory '
@@ -124,6 +130,11 @@ def main(argv):
parser.add_argument('--output', type=pathlib.Path, required=True,
help='Path where the resulting LNT JSON report is written. It is an error for the file to '
'already exist.')
+ parser.add_argument('--main-branch', type=str, default='origin/main',
+ help='The main branch of the repository. Only commits on that branch are given an ordinal.')
+ parser.add_argument('--run-parameter', type=key_value, action='append', default=[], metavar='KEY=VALUE',
+ help='Additional information about the run to include in the report (e.g. the URL of the CI job). '
+ 'This can be passed multiple times.')
parser.add_argument('lit_options', nargs=argparse.REMAINDER,
help='Optional arguments passed to Lit when running the benchmarks. Should be provided last and '
'separated from other arguments with a `--`.')
@@ -187,8 +198,8 @@ def main(argv):
sys.exit(f'error: output report {args.output} already exists; not overwriting it')
if args.artifacts_dir is not None and args.artifacts_dir.exists():
sys.exit(f'error: artifacts directory {args.artifacts_dir} already exists; not overwriting it')
- if shutil.which('lnt') is None:
- sys.exit('error: cannot find `lnt`; install libcxx/utils/ci/lnt/requirements.txt')
+ # Gather the information about the commit up front, since it doesn't depend on the results and can fail.
+ commit = None if args.dry_run else gather_commit_information(args)
with contextlib.ExitStack() as stack:
# The artifacts are kept in --artifacts-dir when it is given, and stored in a temporary directory otherwise.
@@ -221,27 +232,24 @@ def main(argv):
if args.filter is not None:
cmd += ['--filter', args.filter]
run(cmd, enforce_success=False) # some benchmarks may fail to build/run at some commits, and that's okay
+ measured_at = datetime.datetime.now(datetime.timezone.utc)
consolidate = [args.git_repo / 'libcxx/utils/consolidate-benchmarks', artifacts / 'benchmarks-build']
if args.dry_run:
run(consolidate)
- else:
- with open(artifacts / 'benchmarks.lnt', 'w') as f:
- run(consolidate, stdout=f)
+ logging.info('Dry run: not creating a LNT report')
+ return
+ with open(artifacts / 'benchmarks.lnt', 'w') as f:
+ run(consolidate, stdout=f)
logging.info('Creating JSON report for LNT')
- order = len(subprocess.check_output(['git', '-C', args.git_repo, 'rev-list', args.benchmark_commit]).splitlines())
- importreport = ['lnt', 'importreport', '--order', str(order), '--machine', args.machine]
- for arg in dict_to_params(gather_run_information(args)):
- importreport += ['--run-info', arg]
- machine_info = {**gather_machine_information(args),
- **gather_lit_features(artifacts / 'benchmarks-build/bin/llvm-lit', lit_args,
- artifacts / 'benchmarks-build/libcxx/test/benchmarks')}
- for arg in dict_to_params(machine_info):
- importreport += ['--machine-info', arg]
+ machine_fields = {**gather_machine_information(args),
+ **gather_lit_features(artifacts / 'benchmarks-build/bin/llvm-lit', lit_args,
+ artifacts / 'benchmarks-build/libcxx/test/benchmarks')}
+ tests = common.parse_lnt_results((artifacts / 'benchmarks.lnt').read_text().splitlines())
output = args.output.resolve()
- importreport += [artifacts / 'benchmarks.lnt', output]
- run(importreport)
+ common.write_lnt_report(output, args.machine, machine_fields, commit, tests, measured_at,
+ run_parameters=dict(args.run_parameter))
logging.info(f'Report written to {output}')
diff --git a/libcxx/utils/ci/lnt/run-libstdcxx-benchmarks b/libcxx/utils/ci/lnt/run-libstdcxx-benchmarks
index 77697dc8e3a41..213b626f692df 100755
--- a/libcxx/utils/ci/lnt/run-libstdcxx-benchmarks
+++ b/libcxx/utils/ci/lnt/run-libstdcxx-benchmarks
@@ -8,6 +8,7 @@
# ===----------------------------------------------------------------------===##
import argparse
+import common
import datetime
import json
import pathlib
@@ -23,7 +24,7 @@ def gather_machine_information(args):
"""
Gather the machine information to upload to LNT as part of the submission.
"""
- info = {'name': args.machine}
+ info = {}
if platform.system() == 'Darwin':
profiler_info = json.loads(subprocess.check_output(['system_profiler', 'SPHardwareDataType', 'SPSoftwareDataType', '-json']).decode())
info['hardware'] = profiler_info['SPHardwareDataType'][0]['chip_type']
@@ -34,20 +35,18 @@ def gather_machine_information(args):
info['test_suite_commit'] = subprocess.check_output(['git', '-C', MONOREPO_ROOT, 'rev-parse', args.test_suite_commit]).decode().strip()
return info
-def parse_lnt_results(results):
+def version_ordinal(version):
"""
- Parse `<test>.<metric> <value>` lines (as produced by consolidate-benchmarks) into the list of tests
- expected in a LNT JSON report.
+ Return the ordinal of a GCC version (e.g. 16.2.0) in the libstdcxx LNT test suite. LNT orders commits
+ by their ordinal, so this must sort versions correctly.
"""
- tests = {}
- for line in results.splitlines():
- if not line.strip():
- continue
- (key, value) = line.split()
- (name, metric) = key.rsplit('.', 1)
- tests.setdefault(name, {}).setdefault(metric, []).append(float(value))
- return [{'name': name, **{m: v if len(v) > 1 else v[0] for (m, v) in metrics.items()}}
- for (name, metrics) in tests.items()]
+ parts = version.split('.')
+ if len(parts) != 3 or not all(p.isdigit() for p in parts):
+ raise ValueError(f'expected a...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/232866
More information about the libcxx-commits
mailing list