[libcxx-commits] [libcxx] [libc++] Split benchmark generation and submission into two scripts (PR #191211)
via libcxx-commits
libcxx-commits at lists.llvm.org
Thu Apr 9 08:10:21 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-libcxx
Author: Louis Dionne (ldionne)
<details>
<summary>Changes</summary>
This makes it easier to locally replicate perf CI setups locally without actually making submissions to the LNT instance.
---
Full diff: https://github.com/llvm/llvm-project/pull/191211.diff
5 Files Affected:
- (modified) libcxx/utils/ci/lnt/README.md (+5-2)
- (modified) libcxx/utils/ci/lnt/run-benchmarks (+7-12)
- (modified) libcxx/utils/ci/lnt/runners/apple-m5-clang21 (+9-3)
- (modified) libcxx/utils/ci/lnt/runners/apple-m5-xcode26 (+9-3)
- (added) libcxx/utils/ci/lnt/submit-benchmarks (+72)
``````````diff
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index 2f2f42e89d6db..ffca01c70e606 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -22,10 +22,13 @@ libcxx/utils/ci/lnt/commit-watch --lnt-url http://localhost:8000 --test-suite li
while read commit; do \
libcxx/utils/ci/lnt/run-benchmarks \
--test-suite-commit abcdef09 \
- --lnt-url http://localhost:8000 \
--machine my-laptop \
- --test-suite libcxx \
--compiler clang++ \
--benchmark-commit ${commit} \
+ --output results.json \
+ && libcxx/utils/ci/lnt/submit-benchmarks \
+ --lnt-url http://localhost:8000 \
+ --test-suite libcxx \
+ results.json \
done
```
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index ad2175b380fd4..e0d2c153f4f27 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -63,7 +63,7 @@ def dict_to_params(d):
def main(argv):
parser = argparse.ArgumentParser(
prog='run-benchmarks',
- description='Benchmark libc++ at the given commit and submit to LNT.')
+ description='Benchmark libc++ at the given commit and produce a LNT JSON report.')
parser.add_argument('--benchmark-commit', type=str, required=True,
help='The SHA representing the version of the library to benchmark.')
parser.add_argument('--test-suite-commit', type=str, required=True,
@@ -72,10 +72,8 @@ def main(argv):
help='Path to the compiler to use for testing.')
parser.add_argument('--machine', type=str, required=True,
help='The name of the machine for reporting LNT results.')
- parser.add_argument('--test-suite', type=str, required=True,
- help='The name of the test suite for reporting LNT results.')
- parser.add_argument('--lnt-url', type=str, required=True,
- help='The URL of the LNT instance to submit results to.')
+ parser.add_argument('--output', type=pathlib.Path, required=True,
+ help='Path where the resulting LNT JSON report is written. The file is overwritten if it already exists.')
parser.add_argument('--filter', type=str, required=False,
help="Optional test filter to pass to lit when running the benchmarks. This allows "
"running only a subset of the benchmarks.")
@@ -123,7 +121,7 @@ def main(argv):
build_dir = pathlib.Path(build_dir)
logging.info('Installing LNT')
- run(['python', '-m', 'venv', build_dir / '.venv'])
+ run(['python3', '-m', 'venv', build_dir / '.venv'])
run([build_dir / '.venv/bin/pip', 'install', 'llvm-lnt'])
logging.info(f'Building libc++ at commit {args.benchmark_commit}')
@@ -161,13 +159,10 @@ def main(argv):
importreport += ['--run-info', arg]
for arg in dict_to_params(gather_machine_information(args)):
importreport += ['--machine-info', arg]
- importreport += [build_dir / 'benchmarks.lnt', build_dir / 'benchmarks.json']
+ output = args.output.resolve()
+ importreport += [build_dir / 'benchmarks.lnt', output]
run(importreport)
-
- logging.info(f'Submitting results to {args.lnt_url}')
- submission_url = f'{args.lnt_url}/db_default/v4/{args.test_suite}/submitRun'
- run([build_dir / '.venv/bin/lnt', 'submit', '--ignore-regressions', '--merge', 'append',
- submission_url, build_dir / 'benchmarks.json'])
+ logging.info(f'Report written to {output}')
if __name__ == '__main__':
diff --git a/libcxx/utils/ci/lnt/runners/apple-m5-clang21 b/libcxx/utils/ci/lnt/runners/apple-m5-clang21
index 9b5bc44f720a2..01e4027fcb4be 100755
--- a/libcxx/utils/ci/lnt/runners/apple-m5-clang21
+++ b/libcxx/utils/ci/lnt/runners/apple-m5-clang21
@@ -25,11 +25,17 @@ run_benchmarks() {
${MONOREPO_DIR}/libcxx/utils/ci/lnt/run-benchmarks \
--test-suite-commit 0eefb2682bf8c04954c46e91916b5164d8424702 \
--git-repo ${MONOREPO_DIR} \
- --lnt-url http://lnt.llvm.org \
- --test-suite libcxx2 \
--machine apple-m5-clang21 \
--compiler ${COMPILER} \
- --benchmark-commit ${1}
+ --benchmark-commit ${1} \
+ --output benchmarks.json
+
+ ${MONOREPO_DIR}/libcxx/utils/ci/lnt/submit-benchmarks \
+ --lnt-url http://lnt.llvm.org \
+ --test-suite libcxx2 \
+ benchmarks.json
+
+ rm benchmarks.json
}
if [ ${#COMMITS[@]} -gt 0 ]; then
diff --git a/libcxx/utils/ci/lnt/runners/apple-m5-xcode26 b/libcxx/utils/ci/lnt/runners/apple-m5-xcode26
index 6120b71e5595a..18ecc2cafc8f0 100755
--- a/libcxx/utils/ci/lnt/runners/apple-m5-xcode26
+++ b/libcxx/utils/ci/lnt/runners/apple-m5-xcode26
@@ -21,11 +21,17 @@ run_benchmarks() {
${MONOREPO_DIR}/libcxx/utils/ci/lnt/run-benchmarks \
--test-suite-commit 0eefb2682bf8c04954c46e91916b5164d8424702 \
--git-repo ${MONOREPO_DIR} \
- --lnt-url http://lnt.llvm.org \
- --test-suite libcxx2 \
--machine apple-m5-xcode26 \
--compiler clang++ \
- --benchmark-commit ${1}
+ --benchmark-commit ${1} \
+ --output benchmarks.json
+
+ ${MONOREPO_DIR}/libcxx/utils/ci/lnt/submit-benchmarks \
+ --lnt-url http://lnt.llvm.org \
+ --test-suite libcxx2 \
+ benchmarks.json
+
+ rm benchmarks.json
}
if [ ${#COMMITS[@]} -gt 0 ]; then
diff --git a/libcxx/utils/ci/lnt/submit-benchmarks b/libcxx/utils/ci/lnt/submit-benchmarks
new file mode 100755
index 0000000000000..5f25e6fef9fc3
--- /dev/null
+++ b/libcxx/utils/ci/lnt/submit-benchmarks
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import argparse
+import logging
+import pathlib
+import subprocess
+import sys
+import tempfile
+
+
+def main(argv):
+ parser = argparse.ArgumentParser(
+ prog='submit-benchmarks',
+ description='Submit a LNT JSON report to a LNT server.')
+ parser.add_argument('report', type=pathlib.Path,
+ help='Path to the LNT JSON report to submit.')
+ parser.add_argument('--lnt-url', type=str, required=True,
+ help='The URL of the LNT instance to submit results to.')
+ parser.add_argument('--test-suite', type=str, required=True,
+ help='The name of the test suite for reporting LNT results.')
+ parser.add_argument('--dry-run', action='store_true',
+ help='Do not actually perform any action. Use with -vv to see what would be executed.')
+ parser.add_argument('-v', '--verbose', action='count', default=0,
+ help='Verbosity level: passing the option multiple times increases the level.')
+ args = parser.parse_args(argv)
+
+ if args.verbose == 0:
+ logging.basicConfig(level=logging.INFO)
+ elif args.verbose >= 1:
+ logging.basicConfig(level=logging.DEBUG)
+
+ def run(command, **kwargs):
+ command = [str(c) for c in command]
+ logging.debug(f'$ {" ".join(command)}')
+ if args.dry_run:
+ return
+ try:
+ if not args.verbose:
+ if 'stdout' not in kwargs:
+ kwargs['stdout'] = subprocess.PIPE
+ if 'stderr' not in kwargs:
+ kwargs['stderr'] = subprocess.PIPE
+ subprocess.run(command, check=True, **kwargs)
+ except subprocess.CalledProcessError as e:
+ if e.stdout:
+ sys.stdout.write(e.stdout.decode())
+ if e.stderr:
+ sys.stderr.write(e.stderr.decode())
+ raise
+
+ with tempfile.TemporaryDirectory() as tmp:
+ tmp = pathlib.Path(tmp)
+
+ logging.info('Installing LNT')
+ run(['python3', '-m', 'venv', tmp / '.venv'])
+ run([tmp / '.venv/bin/pip', 'install', 'llvm-lnt'])
+
+ logging.info(f'Submitting results to {args.lnt_url}')
+ submission_url = f'{args.lnt_url}/db_default/v4/{args.test_suite}/submitRun'
+ run([tmp / '.venv/bin/lnt', 'submit', '--ignore-regressions', '--merge', 'append',
+ submission_url, args.report.resolve()])
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
``````````
</details>
https://github.com/llvm/llvm-project/pull/191211
More information about the libcxx-commits
mailing list