[libcxx-commits] [libcxx] 1ecd855 - [libc++] Replace individual LNT runner scripts by run-benchbot (#191221)
via libcxx-commits
libcxx-commits at lists.llvm.org
Thu Apr 9 08:48:40 PDT 2026
Author: Louis Dionne
Date: 2026-04-09T11:48:35-04:00
New Revision: 1ecd855a4d95b1295c17a36c13d911dc4ca0f0b9
URL: https://github.com/llvm/llvm-project/commit/1ecd855a4d95b1295c17a36c13d911dc4ca0f0b9
DIFF: https://github.com/llvm/llvm-project/commit/1ecd855a4d95b1295c17a36c13d911dc4ca0f0b9.diff
LOG: [libc++] Replace individual LNT runner scripts by run-benchbot (#191221)
The run-benchbot script is similar to the run-buildbot script. Its goal
is to provide a unified entry point for all the libc++ LNT runners.
Added:
libcxx/utils/ci/lnt/run-benchbot
Modified:
libcxx/utils/ci/lnt/README.md
Removed:
libcxx/utils/ci/lnt/runners/README.md
libcxx/utils/ci/lnt/runners/apple-m5-clang21
libcxx/utils/ci/lnt/runners/apple-m5-xcode26
################################################################################
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index ffca01c70e606..eedd98a082c2f 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -1,7 +1,26 @@
This directory contains utilities for continuous benchmarking of libc++ with LNT.
This can be done locally using a local instance, or using a public instance like http://lnt.llvm.org.
-Example for running locally:
+## Running a benchmark bot
+
+The `run-benchbot` script is the main entry point for running benchmarks. That script
+is where libc++'s pre-defined LNT bot configurations are defined. To benchmark specific
+commits:
+
+```
+libcxx/utils/ci/lnt/run-benchbot --llvm-root <monorepo> <builder> -- <commit1> <commit2> ...
+```
+
+Results are stored as JSON files in `<llvm-root>/build/<builder>/` by default. Use
+`--build-dir <dir>` to override the output directory.
+
+To continuously poll for un-benchmarked commits and submit results to a LNT instance:
+
+```
+libcxx/utils/ci/lnt/run-benchbot --llvm-root <monorepo> --lnt-url http://lnt.llvm.org <builder>
+```
+
+## Setting up a local LNT instance
```
# Create an instance and run a server
@@ -17,18 +36,6 @@ auth_token: example_token
EOF
lnt admin --config lnt-admin-config.yaml --testsuite libcxx test-suite add libcxx/utils/ci/lnt/schema.yaml
-# Then, watch for libc++ commits and submit benchmark results to the locally-running instance
-libcxx/utils/ci/lnt/commit-watch --lnt-url http://localhost:8000 --test-suite libcxx --machine my-laptop | \
- while read commit; do \
- libcxx/utils/ci/lnt/run-benchmarks \
- --test-suite-commit abcdef09 \
- --machine my-laptop \
- --compiler clang++ \
- --benchmark-commit ${commit} \
- --output results.json \
- && libcxx/utils/ci/lnt/submit-benchmarks \
- --lnt-url http://localhost:8000 \
- --test-suite libcxx \
- results.json \
- done
+# Then run the benchbot against the local instance
+libcxx/utils/ci/lnt/run-benchbot --llvm-root <monorepo> --lnt-url http://localhost:8000 <builder>
```
diff --git a/libcxx/utils/ci/lnt/run-benchbot b/libcxx/utils/ci/lnt/run-benchbot
new file mode 100755
index 0000000000000..0b501133ed209
--- /dev/null
+++ b/libcxx/utils/ci/lnt/run-benchbot
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+set -e
+set -o pipefail
+
+PROGNAME="$(basename "${0}")"
+
+function usage() {
+cat <<EOF
+Usage:
+${PROGNAME} [options] <BUILDER> [-- commit ...]
+
+[-h|--help] Display this help and exit.
+
+--llvm-root <DIR> Path to the root of the LLVM monorepo. By default, we try
+ to figure it out based on the current working directory.
+
+--build-dir <DIR> The directory to use for storing benchmark results. By default,
+ this is '<llvm-root>/build/<builder>'. Note that intermediate
+ build results are never kept around.
+
+[--lnt-url <URL>] The optional URL of the LNT instance to submit results to. By
+ default, results are not submitted to any LNT instance.
+
+If commits are provided after --, only those commits are benchmarked and the
+runner exits. Otherwise, the runner polls the given LNT url (which must be
+present) to discover un-benchmarked commits and runs indefinitely.
+EOF
+}
+
+if [[ $# == 0 ]]; then
+ usage
+ exit 0
+fi
+
+while [[ $# -gt 0 ]]; do
+ case ${1} in
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ --llvm-root)
+ MONOREPO_ROOT="${2}"
+ shift; shift
+ ;;
+ --build-dir)
+ BUILD_DIR="${2}"
+ shift; shift
+ ;;
+ --lnt-url)
+ LNT_URL="${2}"
+ shift; shift
+ ;;
+ --)
+ shift
+ COMMITS=("$@")
+ break
+ ;;
+ *)
+ BUILDER="${1}"
+ shift
+ ;;
+ esac
+done
+
+MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
+BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/${BUILDER}}"
+
+case "${BUILDER}" in
+apple-m5-clang21)
+ export SDKROOT=$(xcrun --show-sdk-path)
+ COMPILER=$(brew --prefix)/opt/llvm/bin/clang++
+ BENCHMARK_SUITE_VERSION=0eefb2682bf8c04954c46e91916b5164d8424702
+;;
+apple-m5-xcode26)
+ COMPILER=clang++
+ BENCHMARK_SUITE_VERSION=0eefb2682bf8c04954c46e91916b5164d8424702
+;;
+*)
+ echo "${BUILDER} is not a known configuration"
+ exit 1
+;;
+esac
+
+echo "***********************************************"
+echo "Diagnose tools in use"
+cmake --version
+ninja --version
+${COMPILER} --version
+echo "Benchmark suite version: ${BENCHMARK_SUITE_VERSION}"
+echo "***********************************************"
+
+LNT_TEST_SUITE=libcxx2
+
+mkdir -p "${BUILD_DIR}"
+
+run_benchmarks() {
+ local output="${BUILD_DIR}/${1}.json"
+ ${MONOREPO_ROOT}/libcxx/utils/ci/lnt/run-benchmarks \
+ --test-suite-commit ${BENCHMARK_SUITE_VERSION} \
+ --git-repo ${MONOREPO_ROOT} \
+ --machine ${BUILDER} \
+ --compiler ${COMPILER} \
+ --benchmark-commit ${1} \
+ --output ${output}
+
+ if [ -n "${LNT_URL}" ]; then
+ ${MONOREPO_ROOT}/libcxx/utils/ci/lnt/submit-benchmarks \
+ --lnt-url ${LNT_URL} \
+ --test-suite ${LNT_TEST_SUITE} \
+ ${output}
+ fi
+}
+
+if [ ${#COMMITS[@]} -gt 0 ]; then
+ for commit in "${COMMITS[@]}"; do
+ run_benchmarks ${commit}
+ done
+else
+ if [ -z "${LNT_URL}" ]; then
+ echo "error: --lnt-url is required when polling for commits (it is used to discover un-benchmarked commits)"
+ exit 1
+ fi
+ while true; do
+ ${MONOREPO_ROOT}/libcxx/utils/ci/lnt/commit-watch --git-repo ${MONOREPO_ROOT} \
+ --lnt-url ${LNT_URL} --test-suite ${LNT_TEST_SUITE} --machine ${BUILDER} | \
+ while read commit; do \
+ run_benchmarks ${commit}
+ done
+ sleep 60 # To avoid busy looping in case something goes really wrong
+ done
+fi
diff --git a/libcxx/utils/ci/lnt/runners/README.md b/libcxx/utils/ci/lnt/runners/README.md
deleted file mode 100644
index 08bbc9d7913b1..0000000000000
--- a/libcxx/utils/ci/lnt/runners/README.md
+++ /dev/null
@@ -1,10 +0,0 @@
-## Libc++ LNT runners
-
-This directory defines some LNT runners for tracking libc++ performance. A runner can be run with
-
-```
-bash <(curl -Ls https://raw.githubusercontent.com/llvm/llvm-project/main/libcxx/utils/ci/lnt/runners/RUNNER) <path-to-llvm-monorepo> [-- commit ...]
-```
-
-By default, runners poll `lnt.llvm.org` to discover un-benchmarked commits. If commits are provided
-after `--`, only those commits are benchmarked and the runner exits.
diff --git a/libcxx/utils/ci/lnt/runners/apple-m5-clang21 b/libcxx/utils/ci/lnt/runners/apple-m5-clang21
deleted file mode 100755
index 01e4027fcb4be..0000000000000
--- a/libcxx/utils/ci/lnt/runners/apple-m5-clang21
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Run on macOS against Homebrew Clang
-#
-
-if [ -z "${1}" ] || [ ! -d "${1}" ]; then
- echo "usage: ${0} <path-to-llvm-monorepo> [-- commit ...]"
- echo "error: Please provide a valid path to the LLVM monorepo."
- exit 1
-fi
-
-MONOREPO_DIR=$(cd "${1}" && pwd)
-shift
-if [ "${1}" = "--" ]; then
- shift
-fi
-COMMITS=("$@")
-
-export SDKROOT=$(xcrun --show-sdk-path)
-
-COMPILER=$(brew --prefix)/opt/llvm/bin/clang++
-
-run_benchmarks() {
- ${MONOREPO_DIR}/libcxx/utils/ci/lnt/run-benchmarks \
- --test-suite-commit 0eefb2682bf8c04954c46e91916b5164d8424702 \
- --git-repo ${MONOREPO_DIR} \
- --machine apple-m5-clang21 \
- --compiler ${COMPILER} \
- --benchmark-commit ${1} \
- --output benchmarks.json
-
- ${MONOREPO_DIR}/libcxx/utils/ci/lnt/submit-benchmarks \
- --lnt-url http://lnt.llvm.org \
- --test-suite libcxx2 \
- benchmarks.json
-
- rm benchmarks.json
-}
-
-if [ ${#COMMITS[@]} -gt 0 ]; then
- for commit in "${COMMITS[@]}"; do
- run_benchmarks ${commit}
- done
-else
- while true; do
- ${MONOREPO_DIR}/libcxx/utils/ci/lnt/commit-watch --git-repo ${MONOREPO_DIR} \
- --lnt-url http://lnt.llvm.org --test-suite libcxx2 --machine apple-m5-clang21 | \
- while read commit; do \
- run_benchmarks ${commit}
- done
- sleep 60 # To avoid busy looping in case something goes really wrong
- done
-fi
diff --git a/libcxx/utils/ci/lnt/runners/apple-m5-xcode26 b/libcxx/utils/ci/lnt/runners/apple-m5-xcode26
deleted file mode 100755
index 18ecc2cafc8f0..0000000000000
--- a/libcxx/utils/ci/lnt/runners/apple-m5-xcode26
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Run on macOS against Xcode-provided Clang
-#
-
-if [ -z "${1}" ] || [ ! -d "${1}" ]; then
- echo "usage: ${0} <path-to-llvm-monorepo> [-- commit ...]"
- echo "error: Please provide a valid path to the LLVM monorepo."
- exit 1
-fi
-
-MONOREPO_DIR=$(cd "${1}" && pwd)
-shift
-if [ "${1}" = "--" ]; then
- shift
-fi
-COMMITS=("$@")
-
-run_benchmarks() {
- ${MONOREPO_DIR}/libcxx/utils/ci/lnt/run-benchmarks \
- --test-suite-commit 0eefb2682bf8c04954c46e91916b5164d8424702 \
- --git-repo ${MONOREPO_DIR} \
- --machine apple-m5-xcode26 \
- --compiler clang++ \
- --benchmark-commit ${1} \
- --output benchmarks.json
-
- ${MONOREPO_DIR}/libcxx/utils/ci/lnt/submit-benchmarks \
- --lnt-url http://lnt.llvm.org \
- --test-suite libcxx2 \
- benchmarks.json
-
- rm benchmarks.json
-}
-
-if [ ${#COMMITS[@]} -gt 0 ]; then
- for commit in "${COMMITS[@]}"; do
- run_benchmarks ${commit}
- done
-else
- while true; do
- ${MONOREPO_DIR}/libcxx/utils/ci/lnt/commit-watch --git-repo ${MONOREPO_DIR} \
- --lnt-url http://lnt.llvm.org --test-suite libcxx2 --machine apple-m5-xcode26 | \
- while read commit; do \
- run_benchmarks ${commit}
- done
- sleep 60 # To avoid busy looping in case something goes really wrong
- done
-fi
More information about the libcxx-commits
mailing list