[libcxx-commits] [libcxx] [libc++] Remove legacy LNT benchmarking infrastructure (PR #215612)
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Tue Aug 11 09:51:05 PDT 2026
https://github.com/ldionne created https://github.com/llvm/llvm-project/pull/215612
We had a few scripts that were useful to run benchmarking "servers" locally. They are no longer useful now that we have proper tooling integrated with Github workflows.
Note that running benchmarks locally is still useful, and we'll probably want to add new tooling to make that easier, but that should be built on top of the existing select/plan/dispatch pipeline that we introduced recently instead of reinventing the wheel.
>From c0e14b0e3beacdd462b1f5519e2a41d67ef4b6de Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Tue, 11 Aug 2026 12:27:26 -0400
Subject: [PATCH] [libc++] Remove legacy LNT benchmarking infrastructure
We had a few scripts that were useful to run benchmarking "servers"
locally. They are no longer useful now that we have proper tooling
integrated with Github workflows.
Note that running benchmarks locally is still useful, and we'll
probably want to add new tooling to make that easier, but that
should be built on top of the existing select/plan/dispatch
pipeline that we introduced recently instead of reinventing the
wheel.
---
libcxx/utils/ci/lnt/commit-watch | 132 ---------------------
libcxx/utils/ci/lnt/run-benchbot | 197 -------------------------------
2 files changed, 329 deletions(-)
delete mode 100755 libcxx/utils/ci/lnt/commit-watch
delete mode 100755 libcxx/utils/ci/lnt/run-benchbot
diff --git a/libcxx/utils/ci/lnt/commit-watch b/libcxx/utils/ci/lnt/commit-watch
deleted file mode 100755
index 6aec4712d4036..0000000000000
--- a/libcxx/utils/ci/lnt/commit-watch
+++ /dev/null
@@ -1,132 +0,0 @@
-#!/usr/bin/env python3
-# ===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# ===----------------------------------------------------------------------===##
-
-from typing import List, Optional, Set
-import argparse
-import json
-import logging
-import os
-import pathlib
-import subprocess
-import sys
-import time
-
-
-def directory_path(string):
- if os.path.isdir(string):
- return pathlib.Path(string)
- else:
- raise NotADirectoryError(string)
-
-def api(lnt_url: str, test_suite: str, endpoint: str):
- url = f'{lnt_url}/api/db_default/v4/{test_suite}{endpoint}'
- logging.debug(f'Querying {url}')
- result = json.loads(subprocess.check_output(['curl', '-sS', url]).decode())
- return result
-
-def get_benchmarked_commits(lnt_url: str, test_suite: str, machine: str) -> Set[str]:
- """
- Return the set of commits that have already been benchmarked on the given LNT
- instance, test suite and machine.
- """
- result = api(lnt_url, test_suite, f'/machines/{machine}')
- commits = set()
- if 'runs' not in result: # there is no such machine
- return set()
- for run in result['runs']:
- if 'git_sha' not in run:
- raise ValueError(f'Found run without a git_sha field: are you using the right LNT test suite? {run}')
- commits.add(run['git_sha'])
- return commits
-
-def git_rev_list(git_repo: str, paths: List[str] = []) -> List[str]:
- """
- Return the list of all revisions in the given Git repository. Older commits are earlier in the list.
- """
- cli = ['git', '-C', git_repo, 'rev-list', 'origin', '--', *paths]
- rev_list = subprocess.check_output(cli).decode().strip().splitlines()
- return list(reversed(rev_list))
-
-def git_sort_revlist(git_repo: str, commits: List[str]) -> List[str]:
- """
- Return the list of commits sorted by their chronological order (from oldest to newest) in the
- provided Git repository. Items earlier in the list are older than items later in the list.
- """
- revlist_cmd = ['git', '-C', git_repo, 'rev-list', '--no-walk'] + list(commits)
- revlist = subprocess.check_output(revlist_cmd, text=True).strip().splitlines()
- return list(reversed(revlist))
-
-def get_all_libcxx_commits(git_repo: str) -> Set[str]:
- """
- Return the set of commits available to benchmark for libc++: this is the list of
- commits that touch code in libc++ that we care to benchmark.
- """
- logging.debug(f'Fetching {git_repo}')
- subprocess.check_call(['git', '-C', git_repo, 'fetch', '--quiet', 'origin'])
- return set(git_rev_list(git_repo, paths=['libcxx/include', 'libcxx/src']))
-
-
-def main(argv):
- parser = argparse.ArgumentParser(
- prog='commit-watch',
- description='Watch for libc++ commits to run benchmarks on and print them to standard output.',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument('--lnt-url', type=str, required=True,
- help='The URL of the LNT instance to use as the source of truth for finding already-benchmarked commits.')
- parser.add_argument('--test-suite', type=str, required=True,
- help='The name of the test suite that we are producing results for on the LNT instance.')
- parser.add_argument('--machine', type=str, required=True,
- help='The name of the machine that we are producing results for on the LNT instance.')
- parser.add_argument('-d', '--delay', type=int, required=False, default=60,
- help='The delay (in seconds) to wait before successive polls.')
- parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
- help='Optional path to the Git repository to use. By default, the current working directory is used.')
- parser.add_argument('-v', '--verbose', action='count', default=0,
- help='Verbosity level: passing the option multiple times increases the level.')
- args = parser.parse_args(argv)
-
- VERY_VERBOSE = logging.DEBUG - 1
- if args.verbose == 1:
- logging.basicConfig(level=logging.INFO)
- elif args.verbose == 2:
- logging.basicConfig(level=logging.DEBUG)
- elif args.verbose > 2:
- logging.basicConfig(level=VERY_VERBOSE)
-
- last_commit: Optional[str] = None # last commit printed
- while True:
- logging.info(f'Fetching all libc++ commits in {args.git_repo}')
- all_commits = get_all_libcxx_commits(args.git_repo)
- logging.log(VERY_VERBOSE, f'Found all libc++ commits: {all_commits}')
-
- logging.info(f'Getting benchmarked commits for {args.machine} from {args.lnt_url}')
- benchmarked_commits = get_benchmarked_commits(args.lnt_url, args.test_suite, args.machine)
- logging.log(VERY_VERBOSE, f'Found benchmarked commits: {benchmarked_commits}')
-
- # Print the next commit that should be benchmarked, but only when it changes over the one
- # that we just printed. This ensures that we don't repeatedly print that the same benchmark
- # should be benchmarked, which would otherwise happen when we are running the benchmarks but
- # have not submitted to LNT yet.
- commits_to_benchmark = all_commits - benchmarked_commits
- if not commits_to_benchmark:
- logging.info('No more commits to benchmark')
- else:
- commits_to_benchmark = git_sort_revlist(args.git_repo, list(commits_to_benchmark))
- most_recent = commits_to_benchmark[-1]
- if most_recent != last_commit:
- logging.info(f'Selected {most_recent}')
- print(most_recent, flush=True)
- last_commit = most_recent
-
- logging.debug(f'Waiting {args.delay}s before next poll')
- time.sleep(args.delay)
-
-
-if __name__ == '__main__':
- main(sys.argv[1:])
diff --git a/libcxx/utils/ci/lnt/run-benchbot b/libcxx/utils/ci/lnt/run-benchbot
deleted file mode 100755
index ce198080e8993..0000000000000
--- a/libcxx/utils/ci/lnt/run-benchbot
+++ /dev/null
@@ -1,197 +0,0 @@
-#!/usr/bin/env bash
-# ===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# ===----------------------------------------------------------------------===##
-
-set -e
-set -o pipefail
-
-PROGNAME="$(basename "${0}")"
-
-function usage() {
-cat <<EOF
-Usage:
-${PROGNAME} [options] <BUILDER> [-- commit ...]
-
-[-h|--help] Display this help and exit.
-
---llvm-root <DIR> Path to the root of the LLVM monorepo. By default, we try
- to figure it out based on the current working directory.
-
---results-dir <DIR> The directory to use for storing benchmark results (LNT JSON files).
- By default, this is '<llvm-root>/build/<builder>'.
-
---build-dir <DIR> Optional directory in which to keep build artifacts. Artifacts for each
- run are stored under '<build-dir>/<builder>/<commit>'. By default, build
- artifacts are stored in a temporary directory and discarded after each run.
-
-[--lnt-url <URL>] The optional URL of the LNT instance to submit results to. By
- default, results are not submitted to any LNT instance.
-
-[--filter <FILTER>] Optional filter to pass to lit to run only a subset of benchmarks.
-
-Environment variables
-SPEC_DIR Optional path to a SPEC installation. If set and the directory
- exists, SPEC benchmarks are included in the run.
-
-If commits are provided after --, only those commits are benchmarked and the
-runner exits. Otherwise, the runner polls the given LNT url (which must be
-present) to discover un-benchmarked commits and runs indefinitely.
-EOF
-}
-
-if [[ $# == 0 ]]; then
- usage
- exit 0
-fi
-
-while [[ $# -gt 0 ]]; do
- case ${1} in
- -h|--help)
- usage
- exit 0
- ;;
- --llvm-root)
- MONOREPO_ROOT="${2}"
- shift; shift
- ;;
- --results-dir)
- RESULTS_DIR="${2}"
- shift; shift
- ;;
- --build-dir)
- BUILD_DIR="${2}"
- shift; shift
- ;;
- --lnt-url)
- LNT_URL="${2}"
- shift; shift
- ;;
- --filter)
- FILTER="${2}"
- shift; shift
- ;;
- --)
- shift
- COMMITS=("$@")
- break
- ;;
- *)
- BUILDER="${1}"
- shift
- ;;
- esac
-done
-
-MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
-RESULTS_DIR="${RESULTS_DIR:=${MONOREPO_ROOT}/build/${BUILDER}}"
-
-case "${BUILDER}" in
-apple-m5-clang21)
- export SDKROOT=$(xcrun --show-sdk-path)
- COMPILER=$(brew --prefix)/opt/llvm/bin/clang++
- BENCHMARK_SUITE_VERSION=0eefb2682bf8c04954c46e91916b5164d8424702
-;;
-apple-m5-xcode26)
- COMPILER=clang++
- BENCHMARK_SUITE_VERSION=0eefb2682bf8c04954c46e91916b5164d8424702
-;;
-*)
- echo "${BUILDER} is not a known configuration"
- exit 1
-;;
-esac
-
-echo "***********************************************"
-echo "Diagnose tools in use"
-cmake --version
-ninja --version
-${COMPILER} --version
-echo "Benchmark suite version: ${BENCHMARK_SUITE_VERSION}"
-echo "***********************************************"
-
-LNT_TEST_SUITE=libcxx
-
-mkdir -p "${RESULTS_DIR}"
-
-# Find the smallest suffix such that both the artifacts and the results directory names are available.
-# This ensures that we can always match artifact directories with their results.
-# The first candidate is unsuffixed ('foo.json' / 'foo'), then '.1', '.2', and so on.
-select_run_suffix() {
- local results_base="${1}" # e.g. <results-dir>/<commit>
- local artifacts_base="${2}" # e.g. <build-dir>/<builder>/<commit>, or empty when not keeping artifacts
- local suffix=""
- local i=1
- while [ -f "${results_base}${suffix}.json" ] || { [ -n "${artifacts_base}" ] && [ -e "${artifacts_base}${suffix}" ]; }; do
- suffix=".${i}"
- i=$((i + 1))
- done
- echo "${suffix}"
-}
-
-run_benchmarks() {
- local spec_arg=""
- if [ -d "${SPEC_DIR}" ]; then
- spec_arg="--spec-dir ${SPEC_DIR}"
- fi
-
- local filter_arg=""
- if [ -n "${FILTER}" ]; then
- filter_arg="--filter ${FILTER}"
- fi
-
- # Pick a suffix that is free for both the result report and (when keeping build artifacts) the
- # artifacts directory, so 'foo.N.json' and its build artifacts in 'foo.N' always correspond.
- local artifacts_base=""
- if [ -n "${BUILD_DIR}" ]; then
- artifacts_base="${BUILD_DIR}/${BUILDER}/${1}"
- fi
- local suffix=$(select_run_suffix "${RESULTS_DIR}/${1}" "${artifacts_base}")
-
- local build_dir_arg=""
- if [ -n "${BUILD_DIR}" ]; then
- build_dir_arg="--build-dir ${artifacts_base}${suffix}"
- fi
-
- local output="${RESULTS_DIR}/${1}${suffix}.json"
- ${MONOREPO_ROOT}/libcxx/utils/ci/lnt/run-benchmarks \
- --test-suite-commit ${BENCHMARK_SUITE_VERSION} \
- --git-repo ${MONOREPO_ROOT} \
- --machine ${BUILDER} \
- --compiler ${COMPILER} \
- --benchmark-commit ${1} \
- ${spec_arg} \
- ${filter_arg} \
- ${build_dir_arg} \
- --output ${output}
-
- if [ -n "${LNT_URL}" ]; then
- ${MONOREPO_ROOT}/libcxx/utils/ci/lnt/submit-benchmarks \
- --lnt-url ${LNT_URL} \
- --test-suite ${LNT_TEST_SUITE} \
- ${output}
- fi
-}
-
-if [ ${#COMMITS[@]} -gt 0 ]; then
- for commit in "${COMMITS[@]}"; do
- run_benchmarks ${commit}
- done
-else
- if [ -z "${LNT_URL}" ]; then
- echo "error: --lnt-url is required when polling for commits (it is used to discover un-benchmarked commits)"
- exit 1
- fi
- while true; do
- ${MONOREPO_ROOT}/libcxx/utils/ci/lnt/commit-watch --git-repo ${MONOREPO_ROOT} \
- --lnt-url ${LNT_URL} --test-suite ${LNT_TEST_SUITE} --machine ${BUILDER} | \
- while read commit; do \
- run_benchmarks ${commit}
- done
- sleep 60 # To avoid busy looping in case something goes really wrong
- done
-fi
More information about the libcxx-commits
mailing list