[libcxx-commits] [libcxx] 08d766b - [libc++][lnt] Allow retaining build artifacts in run-benchbot (#205146)
via libcxx-commits
libcxx-commits at lists.llvm.org
Mon Jun 22 10:13:19 PDT 2026
Author: Louis Dionne
Date: 2026-06-22T13:13:14-04:00
New Revision: 08d766ba6d1ffdeb894c0c935465f7134aef6dfc
URL: https://github.com/llvm/llvm-project/commit/08d766ba6d1ffdeb894c0c935465f7134aef6dfc
DIFF: https://github.com/llvm/llvm-project/commit/08d766ba6d1ffdeb894c0c935465f7134aef6dfc.diff
LOG: [libc++][lnt] Allow retaining build artifacts in run-benchbot (#205146)
Also, as a drive-by, introduce `--results-dir` to specify where to put
the JSON results instead of using `--build-dir` for that.
Assisted by Claude
Added:
Modified:
libcxx/utils/ci/lnt/README.md
libcxx/utils/ci/lnt/run-benchbot
libcxx/utils/ci/lnt/run-benchmarks
Removed:
################################################################################
diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md
index eedd98a082c2f..ac5cbc11f4c40 100644
--- a/libcxx/utils/ci/lnt/README.md
+++ b/libcxx/utils/ci/lnt/README.md
@@ -11,8 +11,12 @@ commits:
libcxx/utils/ci/lnt/run-benchbot --llvm-root <monorepo> <builder> -- <commit1> <commit2> ...
```
-Results are stored as JSON files in `<llvm-root>/build/<builder>/` by default. Use
-`--build-dir <dir>` to override the output directory.
+Results are stored as LNT JSON files in `<llvm-root>/build/<builder>/` by default.
+Use `--results-dir <dir>` to override where these reports are written.
+
+By default, build artifacts are stored in a temporary directory and discarded after
+each run. Pass `--build-dir <dir>` to keep them; artifacts for each run are then stored
+under `<dir>/<builder>/<commit>`.
To continuously poll for un-benchmarked commits and submit results to a LNT instance:
diff --git a/libcxx/utils/ci/lnt/run-benchbot b/libcxx/utils/ci/lnt/run-benchbot
index 52fb813b9321e..387853ac52b17 100755
--- a/libcxx/utils/ci/lnt/run-benchbot
+++ b/libcxx/utils/ci/lnt/run-benchbot
@@ -22,9 +22,12 @@ ${PROGNAME} [options] <BUILDER> [-- commit ...]
--llvm-root <DIR> Path to the root of the LLVM monorepo. By default, we try
to figure it out based on the current working directory.
---build-dir <DIR> The directory to use for storing benchmark results. By default,
- this is '<llvm-root>/build/<builder>'. Note that intermediate
- build results are never kept around.
+--results-dir <DIR> The directory to use for storing benchmark results (LNT JSON files).
+ By default, this is '<llvm-root>/build/<builder>'.
+
+--build-dir <DIR> Optional directory in which to keep build artifacts. Artifacts for each
+ run are stored under '<build-dir>/<builder>/<commit>'. By default, build
+ artifacts are stored in a temporary directory and discarded after each run.
[--lnt-url <URL>] The optional URL of the LNT instance to submit results to. By
default, results are not submitted to any LNT instance.
@@ -56,6 +59,10 @@ while [[ $# -gt 0 ]]; do
MONOREPO_ROOT="${2}"
shift; shift
;;
+ --results-dir)
+ RESULTS_DIR="${2}"
+ shift; shift
+ ;;
--build-dir)
BUILD_DIR="${2}"
shift; shift
@@ -81,7 +88,7 @@ while [[ $# -gt 0 ]]; do
done
MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
-BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/${BUILDER}}"
+RESULTS_DIR="${RESULTS_DIR:=${MONOREPO_ROOT}/build/${BUILDER}}"
case "${BUILDER}" in
apple-m5-clang21)
@@ -109,19 +116,21 @@ echo "***********************************************"
LNT_TEST_SUITE=libcxx2
-mkdir -p "${BUILD_DIR}"
+mkdir -p "${RESULTS_DIR}"
-# Given path/to/abcdef.json, figures out the next available path,
-# considering path/to/abcdef.{1,2,3,etc}.json.
-next_available_path() {
- local path="${1}"
- local base="${path%.json}"
+# Find the smallest suffix such that both the artifacts and the results directory names are available.
+# This ensures that we can always match artifact directories with their results.
+# The first candidate is unsuffixed ('foo.json' / 'foo'), then '.1', '.2', and so on.
+select_run_suffix() {
+ local results_base="${1}" # e.g. <results-dir>/<commit>
+ local artifacts_base="${2}" # e.g. <build-dir>/<builder>/<commit>, or empty when not keeping artifacts
+ local suffix=""
local i=1
- while [ -f "${path}" ]; do
- path="${base}.${i}.json"
+ while [ -f "${results_base}${suffix}.json" ] || { [ -n "${artifacts_base}" ] && [ -e "${artifacts_base}${suffix}" ]; }; do
+ suffix=".${i}"
i=$((i + 1))
done
- echo "${path}"
+ echo "${suffix}"
}
run_benchmarks() {
@@ -135,7 +144,20 @@ run_benchmarks() {
filter_arg="--filter ${FILTER}"
fi
- local output=$(next_available_path "${BUILD_DIR}/${1}.json")
+ # Pick a suffix that is free for both the result report and (when keeping build artifacts) the
+ # artifacts directory, so 'foo.N.json' and its build artifacts in 'foo.N' always correspond.
+ local artifacts_base=""
+ if [ -n "${BUILD_DIR}" ]; then
+ artifacts_base="${BUILD_DIR}/${BUILDER}/${1}"
+ fi
+ local suffix=$(select_run_suffix "${RESULTS_DIR}/${1}" "${artifacts_base}")
+
+ local build_dir_arg=""
+ if [ -n "${BUILD_DIR}" ]; then
+ build_dir_arg="--build-dir ${artifacts_base}${suffix}"
+ fi
+
+ local output="${RESULTS_DIR}/${1}${suffix}.json"
${MONOREPO_ROOT}/libcxx/utils/ci/lnt/run-benchmarks \
--test-suite-commit ${BENCHMARK_SUITE_VERSION} \
--git-repo ${MONOREPO_ROOT} \
@@ -144,6 +166,7 @@ run_benchmarks() {
--benchmark-commit ${1} \
${spec_arg} \
${filter_arg} \
+ ${build_dir_arg} \
--output ${output}
if [ -n "${LNT_URL}" ]; then
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index 611bd9c4d45b9..ce4705c0fafa1 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -8,6 +8,7 @@
# ===----------------------------------------------------------------------===##
import argparse
+import contextlib
import json
import logging
import os
@@ -77,12 +78,17 @@ def main(argv):
parser.add_argument('--machine', type=str, required=True,
help='The name of the machine for reporting LNT results.')
parser.add_argument('--output', type=pathlib.Path, required=True,
- help='Path where the resulting LNT JSON report is written. The file is overwritten if it already exists.')
+ help='Path where the resulting LNT JSON report is written. It is an error for the file to '
+ 'already exist.')
parser.add_argument('--filter', type=str, required=False,
help="Optional test filter to pass to lit when running the benchmarks. This allows "
"running only a subset of the benchmarks.")
parser.add_argument('--spec-dir', type=pathlib.Path, required=False,
help='Optional path to a SPEC installation to use for benchmarking.')
+ parser.add_argument('--build-dir', type=pathlib.Path, required=False,
+ help='Optional directory in which to keep build artifacts. By default, a temporary directory '
+ 'is used and the build artifacts are discarded after the run. It is an error to specify '
+ 'a build directory that already exists.')
parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
help='Optional path to the Git repository to use. By default, the current working directory is used.')
parser.add_argument('--dry-run', action='store_true',
@@ -121,17 +127,36 @@ def main(argv):
if enforce_success:
raise
- with tempfile.TemporaryDirectory() as build_dir:
- build_dir = pathlib.Path(build_dir)
+ # Fail fast before doing any expensive work: refuse to overwrite existing results or artifacts.
+ if args.output.exists():
+ sys.exit(f'error: output report {args.output} already exists; not overwriting it')
+ if args.build_dir is not None and args.build_dir.exists():
+ sys.exit(f'error: build directory {args.build_dir} already exists; not overwriting it')
+
+ with contextlib.ExitStack() as stack:
+ # LNT tooling is throwaway: always install it into a temporary directory.
+ venv = pathlib.Path(stack.enter_context(tempfile.TemporaryDirectory()))
+
+ # The build artifacts are kept in --build-dir when it is given, and stored in a temporary directory
+ # otherwise.
+ if args.build_dir is not None:
+ artifacts = args.build_dir
+ else:
+ artifacts = pathlib.Path(stack.enter_context(tempfile.TemporaryDirectory()))
+ if not args.dry_run:
+ artifacts.mkdir(parents=True, exist_ok=True)
+ logging.info(f'Storing build artifacts in {artifacts}')
logging.info('Installing LNT')
- run(['python3', '-m', 'venv', build_dir / '.venv'])
- run([build_dir / '.venv/bin/pip', 'install', 'llvm-lnt'])
+ run(['python3', '-m', 'venv', venv])
+ run([venv / 'bin/pip', 'install', 'llvm-lnt'])
logging.info(f'Building libc++ at commit {args.benchmark_commit}')
build_cmd = [args.git_repo / 'libcxx/utils/build-at-commit',
'--git-repo', args.git_repo,
- '--install-dir', build_dir / 'install',
+ '--install-dir', artifacts / 'libcxx-install',
+ '--tmp-src-dir', artifacts / 'libcxx-src',
+ '--tmp-build-dir', artifacts / 'libcxx-build',
'--commit', args.benchmark_commit,
'--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo', f'-DCMAKE_CXX_COMPILER={args.compiler}']
run(build_cmd, enforce_success=False) # if the build fails, carry on: we'll fail later and submit empty LNT results
@@ -139,32 +164,36 @@ def main(argv):
logging.info(f'Running benchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}')
cmd = [args.git_repo / 'libcxx/utils/test-at-commit',
'--git-repo', args.git_repo,
- '--build-dir', build_dir / 'bench',
+ '--build-dir', artifacts / 'benchmarks',
'--test-suite-commit', args.test_suite_commit,
- '--libcxx-installation', build_dir / 'install',
+ '--libcxx-installation', artifacts / 'libcxx-install',
'--',
'-j1', '--time-tests', '--test-output=failed',
'--param', f'compiler={args.compiler}',
'--param', 'optimization=speed',
'--param', 'std=c++26',
- build_dir / 'bench/libcxx/test/benchmarks']
+ artifacts / 'benchmarks/libcxx/test/benchmarks']
if args.spec_dir is not None:
cmd += ['--param', f'spec_dir={args.spec_dir}']
if args.filter is not None:
cmd += ['--filter', args.filter]
run(cmd, enforce_success=False) # some benchmarks may fail to build/run at some commits, and that's okay
- with open(build_dir / 'benchmarks.lnt', 'w') as f:
- run([args.git_repo / 'libcxx/utils/consolidate-benchmarks', build_dir / 'bench'], stdout=f)
+ consolidate = [args.git_repo / 'libcxx/utils/consolidate-benchmarks', artifacts / 'benchmarks']
+ if args.dry_run:
+ run(consolidate)
+ else:
+ with open(artifacts / 'benchmarks.lnt', 'w') as f:
+ run(consolidate, stdout=f)
logging.info('Creating JSON report for LNT')
order = len(subprocess.check_output(['git', '-C', args.git_repo, 'rev-list', args.benchmark_commit]).splitlines())
- importreport = [build_dir / '.venv/bin/lnt', 'importreport', '--order', str(order), '--machine', args.machine]
+ importreport = [venv / 'bin/lnt', 'importreport', '--order', str(order), '--machine', args.machine]
for arg in dict_to_params(gather_run_information(args)):
importreport += ['--run-info', arg]
for arg in dict_to_params(gather_machine_information(args)):
importreport += ['--machine-info', arg]
output = args.output.resolve()
- importreport += [build_dir / 'benchmarks.lnt', output]
+ importreport += [artifacts / 'benchmarks.lnt', output]
run(importreport)
logging.info(f'Report written to {output}')
More information about the libcxx-commits
mailing list