[libcxx-commits] [libcxx] 5ea70b6 - [libc++] Simplify and optimize the run-benchmarks script (#181382)
via libcxx-commits
libcxx-commits at lists.llvm.org
Mon Feb 23 06:22:05 PST 2026
Author: Louis Dionne
Date: 2026-02-23T09:22:00-05:00
New Revision: 5ea70b62bdfb91f64472dd6473e4880cc41ece4f
URL: https://github.com/llvm/llvm-project/commit/5ea70b62bdfb91f64472dd6473e4880cc41ece4f
DIFF: https://github.com/llvm/llvm-project/commit/5ea70b62bdfb91f64472dd6473e4880cc41ece4f.diff
LOG: [libc++] Simplify and optimize the run-benchmarks script (#181382)
Instead of configuring and running the benchmark suite once for SPEC and
once for the microbenchmarks, run it only once for everything. This
saves a configuration of the test suite (which includes building Google
Benchmark).
To replicate the functionality we had with --disable-microbenchmarks
(whose goal was mostly to run only SPEC), introduce a --filter argument
that can be used to select exactly which benchmarks are run. This is
simpler and more powerful.
Making this work requires hardcoding the only C++ standard that works
for SPEC (C++17) inside spec.gen.py instead of expecting it to be set
correctly when running the test suite.
Added:
Modified:
libcxx/test/benchmarks/spec.gen.py
libcxx/utils/ci/lnt/run-benchmarks
Removed:
################################################################################
diff --git a/libcxx/test/benchmarks/spec.gen.py b/libcxx/test/benchmarks/spec.gen.py
index 09d3bce3f9b22..ab546e53e0692 100644
--- a/libcxx/test/benchmarks/spec.gen.py
+++ b/libcxx/test/benchmarks/spec.gen.py
@@ -28,7 +28,8 @@
link_flags = (test_dir / 'link_flags.subs').open().read().strip()
spec_dir = pathlib.Path((test_dir / 'spec_dir.subs').open().read().strip())
-# Setup the configuration file
+# Setup the configuration file. We need to force the usage of C++17 since
+# SPEC doesn't compile with newer standards.
test_dir.mkdir(parents=True, exist_ok=True)
spec_config = test_dir / 'spec-config.cfg'
spec_config.write_text(f"""
@@ -46,7 +47,7 @@
copies = 1
threads = 1
CC = cc -O3 -std=c18 -Wno-implicit-function-declaration
- CXX = {cxx} {compile_flags} {flags} {link_flags} -w # we don't care about warnings in SPEC
+ CXX = {cxx} {compile_flags} {flags} {link_flags} -std=c++17 -w # we don't care about warnings in SPEC
CC_VERSION_OPTION = --version
CXX_VERSION_OPTION = --version
EXTRA_PORTABILITY = -DSPEC_NO_CXX17_SPECIAL_MATH_FUNCTIONS # because libc++ doesn't implement the special math functions yet
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index 8f9ff72ef3166..a28dce84faf9f 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -76,8 +76,9 @@ def main(argv):
help='The name of the test suite for reporting LNT results.')
parser.add_argument('--lnt-url', type=str, required=True,
help='The URL of the LNT instance to submit results to.')
- parser.add_argument('--disable-microbenchmarks', action='store_true',
- help="Do not run the microbenchmarks, only run SPEC (if possible).")
+ parser.add_argument('--filter', type=str, required=False,
+ help="Optional test filter to pass to lit when running the benchmarks. This allows "
+ "running only a subset of the benchmarks.")
parser.add_argument('--spec-dir', type=pathlib.Path, required=False,
help='Optional path to a SPEC installation to use for benchmarking.')
parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
@@ -93,11 +94,6 @@ def main(argv):
elif args.verbose >= 1:
logging.basicConfig(level=logging.DEBUG)
- do_spec = args.spec_dir is not None
- do_micro = not args.disable_microbenchmarks
- if not (do_spec or do_micro):
- raise ValueError("You must run at least the microbenchmarks or SPEC")
-
def run(command, *posargs, enforce_success=True, **kwargs):
command = [str(c) for c in command]
logging.debug(f'$ {" ".join(command)}')
@@ -137,30 +133,10 @@ def main(argv):
'--commit', args.benchmark_commit,
'--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo', f'-DCMAKE_CXX_COMPILER={args.compiler}'])
- if do_spec:
- logging.info(f'Running SPEC benchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}')
- run([args.git_repo / 'libcxx/utils/test-at-commit',
- '--git-repo', args.git_repo,
- '--build-dir', build_dir / 'spec',
- '--test-suite-commit', args.test_suite_commit,
- '--libcxx-installation', build_dir / 'install',
- '--',
- '-j1', '--time-tests',
- '--param', f'compiler={args.compiler}',
- '--param', 'optimization=speed',
- '--param', 'std=c++17',
- '--param', f'spec_dir={args.spec_dir}',
- build_dir / 'spec/libcxx/test',
- '--filter', 'benchmarks/spec.gen.py'],
- enforce_success=False)
- with open(build_dir / 'benchmarks.lnt', 'a') as f:
- run([args.git_repo / 'libcxx/utils/consolidate-benchmarks', build_dir / 'spec'], stdout=f)
-
- if do_micro:
- logging.info(f'Running microbenchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}')
- run([args.git_repo / 'libcxx/utils/test-at-commit',
+ logging.info(f'Running benchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}')
+ cmd = [args.git_repo / 'libcxx/utils/test-at-commit',
'--git-repo', args.git_repo,
- '--build-dir', build_dir / 'micro',
+ '--build-dir', build_dir / 'bench',
'--test-suite-commit', args.test_suite_commit,
'--libcxx-installation', build_dir / 'install',
'--',
@@ -168,10 +144,14 @@ def main(argv):
'--param', f'compiler={args.compiler}',
'--param', 'optimization=speed',
'--param', 'std=c++26',
- build_dir / 'micro/libcxx/test/benchmarks'],
- enforce_success=False)
- with open(build_dir / 'benchmarks.lnt', 'a') as f:
- run([args.git_repo / 'libcxx/utils/consolidate-benchmarks', build_dir / 'micro'], stdout=f)
+ build_dir / 'bench/libcxx/test/benchmarks']
+ if args.spec_dir is not None:
+ cmd += ['--param', f'spec_dir={args.spec_dir}']
+ if args.filter is not None:
+ cmd += ['--filter', args.filter]
+ run(cmd, enforce_success=False) # some benchmarks may fail to build/run at some commits, and that's okay
+ with open(build_dir / 'benchmarks.lnt', 'w') as f:
+ run([args.git_repo / 'libcxx/utils/consolidate-benchmarks', build_dir / 'bench'], stdout=f)
logging.info('Creating JSON report for LNT')
order = len(subprocess.check_output(['git', '-C', args.git_repo, 'rev-list', args.benchmark_commit]).splitlines())
More information about the libcxx-commits
mailing list