[libcxx-commits] [libcxx] 91b0584 - [libc++] Add a tool to produce historical libc++ benchmark results
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Thu Sep 18 14:55:30 PDT 2025
Author: Louis Dionne
Date: 2025-09-18T17:55:25-04:00
New Revision: 91b05845bcb2befd1ed2407531d3a1adb3c6f828
URL: https://github.com/llvm/llvm-project/commit/91b05845bcb2befd1ed2407531d3a1adb3c6f828
DIFF: https://github.com/llvm/llvm-project/commit/91b05845bcb2befd1ed2407531d3a1adb3c6f828.diff
LOG: [libc++] Add a tool to produce historical libc++ benchmark results
This is extremely useful for analysis purposes like finding regressions.
The ability to run such historical analysis locally is extremely useful
for doing quick investigations that may involve non-mainstream libc++
configurations.
Added:
libcxx/utils/benchmark-historical
Modified:
libcxx/utils/test-at-commit
Removed:
################################################################################
diff --git a/libcxx/utils/benchmark-historical b/libcxx/utils/benchmark-historical
new file mode 100755
index 0000000000000..448d7bd59745a
--- /dev/null
+++ b/libcxx/utils/benchmark-historical
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import os
+import pathlib
+import subprocess
+import sys
+import tempfile
+
+PARENT_DIR = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
+
+def directory_path(string):
+ if os.path.isdir(string):
+ return pathlib.Path(string)
+ else:
+ raise NotADirectoryError(string)
+
+def whitespace_separated(stream):
+ """
+ Iterate over a stream, yielding whitespace-delimited elements.
+ """
+ for line in stream:
+ for element in line.split():
+ yield element
+
+def resolve_commit(git_repo, commit):
+ """
+ Resolve the full commit SHA from any tree-ish.
+ """
+ return subprocess.check_output(['git', '-C', git_repo, 'rev-parse', commit], text=True).strip()
+
+
+def main(argv):
+ parser = argparse.ArgumentParser(
+ prog='benchmark-historical',
+ description='Run the libc++ benchmarks against the commits provided on standard input and store the results in '
+ 'LNT format in a directory. This makes it easy to generate historical benchmark results of libc++ '
+ 'for analysis purposes. This script\'s usage is optimized to be run on a set of commits and then '
+ 're-run on a potentially-overlapping set of commits, such as after pulling new commits with Git.')
+ parser.add_argument('--output', '-o', type=pathlib.Path, required=True,
+ help='Path to the directory where the resulting .lnt files are stored.')
+ parser.add_argument('--commit-list', type=argparse.FileType('r'), default=sys.stdin,
+ help='Path to a file containing a whitespace separated list of commits to test. '
+ 'By default, this is read from standard input.')
+ parser.add_argument('--overwrite', action='store_true',
+ help='When the data for a commit already exists in the output directory, the tool normally skips it. '
+ 'This option instructs the tool to generate the data and overwrite it in the output directory.')
+ parser.add_argument('lit_options', nargs=argparse.REMAINDER,
+ help='Optional arguments passed to lit when running the tests. Should be provided last and '
+ 'separated from other arguments with a `--`.')
+ parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()),
+ help='Optional path to the Git repository to use. By default, the current working directory is used.')
+ parser.add_argument('--dry-run', action='store_true',
+ help='Do not actually run anything, just print what would be done.')
+ args = parser.parse_args(argv)
+
+ logging.getLogger().setLevel(logging.INFO)
+
+ # Gather lit options
+ lit_options = []
+ if args.lit_options:
+ if args.lit_options[0] != '--':
+ raise ArgumentError('For clarity, Lit options must be separated from other options by --')
+ lit_options = args.lit_options[1:]
+
+ # Process commits one by one. Commits just need to be whitespace separated: we also handle
+ # the case where there is more than one commit per line.
+ for commit in whitespace_separated(args.commit_list):
+ commit = resolve_commit(args.git_repo, commit) # resolve e.g. HEAD to a real SHA
+
+ output_file = args.output / (commit + '.lnt')
+ if output_file.exists():
+ if args.overwrite:
+ logging.info(f'Will overwrite data for commit {commit} in {output_file}')
+ else:
+ logging.info(f'Data for commit {commit} already exists in {output_file}, skipping')
+ continue
+ else:
+ logging.info(f'Benchmarking commit {commit}')
+
+ with tempfile.TemporaryDirectory() as build_dir:
+ test_cmd = [PARENT_DIR / 'test-at-commit', '--git-repo', args.git_repo,
+ '--build', build_dir,
+ '--commit', commit]
+ test_cmd += ['--'] + lit_options
+
+ if args.dry_run:
+ pretty = ' '.join(str(a) for a in test_cmd)
+ logging.info(f'Running {pretty}')
+ continue
+
+ if subprocess.call(test_cmd) != 0:
+ logging.error(f'Failed to run the tests for commit {commit}, skipping')
+ continue
+
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+ consolidate_cmd = [(PARENT_DIR / 'consolidate-benchmarks'), build_dir, '--output', output_file]
+ subprocess.check_call(consolidate_cmd)
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
diff --git a/libcxx/utils/test-at-commit b/libcxx/utils/test-at-commit
index 5b3fcede48ab2..d62643d703da8 100755
--- a/libcxx/utils/test-at-commit
+++ b/libcxx/utils/test-at-commit
@@ -1,13 +1,13 @@
#!/usr/bin/env python3
import argparse
-import logging
import os
+import pathlib
import subprocess
import sys
import tempfile
-PARENT_DIR = os.path.dirname(os.path.abspath(__file__))
+PARENT_DIR = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
LIT_CONFIG_FILE = """
#
@@ -37,7 +37,7 @@ libcxx.test.config.configure(
def directory_path(string):
if os.path.isdir(string):
- return string
+ return pathlib.Path(string)
else:
raise NotADirectoryError(string)
@@ -50,14 +50,14 @@ def main(argv):
'performance data, bisect issues, and so on. '
'A current limitation of this script is that it assumes the arguments passed to CMake when '
'building the library.')
- parser.add_argument('--build', '-B', type=str, required=True,
+ parser.add_argument('--build', '-B', type=pathlib.Path, required=True,
help='Path to create the build directory for running the test suite at.')
parser.add_argument('--commit', type=str, required=True,
help='Commit to build libc++ at.')
parser.add_argument('lit_options', nargs=argparse.REMAINDER,
help='Optional arguments passed to lit when running the tests. Should be provided last and '
'separated from other arguments with a `--`.')
- parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
+ parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()),
help='Optional path to the Git repository to use. By default, the current working directory is used.')
args = parser.parse_args(argv)
@@ -70,26 +70,26 @@ def main(argv):
with tempfile.TemporaryDirectory() as install_dir:
# Build the library at the baseline
- build_cmd = [os.path.join(PARENT_DIR, 'build-at-commit'), '--git-repo', args.git_repo,
- '--install-dir', install_dir,
- '--commit', args.commit]
+ build_cmd = [PARENT_DIR / 'build-at-commit', '--git-repo', args.git_repo,
+ '--install-dir', install_dir,
+ '--commit', args.commit]
build_cmd += ['--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo']
subprocess.check_call(build_cmd)
# Configure the test suite in the specified build directory
- os.makedirs(args.build)
- lit_cfg = os.path.abspath(os.path.join(args.build, 'temp_lit_cfg.cfg.in'))
+ args.build.mkdir(parents=True, exist_ok=True)
+ lit_cfg = (args.build / 'temp_lit_cfg.cfg.in').absolute()
with open(lit_cfg, 'w') as f:
f.write(LIT_CONFIG_FILE.format(INSTALL_ROOT=install_dir))
- test_suite_cmd = ['cmake', '-B', args.build, '-S', os.path.join(args.git_repo, 'runtimes'), '-G', 'Ninja']
+ test_suite_cmd = ['cmake', '-B', args.build, '-S', args.git_repo / 'runtimes', '-G', 'Ninja']
test_suite_cmd += ['-D', 'LLVM_ENABLE_RUNTIMES=libcxx;libcxxabi']
test_suite_cmd += ['-D', 'LIBCXXABI_USE_LLVM_UNWINDER=OFF']
test_suite_cmd += ['-D', f'LIBCXX_TEST_CONFIG={lit_cfg}']
subprocess.check_call(test_suite_cmd)
# Run the specified tests against the produced baseline installation
- lit_cmd = [os.path.join(PARENT_DIR, 'libcxx-lit'), args.build] + lit_options
+ lit_cmd = [PARENT_DIR / 'libcxx-lit', args.build] + lit_options
subprocess.check_call(lit_cmd)
if __name__ == '__main__':
More information about the libcxx-commits
mailing list