[libcxx-commits] [libcxx] [libc++] Include some Lit features in LNT submissions (PR #218043)
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Fri Aug 21 15:02:18 PDT 2026
https://github.com/ldionne created https://github.com/llvm/llvm-project/pull/218043
The Lit test suite already sniffs out some properties of the library configuration we are testing / benchmarking. This patch augments the run-benchmarks script to pick up a few of these detected features so they can be included in the Machine information contained in submissions.
This is important to help pin down exactly what is being benchmarked.
>From 61cd2739017ad5b554c01a7d866b0b990b2c72e2 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Fri, 21 Aug 2026 17:27:06 -0400
Subject: [PATCH] [libc++] Include some Lit features in LNT submissions
The Lit test suite already sniffs out some properties of the library
configuration we are testing / benchmarking. This patch augments the
run-benchmarks script to pick up a few of these detected features so
they can be included in the Machine information contained in submissions.
This is important to help pin down exactly what is being benchmarked.
---
libcxx/utils/ci/lnt/run-benchmarks | 64 +++++++++++++++++++++++++-----
libcxx/utils/test-at-commit | 12 +++++-
2 files changed, 64 insertions(+), 12 deletions(-)
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index 9d9c63ca4ba37..e76debafa8d75 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -14,6 +14,7 @@ import logging
import os
import pathlib
import platform
+import re
import shutil
import subprocess
import sys
@@ -46,6 +47,43 @@ def gather_machine_information(args):
return info
+def gather_lit_features(lit, lit_params, tests):
+ """
+ Gather the Lit features in effect for the configuration we benchmarked, as a dictionary of
+ feature name to feature value.
+ """
+ # Lit features that we report as part of the machine information, and the names
+ # we use in that machine information.
+ REPORTED_FEATURES = {
+ 'libcpp-abi-version': 'abi_version',
+ 'libcpp-hardening-mode': 'hardening_mode',
+ }
+
+ if not lit.is_file():
+ logging.warning(f'{lit} does not exist: not reporting any Lit feature')
+ return {}
+
+ # Lit prints the features of the configuration it discovered and exits without running anything.
+ command = [str(c) for c in [lit, '--show-suites', *lit_params, tests]]
+ logging.debug(f'$ {" ".join(command)}')
+ try:
+ output = subprocess.check_output(command, stderr=subprocess.PIPE).decode()
+ except subprocess.CalledProcessError as e:
+ logging.warning(f'Could not determine the Lit features in effect: {e.stderr.decode()}')
+ return {}
+
+ matches = re.findall(r'^\s*Available Features:\s*(.+)$', output, flags=re.MULTILINE)
+ if len(matches) != 1:
+ logging.warning(f'Expected exactly one list of features in the output of Lit, got {len(matches)}')
+ return {}
+ parsed = dict(f.split('=', 1) for f in matches[0].split() if '=' in f)
+
+ reported = {}
+ for (name, value) in parsed.items():
+ if name in REPORTED_FEATURES:
+ reported[REPORTED_FEATURES[name]] = value
+ return reported
+
def gather_run_information(args):
"""
Gather the run information to upload to LNT as part of the submission.
@@ -160,25 +198,28 @@ def main(argv):
run(build_cmd, enforce_success=False) # if the build fails, carry on: we'll fail later and submit empty LNT results
logging.info(f'Running benchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}')
+ lit_params = ['--param', f'compiler={args.compiler}',
+ '--param', 'optimization=speed',
+ '--param', 'std=c++26',
+ '--param', 'enable_werror=False'] # older versions of the library trigger new warnings, don't fail
+ if args.spec_dir is not None:
+ lit_params += ['--param', f'spec_dir={args.spec_dir}']
cmd = [args.git_repo / 'libcxx/utils/test-at-commit',
'--git-repo', args.git_repo,
- '--build-dir', artifacts / 'benchmarks',
+ '--build-dir', artifacts / 'benchmarks-build',
'--test-suite-commit', args.test_suite_commit,
+ '--tmp-src-dir', artifacts / 'benchmarks-src',
'--libcxx-installation', artifacts / 'libcxx-install',
'--compiler', args.compiler,
'--',
'-j1', '--time-tests', '--test-output=failed',
- '--param', f'compiler={args.compiler}',
- '--param', 'optimization=speed',
- '--param', 'std=c++26',
- '--param', 'enable_werror=False', # older versions of the library trigger new warnings, don't fail
- artifacts / 'benchmarks/libcxx/test/benchmarks']
- if args.spec_dir is not None:
- cmd += ['--param', f'spec_dir={args.spec_dir}']
+ *lit_params,
+ artifacts / 'benchmarks-build/libcxx/test/benchmarks']
if args.filter is not None:
cmd += ['--filter', args.filter]
run(cmd, enforce_success=False) # some benchmarks may fail to build/run at some commits, and that's okay
- consolidate = [args.git_repo / 'libcxx/utils/consolidate-benchmarks', artifacts / 'benchmarks']
+
+ consolidate = [args.git_repo / 'libcxx/utils/consolidate-benchmarks', artifacts / 'benchmarks-build']
if args.dry_run:
run(consolidate)
else:
@@ -190,7 +231,10 @@ def main(argv):
importreport = ['lnt', 'importreport', '--order', str(order), '--machine', args.machine]
for arg in dict_to_params(gather_run_information(args)):
importreport += ['--run-info', arg]
- for arg in dict_to_params(gather_machine_information(args)):
+ machine_info = {**gather_machine_information(args),
+ **gather_lit_features(artifacts / 'benchmarks-build/bin/llvm-lit', lit_params,
+ artifacts / 'benchmarks-build/libcxx/test/benchmarks')}
+ for arg in dict_to_params(machine_info):
importreport += ['--machine-info', arg]
output = args.output.resolve()
importreport += [artifacts / 'benchmarks.lnt', output]
diff --git a/libcxx/utils/test-at-commit b/libcxx/utils/test-at-commit
index b02c571ca9b1b..83e1e570d3238 100755
--- a/libcxx/utils/test-at-commit
+++ b/libcxx/utils/test-at-commit
@@ -96,6 +96,11 @@ def main(argv):
'separated from other arguments with a `--`.')
parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()),
help='Optional path to the Git repository to use. By default, the current working directory is used.')
+ parser.add_argument('--tmp-src-dir', type=pathlib.Path, required=False,
+ help='Optional path to use for the ephemeral checkout of the test suite. By default, a temporary '
+ 'directory is used and it is cleaned up after the run. If a custom directory is specified, it '
+ 'is not cleaned up automatically. Note that the build directory refers to that checkout, so '
+ 'keeping it around is required in order to use the build directory after the run.')
args = parser.parse_args(argv)
args.build_dir = args.build_dir.resolve()
@@ -115,8 +120,11 @@ def main(argv):
if args.test_suite_commit is None:
test_suite_sources = args.git_repo
else:
- tempdirs.append(tempfile.TemporaryDirectory())
- test_suite_sources = pathlib.Path(tempdirs[-1].name)
+ if args.tmp_src_dir is not None:
+ test_suite_sources = args.tmp_src_dir
+ else:
+ tempdirs.append(tempfile.TemporaryDirectory())
+ test_suite_sources = pathlib.Path(tempdirs[-1].name)
checkout_dirs = [d for d in LIBCXX_REQUIRED_DIRECTORIES if exists_in_commit(args.git_repo, args.test_suite_commit, d)]
checkout_subdirectories(args.git_repo, args.test_suite_commit, checkout_dirs, test_suite_sources)
More information about the libcxx-commits
mailing list