[libcxx] [llvm] [libc++] Also run the benchmarks with Fast hardening mode (PR #218029)
Louis Dionne via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 13:49:00 PDT 2026
https://github.com/ldionne created https://github.com/llvm/llvm-project/pull/218029
This PR adds a LNT machine configuration that runs with Fast hardening on macOS. This is done by making it possible to pass a CMake cache to use for building the library to `run-benchmarks`.
The obvious caveat is that we can't pass parameters to the test suite itself. In the future, I think the cleanest way to do that would be to make the test suite independent from the libc++ build and give it its own configuration options. We could then pass one CMake cache for the build, and one CMake cache for configuring the test suite. However, since this requires a lot more work and the hardening mode is baked into the library once configured, the current patch is sufficient to achieve our goals.
Fixes #218028
>From d3a232bcbb008c94343fa558118ecadcba7e909a Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Fri, 21 Aug 2026 15:32:42 -0400
Subject: [PATCH] [libc++] Also run the benchmarks with Fast hardening mode
This PR adds a LNT machine configuration that runs with Fast hardening
on macOS. This is done by making it possible to pass a CMake cache to
use for building the library to `run-benchmarks`.
The obvious caveat is that we can't pass parameters to the test suite
itself. In the future, I think the cleanest way to do that would be to
make the test suite independent from the libc++ build and give it its
own configuration options. We could then pass one CMake cache for the
build, and one CMake cache for configuring the test suite. However,
since this requires a lot more work and the hardening mode is baked
into the library once configured, the current patch is sufficient to
achieve our goals.
Fixes #218028
---
.github/workflows/libcxx-benchmark-commit.yml | 4 +++-
.github/workflows/libcxx-pr-benchmark.yml | 5 +++--
libcxx/utils/ci/lnt/cmake/generic.cmake | 1 +
libcxx/utils/ci/lnt/cmake/hardened-fast.cmake | 2 ++
libcxx/utils/ci/lnt/machines.json | 18 ++++++++++++++++++
libcxx/utils/ci/lnt/run-benchmarks | 12 ++++++++++--
6 files changed, 37 insertions(+), 5 deletions(-)
create mode 100644 libcxx/utils/ci/lnt/cmake/generic.cmake
create mode 100644 libcxx/utils/ci/lnt/cmake/hardened-fast.cmake
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
index 5db27f0cf4db3..e05cd843b9938 100644
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ b/.github/workflows/libcxx-benchmark-commit.yml
@@ -148,8 +148,9 @@ jobs:
- name: Run the benchmarks
env:
- COMMIT: ${{ inputs.commit }}
BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
+ CMAKE_CACHE: ${{ matrix.cmake-cache }}
+ COMMIT: ${{ inputs.commit }}
FILTER: ${{ inputs.filter }}
LNT_MACHINE: ${{ matrix.lnt-machine }}
run: |
@@ -163,6 +164,7 @@ jobs:
--machine "${LNT_MACHINE}" \
--compiler "${COMPILER}" \
--benchmark-commit "${COMMIT}" \
+ --cmake-cache "${CMAKE_CACHE}" \
"${filter_arg[@]}" \
--output "${COMMIT}.json"
diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml
index 9b5bdb5adb834..bad08ce8afa72 100644
--- a/.github/workflows/libcxx-pr-benchmark.yml
+++ b/.github/workflows/libcxx-pr-benchmark.yml
@@ -110,6 +110,7 @@ jobs:
- extract-info
env:
BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }}
+ CMAKE_CACHE: ${{ matrix.cmake-cache }}
COMPILER: ${{ matrix.cxx }}
LNT_MACHINE: ${{ matrix.lnt-machine }}
PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}
@@ -156,8 +157,8 @@ jobs:
run: |
source .venv/bin/activate
baseline_commit=$(git merge-base $PR_BASE $PR_HEAD)
- ./libcxx/utils/build-at-commit --commit ${baseline_commit} --install-dir install/baseline -- -DCMAKE_CXX_COMPILER="${COMPILER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo
- ./libcxx/utils/build-at-commit --commit $PR_HEAD --install-dir install/candidate -- -DCMAKE_CXX_COMPILER="${COMPILER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo
+ ./libcxx/utils/build-at-commit --commit ${baseline_commit} --install-dir install/baseline -- -DCMAKE_CXX_COMPILER="${COMPILER}" -C "${CMAKE_CACHE}"
+ ./libcxx/utils/build-at-commit --commit $PR_HEAD --install-dir install/candidate -- -DCMAKE_CXX_COMPILER="${COMPILER}" -C "${CMAKE_CACHE}"
- name: Run baseline and candidate interleaved
run: |
diff --git a/libcxx/utils/ci/lnt/cmake/generic.cmake b/libcxx/utils/ci/lnt/cmake/generic.cmake
new file mode 100644
index 0000000000000..5282a90b53087
--- /dev/null
+++ b/libcxx/utils/ci/lnt/cmake/generic.cmake
@@ -0,0 +1 @@
+set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "")
diff --git a/libcxx/utils/ci/lnt/cmake/hardened-fast.cmake b/libcxx/utils/ci/lnt/cmake/hardened-fast.cmake
new file mode 100644
index 0000000000000..88267822a80cb
--- /dev/null
+++ b/libcxx/utils/ci/lnt/cmake/hardened-fast.cmake
@@ -0,0 +1,2 @@
+set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "")
+set(LIBCXX_HARDENING_MODE "fast" CACHE STRING "")
diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json
index e84c93105be10..98708cbf9d731 100644
--- a/libcxx/utils/ci/lnt/machines.json
+++ b/libcxx/utils/ci/lnt/machines.json
@@ -6,6 +6,7 @@
"running-on": "macos",
"xcode-version": "26.5",
"benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
"coverage": {
"since": "2023-01-01",
"every": "week",
@@ -14,12 +15,29 @@
"lnt-url": "https://lnt.llvm.org"
}
},
+ {
+ "lnt-machine": "macos-26.5-arm64-hardenedfast-20260821",
+ "runner": ["self-hosted", "macOS", "26.5", "ARM64", "apple-runners"],
+ "cxx": "clang++",
+ "running-on": "macos",
+ "xcode-version": "26.5",
+ "benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake",
+ "coverage": {
+ "since": "2023-12-01",
+ "every": "week",
+ "samples": 3,
+ "max-in-flight": 4,
+ "lnt-url": "https://lnt.llvm.org"
+ }
+ },
{
"lnt-machine": "linux-x86_64-20260812",
"runner": "llvm-premerge-libcxx-runners",
"cxx": "clang++-22",
"running-on": "linux",
"benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
"coverage": {
"since": "2023-01-01",
"every": "week",
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index a8111d331441e..b8831aa4edda1 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -87,6 +87,10 @@ def main(argv):
"running only a subset of the benchmarks.")
parser.add_argument('--spec-dir', type=pathlib.Path, required=False,
help='Optional path to a SPEC installation to use for benchmarking.')
+ parser.add_argument('--cmake-cache', type=pathlib.Path, required=False, default=pathlib.Path(__file__).resolve().parent / 'cmake' / 'generic.cmake',
+ help='Optional path to a CMake cache to use when building the library. Only used when building '
+ 'the library, not when running the test suite. Defaults to '
+ 'libcxx/utils/ci/lnt/cmake/generic.cmake, which builds the library with optimizations enabled.')
parser.add_argument('--build-dir', type=pathlib.Path, required=False,
help='Optional directory in which to keep build artifacts. By default, a temporary directory '
'is used and the build artifacts are discarded after the run. It is an error to specify '
@@ -135,6 +139,8 @@ def main(argv):
sys.exit(f'error: output report {args.output} already exists; not overwriting it')
if args.build_dir is not None and args.build_dir.exists():
sys.exit(f'error: build directory {args.build_dir} already exists; not overwriting it')
+ if args.cmake_cache is not None and not args.cmake_cache.is_file():
+ sys.exit(f'error: CMake cache {args.cmake_cache} does not exist')
if shutil.which('lnt') is None:
sys.exit('error: cannot find `lnt`; install libcxx/utils/requirements.txt')
@@ -150,13 +156,15 @@ def main(argv):
logging.info(f'Storing build artifacts in {artifacts}')
logging.info(f'Building libc++ at commit {args.benchmark_commit}')
+ cmake_args = [f'-DCMAKE_CXX_COMPILER={args.compiler}']
+ if args.cmake_cache is not None:
+ cmake_args += ['-C', args.cmake_cache.resolve()]
build_cmd = [args.git_repo / 'libcxx/utils/build-at-commit',
'--git-repo', args.git_repo,
'--install-dir', artifacts / 'libcxx-install',
'--tmp-src-dir', artifacts / 'libcxx-src',
'--tmp-build-dir', artifacts / 'libcxx-build',
- '--commit', args.benchmark_commit,
- '--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo', f'-DCMAKE_CXX_COMPILER={args.compiler}']
+ '--commit', args.benchmark_commit, '--', *cmake_args]
run(build_cmd, enforce_success=False) # if the build fails, carry on: we'll fail later and submit empty LNT results
logging.info(f'Running benchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}')
More information about the llvm-commits
mailing list