[libcxx-commits] [libcxx] 8fbe22b - [libc++] Also run the benchmarks with Fast hardening mode (#218029)
via libcxx-commits
libcxx-commits at lists.llvm.org
Sat Aug 22 04:50:31 PDT 2026
Author: Louis Dionne
Date: 2026-08-22T07:50:25-04:00
New Revision: 8fbe22b71c5116023dc9fd738758976f7591dbb6
URL: https://github.com/llvm/llvm-project/commit/8fbe22b71c5116023dc9fd738758976f7591dbb6
DIFF: https://github.com/llvm/llvm-project/commit/8fbe22b71c5116023dc9fd738758976f7591dbb6.diff
LOG: [libc++] Also run the benchmarks with Fast hardening mode (#218029)
This PR adds a LNT machine configuration that runs with Fast hardening
on macOS. This is done by making it possible to pass a CMake cache to
use for building the library to `run-benchmarks`.
The obvious caveat is that we can't pass parameters to the test suite
itself. In the future, I think the cleanest way to do that would be to
make the test suite independent from the libc++ build and give it its
own configuration options. We could then pass one CMake cache for the
build, and one CMake cache for configuring the test suite. However,
since this requires a lot more work and the hardening mode is baked into
the library once configured, the current patch is sufficient to achieve
our goals.
Fixes #218028
Added:
libcxx/utils/ci/lnt/cmake/generic.cmake
libcxx/utils/ci/lnt/cmake/hardened-fast.cmake
Modified:
.github/workflows/libcxx-benchmark-commit.yml
.github/workflows/libcxx-pr-benchmark.yml
libcxx/utils/ci/lnt/machines.json
libcxx/utils/ci/lnt/run-benchmarks
Removed:
################################################################################
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
index ad6a9dd078d64..06f82383437a9 100644
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ b/.github/workflows/libcxx-benchmark-commit.yml
@@ -148,8 +148,9 @@ jobs:
- name: Run the benchmarks
env:
- COMMIT: ${{ inputs.commit }}
BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }}
+ CMAKE_CACHE: ${{ matrix.cmake-cache }}
+ COMMIT: ${{ inputs.commit }}
FILTER: ${{ inputs.filter }}
LNT_MACHINE: ${{ matrix.lnt-machine }}
run: |
@@ -163,6 +164,7 @@ jobs:
--machine "${LNT_MACHINE}" \
--compiler "${COMPILER}" \
--benchmark-commit "${COMMIT}" \
+ --cmake-cache "${CMAKE_CACHE}" \
"${filter_arg[@]}" \
--output "${COMMIT}.json"
diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml
index 9b5bdb5adb834..bad08ce8afa72 100644
--- a/.github/workflows/libcxx-pr-benchmark.yml
+++ b/.github/workflows/libcxx-pr-benchmark.yml
@@ -110,6 +110,7 @@ jobs:
- extract-info
env:
BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }}
+ CMAKE_CACHE: ${{ matrix.cmake-cache }}
COMPILER: ${{ matrix.cxx }}
LNT_MACHINE: ${{ matrix.lnt-machine }}
PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}
@@ -156,8 +157,8 @@ jobs:
run: |
source .venv/bin/activate
baseline_commit=$(git merge-base $PR_BASE $PR_HEAD)
- ./libcxx/utils/build-at-commit --commit ${baseline_commit} --install-dir install/baseline -- -DCMAKE_CXX_COMPILER="${COMPILER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo
- ./libcxx/utils/build-at-commit --commit $PR_HEAD --install-dir install/candidate -- -DCMAKE_CXX_COMPILER="${COMPILER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo
+ ./libcxx/utils/build-at-commit --commit ${baseline_commit} --install-dir install/baseline -- -DCMAKE_CXX_COMPILER="${COMPILER}" -C "${CMAKE_CACHE}"
+ ./libcxx/utils/build-at-commit --commit $PR_HEAD --install-dir install/candidate -- -DCMAKE_CXX_COMPILER="${COMPILER}" -C "${CMAKE_CACHE}"
- name: Run baseline and candidate interleaved
run: |
diff --git a/libcxx/utils/ci/lnt/cmake/generic.cmake b/libcxx/utils/ci/lnt/cmake/generic.cmake
new file mode 100644
index 0000000000000..5282a90b53087
--- /dev/null
+++ b/libcxx/utils/ci/lnt/cmake/generic.cmake
@@ -0,0 +1 @@
+set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "")
diff --git a/libcxx/utils/ci/lnt/cmake/hardened-fast.cmake b/libcxx/utils/ci/lnt/cmake/hardened-fast.cmake
new file mode 100644
index 0000000000000..88267822a80cb
--- /dev/null
+++ b/libcxx/utils/ci/lnt/cmake/hardened-fast.cmake
@@ -0,0 +1,2 @@
+set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "")
+set(LIBCXX_HARDENING_MODE "fast" CACHE STRING "")
diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json
index e84c93105be10..98708cbf9d731 100644
--- a/libcxx/utils/ci/lnt/machines.json
+++ b/libcxx/utils/ci/lnt/machines.json
@@ -6,6 +6,7 @@
"running-on": "macos",
"xcode-version": "26.5",
"benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
"coverage": {
"since": "2023-01-01",
"every": "week",
@@ -14,12 +15,29 @@
"lnt-url": "https://lnt.llvm.org"
}
},
+ {
+ "lnt-machine": "macos-26.5-arm64-hardenedfast-20260821",
+ "runner": ["self-hosted", "macOS", "26.5", "ARM64", "apple-runners"],
+ "cxx": "clang++",
+ "running-on": "macos",
+ "xcode-version": "26.5",
+ "benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake",
+ "coverage": {
+ "since": "2023-12-01",
+ "every": "week",
+ "samples": 3,
+ "max-in-flight": 4,
+ "lnt-url": "https://lnt.llvm.org"
+ }
+ },
{
"lnt-machine": "linux-x86_64-20260812",
"runner": "llvm-premerge-libcxx-runners",
"cxx": "clang++-22",
"running-on": "linux",
"benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0",
+ "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake",
"coverage": {
"since": "2023-01-01",
"every": "week",
diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks
index 9d9c63ca4ba37..8dfb3a449c8c0 100755
--- a/libcxx/utils/ci/lnt/run-benchmarks
+++ b/libcxx/utils/ci/lnt/run-benchmarks
@@ -87,6 +87,10 @@ def main(argv):
"running only a subset of the benchmarks.")
parser.add_argument('--spec-dir', type=pathlib.Path, required=False,
help='Optional path to a SPEC installation to use for benchmarking.')
+ parser.add_argument('--cmake-cache', type=pathlib.Path, required=False, default=pathlib.Path(__file__).resolve().parent / 'cmake' / 'generic.cmake',
+ help='Optional path to a CMake cache to use when building the library. Only used when building '
+ 'the library, not when running the test suite. Defaults to '
+ 'libcxx/utils/ci/lnt/cmake/generic.cmake, which builds the library with optimizations enabled.')
parser.add_argument('--build-dir', type=pathlib.Path, required=False,
help='Optional directory in which to keep build artifacts. By default, a temporary directory '
'is used and the build artifacts are discarded after the run. It is an error to specify '
@@ -135,6 +139,8 @@ def main(argv):
sys.exit(f'error: output report {args.output} already exists; not overwriting it')
if args.build_dir is not None and args.build_dir.exists():
sys.exit(f'error: build directory {args.build_dir} already exists; not overwriting it')
+ if args.cmake_cache is not None and not args.cmake_cache.is_file():
+ sys.exit(f'error: CMake cache {args.cmake_cache} does not exist')
if shutil.which('lnt') is None:
sys.exit('error: cannot find `lnt`; install libcxx/utils/ci/lnt/requirements.txt')
@@ -150,13 +156,15 @@ def main(argv):
logging.info(f'Storing build artifacts in {artifacts}')
logging.info(f'Building libc++ at commit {args.benchmark_commit}')
+ cmake_args = [f'-DCMAKE_CXX_COMPILER={args.compiler}']
+ if args.cmake_cache is not None:
+ cmake_args += ['-C', args.cmake_cache.resolve()]
build_cmd = [args.git_repo / 'libcxx/utils/build-at-commit',
'--git-repo', args.git_repo,
'--install-dir', artifacts / 'libcxx-install',
'--tmp-src-dir', artifacts / 'libcxx-src',
'--tmp-build-dir', artifacts / 'libcxx-build',
- '--commit', args.benchmark_commit,
- '--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo', f'-DCMAKE_CXX_COMPILER={args.compiler}']
+ '--commit', args.benchmark_commit, '--', *cmake_args]
run(build_cmd, enforce_success=False) # if the build fails, carry on: we'll fail later and submit empty LNT results
logging.info(f'Running benchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}')
More information about the libcxx-commits
mailing list