[libcxx-commits] [libcxx] [llvm] [libc++] Unify the benchmarks with the test suite (PR #101399)
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Jul 31 13:04:08 PDT 2024
https://github.com/ldionne created https://github.com/llvm/llvm-project/pull/101399
Instead of building the benchmarks separately via CMake and running them
separately from the test suite, this patch merges the benchmarks into
the test suite and handles both uniformly.
As a result:
- It is now possible to run individual benchmarks like we run tests
(e.g. using libcxx-lit), which is a huge quality-of-life improvement.
- The benchmarks will be run under exactly the same configuration as
the rest of the tests, which is a nice simplification. This does
mean that one has to be careful to enable the desired optimization
flags when running benchmarks, but that is easy with e.g.
`libcxx-lit <...> --param optimization=speed`.
- Benchmarks can use the same annotations as the rest of the test
suite, such as `// UNSUPPORTED` & friends.
When running the tests via `check-cxx`, we only compile the benchmarks
because running them would be too time consuming. This introduces a bit
of complexity in the testing setup, and instead it would be better to
allow passing a --dry-run flag to GoogleBenchmark executables, which is
the topic of https://github.com/google/benchmark/issues/1827.
I am not really satisfied with the layering violation of adding the
%{benchmark_flags} substitution to cmake-bridge, however I believe
this can be improved in the future.
>From f5f070b7a2237c43d69c4f43a6cd41895ccd1876 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Wed, 31 Jul 2024 13:00:25 -0400
Subject: [PATCH 1/2] [libc++] Make benchmarks forward-compatible with the test
suite
This patch fixes a few warnings and errors when running the benchmarks
as part of the test suite. It also adds most of the necessary Lit
annotations to make it pass on various configurations.
---
libcxx/test/benchmarks/CartesianBenchmarks.h | 6 ++---
libcxx/test/benchmarks/ContainerBenchmarks.h | 10 ++++----
.../algorithms.partition_point.bench.cpp | 2 ++
.../benchmarks/algorithms/count.bench.cpp | 2 ++
.../benchmarks/algorithms/equal.bench.cpp | 2 ++
.../test/benchmarks/algorithms/fill.bench.cpp | 2 ++
.../test/benchmarks/algorithms/find.bench.cpp | 2 ++
.../benchmarks/algorithms/for_each.bench.cpp | 2 ++
.../algorithms/lower_bound.bench.cpp | 2 ++
.../benchmarks/algorithms/make_heap.bench.cpp | 2 ++
.../make_heap_then_sort_heap.bench.cpp | 2 ++
.../test/benchmarks/algorithms/min.bench.cpp | 2 ++
.../algorithms/min_max_element.bench.cpp | 2 ++
.../benchmarks/algorithms/minmax.bench.cpp | 2 ++
.../benchmarks/algorithms/mismatch.bench.cpp | 2 ++
.../benchmarks/algorithms/pop_heap.bench.cpp | 2 ++
.../algorithms/pstl.stable_sort.bench.cpp | 2 ++
.../benchmarks/algorithms/push_heap.bench.cpp | 2 ++
.../algorithms/ranges_contains.bench.cpp | 2 ++
.../algorithms/ranges_ends_with.bench.cpp | 2 ++
.../algorithms/ranges_make_heap.bench.cpp | 2 ++
.../ranges_make_heap_then_sort_heap.bench.cpp | 2 ++
.../algorithms/ranges_pop_heap.bench.cpp | 2 ++
.../algorithms/ranges_push_heap.bench.cpp | 2 ++
.../algorithms/ranges_sort.bench.cpp | 2 ++
.../algorithms/ranges_sort_heap.bench.cpp | 2 ++
.../algorithms/ranges_stable_sort.bench.cpp | 2 ++
.../algorithms/set_intersection.bench.cpp | 2 ++
.../test/benchmarks/algorithms/sort.bench.cpp | 2 ++
.../benchmarks/algorithms/sort_heap.bench.cpp | 2 ++
.../algorithms/stable_sort.bench.cpp | 2 ++
libcxx/test/benchmarks/allocation.bench.cpp | 4 ++++
libcxx/test/benchmarks/atomic_wait.bench.cpp | 2 ++
.../atomic_wait_vs_mutex_lock.bench.cpp | 2 ++
libcxx/test/benchmarks/deque.bench.cpp | 2 ++
.../test/benchmarks/deque_iterator.bench.cpp | 2 ++
libcxx/test/benchmarks/filesystem.bench.cpp | 2 ++
libcxx/test/benchmarks/format.bench.cpp | 2 ++
libcxx/test/benchmarks/format_to.bench.cpp | 2 ++
libcxx/test/benchmarks/format_to_n.bench.cpp | 2 ++
.../test/benchmarks/formatted_size.bench.cpp | 2 ++
.../test/benchmarks/formatter_float.bench.cpp | 3 +++
.../test/benchmarks/formatter_int.bench.cpp | 2 ++
libcxx/test/benchmarks/function.bench.cpp | 4 +++-
libcxx/test/benchmarks/join_view.bench.cpp | 2 ++
...exicographical_compare_three_way.bench.cpp | 2 ++
.../dynamic_cast_old_stress.bench.cpp | 2 ++
libcxx/test/benchmarks/map.bench.cpp | 2 ++
.../benchmarks/monotonic_buffer.bench.cpp | 2 ++
libcxx/test/benchmarks/numeric/gcd.bench.cpp | 5 +++-
libcxx/test/benchmarks/ordered_set.bench.cpp | 6 +++--
.../shared_mutex_vs_mutex.bench.cpp | 2 ++
.../std_format_spec_string_unicode.bench.cpp | 2 ++
...ormat_spec_string_unicode_escape.bench.cpp | 2 ++
libcxx/test/benchmarks/stop_token.bench.cpp | 2 ++
libcxx/test/benchmarks/string.bench.cpp | 23 +++++++++++--------
libcxx/test/benchmarks/stringstream.bench.cpp | 2 ++
libcxx/test/benchmarks/to_chars.bench.cpp | 2 ++
.../unordered_set_operations.bench.cpp | 2 ++
.../test/benchmarks/variant_visit_1.bench.cpp | 2 ++
.../test/benchmarks/variant_visit_2.bench.cpp | 2 ++
.../test/benchmarks/variant_visit_3.bench.cpp | 2 ++
.../benchmarks/vector_operations.bench.cpp | 2 ++
63 files changed, 149 insertions(+), 22 deletions(-)
diff --git a/libcxx/test/benchmarks/CartesianBenchmarks.h b/libcxx/test/benchmarks/CartesianBenchmarks.h
index eca4e15cd009b..c712230843ebf 100644
--- a/libcxx/test/benchmarks/CartesianBenchmarks.h
+++ b/libcxx/test/benchmarks/CartesianBenchmarks.h
@@ -27,11 +27,11 @@ constexpr auto makeEnumValueTuple(std::index_sequence<Idxs...>) {
}
template <class B>
-static auto skip(const B& Bench, int) -> decltype(Bench.skip()) {
+auto skip(const B& Bench, int) -> decltype(Bench.skip()) {
return Bench.skip();
}
template <class B>
-static auto skip(const B& Bench, char) {
+auto skip(const B&, char) {
return false;
}
@@ -51,7 +51,7 @@ void makeBenchmarkFromValues(const std::vector<std::tuple<Args...> >& A) {
}
template <template <class...> class B, class Args, class... U>
-void makeBenchmarkImpl(const Args& A, std::tuple<U...> t) {
+void makeBenchmarkImpl(const Args& A, std::tuple<U...>) {
makeBenchmarkFromValues<B<U...> >(A);
}
diff --git a/libcxx/test/benchmarks/ContainerBenchmarks.h b/libcxx/test/benchmarks/ContainerBenchmarks.h
index 744505b439985..e572792d9fe5f 100644
--- a/libcxx/test/benchmarks/ContainerBenchmarks.h
+++ b/libcxx/test/benchmarks/ContainerBenchmarks.h
@@ -150,7 +150,7 @@ void BM_EmplaceDuplicate(benchmark::State& st, Container c, GenInputs gen) {
}
template <class Container, class GenInputs>
-static void BM_Find(benchmark::State& st, Container c, GenInputs gen) {
+void BM_Find(benchmark::State& st, Container c, GenInputs gen) {
auto in = gen(st.range(0));
c.insert(in.begin(), in.end());
benchmark::DoNotOptimize(&(*c.begin()));
@@ -164,7 +164,7 @@ static void BM_Find(benchmark::State& st, Container c, GenInputs gen) {
}
template <class Container, class GenInputs>
-static void BM_FindRehash(benchmark::State& st, Container c, GenInputs gen) {
+void BM_FindRehash(benchmark::State& st, Container c, GenInputs gen) {
c.rehash(8);
auto in = gen(st.range(0));
c.insert(in.begin(), in.end());
@@ -179,7 +179,7 @@ static void BM_FindRehash(benchmark::State& st, Container c, GenInputs gen) {
}
template <class Container, class GenInputs>
-static void BM_Rehash(benchmark::State& st, Container c, GenInputs gen) {
+void BM_Rehash(benchmark::State& st, Container c, GenInputs gen) {
auto in = gen(st.range(0));
c.max_load_factor(3.0);
c.insert(in.begin(), in.end());
@@ -193,7 +193,7 @@ static void BM_Rehash(benchmark::State& st, Container c, GenInputs gen) {
}
template <class Container, class GenInputs>
-static void BM_Compare_same_container(benchmark::State& st, Container, GenInputs gen) {
+void BM_Compare_same_container(benchmark::State& st, Container, GenInputs gen) {
auto in = gen(st.range(0));
Container c1(in.begin(), in.end());
Container c2 = c1;
@@ -208,7 +208,7 @@ static void BM_Compare_same_container(benchmark::State& st, Container, GenInputs
}
template <class Container, class GenInputs>
-static void BM_Compare_different_containers(benchmark::State& st, Container, GenInputs gen) {
+void BM_Compare_different_containers(benchmark::State& st, Container, GenInputs gen) {
auto in1 = gen(st.range(0));
auto in2 = gen(st.range(0));
Container c1(in1.begin(), in1.end());
diff --git a/libcxx/test/benchmarks/algorithms.partition_point.bench.cpp b/libcxx/test/benchmarks/algorithms.partition_point.bench.cpp
index ed2e337fa6ea3..42ebce8ad2f4a 100644
--- a/libcxx/test/benchmarks/algorithms.partition_point.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms.partition_point.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include <array>
#include <cassert>
diff --git a/libcxx/test/benchmarks/algorithms/count.bench.cpp b/libcxx/test/benchmarks/algorithms/count.bench.cpp
index 7370293fd6efd..46b85e909efa5 100644
--- a/libcxx/test/benchmarks/algorithms/count.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/count.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include <benchmark/benchmark.h>
#include <cstring>
diff --git a/libcxx/test/benchmarks/algorithms/equal.bench.cpp b/libcxx/test/benchmarks/algorithms/equal.bench.cpp
index 6d63d8c48ce1e..2dc11585c15c7 100644
--- a/libcxx/test/benchmarks/algorithms/equal.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/equal.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include <benchmark/benchmark.h>
#include <vector>
diff --git a/libcxx/test/benchmarks/algorithms/fill.bench.cpp b/libcxx/test/benchmarks/algorithms/fill.bench.cpp
index 40f37425c394c..c157b5e5c9862 100644
--- a/libcxx/test/benchmarks/algorithms/fill.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/fill.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
#include <algorithm>
#include <benchmark/benchmark.h>
#include <vector>
diff --git a/libcxx/test/benchmarks/algorithms/find.bench.cpp b/libcxx/test/benchmarks/algorithms/find.bench.cpp
index 6ff2d95ab4353..43d103474ebdf 100644
--- a/libcxx/test/benchmarks/algorithms/find.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/find.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include <benchmark/benchmark.h>
#include <cstring>
diff --git a/libcxx/test/benchmarks/algorithms/for_each.bench.cpp b/libcxx/test/benchmarks/algorithms/for_each.bench.cpp
index 7019dc13ca601..554c9ec993043 100644
--- a/libcxx/test/benchmarks/algorithms/for_each.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/for_each.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include <benchmark/benchmark.h>
#include <deque>
diff --git a/libcxx/test/benchmarks/algorithms/lower_bound.bench.cpp b/libcxx/test/benchmarks/algorithms/lower_bound.bench.cpp
index 3be5010f5bdf2..d9d57969df67a 100644
--- a/libcxx/test/benchmarks/algorithms/lower_bound.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/lower_bound.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include <numeric>
#include <random>
diff --git a/libcxx/test/benchmarks/algorithms/make_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/make_heap.bench.cpp
index dade7b8273107..b7320e17c3e50 100644
--- a/libcxx/test/benchmarks/algorithms/make_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/make_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
index 48f34f8651b97..5991d2846aee4 100644
--- a/libcxx/test/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/min.bench.cpp b/libcxx/test/benchmarks/algorithms/min.bench.cpp
index a09bd530d0120..e1e7c4818a00e 100644
--- a/libcxx/test/benchmarks/algorithms/min.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/min.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include <cassert>
diff --git a/libcxx/test/benchmarks/algorithms/min_max_element.bench.cpp b/libcxx/test/benchmarks/algorithms/min_max_element.bench.cpp
index e2c6423656104..4dbfed66d0c2c 100644
--- a/libcxx/test/benchmarks/algorithms/min_max_element.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/min_max_element.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/minmax.bench.cpp b/libcxx/test/benchmarks/algorithms/minmax.bench.cpp
index ca1cdb441deef..7d7c74c9c96aa 100644
--- a/libcxx/test/benchmarks/algorithms/minmax.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/minmax.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include <cassert>
diff --git a/libcxx/test/benchmarks/algorithms/mismatch.bench.cpp b/libcxx/test/benchmarks/algorithms/mismatch.bench.cpp
index 791782879011e..348009a230d6e 100644
--- a/libcxx/test/benchmarks/algorithms/mismatch.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/mismatch.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include <benchmark/benchmark.h>
#include <random>
diff --git a/libcxx/test/benchmarks/algorithms/pop_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/pop_heap.bench.cpp
index 26cdd25cce90d..5fef52284239d 100644
--- a/libcxx/test/benchmarks/algorithms/pop_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/pop_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/pstl.stable_sort.bench.cpp b/libcxx/test/benchmarks/algorithms/pstl.stable_sort.bench.cpp
index 72541f70640f5..c79f732781003 100644
--- a/libcxx/test/benchmarks/algorithms/pstl.stable_sort.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/pstl.stable_sort.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include <execution>
diff --git a/libcxx/test/benchmarks/algorithms/push_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/push_heap.bench.cpp
index ba96fa1469e60..89d8122bd1dbe 100644
--- a/libcxx/test/benchmarks/algorithms/push_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/push_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/ranges_contains.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_contains.bench.cpp
index f36ebff900958..b98e17a00ef83 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_contains.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_contains.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
#include <algorithm>
#include <benchmark/benchmark.h>
#include <iterator>
diff --git a/libcxx/test/benchmarks/algorithms/ranges_ends_with.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_ends_with.bench.cpp
index 049af7c2e15a3..c975d164c16d4 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_ends_with.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_ends_with.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
#include <algorithm>
#include <benchmark/benchmark.h>
#include <iterator>
diff --git a/libcxx/test/benchmarks/algorithms/ranges_make_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_make_heap.bench.cpp
index 66a8335a8710c..c04ea369fea0c 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_make_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_make_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp
index 01632c807c178..b84d3c2cea765 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/ranges_pop_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_pop_heap.bench.cpp
index bcc7a834a655a..ab3ae6f7c30ae 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_pop_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_pop_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/ranges_push_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_push_heap.bench.cpp
index 902f481228e46..8139ba32cb974 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_push_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_push_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/ranges_sort.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_sort.bench.cpp
index aeb2aedd34af3..d145a159a21fd 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_sort.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_sort.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/ranges_sort_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_sort_heap.bench.cpp
index 62c607cf7a5ed..90e9b9a6933f5 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_sort_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_sort_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/ranges_stable_sort.bench.cpp b/libcxx/test/benchmarks/algorithms/ranges_stable_sort.bench.cpp
index 88327483cf43c..acc2f3f755fb8 100644
--- a/libcxx/test/benchmarks/algorithms/ranges_stable_sort.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/ranges_stable_sort.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/set_intersection.bench.cpp b/libcxx/test/benchmarks/algorithms/set_intersection.bench.cpp
index b3fb15fc77b31..9bde4bb29dc22 100644
--- a/libcxx/test/benchmarks/algorithms/set_intersection.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/set_intersection.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include <cstdlib>
#include <iterator>
diff --git a/libcxx/test/benchmarks/algorithms/sort.bench.cpp b/libcxx/test/benchmarks/algorithms/sort.bench.cpp
index f87434b779712..899272e34795f 100644
--- a/libcxx/test/benchmarks/algorithms/sort.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/sort.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/sort_heap.bench.cpp b/libcxx/test/benchmarks/algorithms/sort_heap.bench.cpp
index 1372b4d5c9ab7..ee4b6bfc7387b 100644
--- a/libcxx/test/benchmarks/algorithms/sort_heap.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/sort_heap.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/algorithms/stable_sort.bench.cpp b/libcxx/test/benchmarks/algorithms/stable_sort.bench.cpp
index 024a036e0c258..c68f73838c319 100644
--- a/libcxx/test/benchmarks/algorithms/stable_sort.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/stable_sort.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include "common.h"
diff --git a/libcxx/test/benchmarks/allocation.bench.cpp b/libcxx/test/benchmarks/allocation.bench.cpp
index 1d0c71f5bec19..189544ff173e9 100644
--- a/libcxx/test/benchmarks/allocation.bench.cpp
+++ b/libcxx/test/benchmarks/allocation.bench.cpp
@@ -6,9 +6,13 @@
//
//===----------------------------------------------------------------------===//
+// REQUIRES: -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS: -fsized-deallocation
+
#include "benchmark/benchmark.h"
#include <cassert>
+#include <cstdlib>
#include <new>
#include <vector>
diff --git a/libcxx/test/benchmarks/atomic_wait.bench.cpp b/libcxx/test/benchmarks/atomic_wait.bench.cpp
index dd541b445448a..844307b4892f5 100644
--- a/libcxx/test/benchmarks/atomic_wait.bench.cpp
+++ b/libcxx/test/benchmarks/atomic_wait.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <atomic>
#include <numeric>
#include <thread>
diff --git a/libcxx/test/benchmarks/atomic_wait_vs_mutex_lock.bench.cpp b/libcxx/test/benchmarks/atomic_wait_vs_mutex_lock.bench.cpp
index 1a52e5dddb8e9..53f49d9a8ed97 100644
--- a/libcxx/test/benchmarks/atomic_wait_vs_mutex_lock.bench.cpp
+++ b/libcxx/test/benchmarks/atomic_wait_vs_mutex_lock.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
// To run this test, build libcxx and cxx-benchmarks targets
// cd third-party/benchmark/tools
// ./compare.py filters ../../../build/libcxx/benchmarks/atomic_wait_vs_mutex_lock.libcxx.out BM_atomic_wait BM_mutex
diff --git a/libcxx/test/benchmarks/deque.bench.cpp b/libcxx/test/benchmarks/deque.bench.cpp
index d6dadaa3f23e4..b8f3b76dd27ee 100644
--- a/libcxx/test/benchmarks/deque.bench.cpp
+++ b/libcxx/test/benchmarks/deque.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
#include <deque>
#include "benchmark/benchmark.h"
diff --git a/libcxx/test/benchmarks/deque_iterator.bench.cpp b/libcxx/test/benchmarks/deque_iterator.bench.cpp
index 0eb23f2f3bea5..d1db8ed358c0b 100644
--- a/libcxx/test/benchmarks/deque_iterator.bench.cpp
+++ b/libcxx/test/benchmarks/deque_iterator.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include <deque>
diff --git a/libcxx/test/benchmarks/filesystem.bench.cpp b/libcxx/test/benchmarks/filesystem.bench.cpp
index 19f9586d08528..83a87c86d3de0 100644
--- a/libcxx/test/benchmarks/filesystem.bench.cpp
+++ b/libcxx/test/benchmarks/filesystem.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <filesystem>
#include "GenerateInput.h"
diff --git a/libcxx/test/benchmarks/format.bench.cpp b/libcxx/test/benchmarks/format.bench.cpp
index d6cb0467b81dd..3fc55322dc065 100644
--- a/libcxx/test/benchmarks/format.bench.cpp
+++ b/libcxx/test/benchmarks/format.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <format>
#include <string>
diff --git a/libcxx/test/benchmarks/format_to.bench.cpp b/libcxx/test/benchmarks/format_to.bench.cpp
index 48cb3da498235..d0b8b25bb172d 100644
--- a/libcxx/test/benchmarks/format_to.bench.cpp
+++ b/libcxx/test/benchmarks/format_to.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <format>
#include <algorithm>
diff --git a/libcxx/test/benchmarks/format_to_n.bench.cpp b/libcxx/test/benchmarks/format_to_n.bench.cpp
index 99723c7c6f695..d3575fd88e376 100644
--- a/libcxx/test/benchmarks/format_to_n.bench.cpp
+++ b/libcxx/test/benchmarks/format_to_n.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <format>
#include <algorithm>
diff --git a/libcxx/test/benchmarks/formatted_size.bench.cpp b/libcxx/test/benchmarks/formatted_size.bench.cpp
index 2945df24a1172..d15d4679f3576 100644
--- a/libcxx/test/benchmarks/formatted_size.bench.cpp
+++ b/libcxx/test/benchmarks/formatted_size.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <format>
#include <string>
diff --git a/libcxx/test/benchmarks/formatter_float.bench.cpp b/libcxx/test/benchmarks/formatter_float.bench.cpp
index d1da58598ea46..7a46ee64010f8 100644
--- a/libcxx/test/benchmarks/formatter_float.bench.cpp
+++ b/libcxx/test/benchmarks/formatter_float.bench.cpp
@@ -6,9 +6,12 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <format>
#include <array>
+#include <bit>
#include <limits>
#include <random>
#include <string>
diff --git a/libcxx/test/benchmarks/formatter_int.bench.cpp b/libcxx/test/benchmarks/formatter_int.bench.cpp
index 6c624e9e0379c..e2df163c77c3e 100644
--- a/libcxx/test/benchmarks/formatter_int.bench.cpp
+++ b/libcxx/test/benchmarks/formatter_int.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <array>
#include <format>
#include <random>
diff --git a/libcxx/test/benchmarks/function.bench.cpp b/libcxx/test/benchmarks/function.bench.cpp
index dd397bcfa5b07..dd8edec7110bc 100644
--- a/libcxx/test/benchmarks/function.bench.cpp
+++ b/libcxx/test/benchmarks/function.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <cstdint>
#include <functional>
#include <memory>
@@ -179,7 +181,7 @@ template <class FunctionType>
struct Invoke {
static void run(benchmark::State& state) {
S s;
- const auto value = MakeFunction(FunctionType());
+ auto value = MakeFunction(FunctionType());
for (auto _ : state) {
benchmark::DoNotOptimize(value);
benchmark::DoNotOptimize(value(&s));
diff --git a/libcxx/test/benchmarks/join_view.bench.cpp b/libcxx/test/benchmarks/join_view.bench.cpp
index c789a39a8e208..9f6db4a3766af 100644
--- a/libcxx/test/benchmarks/join_view.bench.cpp
+++ b/libcxx/test/benchmarks/join_view.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include <deque>
#include <ranges>
diff --git a/libcxx/test/benchmarks/lexicographical_compare_three_way.bench.cpp b/libcxx/test/benchmarks/lexicographical_compare_three_way.bench.cpp
index 03f20d05ed699..878cf1842f76f 100644
--- a/libcxx/test/benchmarks/lexicographical_compare_three_way.bench.cpp
+++ b/libcxx/test/benchmarks/lexicographical_compare_three_way.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <algorithm>
#include "benchmark/benchmark.h"
diff --git a/libcxx/test/benchmarks/libcxxabi/dynamic_cast_old_stress.bench.cpp b/libcxx/test/benchmarks/libcxxabi/dynamic_cast_old_stress.bench.cpp
index df4daf7409b8f..e79d865c2ca30 100644
--- a/libcxx/test/benchmarks/libcxxabi/dynamic_cast_old_stress.bench.cpp
+++ b/libcxx/test/benchmarks/libcxxabi/dynamic_cast_old_stress.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11
+
#include <cassert>
#include <cstddef>
#include <utility>
diff --git a/libcxx/test/benchmarks/map.bench.cpp b/libcxx/test/benchmarks/map.bench.cpp
index 255164b22aa80..81bdc5077f026 100644
--- a/libcxx/test/benchmarks/map.bench.cpp
+++ b/libcxx/test/benchmarks/map.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include <cstdint>
#include <map>
diff --git a/libcxx/test/benchmarks/monotonic_buffer.bench.cpp b/libcxx/test/benchmarks/monotonic_buffer.bench.cpp
index 39bb853ccba5d..66ab5809985b5 100644
--- a/libcxx/test/benchmarks/monotonic_buffer.bench.cpp
+++ b/libcxx/test/benchmarks/monotonic_buffer.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <list>
#include <memory_resource>
diff --git a/libcxx/test/benchmarks/numeric/gcd.bench.cpp b/libcxx/test/benchmarks/numeric/gcd.bench.cpp
index f8b6a856cd0d5..755ff6543a905 100644
--- a/libcxx/test/benchmarks/numeric/gcd.bench.cpp
+++ b/libcxx/test/benchmarks/numeric/gcd.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <array>
#include <benchmark/benchmark.h>
#include <cstring>
@@ -41,7 +43,8 @@ static void bm_gcd_trivial(benchmark::State& state) {
BENCHMARK(bm_gcd_trivial);
static void bm_gcd_complex(benchmark::State& state) {
- int lhs = 2971215073, rhs = 1836311903;
+ long lhs = 2971215073;
+ long rhs = 1836311903;
for (auto _ : state) {
benchmark::DoNotOptimize(lhs);
benchmark::DoNotOptimize(rhs);
diff --git a/libcxx/test/benchmarks/ordered_set.bench.cpp b/libcxx/test/benchmarks/ordered_set.bench.cpp
index 22540d8ab7e0c..6871eca4ac4df 100644
--- a/libcxx/test/benchmarks/ordered_set.bench.cpp
+++ b/libcxx/test/benchmarks/ordered_set.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <algorithm>
#include <cstdint>
#include <memory>
@@ -181,7 +183,7 @@ struct IterateRangeFor : Base {
while (State.KeepRunningBatch(TableSize * NumTables)) {
for (auto& Set : Data.Sets) {
for (auto& V : Set) {
- benchmark::DoNotOptimize(V);
+ benchmark::DoNotOptimize(const_cast<std::set<uint64_t>::reference>(V));
}
}
}
@@ -199,7 +201,7 @@ struct IterateBeginEnd : Base {
while (State.KeepRunningBatch(TableSize * NumTables)) {
for (auto& Set : Data.Sets) {
for (auto it = Set.begin(); it != Set.end(); ++it) {
- benchmark::DoNotOptimize(*it);
+ benchmark::DoNotOptimize(const_cast<std::set<uint64_t>::reference>(*it));
}
}
}
diff --git a/libcxx/test/benchmarks/shared_mutex_vs_mutex.bench.cpp b/libcxx/test/benchmarks/shared_mutex_vs_mutex.bench.cpp
index 548293507ac03..84a49a8d07d04 100644
--- a/libcxx/test/benchmarks/shared_mutex_vs_mutex.bench.cpp
+++ b/libcxx/test/benchmarks/shared_mutex_vs_mutex.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
// This benchmark compares the performance of std::mutex and std::shared_mutex in contended scenarios.
// it's meant to establish a baseline overhead for std::shared_mutex and std::mutex, and to help inform decisions about
// which mutex to use when selecting a mutex type for a given use case.
diff --git a/libcxx/test/benchmarks/std_format_spec_string_unicode.bench.cpp b/libcxx/test/benchmarks/std_format_spec_string_unicode.bench.cpp
index 3769e964269d8..730704f39f6c8 100644
--- a/libcxx/test/benchmarks/std_format_spec_string_unicode.bench.cpp
+++ b/libcxx/test/benchmarks/std_format_spec_string_unicode.bench.cpp
@@ -4,6 +4,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#ifndef _LIBCPP_HAS_NO_UNICODE
# include <format>
diff --git a/libcxx/test/benchmarks/std_format_spec_string_unicode_escape.bench.cpp b/libcxx/test/benchmarks/std_format_spec_string_unicode_escape.bench.cpp
index 3b5a1c4340c35..a33826269c09c 100644
--- a/libcxx/test/benchmarks/std_format_spec_string_unicode_escape.bench.cpp
+++ b/libcxx/test/benchmarks/std_format_spec_string_unicode_escape.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
// This test formats a larger piece of text in "escaped" mode. It uses several
// datasets to give an impression how the amount of multibyte UTF-8 sequences
// and larger grapheme clusters affect the performance.
diff --git a/libcxx/test/benchmarks/stop_token.bench.cpp b/libcxx/test/benchmarks/stop_token.bench.cpp
index 6be4736c3aec7..201f54c7d4c3a 100644
--- a/libcxx/test/benchmarks/stop_token.bench.cpp
+++ b/libcxx/test/benchmarks/stop_token.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
#include <numeric>
#include <stop_token>
#include <thread>
diff --git a/libcxx/test/benchmarks/string.bench.cpp b/libcxx/test/benchmarks/string.bench.cpp
index 49d37229fa35b..cc52e1d07a83b 100644
--- a/libcxx/test/benchmarks/string.bench.cpp
+++ b/libcxx/test/benchmarks/string.bench.cpp
@@ -6,7 +6,10 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <cstdint>
+#include <cstdlib>
#include <new>
#include <vector>
@@ -170,13 +173,13 @@ template <class Length, bool MeasureCopy, bool MeasureDestroy>
static void StringCopyAndDestroy(benchmark::State& state) {
static constexpr size_t NumStrings = 1024;
auto Orig = makeString(Length());
- std::aligned_storage<sizeof(std::string)>::type Storage[NumStrings];
+ alignas(std::string) char Storage[NumStrings * sizeof(std::string)];
while (state.KeepRunningBatch(NumStrings)) {
if (!MeasureCopy)
state.PauseTiming();
for (size_t I = 0; I < NumStrings; ++I) {
- ::new (static_cast<void*>(Storage + I)) std::string(Orig);
+ ::new (static_cast<std::string*>(Storage) + I) std::string(Orig);
}
if (!MeasureCopy)
state.ResumeTiming();
@@ -184,7 +187,7 @@ static void StringCopyAndDestroy(benchmark::State& state) {
state.PauseTiming();
for (size_t I = 0; I < NumStrings; ++I) {
using S = std::string;
- reinterpret_cast<S*>(Storage + I)->~S();
+ (static_cast<S*>(Storage) + I)->~S();
}
if (!MeasureDestroy)
state.ResumeTiming();
@@ -209,16 +212,16 @@ template <class Length>
struct StringMove {
static void run(benchmark::State& state) {
// Keep two object locations and move construct back and forth.
- std::aligned_storage<sizeof(std::string), alignof(std::string)>::type Storage[2];
+ alignas(std::string) char Storage[2 * sizeof(std::string)];
using S = std::string;
size_t I = 0;
- S* newS = new (static_cast<void*>(Storage)) std::string(makeString(Length()));
+ S* newS = new (static_cast<std::string*>(Storage)) std::string(makeString(Length()));
for (auto _ : state) {
// Switch locations.
I ^= 1;
benchmark::DoNotOptimize(Storage);
// Move construct into the new location,
- S* tmpS = new (static_cast<void*>(Storage + I)) S(std::move(*newS));
+ S* tmpS = new (static_cast<std::string*>(Storage) + I) S(std::move(*newS));
// then destroy the old one.
newS->~S();
newS = tmpS;
@@ -481,14 +484,14 @@ struct StringRead {
for (auto _ : state) {
// Jump long enough to defeat cache locality, and use a value that is
// coprime with NumStrings to ensure we visit every element.
- I = (I + 17) % NumStrings;
- const auto& V = Values[I];
+ I = (I + 17) % NumStrings;
+ auto& V = Values[I];
// Read everything first. Escaping data() through DoNotOptimize might
// cause the compiler to have to recalculate information about `V` due to
// aliasing.
- const char* const Data = V.data();
- const size_t Size = V.size();
+ char* Data = V.data();
+ size_t Size = V.size();
benchmark::DoNotOptimize(Data);
benchmark::DoNotOptimize(Size);
if (Depth() == ::Depth::Deep) {
diff --git a/libcxx/test/benchmarks/stringstream.bench.cpp b/libcxx/test/benchmarks/stringstream.bench.cpp
index a333900d6d2fa..b7c50a96ef51e 100644
--- a/libcxx/test/benchmarks/stringstream.bench.cpp
+++ b/libcxx/test/benchmarks/stringstream.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include "benchmark/benchmark.h"
#include "test_macros.h"
diff --git a/libcxx/test/benchmarks/to_chars.bench.cpp b/libcxx/test/benchmarks/to_chars.bench.cpp
index 2e3c59f471826..4d21537da0fd2 100644
--- a/libcxx/test/benchmarks/to_chars.bench.cpp
+++ b/libcxx/test/benchmarks/to_chars.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include <array>
#include <charconv>
#include <random>
diff --git a/libcxx/test/benchmarks/unordered_set_operations.bench.cpp b/libcxx/test/benchmarks/unordered_set_operations.bench.cpp
index bcf6adda2e021..818b86db6de1a 100644
--- a/libcxx/test/benchmarks/unordered_set_operations.bench.cpp
+++ b/libcxx/test/benchmarks/unordered_set_operations.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
#include <cstdint>
#include <cstdlib>
#include <cstring>
diff --git a/libcxx/test/benchmarks/variant_visit_1.bench.cpp b/libcxx/test/benchmarks/variant_visit_1.bench.cpp
index fa9b246a0a411..94d2103d7c3ac 100644
--- a/libcxx/test/benchmarks/variant_visit_1.bench.cpp
+++ b/libcxx/test/benchmarks/variant_visit_1.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include "benchmark/benchmark.h"
#include "VariantBenchmarks.h"
diff --git a/libcxx/test/benchmarks/variant_visit_2.bench.cpp b/libcxx/test/benchmarks/variant_visit_2.bench.cpp
index 84e26f9c61e40..aa20e4bee57f0 100644
--- a/libcxx/test/benchmarks/variant_visit_2.bench.cpp
+++ b/libcxx/test/benchmarks/variant_visit_2.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include "benchmark/benchmark.h"
#include "VariantBenchmarks.h"
diff --git a/libcxx/test/benchmarks/variant_visit_3.bench.cpp b/libcxx/test/benchmarks/variant_visit_3.bench.cpp
index 1b4e903ad8ff8..a6756ccfcf16b 100644
--- a/libcxx/test/benchmarks/variant_visit_3.bench.cpp
+++ b/libcxx/test/benchmarks/variant_visit_3.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14
+
#include "benchmark/benchmark.h"
#include "VariantBenchmarks.h"
diff --git a/libcxx/test/benchmarks/vector_operations.bench.cpp b/libcxx/test/benchmarks/vector_operations.bench.cpp
index 8698e45c00671..b0dffe35ec6e1 100644
--- a/libcxx/test/benchmarks/vector_operations.bench.cpp
+++ b/libcxx/test/benchmarks/vector_operations.bench.cpp
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
#include <cstdint>
#include <cstdlib>
#include <cstring>
>From 66650b256022c4781850e8334d8c2f16e10cbe65 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 25 Jul 2024 10:41:10 -0400
Subject: [PATCH 2/2] [libc++] Unify the benchmarks with the test suite
Instead of building the benchmarks separately via CMake and running them
separately from the test suite, this patch merges the benchmarks into
the test suite and handles both uniformly.
As a result:
- It is now possible to run individual benchmarks like we run tests
(e.g. using libcxx-lit), which is a huge quality-of-life improvement.
- The benchmarks will be run under exactly the same configuration as
the rest of the tests, which is a nice simplification. This does
mean that one has to be careful to enable the desired optimization
flags when running benchmarks, but that is easy with e.g.
`libcxx-lit <...> --param optimization=speed`.
- Benchmarks can use the same annotations as the rest of the test
suite, such as `// UNSUPPORTED` & friends.
When running the tests via `check-cxx`, we only compile the benchmarks
because running them would be too time consuming. This introduces a bit
of complexity in the testing setup, and instead it would be better to
allow passing a --dry-run flag to GoogleBenchmark executables, which is
the topic of https://github.com/google/benchmark/issues/1827.
I am not really satisfied with the layering violation of adding the
%{benchmark_flags} substitution to cmake-bridge, however I believe
this can be improved in the future.
---
.github/workflows/libcxx-build-and-test.yaml | 4 +-
libcxx/CMakeLists.txt | 5 -
libcxx/docs/BuildingLibcxx.rst | 12 +-
libcxx/docs/TestingLibcxx.rst | 53 ++----
libcxx/test/CMakeLists.txt | 15 +-
libcxx/test/benchmarks/CMakeLists.txt | 171 +------------------
libcxx/test/benchmarks/lit.cfg.py.in | 23 ---
libcxx/test/benchmarks/lit.site.cfg.py.in | 10 --
libcxx/test/configs/cmake-bridge.cfg.in | 1 +
libcxx/utils/ci/run-buildbot | 10 --
libcxx/utils/libcxx/test/config.py | 2 +-
libcxx/utils/libcxx/test/format.py | 31 +++-
libcxx/utils/libcxx/test/googlebenchmark.py | 125 --------------
libcxx/utils/libcxx/test/params.py | 10 ++
14 files changed, 67 insertions(+), 405 deletions(-)
delete mode 100644 libcxx/test/benchmarks/lit.cfg.py.in
delete mode 100644 libcxx/test/benchmarks/lit.site.cfg.py.in
delete mode 100644 libcxx/utils/libcxx/test/googlebenchmark.py
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 1456f245cf7c0..94889bb243f3c 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -154,9 +154,7 @@ jobs:
'generic-no-rtti',
'generic-optimized-speed',
'generic-static',
- # TODO Find a better place for the benchmark and bootstrapping builds to live. They're either very expensive
- # or don't provide much value since the benchmark run results are too noise on the bots.
- 'benchmarks',
+ # TODO Find a better place for the bootstrapping build to live, since it's very expensive.
'bootstrapping-build'
]
machine: [ 'libcxx-runners-8-set' ]
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 6168c76bff6d9..d58c485110bd6 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -156,13 +156,8 @@ message(STATUS "Using libc++ testing configuration: ${LIBCXX_TEST_CONFIG}")
set(LIBCXX_TEST_PARAMS "" CACHE STRING
"A list of parameters to run the Lit test suite with.")
-# Benchmark options -----------------------------------------------------------
option(LIBCXX_INCLUDE_BENCHMARKS "Build the libc++ benchmarks and their dependencies" ON)
-set(LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT --benchmark_min_time=0.01)
-set(LIBCXX_BENCHMARK_TEST_ARGS "${LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT}" CACHE STRING
- "Arguments to pass when running the benchmarks using check-cxx-benchmarks")
-
option(LIBCXX_INCLUDE_DOCS "Build the libc++ documentation." ${LLVM_INCLUDE_DOCS})
set(LIBCXX_LIBDIR_SUFFIX "${LLVM_LIBDIR_SUFFIX}" CACHE STRING
"Define suffix of library directory name (32/64)")
diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst
index 5c224689e0f9f..ebc5513d40d67 100644
--- a/libcxx/docs/BuildingLibcxx.rst
+++ b/libcxx/docs/BuildingLibcxx.rst
@@ -381,7 +381,8 @@ libc++ Feature Options
**Default**: ``ON`` (or value of ``LLVM_INCLUDE_TESTS``)
- Build the libc++ tests.
+ Build the libc++ test suite, which includes various types of tests like conformance
+ tests, vendor-specific tests and benchmarks.
.. option:: LIBCXX_INCLUDE_BENCHMARKS:BOOL
@@ -390,15 +391,6 @@ libc++ Feature Options
Build the libc++ benchmark tests and the Google Benchmark library needed
to support them.
-.. option:: LIBCXX_BENCHMARK_TEST_ARGS:STRING
-
- **Default**: ``--benchmark_min_time=0.01``
-
- A semicolon list of arguments to pass when running the libc++ benchmarks using the
- ``check-cxx-benchmarks`` rule. By default we run the benchmarks for a very short amount of time,
- since the primary use of ``check-cxx-benchmarks`` is to get test and sanitizer coverage, not to
- get accurate measurements.
-
.. option:: LIBCXX_ASSERTION_HANDLER_FILE:PATH
**Default**:: ``"${CMAKE_CURRENT_SOURCE_DIR}/vendor/llvm/default_assertion_handler.in"``
diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index 65a7e1ec30c96..6ee8b476bba1d 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -392,6 +392,10 @@ Test Filenames`_ when determining the names for new test files.
of Lit test to be executed. This can be used to generate multiple Lit tests from a single source file, which is useful for testing repetitive properties
in the library. Be careful not to abuse this since this is not a replacement for usual code reuse techniques.
+ * - ``FOO.bench.cpp``
+ - A benchmark test. These tests are linked against the GoogleBenchmark library and generally contain micro-benchmarks of individual
+ components of the library.
+
libc++-Specific Lit Features
----------------------------
@@ -438,48 +442,20 @@ Libc++ contains benchmark tests separately from the test of the test suite.
The benchmarks are written using the `Google Benchmark`_ library, a copy of which
is stored in the libc++ repository.
-For more information about using the Google Benchmark library see the
+For more information about using the Google Benchmark library, see the
`official documentation <https://github.com/google/benchmark>`_.
-.. _`Google Benchmark`: https://github.com/google/benchmark
-
-Building Benchmarks
--------------------
-
-The benchmark tests are not built by default. The benchmarks can be built using
-the ``cxx-benchmarks`` target.
-
-An example build would look like:
-
-.. code-block:: bash
-
- $ ninja -C build cxx-benchmarks
-
-This will build all of the benchmarks under ``<libcxx>/test/benchmarks`` to be
-built against the just-built libc++. The compiled tests are output into
-``build/libcxx/test/benchmarks``.
+The benchmarks are located under ``libcxx/test/benchmarks``. Running a benchmark
+works in the same way as running a test. Both the benchmarks and the tests share
+the same configuration, so make sure to enable the relevant optimization level
+when running the benchmarks.
-Also See:
-
- * :ref:`Building Libc++ <build instructions>`
- * :ref:`CMake Options`
-
-Running Benchmarks
-------------------
-
-The benchmarks must be run manually by the user. Currently there is no way
-to run them as part of the build.
-
-For example:
-
-.. code-block:: bash
-
- $ cd build/libcxx/test/benchmarks
- $ ./find.bench.out # Runs all the benchmarks
- $ ./find.bench.out --benchmark_filter="bm_ranges_find<std::vector<char>>" # Only runs that specific benchmark
-
-For more information about running benchmarks see `Google Benchmark`_.
+Note that benchmarks are only dry-run when run via the ``check-cxx`` target since
+we only want to make sure they don't rot. Do not rely on the results of benchmarks
+run through ``check-cxx`` for anything, instead run the benchmarks manually using
+the instructions for running individual tests.
+.. _`Google Benchmark`: https://github.com/google/benchmark
.. _testing-hardening-assertions:
@@ -523,4 +499,3 @@ A toy example:
Note that error messages are only tested (matched) if the ``debug``
hardening mode is used.
-
diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt
index b25712b0e363b..ddff951e435d5 100644
--- a/libcxx/test/CMakeLists.txt
+++ b/libcxx/test/CMakeLists.txt
@@ -1,15 +1,21 @@
include(HandleLitArguments)
add_subdirectory(tools)
-if (LIBCXX_INCLUDE_BENCHMARKS)
- add_subdirectory(benchmarks)
-endif()
-
set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!")
set(SERIALIZED_LIT_PARAMS "# Lit parameters serialized here for llvm-lit to pick them up\n")
serialize_lit_string_param(SERIALIZED_LIT_PARAMS compiler "${CMAKE_CXX_COMPILER}")
+if (LIBCXX_INCLUDE_BENCHMARKS
+ AND LIBCXX_ENABLE_LOCALIZATION AND LIBCXX_ENABLE_THREADS AND LIBCXX_ENABLE_FILESYSTEM AND LIBCXX_ENABLE_RANDOM_DEVICE
+ AND LIBCXX_ENABLE_EXCEPTIONS AND LIBCXX_ENABLE_RTTI) # TODO: The benchmarks should work with exceptions/RTTI disabled
+ add_subdirectory(benchmarks)
+ set(_libcxx_benchmark_mode "dry-run")
+else()
+ serialize_lit_string_param(SERIALIZED_LIT_PARAMS enable_benchmarks "no")
+ set(_libcxx_benchmark_mode "no")
+endif()
+
if (NOT LIBCXX_ENABLE_EXCEPTIONS)
serialize_lit_param(SERIALIZED_LIT_PARAMS enable_exceptions False)
endif()
@@ -46,4 +52,5 @@ configure_lit_site_cfg(
add_lit_testsuite(check-cxx
"Running libcxx tests"
${CMAKE_CURRENT_BINARY_DIR}
+ PARAMS enable_benchmarks="${_libcxx_benchmark_mode}"
DEPENDS cxx-test-depends)
diff --git a/libcxx/test/benchmarks/CMakeLists.txt b/libcxx/test/benchmarks/CMakeLists.txt
index d61367a367738..a505bdb0cacb6 100644
--- a/libcxx/test/benchmarks/CMakeLists.txt
+++ b/libcxx/test/benchmarks/CMakeLists.txt
@@ -1,10 +1,8 @@
-include(ExternalProject)
-include(CheckCXXCompilerFlag)
-
#==============================================================================
# Build Google Benchmark
#==============================================================================
+include(ExternalProject)
set(BENCHMARK_COMPILE_FLAGS
-Wno-unused-command-line-argument
-nostdinc++
@@ -39,169 +37,4 @@ ExternalProject_Add(google-benchmark
-DBENCHMARK_USE_LIBCXX:BOOL=ON
-DBENCHMARK_ENABLE_TESTING:BOOL=OFF)
-#==============================================================================
-# Benchmark tests configuration
-#==============================================================================
-add_custom_target(cxx-benchmarks)
-set(BENCHMARK_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(BENCHMARK_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/google-benchmark)
-
-add_library( cxx-benchmarks-flags INTERFACE)
-
-# TODO(cmake): remove. This is a workaround to prevent older versions of GCC
-# from failing the configure step because they don't support C++23.
-if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "13.0")
- return()
-endif()
-#TODO(cmake): remove the `add_compile_options`. Currently we have to explicitly
-# pass the `std:c++latest` flag on Windows to work around an issue where
-# requesting `cxx_std_23` results in an error -- somehow CMake fails to
-# translate the `c++23` flag into `c++latest`, and the highest numbered C++
-# version that MSVC flags support is C++20.
-if (MSVC)
- add_compile_options(/std:c++latest)
-# ibm-clang does not recognize the cxx_std_23 flag, so use this as a temporary
-# workaround on AIX as well.
-elseif (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
- add_compile_options(-std=c++23)
-else()
- target_compile_features( cxx-benchmarks-flags INTERFACE cxx_std_23)
-endif()
-
-target_compile_options(cxx-benchmarks-flags INTERFACE -fsized-deallocation -nostdinc++
- ${SANITIZER_FLAGS} -Wno-user-defined-literals -Wno-suggest-override)
-target_include_directories(cxx-benchmarks-flags INTERFACE "${LIBCXX_GENERATED_INCLUDE_DIR}"
- INTERFACE "${BENCHMARK_INSTALL_DIR}/include"
- INTERFACE "${LIBCXX_SOURCE_DIR}/test/support")
-target_link_options(cxx-benchmarks-flags INTERFACE -lm -nostdlib++
- "-L${BENCHMARK_INSTALL_DIR}/lib" "-L${BENCHMARK_INSTALL_DIR}/lib64"
- ${SANITIZER_FLAGS})
-
-set(libcxx_benchmark_targets)
-
-function(add_benchmark_test name source_file)
- set(libcxx_target ${name}_libcxx)
- list(APPEND libcxx_benchmark_targets ${libcxx_target})
- add_executable(${libcxx_target} EXCLUDE_FROM_ALL ${source_file})
- target_link_libraries(${libcxx_target} PRIVATE cxx-benchmarks-flags)
- add_dependencies(${libcxx_target} cxx google-benchmark)
- add_dependencies(cxx-benchmarks ${libcxx_target})
- if (LIBCXX_ENABLE_SHARED)
- target_link_libraries(${libcxx_target} PRIVATE cxx_shared)
- else()
- target_link_libraries(${libcxx_target} PRIVATE cxx_static)
- endif()
- target_link_libraries(${libcxx_target} PRIVATE cxx_experimental benchmark)
- if (LLVM_USE_SANITIZER)
- target_link_libraries(${libcxx_target} PRIVATE -ldl)
- endif()
- set_target_properties(${libcxx_target}
- PROPERTIES
- OUTPUT_NAME "${name}.bench.out"
- RUNTIME_OUTPUT_DIRECTORY "${BENCHMARK_OUTPUT_DIR}"
- CXX_EXTENSIONS NO)
- cxx_link_system_libraries(${libcxx_target})
-endfunction()
-
-
-#==============================================================================
-# Register Benchmark tests
-#==============================================================================
-set(BENCHMARK_TESTS
- algorithms.partition_point.bench.cpp
- algorithms/count.bench.cpp
- algorithms/equal.bench.cpp
- algorithms/find.bench.cpp
- algorithms/fill.bench.cpp
- algorithms/for_each.bench.cpp
- algorithms/lower_bound.bench.cpp
- algorithms/make_heap.bench.cpp
- algorithms/make_heap_then_sort_heap.bench.cpp
- algorithms/min.bench.cpp
- algorithms/minmax.bench.cpp
- algorithms/min_max_element.bench.cpp
- algorithms/mismatch.bench.cpp
- algorithms/pop_heap.bench.cpp
- algorithms/pstl.stable_sort.bench.cpp
- algorithms/push_heap.bench.cpp
- algorithms/ranges_contains.bench.cpp
- algorithms/ranges_ends_with.bench.cpp
- algorithms/ranges_make_heap.bench.cpp
- algorithms/ranges_make_heap_then_sort_heap.bench.cpp
- algorithms/ranges_pop_heap.bench.cpp
- algorithms/ranges_push_heap.bench.cpp
- algorithms/ranges_sort.bench.cpp
- algorithms/ranges_sort_heap.bench.cpp
- algorithms/ranges_stable_sort.bench.cpp
- algorithms/set_intersection.bench.cpp
- algorithms/sort.bench.cpp
- algorithms/sort_heap.bench.cpp
- algorithms/stable_sort.bench.cpp
- atomic_wait.bench.cpp
- atomic_wait_vs_mutex_lock.bench.cpp
- libcxxabi/dynamic_cast.bench.cpp
- libcxxabi/dynamic_cast_old_stress.bench.cpp
- allocation.bench.cpp
- deque.bench.cpp
- deque_iterator.bench.cpp
- exception_ptr.bench.cpp
- filesystem.bench.cpp
- format_to_n.bench.cpp
- format_to.bench.cpp
- format.bench.cpp
- formatted_size.bench.cpp
- formatter_float.bench.cpp
- formatter_int.bench.cpp
- function.bench.cpp
- join_view.bench.cpp
- lexicographical_compare_three_way.bench.cpp
- map.bench.cpp
- monotonic_buffer.bench.cpp
- numeric/gcd.bench.cpp
- ordered_set.bench.cpp
- shared_mutex_vs_mutex.bench.cpp
- stop_token.bench.cpp
- std_format_spec_string_unicode.bench.cpp
- std_format_spec_string_unicode_escape.bench.cpp
- string.bench.cpp
- stringstream.bench.cpp
- system_error.bench.cpp
- to_chars.bench.cpp
- unordered_set_operations.bench.cpp
- util_smartptr.bench.cpp
- variant_visit_1.bench.cpp
- variant_visit_2.bench.cpp
- variant_visit_3.bench.cpp
- vector_operations.bench.cpp
- )
-
-foreach(test_path ${BENCHMARK_TESTS})
- get_filename_component(test_file "${test_path}" NAME)
- string(REPLACE ".bench.cpp" "" test_name "${test_file}")
- if (NOT DEFINED ${test_name}_REPORTED)
- message(STATUS "Adding Benchmark: ${test_file}")
- # Only report the adding of the benchmark once.
- set(${test_name}_REPORTED ON CACHE INTERNAL "")
- endif()
- add_benchmark_test(${test_name} ${test_path})
-endforeach()
-
-if (LIBCXX_INCLUDE_TESTS)
- include(AddLLVM)
-
- configure_lit_site_cfg(
- ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py.in
- ${CMAKE_CURRENT_BINARY_DIR}/lit.cfg.py)
-
- configure_lit_site_cfg(
- ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
- ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py)
-
- set(BENCHMARK_LIT_ARGS "--show-all --show-xfail --show-unsupported ${LIT_ARGS_DEFAULT}")
-
- add_lit_target(check-cxx-benchmarks
- "Running libcxx benchmarks tests"
- ${CMAKE_CURRENT_BINARY_DIR}
- DEPENDS cxx-benchmarks cxx-test-depends
- ARGS ${BENCHMARK_LIT_ARGS})
-endif()
+add_dependencies(cxx-test-depends google-benchmark)
diff --git a/libcxx/test/benchmarks/lit.cfg.py.in b/libcxx/test/benchmarks/lit.cfg.py.in
deleted file mode 100644
index a3ce2477b9a85..0000000000000
--- a/libcxx/test/benchmarks/lit.cfg.py.in
+++ /dev/null
@@ -1,23 +0,0 @@
-# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
-# Configuration file for the 'lit' test runner.
-import os
-import site
-
-site.addsitedir(os.path.join("@LIBCXX_SOURCE_DIR@", "utils"))
-from libcxx.test.googlebenchmark import GoogleBenchmark
-
-# Tell pylint that we know config and lit_config exist somewhere.
-if "PYLINT_IMPORT" in os.environ:
- config = object()
- lit_config = object()
-
-# name: The name of this test suite.
-config.name = "libc++ benchmarks"
-config.suffixes = []
-
-config.test_exec_root = "@CMAKE_CURRENT_BINARY_DIR@"
-config.test_source_root = "@CMAKE_CURRENT_BINARY_DIR@"
-
-config.test_format = GoogleBenchmark(
- test_sub_dirs=".", test_suffix=".bench.out", benchmark_args=config.benchmark_args
-)
diff --git a/libcxx/test/benchmarks/lit.site.cfg.py.in b/libcxx/test/benchmarks/lit.site.cfg.py.in
deleted file mode 100644
index 6d4b0ca3dae0e..0000000000000
--- a/libcxx/test/benchmarks/lit.site.cfg.py.in
+++ /dev/null
@@ -1,10 +0,0 @@
- at LIT_SITE_CFG_IN_HEADER@
-
-import sys
-
-config.libcxx_src_root = "@LIBCXX_SOURCE_DIR@"
-config.libcxx_obj_root = "@LIBCXX_BINARY_DIR@"
-config.benchmark_args = "@LIBCXX_BENCHMARK_TEST_ARGS@".split(';')
-
-# Let the main config do the real work.
-lit_config.load_config(config, "@CMAKE_CURRENT_BINARY_DIR@/lit.cfg.py")
\ No newline at end of file
diff --git a/libcxx/test/configs/cmake-bridge.cfg.in b/libcxx/test/configs/cmake-bridge.cfg.in
index 78d0cb5a25748..551a8d067c109 100644
--- a/libcxx/test/configs/cmake-bridge.cfg.in
+++ b/libcxx/test/configs/cmake-bridge.cfg.in
@@ -29,3 +29,4 @@ config.substitutions.append(('%{target-include-dir}', '@LIBCXX_GENERATED_INCLUDE
config.substitutions.append(('%{lib-dir}', '@LIBCXX_LIBRARY_DIR@'))
config.substitutions.append(('%{module-dir}', '@LIBCXX_GENERATED_MODULE_DIR@'))
config.substitutions.append(('%{test-tools-dir}', '@LIBCXX_TEST_TOOLS_PATH@'))
+config.substitutions.append(('%{benchmark_flags}', '-I @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/include -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib -l benchmark'))
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index f1c20b9d72190..d982203e66037 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -180,11 +180,6 @@ function check-abi-list() {
)
}
-function check-cxx-benchmarks() {
- echo "--- Running the benchmarks"
- ${NINJA} -vC "${BUILD_DIR}" check-cxx-benchmarks
-}
-
function test-armv7m-picolibc() {
clean
@@ -605,11 +600,6 @@ apple-system-backdeployment-*)
check-runtimes
;;
-benchmarks)
- clean
- generate-cmake
- check-cxx-benchmarks
-;;
aarch64)
clean
generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/AArch64.cmake"
diff --git a/libcxx/utils/libcxx/test/config.py b/libcxx/utils/libcxx/test/config.py
index 9a71c494030d7..0840c46d7bfae 100644
--- a/libcxx/utils/libcxx/test/config.py
+++ b/libcxx/utils/libcxx/test/config.py
@@ -52,7 +52,7 @@ def configure(parameters, features, config, lit_config):
)
# Print the basic substitutions
- for sub in ("%{cxx}", "%{flags}", "%{compile_flags}", "%{link_flags}", "%{exec}"):
+ for sub in ("%{cxx}", "%{flags}", "%{compile_flags}", "%{link_flags}", "%{benchmark_flags}", "%{exec}"):
note("Using {} substitution: '{}'".format(sub, _getSubstitution(sub, config)))
# Print all available features
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index 7e5281c0b7406..ccd27170e197b 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -29,7 +29,7 @@ def _getTempPaths(test):
def _checkBaseSubstitutions(substitutions):
substitutions = [s for (s, _) in substitutions]
- for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
+ for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{benchmark_flags}", "%{flags}", "%{exec}"]:
assert s in substitutions, "Required substitution {} was not provided".format(s)
def _executeScriptInternal(test, litConfig, commands):
@@ -220,11 +220,15 @@ class CxxStandardLibraryTest(lit.formats.FileBasedTest):
The test format operates by assuming that each test's configuration provides
the following substitutions, which it will reuse in the shell scripts it
constructs:
- %{cxx} - A command that can be used to invoke the compiler
- %{compile_flags} - Flags to use when compiling a test case
- %{link_flags} - Flags to use when linking a test case
- %{flags} - Flags to use either when compiling or linking a test case
- %{exec} - A command to prefix the execution of executables
+ %{cxx} - A command that can be used to invoke the compiler
+ %{compile_flags} - Flags to use when compiling a test case
+ %{link_flags} - Flags to use when linking a test case
+ %{flags} - Flags to use either when compiling or linking a test case
+ %{benchmark_flags} - Flags to use when compiling benchmarks. These flags should provide access to
+ GoogleBenchmark but shouldn't hardcode any optimization level or other settings,
+ since the benchmarks should be run under the same configuration as the rest of
+ the test suite.
+ %{exec} - A command to prefix the execution of executables
Note that when building an executable (as opposed to only compiling a source
file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used
@@ -254,6 +258,7 @@ class CxxStandardLibraryTest(lit.formats.FileBasedTest):
def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig):
SUPPORTED_SUFFIXES = [
+ "[.]bench[.]cpp$",
"[.]pass[.]cpp$",
"[.]pass[.]mm$",
"[.]compile[.]pass[.]cpp$",
@@ -331,6 +336,20 @@ def execute(self, test, litConfig):
"%dbg(EXECUTED AS) %{exec} %t.exe",
]
return self._executeShTest(test, litConfig, steps)
+ elif filename.endswith(".bench.cpp"):
+ if "enable-benchmarks=no" in test.config.available_features:
+ return lit.Test.Result(
+ lit.Test.UNSUPPORTED,
+ "Test {} requires support for benchmarks, which isn't supported by this configuration".format(
+ test.getFullName()
+ ),
+ )
+ steps = [
+ "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} %{benchmark_flags} -o %t.exe",
+ ]
+ if "enable-benchmarks=run" in test.config.available_features:
+ steps += ["%dbg(EXECUTED AS) %{exec} %t.exe"]
+ return self._executeShTest(test, litConfig, steps)
else:
return lit.Test.Result(
lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)
diff --git a/libcxx/utils/libcxx/test/googlebenchmark.py b/libcxx/utils/libcxx/test/googlebenchmark.py
deleted file mode 100644
index d260484c83770..0000000000000
--- a/libcxx/utils/libcxx/test/googlebenchmark.py
+++ /dev/null
@@ -1,125 +0,0 @@
-from __future__ import absolute_import
-import os
-import subprocess
-import sys
-
-import lit.Test
-import lit.TestRunner
-import lit.util
-from lit.formats.base import TestFormat
-
-kIsWindows = sys.platform in ["win32", "cygwin"]
-
-
-class GoogleBenchmark(TestFormat):
- def __init__(self, test_sub_dirs, test_suffix, benchmark_args=[]):
- self.benchmark_args = list(benchmark_args)
- self.test_sub_dirs = os.path.normcase(str(test_sub_dirs)).split(";")
-
- # On Windows, assume tests will also end in '.exe'.
- exe_suffix = str(test_suffix)
- if kIsWindows:
- exe_suffix += ".exe"
-
- # Also check for .py files for testing purposes.
- self.test_suffixes = {exe_suffix, test_suffix + ".py"}
-
- def getBenchmarkTests(self, path, litConfig, localConfig):
- """getBenchmarkTests(path) - [name]
-
- Return the tests available in gtest executable.
-
- Args:
- path: String path to a gtest executable
- litConfig: LitConfig instance
- localConfig: TestingConfig instance"""
-
- # TODO: allow splitting tests according to the "benchmark family" so
- # the output for a single family of tests all belongs to the same test
- # target.
- list_test_cmd = [path, "--benchmark_list_tests"]
- try:
- output = subprocess.check_output(list_test_cmd, env=localConfig.environment)
- except subprocess.CalledProcessError as exc:
- litConfig.warning(
- "unable to discover google-benchmarks in %r: %s. Process output: %s"
- % (path, sys.exc_info()[1], exc.output)
- )
- raise StopIteration
-
- nested_tests = []
- for ln in output.splitlines(False): # Don't keep newlines.
- ln = lit.util.to_string(ln)
- if not ln.strip():
- continue
-
- index = 0
- while ln[index * 2 : index * 2 + 2] == " ":
- index += 1
- while len(nested_tests) > index:
- nested_tests.pop()
-
- ln = ln[index * 2 :]
- if ln.endswith("."):
- nested_tests.append(ln)
- elif any([name.startswith("DISABLED_") for name in nested_tests + [ln]]):
- # Gtest will internally skip these tests. No need to launch a
- # child process for it.
- continue
- else:
- yield "".join(nested_tests) + ln
-
- def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
- source_path = testSuite.getSourcePath(path_in_suite)
- for subdir in self.test_sub_dirs:
- dir_path = os.path.join(source_path, subdir)
- if not os.path.isdir(dir_path):
- continue
- for fn in lit.util.listdir_files(dir_path, suffixes=self.test_suffixes):
- # Discover the tests in this executable.
- execpath = os.path.join(source_path, subdir, fn)
- testnames = self.getBenchmarkTests(execpath, litConfig, localConfig)
- for testname in testnames:
- testPath = path_in_suite + (subdir, fn, testname)
- yield lit.Test.Test(
- testSuite, testPath, localConfig, file_path=execpath
- )
-
- def execute(self, test, litConfig):
- testPath, testName = os.path.split(test.getSourcePath())
- while not os.path.exists(testPath):
- # Handle GTest parametrized and typed tests, whose name includes
- # some '/'s.
- testPath, namePrefix = os.path.split(testPath)
- testName = namePrefix + "/" + testName
-
- cmd = [testPath, "--benchmark_filter=%s$" % testName] + self.benchmark_args
-
- if litConfig.noExecute:
- return lit.Test.PASS, ""
-
- try:
- out, err, exitCode = lit.util.executeCommand(
- cmd,
- env=test.config.environment,
- timeout=litConfig.maxIndividualTestTime,
- )
- except lit.util.ExecuteCommandTimeoutException:
- return (
- lit.Test.TIMEOUT,
- "Reached timeout of {} seconds".format(litConfig.maxIndividualTestTime),
- )
-
- if exitCode:
- return lit.Test.FAIL, ("exit code: %d\n" % exitCode) + out + err
-
- passing_test_line = testName
- if passing_test_line not in out:
- msg = "Unable to find %r in google benchmark output:\n\n%s%s" % (
- passing_test_line,
- out,
- err,
- )
- return lit.Test.UNRESOLVED, msg
-
- return lit.Test.PASS, err + out
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index 13c7297fd7304..d5df5f6d5191e 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -365,6 +365,16 @@ def getSuitableClangTidy(cfg):
AddFeature("libcpp-has-no-experimental-syncstream"),
],
),
+ # TODO: This can be improved once GoogleBenchmark supports a mode to dry-run benchmarks instead.
+ # See https://github.com/google/benchmark/issues/1827.
+ Parameter(
+ name="enable_benchmarks",
+ choices=["no", "run", "dry-run"],
+ type=str,
+ default="run",
+ help="Whether to run the benchmarks in the test suite, to only dry-run them or to disable them entirely.",
+ actions=lambda mode: [AddFeature(f"enable-benchmarks={mode}")],
+ ),
Parameter(
name="long_tests",
choices=[True, False],
More information about the libcxx-commits
mailing list