[libcxx-commits] [libcxx] [libc++] Optimize vector::__move_range for trivially copyable types (PR #207384)
Nikolas Klauser via libcxx-commits
libcxx-commits at lists.llvm.org
Tue Aug 4 22:58:47 PDT 2026
https://github.com/philnik777 updated https://github.com/llvm/llvm-project/pull/207384
>From c533c9e1f712cb45a61e0687e350d0a3ff4e7e2f Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser at berlin.de>
Date: Fri, 3 Jul 2026 13:34:50 +0200
Subject: [PATCH 1/2] [libc++] Optimize vector::__move_range for trivially
copyable types
This is a code size rather than a performance optimization.
For example, the number of instructions in `__insert_with_iter` is
almost cut in half.
---
libcxx/include/__vector/vector.h | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 5e9fa4a7d0030..5cbc95c323d95 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -47,6 +47,7 @@
#include <__ranges/container_compatible_range.h>
#include <__ranges/from_range.h>
#include <__split_buffer>
+#include <__string/constexpr_c_functions.h>
#include <__type_traits/conditional.h>
#include <__type_traits/enable_if.h>
#include <__type_traits/is_allocator.h>
@@ -57,6 +58,7 @@
#include <__type_traits/is_pointer.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_copyable.h>
#include <__type_traits/is_trivially_relocatable.h>
#include <__type_traits/type_identity.h>
#include <__utility/declval.h>
@@ -1118,6 +1120,18 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void
vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointer __to) {
pointer __old_last = __layout_.__end_ptr();
difference_type __n = __old_last - __to;
+#ifndef _LIBCPP_CXX03_LANG
+ if constexpr (__allocator_has_trivial_move_construct_v<allocator_type, value_type> &&
+ is_trivially_copyable<value_type>::value) {
+ if (!__libcpp_is_constant_evaluated()) {
+ auto __count = __from_e - __from_s;
+ __annotate_increase(__count - __n);
+ std::__constexpr_memmove(std::__to_address(__to), std::__to_address(__from_s), __element_count(__count));
+ __layout_.__set_bound_using_pointer(__to + __count);
+ return;
+ }
+ }
+#endif
{
pointer __i = __from_s + __n;
_ConstructTransaction __tx(*this, __from_e - __i);
>From 31da75eaa9ce84bf090570ff7ce3dfbcd6c13e8b Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser at berlin.de>
Date: Tue, 4 Aug 2026 15:25:21 +0200
Subject: [PATCH 2/2] [libc++] Try to make vector benchmarks less dependent on
surrounding ones
---
.../sequence/sequence_container_benchmarks.h | 42 ++++++++++---------
1 file changed, 23 insertions(+), 19 deletions(-)
diff --git a/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h b/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h
index 5210008989ff7..c3c4c7533f693 100644
--- a/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h
+++ b/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h
@@ -35,6 +35,9 @@ void DoNotOptimizeData(Container& c) {
}
}
+// Align benchmark functions and never inline them to reduce changes in performance in unrelated benchmarks
+#define BENCHMARK_ATTRS [[gnu::aligned(128), gnu::noinline]]
+
template <class Container>
void sequence_container_benchmarks(std::string container) {
using ValueType = typename Container::value_type;
@@ -65,7 +68,7 @@ void sequence_container_benchmarks(std::string container) {
/////////////////////////
if constexpr (std::is_constructible_v<Container, std::size_t>) {
// not all containers provide this constructor
- bench("ctor(size_type)", [](auto& st) {
+ bench("ctor(size_type)", [] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
for ([[maybe_unused]] auto _ : st) {
@@ -76,7 +79,7 @@ void sequence_container_benchmarks(std::string container) {
}
for (auto gen : generators)
- bench("ctor(size_type, const value_type&)" + tostr(gen), [gen](auto& st) {
+ bench("ctor(size_type, const value_type&)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
ValueType value = gen();
benchmark::DoNotOptimize(value);
@@ -88,7 +91,7 @@ void sequence_container_benchmarks(std::string container) {
});
for (auto gen : generators)
- bench("ctor(Iterator, Iterator)" + tostr(gen), [gen](auto& st) {
+ bench("ctor(Iterator, Iterator)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -104,7 +107,7 @@ void sequence_container_benchmarks(std::string container) {
#if defined(__cpp_lib_containers_ranges) && __cpp_lib_containers_ranges >= 202202L
for (auto gen : generators)
- bench("ctor(Range)" + tostr(gen), [gen](auto& st) {
+ bench("ctor(Range)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -118,7 +121,7 @@ void sequence_container_benchmarks(std::string container) {
#endif
for (auto gen : generators)
- bench("ctor(const Self&)" + tostr(gen), [gen](auto& st) {
+ bench("ctor(const Self&)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
Container in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -135,7 +138,7 @@ void sequence_container_benchmarks(std::string container) {
// Assignment
/////////////////////////
for (auto gen : generators)
- bench("operator=(const Self&)" + tostr(gen), [gen](auto& st) {
+ bench("operator=(const Self&)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
Container in1, in2;
std::generate_n(std::back_inserter(in1), size, gen);
@@ -161,7 +164,7 @@ void sequence_container_benchmarks(std::string container) {
// implementation basically creates a new container from scratch or manages to reuse the
// pre-existing storage.
for (auto gen : generators)
- bench("assign(input-iter, input-iter) (full container)" + tostr(gen), [gen](auto& st) {
+ bench("assign(input-iter, input-iter) (full container)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in1, in2;
std::generate_n(std::back_inserter(in1), size, gen);
@@ -185,7 +188,7 @@ void sequence_container_benchmarks(std::string container) {
// Insertion
/////////////////////////
for (auto gen : generators)
- bench("insert(begin)" + tostr(gen), [gen](auto& st) {
+ bench("insert(begin)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -207,7 +210,7 @@ void sequence_container_benchmarks(std::string container) {
if constexpr (std::random_access_iterator<typename Container::iterator>) {
for (auto gen : generators)
- bench("insert(middle)" + tostr(gen), [gen](auto& st) {
+ bench("insert(middle)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -233,7 +236,7 @@ void sequence_container_benchmarks(std::string container) {
// Insert at the start of a vector in a scenario where the vector already
// has enough capacity to hold all the elements we are inserting.
for (auto gen : generators)
- bench("insert(begin, input-iter, input-iter) (no realloc)" + tostr(gen), [gen](auto& st) {
+ bench("insert(begin, input-iter, input-iter) (no realloc)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -260,7 +263,7 @@ void sequence_container_benchmarks(std::string container) {
// has almost enough capacity to hold all the elements we are inserting,
// but does need to reallocate.
for (auto gen : generators)
- bench("insert(begin, input-iter, input-iter) (half filled)" + tostr(gen), [gen](auto& st) {
+ bench("insert(begin, input-iter, input-iter) (half filled)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -286,7 +289,7 @@ void sequence_container_benchmarks(std::string container) {
// more elements, but needs to reallocate almost immediately to fit the remaining
// elements.
for (auto gen : generators)
- bench("insert(begin, input-iter, input-iter) (near full)" + tostr(gen), [gen](auto& st) {
+ bench("insert(begin, input-iter, input-iter) (near full)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -320,7 +323,7 @@ void sequence_container_benchmarks(std::string container) {
// For containers where we can observe capacity(), push_back a single element
// without reserving to ensure the container needs to grow
for (auto gen : generators)
- bench("push_back() (growing)" + tostr(gen), [gen](auto& st) {
+ bench("push_back() (growing)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -355,7 +358,7 @@ void sequence_container_benchmarks(std::string container) {
// ensure the container doesn't grow
if constexpr (has_reserve) {
for (auto gen : generators)
- bench("push_back() (with reserve)" + tostr(gen), [gen](auto& st) {
+ bench("push_back() (with reserve)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -379,7 +382,7 @@ void sequence_container_benchmarks(std::string container) {
// push_back many elements: this is amortized constant for std::vector but not all containers
for (auto gen : generators)
- bench("push_back() (many elements)" + tostr(gen), [gen](auto& st) {
+ bench("push_back() (many elements)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -401,7 +404,7 @@ void sequence_container_benchmarks(std::string container) {
#if TEST_STD_VER >= 23
for (auto gen : generators)
- bench("append_range() (into empty container)" + tostr(gen), [gen](auto& state) {
+ bench("append_range() (into empty container)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& state) {
auto const size = state.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -428,8 +431,9 @@ void sequence_container_benchmarks(std::string container) {
if constexpr (has_prepend_range) {
for (auto gen : generators)
- bench("prepend_range() (into empty container)" + tostr(gen), [gen](auto& state) {
+ bench("prepend_range() (into empty container)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& state) {
auto const size = state.range(0);
+ BENCHMARK_ATTRS
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
DoNotOptimizeData(in);
@@ -451,7 +455,7 @@ void sequence_container_benchmarks(std::string container) {
// Erasure
/////////////////////////
for (auto gen : generators)
- bench("erase(begin)" + tostr(gen), [gen](auto& st) {
+ bench("erase(begin)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
@@ -473,7 +477,7 @@ void sequence_container_benchmarks(std::string container) {
if constexpr (std::random_access_iterator<typename Container::iterator>) {
for (auto gen : generators)
- bench("erase(middle)" + tostr(gen), [gen](auto& st) {
+ bench("erase(middle)" + tostr(gen), [gen] BENCHMARK_ATTRS(auto& st) {
auto const size = st.range(0);
std::vector<ValueType> in;
std::generate_n(std::back_inserter(in), size, gen);
More information about the libcxx-commits
mailing list