[libcxx-commits] [libcxx] [libc++] Optimize __tree copy/move constructor/assignment with allocator (PR #163558)
Nikolas Klauser via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Oct 22 03:03:22 PDT 2025
https://github.com/philnik777 updated https://github.com/llvm/llvm-project/pull/163558
>From dddd4fba3061633a72b8b6825cda78de76b7732b Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser at berlin.de>
Date: Thu, 21 Aug 2025 13:47:34 +0200
Subject: [PATCH] [libc++] Optimize __tree copy/move constructor/assignment
with allocator
---
libcxx/include/__tree | 156 +++++++++++-------
libcxx/include/map | 39 +----
libcxx/include/set | 40 +----
.../associative_container_benchmarks.h | 59 +++++++
.../containers/associative/flat_map.bench.cpp | 8 +
.../associative/flat_multimap.bench.cpp | 8 +
.../containers/associative/map.bench.cpp | 3 +
.../containers/associative/multimap.bench.cpp | 3 +
.../containers/associative/multiset.bench.cpp | 3 +
.../containers/associative/set.bench.cpp | 3 +
.../associative/unordered_map.bench.cpp | 3 +
.../associative/unordered_multimap.bench.cpp | 3 +
.../associative/unordered_multiset.bench.cpp | 3 +
.../associative/unordered_set.bench.cpp | 3 +
14 files changed, 202 insertions(+), 132 deletions(-)
diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index 0738c8c6a5e2b..eb833b97af2db 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -899,6 +899,18 @@ public:
}
_LIBCPP_HIDE_FROM_ABI __tree(const __tree& __t);
+
+ _LIBCPP_HIDE_FROM_ABI __tree(const __tree& __other, const allocator_type& __alloc)
+ : __begin_node_(__end_node()), __node_alloc_(__alloc), __size_(0), __value_comp_(__other.value_comp()) {
+ if (__other.size() == 0)
+ return;
+
+ *__root_ptr() = static_cast<__node_base_pointer>(__copy_construct_tree(__other.__root()));
+ __root()->__parent_ = __end_node();
+ __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_));
+ __size_ = __other.size();
+ }
+
_LIBCPP_HIDE_FROM_ABI __tree& operator=(const __tree& __t);
template <class _ForwardIterator>
_LIBCPP_HIDE_FROM_ABI void __assign_unique(_ForwardIterator __first, _ForwardIterator __last);
@@ -1007,27 +1019,6 @@ public:
std::forward<_Args>(__args)...);
}
- template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
- _LIBCPP_HIDE_FROM_ABI void
- __insert_unique_from_orphaned_node(const_iterator __p, __get_node_value_type_t<_Tp>&& __value) {
- __emplace_hint_unique(__p, const_cast<key_type&&>(__value.first), std::move(__value.second));
- }
-
- template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
- _LIBCPP_HIDE_FROM_ABI void __insert_unique_from_orphaned_node(const_iterator __p, _Tp&& __value) {
- __emplace_hint_unique(__p, std::move(__value));
- }
-
- template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
- _LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(const_iterator __p, value_type&& __value) {
- __emplace_hint_multi(__p, const_cast<key_type&&>(__value.first), std::move(__value.second));
- }
-
- template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
- _LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(const_iterator __p, _Tp&& __value) {
- __emplace_hint_multi(__p, std::move(__value));
- }
-
template <class _InIter, class _Sent>
_LIBCPP_HIDE_FROM_ABI void __insert_range_multi(_InIter __first, _Sent __last) {
if (__first == __last)
@@ -1400,19 +1391,19 @@ private:
// copy the exact structure 1:1. Since this is for copy construction _only_ we know that we get a correct tree. If we
// didn't get a correct tree, the invariants of __tree are broken and we have a much bigger problem than an improperly
// balanced tree.
+ template <class _NodeConstructor>
#ifdef _LIBCPP_COMPILER_CLANG_BASED // FIXME: GCC complains about not being able to always_inline a recursive function
_LIBCPP_HIDE_FROM_ABI
#endif
- __node_pointer
- __copy_construct_tree(__node_pointer __src) {
+ __node_pointer __construct_from_tree(__node_pointer __src, _NodeConstructor __construct) {
if (!__src)
return nullptr;
- __node_holder __new_node = __construct_node(__src->__get_value());
+ __node_holder __new_node = __construct(__src->__get_value());
unique_ptr<__node, __tree_deleter> __left(
- __copy_construct_tree(static_cast<__node_pointer>(__src->__left_)), __node_alloc_);
- __node_pointer __right = __copy_construct_tree(static_cast<__node_pointer>(__src->__right_));
+ __construct_from_tree(static_cast<__node_pointer>(__src->__left_), __construct), __node_alloc_);
+ __node_pointer __right = __construct_from_tree(static_cast<__node_pointer>(__src->__right_), __construct);
__node_pointer __new_node_ptr = __new_node.release();
@@ -1426,46 +1417,85 @@ private:
return __new_node_ptr;
}
+ _LIBCPP_HIDE_FROM_ABI __node_pointer __copy_construct_tree(__node_pointer __src) {
+ return __construct_from_tree(__src, [this](const value_type& __val) { return __construct_node(__val); });
+ }
+
+ template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
+ _LIBCPP_HIDE_FROM_ABI __node_pointer __move_construct_tree(__node_pointer __src) {
+ return __construct_from_tree(__src, [this](value_type& __val) {
+ return __construct_node(const_cast<key_type&&>(__val.first), std::move(__val.second));
+ });
+ }
+
+ template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
+ _LIBCPP_HIDE_FROM_ABI __node_pointer __move_construct_tree(__node_pointer __src) {
+ return __construct_from_tree(__src, [this](value_type& __val) { return __construct_node(std::move(__val)); });
+ }
+
+ template <class _Assignment, class _ConstructionAlg>
// This copy assignment will always produce a correct red-black-tree assuming the incoming tree is correct, since our
// own tree is a red-black-tree and the incoming tree is a red-black-tree. The invariants of a red-black-tree are
// temporarily not met until all of the incoming red-black tree is copied.
#ifdef _LIBCPP_COMPILER_CLANG_BASED // FIXME: GCC complains about not being able to always_inline a recursive function
_LIBCPP_HIDE_FROM_ABI
#endif
- __node_pointer
- __copy_assign_tree(__node_pointer __dest, __node_pointer __src) {
+ __node_pointer __assign_from_tree(
+ __node_pointer __dest, __node_pointer __src, _Assignment __assign, _ConstructionAlg __continue_with_construct) {
if (!__src) {
destroy(__dest);
return nullptr;
}
- __assign_value(__dest->__get_value(), __src->__get_value());
+ __assign(__dest->__get_value(), __src->__get_value());
__dest->__is_black_ = __src->__is_black_;
// If we already have a left node in the destination tree, reuse it and copy-assign recursively
if (__dest->__left_) {
- __dest->__left_ = static_cast<__node_base_pointer>(__copy_assign_tree(
- static_cast<__node_pointer>(__dest->__left_), static_cast<__node_pointer>(__src->__left_)));
+ __dest->__left_ = static_cast<__node_base_pointer>(__assign_from_tree(
+ static_cast<__node_pointer>(__dest->__left_),
+ static_cast<__node_pointer>(__src->__left_),
+ __assign,
+ __continue_with_construct));
// Otherwise, we must create new nodes; copy-construct from here on
} else if (__src->__left_) {
- auto __new_left = __copy_construct_tree(static_cast<__node_pointer>(__src->__left_));
+ auto __new_left = __continue_with_construct(static_cast<__node_pointer>(__src->__left_));
__dest->__left_ = static_cast<__node_base_pointer>(__new_left);
__new_left->__parent_ = static_cast<__end_node_pointer>(__dest);
}
// Identical to the left case above, just for the right nodes
if (__dest->__right_) {
- __dest->__right_ = static_cast<__node_base_pointer>(__copy_assign_tree(
- static_cast<__node_pointer>(__dest->__right_), static_cast<__node_pointer>(__src->__right_)));
+ __dest->__right_ = static_cast<__node_base_pointer>(__assign_from_tree(
+ static_cast<__node_pointer>(__dest->__right_),
+ static_cast<__node_pointer>(__src->__right_),
+ __assign,
+ __continue_with_construct));
} else if (__src->__right_) {
- auto __new_right = __copy_construct_tree(static_cast<__node_pointer>(__src->__right_));
+ auto __new_right = __continue_with_construct(static_cast<__node_pointer>(__src->__right_));
__dest->__right_ = static_cast<__node_base_pointer>(__new_right);
__new_right->__parent_ = static_cast<__end_node_pointer>(__dest);
}
return __dest;
}
+
+ _LIBCPP_HIDE_FROM_ABI __node_pointer __copy_assign_tree(__node_pointer __dest, __node_pointer __src) {
+ return __assign_from_tree(
+ __dest,
+ __src,
+ [](value_type& __lhs, const value_type& __rhs) { __assign_value(__lhs, std::move(__rhs)); },
+ [this](__node_pointer __nd) { return __copy_construct_tree(__nd); });
+ }
+
+ _LIBCPP_HIDE_FROM_ABI __node_pointer __move_assign_tree(__node_pointer __dest, __node_pointer __src) {
+ return __assign_from_tree(
+ __dest,
+ __src,
+ [](value_type& __lhs, value_type& __rhs) { __assign_value(__lhs, std::move(__rhs)); },
+ [this](__node_pointer __nd) { return __move_construct_tree(__nd); });
+ }
};
// Precondition: __size_ != 0
@@ -1606,21 +1636,26 @@ __tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t) _NOEXCEPT_(
template <class _Tp, class _Compare, class _Allocator>
__tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t, const allocator_type& __a)
- : __node_alloc_(__node_allocator(__a)), __size_(0), __value_comp_(std::move(__t.value_comp())) {
+ : __begin_node_(__end_node()),
+ __node_alloc_(__node_allocator(__a)),
+ __size_(0),
+ __value_comp_(std::move(__t.value_comp())) {
+ if (__t.size() == 0)
+ return;
if (__a == __t.__alloc()) {
- if (__t.__size_ == 0)
- __begin_node_ = __end_node();
- else {
- __begin_node_ = __t.__begin_node_;
- __end_node()->__left_ = __t.__end_node()->__left_;
- __end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
- __size_ = __t.__size_;
- __t.__begin_node_ = __t.__end_node();
- __t.__end_node()->__left_ = nullptr;
- __t.__size_ = 0;
- }
+ __begin_node_ = __t.__begin_node_;
+ __end_node()->__left_ = __t.__end_node()->__left_;
+ __end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
+ __size_ = __t.__size_;
+ __t.__begin_node_ = __t.__end_node();
+ __t.__end_node()->__left_ = nullptr;
+ __t.__size_ = 0;
} else {
- __begin_node_ = __end_node();
+ *__root_ptr() = static_cast<__node_base_pointer>(__move_construct_tree(__t.__root()));
+ __root()->__parent_ = __end_node();
+ __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_));
+ __size_ = __t.size();
+ __t.clear(); // Ensure that __t is in a valid state after moving out the keys
}
}
@@ -1645,22 +1680,21 @@ void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, true_type)
template <class _Tp, class _Compare, class _Allocator>
void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, false_type) {
- if (__node_alloc() == __t.__node_alloc())
+ if (__node_alloc() == __t.__node_alloc()) {
__move_assign(__t, true_type());
- else {
- value_comp() = std::move(__t.value_comp());
- const_iterator __e = end();
+ } else {
+ value_comp() = std::move(__t.value_comp());
if (__size_ != 0) {
- _DetachedTreeCache __cache(this);
- while (__cache.__get() != nullptr && __t.__size_ != 0) {
- __assign_value(__cache.__get()->__get_value(), std::move(__t.remove(__t.begin())->__get_value()));
- __node_insert_multi(__cache.__get());
- __cache.__advance();
- }
- }
- while (__t.__size_ != 0) {
- __insert_multi_from_orphaned_node(__e, std::move(__t.remove(__t.begin())->__get_value()));
+ *__root_ptr() = static_cast<__node_base_pointer>(__move_assign_tree(__root(), __t.__root()));
+ } else {
+ *__root_ptr() = static_cast<__node_base_pointer>(__move_construct_tree(__t.__root()));
+ if (__root())
+ __root()->__parent_ = __end_node();
}
+ __begin_node_ =
+ __end_node()->__left_ ? static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_)) : __end_node();
+ __size_ = __t.size();
+ __t.clear(); // Ensure that __t is in a valid state after moving out the keys
}
}
diff --git a/libcxx/include/map b/libcxx/include/map
index 3ff849afcde09..28016374d3551 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -997,7 +997,7 @@ public:
_LIBCPP_HIDE_FROM_ABI map(map&& __m) = default;
- _LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a);
+ _LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a) : __tree_(std::move(__m.__tree_), __a) {}
_LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) = default;
@@ -1025,10 +1025,7 @@ public:
_LIBCPP_HIDE_FROM_ABI explicit map(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}
- _LIBCPP_HIDE_FROM_ABI map(const map& __m, const allocator_type& __a)
- : __tree_(__m.__tree_.value_comp(), typename __base::allocator_type(__a)) {
- insert(__m.begin(), __m.end());
- }
+ _LIBCPP_HIDE_FROM_ABI map(const map& __m, const allocator_type& __alloc) : __tree_(__m.__tree_, __alloc) {}
_LIBCPP_HIDE_FROM_ABI ~map() { static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
@@ -1428,18 +1425,6 @@ map(initializer_list<pair<_Key, _Tp>>, _Allocator)
# endif
# ifndef _LIBCPP_CXX03_LANG
-template <class _Key, class _Tp, class _Compare, class _Allocator>
-map<_Key, _Tp, _Compare, _Allocator>::map(map&& __m, const allocator_type& __a)
- : __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
- if (__a != __m.get_allocator()) {
- const_iterator __e = cend();
- while (!__m.empty()) {
- __tree_.__insert_unique_from_orphaned_node(
- __e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__get_value()));
- }
- }
-}
-
template <class _Key, class _Tp, class _Compare, class _Allocator>
_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
return __tree_.__emplace_unique(std::piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
@@ -1685,7 +1670,7 @@ public:
_LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) = default;
- _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a);
+ _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a) : __tree_(std::move(__m.__tree_), __a) {}
_LIBCPP_HIDE_FROM_ABI multimap& operator=(multimap&& __m) = default;
@@ -1714,10 +1699,7 @@ public:
_LIBCPP_HIDE_FROM_ABI explicit multimap(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}
- _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m, const allocator_type& __a)
- : __tree_(__m.__tree_.value_comp(), typename __base::allocator_type(__a)) {
- insert(__m.begin(), __m.end());
- }
+ _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m, const allocator_type& __a) : __tree_(__m.__tree_, __a) {}
_LIBCPP_HIDE_FROM_ABI ~multimap() {
static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), "");
@@ -1992,19 +1974,6 @@ multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
-> multimap<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
# endif
-# ifndef _LIBCPP_CXX03_LANG
-template <class _Key, class _Tp, class _Compare, class _Allocator>
-multimap<_Key, _Tp, _Compare, _Allocator>::multimap(multimap&& __m, const allocator_type& __a)
- : __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
- if (__a != __m.get_allocator()) {
- const_iterator __e = cend();
- while (!__m.empty())
- __tree_.__insert_multi_from_orphaned_node(
- __e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__get_value()));
- }
-}
-# endif
-
template <class _Key, class _Tp, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
diff --git a/libcxx/include/set b/libcxx/include/set
index 59ed0155c1def..9c04a58a4ffaa 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -673,12 +673,10 @@ public:
_LIBCPP_HIDE_FROM_ABI explicit set(const allocator_type& __a) : __tree_(__a) {}
- _LIBCPP_HIDE_FROM_ABI set(const set& __s, const allocator_type& __a) : __tree_(__s.__tree_.value_comp(), __a) {
- insert(__s.begin(), __s.end());
- }
+ _LIBCPP_HIDE_FROM_ABI set(const set& __s, const allocator_type& __alloc) : __tree_(__s.__tree_, __alloc) {}
# ifndef _LIBCPP_CXX03_LANG
- _LIBCPP_HIDE_FROM_ABI set(set&& __s, const allocator_type& __a);
+ _LIBCPP_HIDE_FROM_ABI set(set&& __s, const allocator_type& __alloc) : __tree_(std::move(__s.__tree_), __alloc) {}
_LIBCPP_HIDE_FROM_ABI set(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
: __tree_(__comp) {
@@ -948,19 +946,6 @@ template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Al
set(initializer_list<_Key>, _Allocator) -> set<_Key, less<_Key>, _Allocator>;
# endif
-# ifndef _LIBCPP_CXX03_LANG
-
-template <class _Key, class _Compare, class _Allocator>
-set<_Key, _Compare, _Allocator>::set(set&& __s, const allocator_type& __a) : __tree_(std::move(__s.__tree_), __a) {
- if (__a != __s.get_allocator()) {
- const_iterator __e = cend();
- while (!__s.empty())
- insert(__e, std::move(__s.__tree_.remove(__s.begin())->__get_value()));
- }
-}
-
-# endif // _LIBCPP_CXX03_LANG
-
template <class _Key, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
@@ -1130,13 +1115,10 @@ public:
# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) = default;
- _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a);
+ _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a) : __tree_(std::move(__s.__tree_), __a) {}
# endif // _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI explicit multiset(const allocator_type& __a) : __tree_(__a) {}
- _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s, const allocator_type& __a)
- : __tree_(__s.__tree_.value_comp(), __a) {
- insert(__s.begin(), __s.end());
- }
+ _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s, const allocator_type& __a) : __tree_(__s.__tree_, __a) {}
# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI multiset(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
@@ -1409,20 +1391,6 @@ template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Al
multiset(initializer_list<_Key>, _Allocator) -> multiset<_Key, less<_Key>, _Allocator>;
# endif
-# ifndef _LIBCPP_CXX03_LANG
-
-template <class _Key, class _Compare, class _Allocator>
-multiset<_Key, _Compare, _Allocator>::multiset(multiset&& __s, const allocator_type& __a)
- : __tree_(std::move(__s.__tree_), __a) {
- if (__a != __s.get_allocator()) {
- const_iterator __e = cend();
- while (!__s.empty())
- insert(__e, std::move(__s.__tree_.remove(__s.begin())->__get_value()));
- }
-}
-
-# endif // _LIBCPP_CXX03_LANG
-
template <class _Key, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
diff --git a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
index 22a6d0d753b0c..f7832a81987ff 100644
--- a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
+++ b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
@@ -11,6 +11,7 @@
#include <algorithm>
#include <iterator>
+#include <memory_resource>
#include <random>
#include <string>
#include <ranges>
@@ -33,6 +34,9 @@ struct adapt_operations {
// using InsertionResult = ...;
// static Container::iterator get_iterator(InsertionResult const&);
+
+ // template <class Allocator>
+ // using rebind_alloc = ...;
};
template <class Container>
@@ -103,6 +107,61 @@ void associative_container_benchmarks(std::string container) {
}
});
+ bench("ctor(const&, alloc)", [=](auto& st) {
+ const std::size_t size = st.range(0);
+ std::vector<Value> in = make_value_types(generate_unique_keys(size));
+ Container src(in.begin(), in.end());
+ ScratchSpace c[BatchSize];
+
+ while (st.KeepRunningBatch(BatchSize)) {
+ for (std::size_t i = 0; i != BatchSize; ++i) {
+ new (c + i) Container(src, typename std::allocator<typename Container::value_type>());
+ benchmark::DoNotOptimize(c + i);
+ benchmark::ClobberMemory();
+ }
+
+ st.PauseTiming();
+ for (std::size_t i = 0; i != BatchSize; ++i) {
+ reinterpret_cast<Container*>(c + i)->~Container();
+ }
+ st.ResumeTiming();
+ }
+ });
+
+ bench("ctor(&&, different allocs)", [=](auto& st) {
+ using PMRContainer = adapt_operations<Container>::template rebind_alloc<
+ std::pmr::polymorphic_allocator<typename Container::value_type>>;
+
+ const std::size_t size = st.range(0);
+ std::vector<Value> in = make_value_types(generate_unique_keys(size));
+ std::pmr::monotonic_buffer_resource rs(524288 * 32);
+ std::vector<PMRContainer> srcs;
+ srcs.reserve(BatchSize);
+ for (size_t i = 0; i != BatchSize; ++i)
+ srcs.emplace_back(&rs).insert(in.begin(), in.end());
+ alignas(PMRContainer) char c[BatchSize * sizeof(PMRContainer)];
+
+ std::pmr::monotonic_buffer_resource rs2(524288 * 32);
+ while (st.KeepRunningBatch(BatchSize)) {
+ for (std::size_t i = 0; i != BatchSize; ++i) {
+ new (c + i * sizeof(PMRContainer)) PMRContainer(std::move(srcs[i]), &rs2);
+ benchmark::DoNotOptimize(c + i);
+ benchmark::ClobberMemory();
+ }
+
+ st.PauseTiming();
+ for (std::size_t i = 0; i != BatchSize; ++i) {
+ reinterpret_cast<PMRContainer*>(c + i * sizeof(PMRContainer))->~PMRContainer();
+ }
+ rs2.release();
+ srcs.clear();
+ for (size_t i = 0; i != BatchSize; ++i)
+ srcs.emplace_back(&rs).insert(in.begin(), in.end());
+
+ st.ResumeTiming();
+ }
+ });
+
bench("ctor(iterator, iterator) (unsorted sequence)", [=](auto& st) {
const std::size_t size = st.range(0);
std::mt19937 randomness;
diff --git a/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp b/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
index f3b86554802ca..407afb14e1e13 100644
--- a/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
@@ -24,6 +24,14 @@ struct support::adapt_operations<std::flat_map<K, V>> {
using InsertionResult = std::pair<typename std::flat_map<K, V>::iterator, bool>;
static auto get_iterator(InsertionResult const& result) { return result.first; }
+
+ template <class Allocator>
+ using rebind_alloc =
+ std::flat_map<K,
+ V,
+ std::less<K>,
+ std::vector<K, typename std::allocator_traits<Allocator>::template rebind_alloc<K>>,
+ std::vector<V, typename std::allocator_traits<Allocator>::template rebind_alloc<V>>>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp b/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
index 80eaa549042c6..4f70d26116b0b 100644
--- a/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
@@ -23,6 +23,14 @@ struct support::adapt_operations<std::flat_multimap<K, V>> {
using InsertionResult = typename std::flat_multimap<K, V>::iterator;
static auto get_iterator(InsertionResult const& result) { return result; }
+
+ template <class Allocator>
+ using rebind_alloc =
+ std::flat_multimap<K,
+ V,
+ std::less<K>,
+ std::vector<K, typename std::allocator_traits<Allocator>::template rebind_alloc<K>>,
+ std::vector<V, typename std::allocator_traits<Allocator>::template rebind_alloc<V>>>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/map.bench.cpp b/libcxx/test/benchmarks/containers/associative/map.bench.cpp
index 142229ae64cad..cc9ffd857caf2 100644
--- a/libcxx/test/benchmarks/containers/associative/map.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/map.bench.cpp
@@ -38,6 +38,9 @@ struct support::adapt_operations<std::map<K, V>> {
using InsertionResult = std::pair<typename std::map<K, V>::iterator, bool>;
static auto get_iterator(InsertionResult const& result) { return result.first; }
+
+ template <class Allocator>
+ using rebind_alloc = std::map<K, V, std::less<K>, Allocator>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/multimap.bench.cpp b/libcxx/test/benchmarks/containers/associative/multimap.bench.cpp
index 15a0b573081bb..8e3abf0b7cf8b 100644
--- a/libcxx/test/benchmarks/containers/associative/multimap.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/multimap.bench.cpp
@@ -24,6 +24,9 @@ struct support::adapt_operations<std::multimap<K, V>> {
using InsertionResult = typename std::multimap<K, V>::iterator;
static auto get_iterator(InsertionResult const& result) { return result; }
+
+ template <class Allocator>
+ using rebind_alloc = std::multimap<K, V, std::less<K>, Allocator>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/multiset.bench.cpp b/libcxx/test/benchmarks/containers/associative/multiset.bench.cpp
index c205e0a4f793f..7bafd0ab52dce 100644
--- a/libcxx/test/benchmarks/containers/associative/multiset.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/multiset.bench.cpp
@@ -22,6 +22,9 @@ struct support::adapt_operations<std::multiset<K>> {
using InsertionResult = typename std::multiset<K>::iterator;
static auto get_iterator(InsertionResult const& result) { return result; }
+
+ template <class Allocator>
+ using rebind_alloc = std::multiset<K, std::less<K>, Allocator>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/set.bench.cpp b/libcxx/test/benchmarks/containers/associative/set.bench.cpp
index 50ee142b6e8b3..e5a6cc58913d2 100644
--- a/libcxx/test/benchmarks/containers/associative/set.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/set.bench.cpp
@@ -23,6 +23,9 @@ struct support::adapt_operations<std::set<K>> {
using InsertionResult = std::pair<typename std::set<K>::iterator, bool>;
static auto get_iterator(InsertionResult const& result) { return result.first; }
+
+ template <class Allocator>
+ using rebind_alloc = std::set<K, std::less<K>, Allocator>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_map.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_map.bench.cpp
index d670c531910ea..ddfc90c306010 100644
--- a/libcxx/test/benchmarks/containers/associative/unordered_map.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/unordered_map.bench.cpp
@@ -37,6 +37,9 @@ struct support::adapt_operations<std::unordered_map<K, V>> {
using InsertionResult = std::pair<typename std::unordered_map<K, V>::iterator, bool>;
static auto get_iterator(InsertionResult const& result) { return result.first; }
+
+ template <class Allocator>
+ using rebind_alloc = std::unordered_map<K, V, std::hash<K>, std::equal_to<K>, Allocator>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_multimap.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_multimap.bench.cpp
index 8738ca4bf9f0c..5d92bd8b2deaf 100644
--- a/libcxx/test/benchmarks/containers/associative/unordered_multimap.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/unordered_multimap.bench.cpp
@@ -23,6 +23,9 @@ struct support::adapt_operations<std::unordered_multimap<K, V>> {
using InsertionResult = typename std::unordered_multimap<K, V>::iterator;
static auto get_iterator(InsertionResult const& result) { return result; }
+
+ template <class Allocator>
+ using rebind_alloc = std::unordered_multimap<K, V, std::hash<K>, std::equal_to<K>, Allocator>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_multiset.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_multiset.bench.cpp
index 4888b01bfeba0..09412fc4aeae7 100644
--- a/libcxx/test/benchmarks/containers/associative/unordered_multiset.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/unordered_multiset.bench.cpp
@@ -22,6 +22,9 @@ struct support::adapt_operations<std::unordered_multiset<K>> {
using InsertionResult = typename std::unordered_multiset<K>::iterator;
static auto get_iterator(InsertionResult const& result) { return result; }
+
+ template <class Allocator>
+ using rebind_alloc = std::unordered_multiset<K, std::hash<K>, std::equal_to<K>, Allocator>;
};
int main(int argc, char** argv) {
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
index 89443a597e85a..1b6663321b43c 100644
--- a/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
@@ -24,6 +24,9 @@ struct support::adapt_operations<std::unordered_set<K>> {
using InsertionResult = std::pair<typename std::unordered_set<K>::iterator, bool>;
static auto get_iterator(InsertionResult const& result) { return result.first; }
+
+ template <class Allocator>
+ using rebind_alloc = std::unordered_set<K, std::hash<K>, std::equal_to<K>, Allocator>;
};
int main(int argc, char** argv) {
More information about the libcxx-commits
mailing list