[libcxx-commits] [libcxx] [libc++] Optimize copy construction and assignment of __tree (PR #151304)
Nikolas Klauser via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Jul 30 03:26:37 PDT 2025
https://github.com/philnik777 created https://github.com/llvm/llvm-project/pull/151304
None
>From 01b932d12567ab81c425ddd72ebf84878e0069db Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser at berlin.de>
Date: Wed, 30 Jul 2025 12:26:00 +0200
Subject: [PATCH] [libc++] Optimize copy construction and assignment of __tree
---
libcxx/include/__tree | 128 ++++++++++++++++--
libcxx/include/map | 8 +-
libcxx/include/set | 8 +-
.../associative_container_benchmarks.h | 17 ++-
.../containers/associative/map.bench.cpp | 1 +
5 files changed, 136 insertions(+), 26 deletions(-)
diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index f8bb4f01b1e29..ec1ea81572fce 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -1213,6 +1213,86 @@ private:
__node_pointer __cache_root_;
__node_pointer __cache_elem_;
};
+
+ class __tree_deleter {
+ __node_allocator& __alloc_;
+
+ public:
+ using pointer = __node_pointer;
+
+ _LIBCPP_HIDE_FROM_ABI __tree_deleter(__node_allocator& __alloc) : __alloc_(__alloc) {}
+
+ void operator()(__node_pointer __ptr) {
+ if (!__ptr)
+ return;
+
+ (*this)(static_cast<__node_pointer>(__ptr->__left_));
+
+ auto __right = __ptr->__right_;
+
+ __node_traits::destroy(__alloc_, std::addressof(__ptr->__value_));
+ __node_traits::deallocate(__alloc_, __ptr, 1);
+
+ (*this)(static_cast<__node_pointer>(__right));
+ }
+ };
+
+ class __allocate_node_builder {
+ __tree& __base_;
+
+ public:
+ __allocate_node_builder(__tree& __base) : __base_(__base) {}
+
+ template <class _Up>
+ __node_holder operator()(_Up&& __v) {
+ return __base_.__construct_node(std::forward<_Up>(__v));
+ }
+ };
+
+ class __cache_node_builder {
+ __tree& __base_;
+ _DetachedTreeCache& __cache_;
+
+ public:
+ __cache_node_builder(__tree& __base, _DetachedTreeCache& __cache) : __base_(__base), __cache_(__cache) {}
+
+ template <class _Up>
+ __node_holder operator()(_Up&& __v) {
+ __node_pointer __ret = __cache_.__get();
+ if (__ret) {
+ __assign_value(__ret->__value_, std::forward<_Up>(__v));
+ __cache_.__advance();
+ }
+ return __node_holder(__ret, _Dp(__base_.__node_alloc_));
+ }
+ };
+
+ template <class _NodeBuilder>
+ _LIBCPP_HIDE_FROM_ABI __node_pointer __copy_tree(_NodeBuilder& __builder, __node_pointer __src) {
+ if (!__src)
+ return nullptr;
+
+ __node_holder __new_node = __builder(__src->__value_);
+ if (!__new_node) {
+ __allocate_node_builder __alloc_builder(*this);
+ return __copy_tree(__alloc_builder, __src);
+ }
+
+ unique_ptr<__node, __tree_deleter> __left(
+ __copy_tree(__builder, static_cast<__node_pointer>(__src->__left_)), __node_alloc_);
+ __node_pointer __right = __copy_tree(__builder, static_cast<__node_pointer>(__src->__right_));
+
+ __node_pointer __new_node_ptr = __new_node.release();
+
+ __new_node_ptr->__is_black_ = __src->__is_black_;
+ __new_node_ptr->__left_ = static_cast<__node_base_pointer>(__left.release());
+ __new_node_ptr->__right_ = static_cast<__node_base_pointer>(__right);
+ if (__new_node_ptr->__left_)
+ __new_node_ptr->__left_->__parent_ = static_cast<__end_node_pointer>(__new_node_ptr);
+ if (__right)
+ __right->__parent_ = static_cast<__end_node_pointer>(__new_node_ptr);
+ return __new_node_ptr;
+ }
};
template <class _Tp, class _Compare, class _Allocator>
@@ -1277,11 +1357,31 @@ __tree<_Tp, _Compare, _Allocator>::_DetachedTreeCache::__detach_next(__node_poin
template <class _Tp, class _Compare, class _Allocator>
__tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(const __tree& __t) {
- if (this != std::addressof(__t)) {
- value_comp() = __t.value_comp();
- __copy_assign_alloc(__t);
- __assign_multi(__t.begin(), __t.end());
+ if (this == std::addressof(__t))
+ return *this;
+
+ value_comp() = __t.value_comp();
+ __copy_assign_alloc(__t);
+
+ if (__t.size() == 0) {
+ clear();
+ return *this;
+ }
+
+ if (__size_ != 0) {
+ _DetachedTreeCache __cache(this);
+ __cache_node_builder __builder(*this, __cache);
+ __end_node_.__left_ =
+ static_cast<__node_base_pointer>(__copy_tree(__builder, static_cast<__node_pointer>(__t.__end_node_.__left_)));
+ } else {
+ __allocate_node_builder __builder(*this);
+ __end_node_.__left_ =
+ static_cast<__node_base_pointer>(__copy_tree(__builder, static_cast<__node_pointer>(__t.__end_node_.__left_)));
}
+ __end_node_.__left_->__parent_ = __end_node();
+ __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(static_cast<__node_base_pointer>(__end_node())));
+ __size_ = __t.__size_;
+
return *this;
}
@@ -1327,11 +1427,19 @@ void __tree<_Tp, _Compare, _Allocator>::__assign_multi(_InputIterator __first, _
template <class _Tp, class _Compare, class _Allocator>
__tree<_Tp, _Compare, _Allocator>::__tree(const __tree& __t)
- : __begin_node_(),
+ : __begin_node_(__end_node()),
__node_alloc_(__node_traits::select_on_container_copy_construction(__t.__node_alloc())),
__size_(0),
__value_comp_(__t.value_comp()) {
- __begin_node_ = __end_node();
+ if (__t.__size_ == 0)
+ return;
+
+ __allocate_node_builder __builder(*this);
+ __end_node_.__left_ =
+ static_cast<__node_base_pointer>(__copy_tree(__builder, static_cast<__node_pointer>(__t.__end_node_.__left_)));
+ __end_node_.__left_->__parent_ = __end_node();
+ __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(static_cast<__node_base_pointer>(__end_node())));
+ __size_ = __t.__size_;
}
template <class _Tp, class _Compare, class _Allocator>
@@ -1430,13 +1538,7 @@ __tree<_Tp, _Compare, _Allocator>::~__tree() {
template <class _Tp, class _Compare, class _Allocator>
void __tree<_Tp, _Compare, _Allocator>::destroy(__node_pointer __nd) _NOEXCEPT {
- if (__nd != nullptr) {
- destroy(static_cast<__node_pointer>(__nd->__left_));
- destroy(static_cast<__node_pointer>(__nd->__right_));
- __node_allocator& __na = __node_alloc();
- __node_traits::destroy(__na, std::addressof(__nd->__value_));
- __node_traits::deallocate(__na, __nd, 1);
- }
+ (__tree_deleter(__node_alloc_))(__nd);
}
template <class _Tp, class _Compare, class _Allocator>
diff --git a/libcxx/include/map b/libcxx/include/map
index 2251565801470..0a43bd09a0b16 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -970,7 +970,7 @@ public:
: map(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
# endif
- _LIBCPP_HIDE_FROM_ABI map(const map& __m) : __tree_(__m.__tree_) { insert(__m.begin(), __m.end()); }
+ _LIBCPP_HIDE_FROM_ABI map(const map& __m) = default;
_LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) = default;
@@ -1637,11 +1637,7 @@ public:
: multimap(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
# endif
- _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m)
- : __tree_(__m.__tree_.value_comp(),
- __alloc_traits::select_on_container_copy_construction(__m.__tree_.__alloc())) {
- insert(__m.begin(), __m.end());
- }
+ _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m) = default;
_LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) = default;
diff --git a/libcxx/include/set b/libcxx/include/set
index 1f2fd7fc91bbb..342a5294c814f 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -660,7 +660,7 @@ public:
: set(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
# endif
- _LIBCPP_HIDE_FROM_ABI set(const set& __s) : __tree_(__s.__tree_) { insert(__s.begin(), __s.end()); }
+ _LIBCPP_HIDE_FROM_ABI set(const set& __s) = default;
_LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) = default;
@@ -1119,11 +1119,7 @@ public:
: multiset(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
# endif
- _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s)
- : __tree_(__s.__tree_.value_comp(),
- __alloc_traits::select_on_container_copy_construction(__s.__tree_.__alloc())) {
- insert(__s.begin(), __s.end());
- }
+ _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s) = default;
_LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) = default;
diff --git a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
index 0ff7f15164d8a..6cf2dcfc59e13 100644
--- a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
+++ b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
@@ -151,7 +151,7 @@ void associative_container_benchmarks(std::string container) {
/////////////////////////
// Assignment
/////////////////////////
- bench("operator=(const&)", [=](auto& st) {
+ bench("operator=(const&) (into cleared Container)", [=](auto& st) {
const std::size_t size = st.range(0);
std::vector<Value> in = make_value_types(generate_unique_keys(size));
Container src(in.begin(), in.end());
@@ -172,6 +172,21 @@ void associative_container_benchmarks(std::string container) {
}
});
+ bench("operator=(const&) (into populated Container)", [=](auto& st) {
+ const std::size_t size = st.range(0);
+ std::vector<Value> in = make_value_types(generate_unique_keys(size));
+ Container src(in.begin(), in.end());
+ Container c[BatchSize];
+
+ while (st.KeepRunningBatch(BatchSize)) {
+ for (std::size_t i = 0; i != BatchSize; ++i) {
+ c[i] = src;
+ benchmark::DoNotOptimize(c[i]);
+ benchmark::ClobberMemory();
+ }
+ }
+ });
+
/////////////////////////
// Insertion
/////////////////////////
diff --git a/libcxx/test/benchmarks/containers/associative/map.bench.cpp b/libcxx/test/benchmarks/containers/associative/map.bench.cpp
index cee669ae0a667..bd664dbb56ee7 100644
--- a/libcxx/test/benchmarks/containers/associative/map.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/map.bench.cpp
@@ -29,6 +29,7 @@ struct support::adapt_operations<std::map<K, V>> {
int main(int argc, char** argv) {
support::associative_container_benchmarks<std::map<int, int>>("std::map<int, int>");
+ support::associative_container_benchmarks<std::map<std::string, int>>("std::map<std::string, int>");
benchmark::Initialize(&argc, argv);
benchmark::RunSpecifiedBenchmarks();
More information about the libcxx-commits
mailing list