[libcxx-commits] [libcxx] [libc++] Optimize copy construction and assignment of __tree (PR #151304)
Nikolas Klauser via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Jul 30 04:36:41 PDT 2025
https://github.com/philnik777 updated https://github.com/llvm/llvm-project/pull/151304
>From 651b570533abd208f1d708b2c24ec7cad3109110 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser at berlin.de>
Date: Wed, 30 Jul 2025 12:26:00 +0200
Subject: [PATCH] [libc++] Optimize copy construction and assignment of __tree
---
libcxx/docs/ReleaseNotes/22.rst | 3 +
libcxx/include/__tree | 118 ++++++++++++++++--
libcxx/include/map | 8 +-
libcxx/include/set | 8 +-
.../associative_container_benchmarks.h | 17 ++-
.../containers/associative/map.bench.cpp | 1 +
6 files changed, 129 insertions(+), 26 deletions(-)
diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst
index 15bf46d44b07f..8b8dce5083149 100644
--- a/libcxx/docs/ReleaseNotes/22.rst
+++ b/libcxx/docs/ReleaseNotes/22.rst
@@ -43,6 +43,9 @@ Implemented Papers
Improvements and New Features
-----------------------------
+- The performance of ``map::map(const map&)`` has been improved up to 2.3x
+- The performance of ``map::operator=(const map&)`` has been improved by up to 11x
+
Deprecations and Removals
-------------------------
diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index f8bb4f01b1e29..c1f667b7ec74f 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -1213,6 +1213,80 @@ private:
__node_pointer __cache_root_;
__node_pointer __cache_elem_;
};
+
+ class __tree_deleter {
+ __node_allocator& __alloc_;
+
+ public:
+ using pointer = __node_pointer;
+
+ _LIBCPP_HIDE_FROM_ABI __tree_deleter(__node_allocator& __alloc) : __alloc_(__alloc) {}
+
+ void operator()(__node_pointer __ptr) {
+ if (!__ptr)
+ return;
+
+ (*this)(static_cast<__node_pointer>(__ptr->__left_));
+
+ auto __right = __ptr->__right_;
+
+ __node_traits::destroy(__alloc_, std::addressof(__ptr->__value_));
+ __node_traits::deallocate(__alloc_, __ptr, 1);
+
+ (*this)(static_cast<__node_pointer>(__right));
+ }
+ };
+
+ _LIBCPP_HIDE_FROM_ABI __node_pointer __copy_construct_tree(__node_pointer __src) {
+ if (!__src)
+ return nullptr;
+
+ __node_holder __new_node = __construct_node(__src->__value_);
+
+ unique_ptr<__node, __tree_deleter> __left(
+ __copy_construct_tree(static_cast<__node_pointer>(__src->__left_)), __node_alloc_);
+ __node_pointer __right = __copy_construct_tree(static_cast<__node_pointer>(__src->__right_));
+
+ __node_pointer __new_node_ptr = __new_node.release();
+
+ __new_node_ptr->__is_black_ = __src->__is_black_;
+ __new_node_ptr->__left_ = static_cast<__node_base_pointer>(__left.release());
+ __new_node_ptr->__right_ = static_cast<__node_base_pointer>(__right);
+ if (__new_node_ptr->__left_)
+ __new_node_ptr->__left_->__parent_ = static_cast<__end_node_pointer>(__new_node_ptr);
+ if (__new_node_ptr->__right_)
+ __new_node_ptr->__right_->__parent_ = static_cast<__end_node_pointer>(__new_node_ptr);
+ return __new_node_ptr;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI __node_pointer __copy_assign_tree(__node_pointer __assign_to, __node_pointer __src) {
+ if (!__src) {
+ destroy(__assign_to);
+ return nullptr;
+ }
+
+ __assign_value(__assign_to->__value_, __src->__value_);
+
+ if (__assign_to->__left_) {
+ __assign_to->__left_ = static_cast<__node_base_pointer>(__copy_assign_tree(
+ static_cast<__node_pointer>(__assign_to->__left_), static_cast<__node_pointer>(__src->__left_)));
+ } else if (__src->__left_) {
+ auto __new_left = __copy_construct_tree(static_cast<__node_pointer>(__src->__left_));
+ __assign_to->__left_ = static_cast<__node_base_pointer>(__new_left);
+ __new_left->__parent_ = static_cast<__end_node_pointer>(__assign_to);
+ }
+
+ if (__assign_to->__right_) {
+ __assign_to->__right_ = static_cast<__node_base_pointer>(__copy_assign_tree(
+ static_cast<__node_pointer>(__assign_to->__right_), static_cast<__node_pointer>(__src->__right_)));
+ } else if (__src->__right_) {
+ auto __new_right = __copy_construct_tree(static_cast<__node_pointer>(__src->__right_));
+ __assign_to->__right_ = static_cast<__node_base_pointer>(__new_right);
+ __new_right->__parent_ = static_cast<__end_node_pointer>(__assign_to);
+ }
+
+ return __assign_to;
+ }
};
template <class _Tp, class _Compare, class _Allocator>
@@ -1277,11 +1351,28 @@ __tree<_Tp, _Compare, _Allocator>::_DetachedTreeCache::__detach_next(__node_poin
template <class _Tp, class _Compare, class _Allocator>
__tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(const __tree& __t) {
- if (this != std::addressof(__t)) {
- value_comp() = __t.value_comp();
- __copy_assign_alloc(__t);
- __assign_multi(__t.begin(), __t.end());
+ if (this == std::addressof(__t))
+ return *this;
+
+ value_comp() = __t.value_comp();
+ __copy_assign_alloc(__t);
+
+ if (__t.size() == 0) {
+ clear();
+ return *this;
+ }
+
+ if (__size_ != 0) {
+ __end_node_.__left_ = static_cast<__node_base_pointer>(__copy_assign_tree(
+ static_cast<__node_pointer>(__end_node_.__left_), static_cast<__node_pointer>(__t.__end_node_.__left_)));
+ } else {
+ __end_node_.__left_ =
+ static_cast<__node_base_pointer>(__copy_construct_tree(static_cast<__node_pointer>(__t.__end_node_.__left_)));
+ __end_node_.__left_->__parent_ = __end_node();
}
+ __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(static_cast<__node_base_pointer>(__end_node())));
+ __size_ = __t.__size_;
+
return *this;
}
@@ -1327,11 +1418,18 @@ void __tree<_Tp, _Compare, _Allocator>::__assign_multi(_InputIterator __first, _
template <class _Tp, class _Compare, class _Allocator>
__tree<_Tp, _Compare, _Allocator>::__tree(const __tree& __t)
- : __begin_node_(),
+ : __begin_node_(__end_node()),
__node_alloc_(__node_traits::select_on_container_copy_construction(__t.__node_alloc())),
__size_(0),
__value_comp_(__t.value_comp()) {
- __begin_node_ = __end_node();
+ if (__t.__size_ == 0)
+ return;
+
+ __end_node_.__left_ =
+ static_cast<__node_base_pointer>(__copy_construct_tree(static_cast<__node_pointer>(__t.__end_node_.__left_)));
+ __end_node_.__left_->__parent_ = __end_node();
+ __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(static_cast<__node_base_pointer>(__end_node())));
+ __size_ = __t.__size_;
}
template <class _Tp, class _Compare, class _Allocator>
@@ -1430,13 +1528,7 @@ __tree<_Tp, _Compare, _Allocator>::~__tree() {
template <class _Tp, class _Compare, class _Allocator>
void __tree<_Tp, _Compare, _Allocator>::destroy(__node_pointer __nd) _NOEXCEPT {
- if (__nd != nullptr) {
- destroy(static_cast<__node_pointer>(__nd->__left_));
- destroy(static_cast<__node_pointer>(__nd->__right_));
- __node_allocator& __na = __node_alloc();
- __node_traits::destroy(__na, std::addressof(__nd->__value_));
- __node_traits::deallocate(__na, __nd, 1);
- }
+ (__tree_deleter(__node_alloc_))(__nd);
}
template <class _Tp, class _Compare, class _Allocator>
diff --git a/libcxx/include/map b/libcxx/include/map
index 2251565801470..0a43bd09a0b16 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -970,7 +970,7 @@ public:
: map(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
# endif
- _LIBCPP_HIDE_FROM_ABI map(const map& __m) : __tree_(__m.__tree_) { insert(__m.begin(), __m.end()); }
+ _LIBCPP_HIDE_FROM_ABI map(const map& __m) = default;
_LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) = default;
@@ -1637,11 +1637,7 @@ public:
: multimap(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
# endif
- _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m)
- : __tree_(__m.__tree_.value_comp(),
- __alloc_traits::select_on_container_copy_construction(__m.__tree_.__alloc())) {
- insert(__m.begin(), __m.end());
- }
+ _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m) = default;
_LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) = default;
diff --git a/libcxx/include/set b/libcxx/include/set
index 1f2fd7fc91bbb..342a5294c814f 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -660,7 +660,7 @@ public:
: set(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
# endif
- _LIBCPP_HIDE_FROM_ABI set(const set& __s) : __tree_(__s.__tree_) { insert(__s.begin(), __s.end()); }
+ _LIBCPP_HIDE_FROM_ABI set(const set& __s) = default;
_LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) = default;
@@ -1119,11 +1119,7 @@ public:
: multiset(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
# endif
- _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s)
- : __tree_(__s.__tree_.value_comp(),
- __alloc_traits::select_on_container_copy_construction(__s.__tree_.__alloc())) {
- insert(__s.begin(), __s.end());
- }
+ _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s) = default;
_LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) = default;
diff --git a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
index 0ff7f15164d8a..6cf2dcfc59e13 100644
--- a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
+++ b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
@@ -151,7 +151,7 @@ void associative_container_benchmarks(std::string container) {
/////////////////////////
// Assignment
/////////////////////////
- bench("operator=(const&)", [=](auto& st) {
+ bench("operator=(const&) (into cleared Container)", [=](auto& st) {
const std::size_t size = st.range(0);
std::vector<Value> in = make_value_types(generate_unique_keys(size));
Container src(in.begin(), in.end());
@@ -172,6 +172,21 @@ void associative_container_benchmarks(std::string container) {
}
});
+ bench("operator=(const&) (into populated Container)", [=](auto& st) {
+ const std::size_t size = st.range(0);
+ std::vector<Value> in = make_value_types(generate_unique_keys(size));
+ Container src(in.begin(), in.end());
+ Container c[BatchSize];
+
+ while (st.KeepRunningBatch(BatchSize)) {
+ for (std::size_t i = 0; i != BatchSize; ++i) {
+ c[i] = src;
+ benchmark::DoNotOptimize(c[i]);
+ benchmark::ClobberMemory();
+ }
+ }
+ });
+
/////////////////////////
// Insertion
/////////////////////////
diff --git a/libcxx/test/benchmarks/containers/associative/map.bench.cpp b/libcxx/test/benchmarks/containers/associative/map.bench.cpp
index cee669ae0a667..bd664dbb56ee7 100644
--- a/libcxx/test/benchmarks/containers/associative/map.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/map.bench.cpp
@@ -29,6 +29,7 @@ struct support::adapt_operations<std::map<K, V>> {
int main(int argc, char** argv) {
support::associative_container_benchmarks<std::map<int, int>>("std::map<int, int>");
+ support::associative_container_benchmarks<std::map<std::string, int>>("std::map<std::string, int>");
benchmark::Initialize(&argc, argv);
benchmark::RunSpecifiedBenchmarks();
More information about the libcxx-commits
mailing list