[libcxx-commits] [libcxx] [libc++] Speed-up and refactor move-assignment operator for vector<bool> (PR #119817)
Peng Liu via libcxx-commits
libcxx-commits at lists.llvm.org
Sat Dec 14 18:15:12 PST 2024
https://github.com/winner245 updated https://github.com/llvm/llvm-project/pull/119817
>From 8683b21c14c7d60c244a09ce217b26bf41ca04da Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245 at hotmail.com>
Date: Thu, 12 Dec 2024 13:05:06 -0500
Subject: [PATCH] Speed-up and refactor move-assignment operator for
vector<bool>
---
libcxx/include/__vector/vector_bool.h | 27 ++-
.../containers/ContainerBenchmarks.h | 14 ++
.../vector_bool_operations.bench.cpp | 77 +++++++++
.../vector.bool/assign_copy.pass.cpp | 83 +++++----
.../vector.bool/assign_move.pass.cpp | 163 +++++++++++-------
5 files changed, 260 insertions(+), 104 deletions(-)
create mode 100644 libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp
diff --git a/libcxx/include/__vector/vector_bool.h b/libcxx/include/__vector/vector_bool.h
index 36eb7f350ac406..17bfe02f8becf7 100644
--- a/libcxx/include/__vector/vector_bool.h
+++ b/libcxx/include/__vector/vector_bool.h
@@ -398,6 +398,8 @@ class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
__guard.__complete();
}
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_by_words(const vector& __v);
+
template <class _Iterator, class _Sentinel>
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
@@ -695,18 +697,25 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v
}
}
+// This function copies each storage word as a whole, which is substantially more efficient than copying each bit
+// individually
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void vector<bool, _Allocator>::__copy_by_words(const vector& __v) {
+ if (__v.__size_) {
+ if (__v.__size_ > capacity()) {
+ __vdeallocate();
+ __vallocate(__v.__size_);
+ }
+ std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
+ }
+ __size_ = __v.__size_;
+}
+
template <class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>& vector<bool, _Allocator>::operator=(const vector& __v) {
if (this != std::addressof(__v)) {
__copy_assign_alloc(__v);
- if (__v.__size_) {
- if (__v.__size_ > capacity()) {
- __vdeallocate();
- __vallocate(__v.__size_);
- }
- std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
- }
- __size_ = __v.__size_;
+ __copy_by_words(__v);
}
return *this;
}
@@ -754,7 +763,7 @@ vector<bool, _Allocator>::operator=(vector&& __v)
template <class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, false_type) {
if (__alloc_ != __c.__alloc_)
- assign(__c.begin(), __c.end());
+ __copy_by_words(__c);
else
__move_assign(__c, true_type());
}
diff --git a/libcxx/test/benchmarks/containers/ContainerBenchmarks.h b/libcxx/test/benchmarks/containers/ContainerBenchmarks.h
index 6d21e12896ec9e..6f565049920139 100644
--- a/libcxx/test/benchmarks/containers/ContainerBenchmarks.h
+++ b/libcxx/test/benchmarks/containers/ContainerBenchmarks.h
@@ -51,6 +51,20 @@ void BM_Assignment(benchmark::State& st, Container) {
}
}
+template <class Container, class Allocator>
+void BM_Move_Assignment(benchmark::State& st, Container, Allocator) {
+ auto size = st.range(0);
+ Container c1(Allocator{1});
+ Container c2(Allocator{2});
+ c1.reserve(size);
+ c2.resize(size);
+ for (auto _ : st) {
+ c1 = std::move(c2);
+ DoNotOptimizeData(c1);
+ DoNotOptimizeData(c2);
+ }
+}
+
template <std::size_t... sz, typename Container, typename GenInputs>
void BM_AssignInputIterIter(benchmark::State& st, Container c, GenInputs gen) {
auto v = gen(1, sz...);
diff --git a/libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp b/libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp
new file mode 100644
index 00000000000000..86340bcffbbbe9
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp
@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <deque>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "ContainerBenchmarks.h"
+#include "../GenerateInput.h"
+
+using namespace ContainerBenchmarks;
+
+template <typename T, typename SIZE_TYPE = std::size_t, typename DIFF_TYPE = std::ptrdiff_t>
+class CustomSizedAllocator {
+ template <typename U, typename Sz, typename Diff>
+ friend class CustomSizedAllocator;
+
+public:
+ using value_type = T;
+ using size_type = SIZE_TYPE;
+ using difference_type = DIFF_TYPE;
+ using propagate_on_container_swap = std::true_type;
+
+ explicit CustomSizedAllocator(int i = 0) : data_(i) {}
+
+ template <typename U, typename Sz, typename Diff>
+ constexpr CustomSizedAllocator(const CustomSizedAllocator<U, Sz, Diff>& a) noexcept : data_(a.data_) {}
+
+ constexpr T* allocate(size_type n) {
+ if (n > max_size())
+ throw std::bad_array_new_length();
+ return std::allocator<T>().allocate(n);
+ }
+
+ constexpr void deallocate(T* p, size_type n) noexcept { std::allocator<T>().deallocate(p, n); }
+
+ constexpr size_type max_size() const noexcept { return std::numeric_limits<size_type>::max() / sizeof(value_type); }
+
+ int get() { return data_; }
+
+private:
+ int data_;
+
+ constexpr friend bool operator==(const CustomSizedAllocator& a, const CustomSizedAllocator& b) {
+ return a.data_ == b.data_;
+ }
+ constexpr friend bool operator!=(const CustomSizedAllocator& a, const CustomSizedAllocator& b) {
+ return a.data_ != b.data_;
+ }
+};
+
+BENCHMARK_CAPTURE(BM_Move_Assignment,
+ vector_bool_uint32_t,
+ std::vector<bool, CustomSizedAllocator<bool, std::uint32_t, std::int32_t>>{},
+ CustomSizedAllocator<bool, std::uint32_t, std::int32_t>{})
+ ->Arg(5140480);
+
+BENCHMARK_CAPTURE(BM_Move_Assignment,
+ vector_bool_uint64_t,
+ std::vector<bool, CustomSizedAllocator<bool, std::uint64_t, std::int64_t>>{},
+ CustomSizedAllocator<bool, std::uint64_t, std::int64_t>{})
+ ->Arg(5140480);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
index c4866ea4c9b45d..889acb88bfcec7 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
@@ -10,46 +10,65 @@
// vector& operator=(const vector& c);
-#include <vector>
#include <cassert>
-#include "test_macros.h"
-#include "test_allocator.h"
+#include <vector>
+
#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
-TEST_CONSTEXPR_CXX20 bool tests()
-{
- {
- std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
- std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
- l2 = l;
- assert(l2 == l);
- assert(l2.get_allocator() == test_allocator<bool>(3));
- }
- {
- std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
- std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
- l2 = l;
- assert(l2 == l);
- assert(l2.get_allocator() == other_allocator<bool>(5));
- }
+TEST_CONSTEXPR_CXX20 bool tests() {
+ // Test with insufficient space where reallocation occurs during assignment
+ {
+ std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == test_allocator<bool>(3));
+ }
+ {
+ std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
+ std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == other_allocator<bool>(5));
+ }
#if TEST_STD_VER >= 11
- {
- std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
- std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
- l2 = l;
- assert(l2 == l);
- assert(l2.get_allocator() == min_allocator<bool>());
- }
+ {
+ std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
+ std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == min_allocator<bool>());
+ }
#endif
- return true;
+ // Test with sufficient size where no reallocation occurs during assignment
+ {
+ std::vector<bool, test_allocator<bool> > l(4, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(5, false, test_allocator<bool>(3));
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == test_allocator<bool>(3));
+ }
+
+ // Test with sufficient spare space where no reallocation occurs during assignment
+ {
+ std::vector<bool, test_allocator<bool> > l(4, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(2, false, test_allocator<bool>(3));
+ l2.reserve(5);
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == test_allocator<bool>(3));
+ }
+
+ return true;
}
-int main(int, char**)
-{
- tests();
+int main(int, char**) {
+ tests();
#if TEST_STD_VER > 17
- static_assert(tests());
+ static_assert(tests());
#endif
- return 0;
+ return 0;
}
diff --git a/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
index 10271efc3f4038..e047807d68f735 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
@@ -12,79 +12,116 @@
// vector& operator=(vector&& c);
-#include <vector>
#include <cassert>
-#include "test_macros.h"
-#include "test_allocator.h"
+#include <vector>
+
#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
-TEST_CONSTEXPR_CXX20 bool tests()
-{
- {
- std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
- std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
- for (int i = 1; i <= 3; ++i)
- {
- l.push_back(i);
- lo.push_back(i);
- }
- std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(5));
- l2 = std::move(l);
- assert(l2 == lo);
- LIBCPP_ASSERT(l.empty());
- assert(l2.get_allocator() == lo.get_allocator());
+TEST_CONSTEXPR_CXX20 bool tests() {
+ //
+ // Testing for O(1) ownership move
+ //
+ { // Test with pocma = true_type, thus performing an ownership move.
+ std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
+ std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
+ std::vector<bool, other_allocator<bool> > l2(other_allocator<bool>(6));
+ for (int i = 1; i <= 3; ++i) {
+ l.push_back(i);
+ lo.push_back(i);
+ }
+ l2 = std::move(l);
+ assert(l2 == lo);
+ assert(l.empty());
+ assert(l2.get_allocator() == lo.get_allocator());
+ }
+ { // Test with pocma = false_type and equal allocators, thus performing an ownership move.
+ std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(5));
+ for (int i = 1; i <= 3; ++i) {
+ l.push_back(i);
+ lo.push_back(i);
}
- {
- std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
- std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
- for (int i = 1; i <= 3; ++i)
- {
- l.push_back(i);
- lo.push_back(i);
- }
- std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(6));
- l2 = std::move(l);
- assert(l2 == lo);
- assert(!l.empty());
- assert(l2.get_allocator() == test_allocator<bool>(6));
+ l2 = std::move(l);
+ assert(l2 == lo);
+ LIBCPP_ASSERT(l.empty());
+ assert(l2.get_allocator() == lo.get_allocator());
+ }
+ { // Test with pocma = false_type and equal allocators, thus performing an ownership move.
+ std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
+ std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
+ std::vector<bool, min_allocator<bool> > l2(min_allocator<bool>{});
+ for (int i = 1; i <= 3; ++i) {
+ l.push_back(i);
+ lo.push_back(i);
}
- {
- std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
- std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
- for (int i = 1; i <= 3; ++i)
- {
- l.push_back(i);
- lo.push_back(i);
- }
- std::vector<bool, other_allocator<bool> > l2(other_allocator<bool>(6));
- l2 = std::move(l);
- assert(l2 == lo);
- assert(l.empty());
- assert(l2.get_allocator() == lo.get_allocator());
+ l2 = std::move(l);
+ assert(l2 == lo);
+ assert(l.empty());
+ assert(l2.get_allocator() == lo.get_allocator());
+ }
+
+ //
+ // Testing for O(n) element-wise move
+ //
+ { // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
+ // Reallocation occurs during the element-wise move due to insufficient space.
+ std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(6));
+ for (int i = 1; i <= 3; ++i) {
+ l.push_back(i);
+ lo.push_back(i);
+ }
+ l2 = std::move(l);
+ assert(l2 == lo);
+ assert(!l.empty());
+ assert(l2.get_allocator() == test_allocator<bool>(6));
+ }
+
+ { // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
+ // No reallocation occurs since source data fits within current size.
+ std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(6));
+ for (int i = 1; i <= 3; ++i) {
+ l.push_back(i);
+ lo.push_back(i);
}
- {
- std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
- std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
- for (int i = 1; i <= 3; ++i)
- {
- l.push_back(i);
- lo.push_back(i);
- }
- std::vector<bool, min_allocator<bool> > l2(min_allocator<bool>{});
- l2 = std::move(l);
- assert(l2 == lo);
- assert(l.empty());
- assert(l2.get_allocator() == lo.get_allocator());
+ for (int i = 1; i <= 5; ++i)
+ l2.push_back(i);
+ l2 = std::move(l);
+ assert(l2 == lo);
+ assert(!l.empty());
+ assert(l2.get_allocator() == test_allocator<bool>(6));
+ }
+ { // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
+ // No reallocation occurs since source data fits within current spare space.
+ std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(6));
+ for (int i = 1; i <= 3; ++i) {
+ l.push_back(i);
+ lo.push_back(i);
}
+ for (int i = 1; i <= 2; ++i)
+ l2.push_back(i);
+ l2.reserve(5);
+ l2 = std::move(l);
+ assert(l2 == lo);
+ assert(!l.empty());
+ assert(l2.get_allocator() == test_allocator<bool>(6));
+ }
- return true;
+ return true;
}
-int main(int, char**)
-{
- tests();
+int main(int, char**) {
+ tests();
#if TEST_STD_VER > 17
- static_assert(tests());
+ static_assert(tests());
#endif
- return 0;
+ return 0;
}
More information about the libcxx-commits
mailing list