[libcxx-commits] [libcxx] [libc++] Speed up and refactor vector<bool> move-assignment operator [2/3] (PR #119817)
Peng Liu via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Jan 22 10:31:59 PST 2025
https://github.com/winner245 updated https://github.com/llvm/llvm-project/pull/119817
>From b7818e10d85857a442733868fcbcd0e1d08e4d65 Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245 at hotmail.com>
Date: Mon, 16 Dec 2024 12:26:49 -0500
Subject: [PATCH] Speed-up and refactor move-assignment operator for
vector<bool>
---
libcxx/include/__vector/vector_bool.h | 26 +--
.../containers/ContainerBenchmarks.h | 14 ++
.../vector_bool_operations.bench.cpp | 46 ++++++
.../vector.bool/assign_copy.pass.cpp | 98 +++++++----
.../vector.bool/assign_move.pass.cpp | 152 ++++++++++--------
libcxx/test/support/sized_allocator.h | 60 +++++++
6 files changed, 284 insertions(+), 112 deletions(-)
create mode 100644 libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp
create mode 100644 libcxx/test/support/sized_allocator.h
diff --git a/libcxx/include/__vector/vector_bool.h b/libcxx/include/__vector/vector_bool.h
index 4f1c442ce0be8d..e2861f3e809e45 100644
--- a/libcxx/include/__vector/vector_bool.h
+++ b/libcxx/include/__vector/vector_bool.h
@@ -417,6 +417,8 @@ class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
__guard.__complete();
}
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_by_words(const vector& __v);
+
template <class _Iterator, class _Sentinel>
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
@@ -716,18 +718,24 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v
}
}
+// This function copies entire storage words instead of individual bits for improved performance.
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void vector<bool, _Allocator>::__copy_by_words(const vector& __v) {
+ if (__v.__size_) {
+ if (__v.__size_ > capacity()) {
+ __vdeallocate();
+ __vallocate(__v.__size_);
+ }
+ std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
+ }
+ __size_ = __v.__size_;
+}
+
template <class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>& vector<bool, _Allocator>::operator=(const vector& __v) {
if (this != std::addressof(__v)) {
__copy_assign_alloc(__v);
- if (__v.__size_) {
- if (__v.__size_ > capacity()) {
- __vdeallocate();
- __vallocate(__v.__size_);
- }
- std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
- }
- __size_ = __v.__size_;
+ __copy_by_words(__v);
}
return *this;
}
@@ -775,7 +783,7 @@ vector<bool, _Allocator>::operator=(vector&& __v)
template <class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, false_type) {
if (__alloc_ != __c.__alloc_)
- assign(__c.begin(), __c.end());
+ __copy_by_words(__c);
else
__move_assign(__c, true_type());
}
diff --git a/libcxx/test/benchmarks/containers/ContainerBenchmarks.h b/libcxx/test/benchmarks/containers/ContainerBenchmarks.h
index 5fc8981619672c..e7d95208ff7324 100644
--- a/libcxx/test/benchmarks/containers/ContainerBenchmarks.h
+++ b/libcxx/test/benchmarks/containers/ContainerBenchmarks.h
@@ -51,6 +51,20 @@ void BM_Assignment(benchmark::State& st, Container) {
}
}
+template <class Container, class Allocator>
+void BM_Move_Assignment(benchmark::State& st, Container, Allocator) {
+ auto size = st.range(0);
+ Container c1(Allocator{1});
+ Container c2(Allocator{2});
+ // c1.reserve(size);
+ c2.resize(size);
+ for (auto _ : st) {
+ c1 = std::move(c2);
+ DoNotOptimizeData(c1);
+ DoNotOptimizeData(c2);
+ }
+}
+
template <std::size_t... sz, typename Container, typename GenInputs>
void BM_AssignInputIterIter(benchmark::State& st, Container c, GenInputs gen) {
auto v = gen(1, sz...);
diff --git a/libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp b/libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp
new file mode 100644
index 00000000000000..33d6e6a75c4bea
--- /dev/null
+++ b/libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <deque>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "ContainerBenchmarks.h"
+#include "../GenerateInput.h"
+#include "sized_allocator.h"
+#include "test_allocator.h"
+
+using namespace ContainerBenchmarks;
+
+BENCHMARK_CAPTURE(BM_Move_Assignment,
+ vector_bool_uint32_t,
+ std::vector<bool, sized_allocator<bool, std::uint32_t, std::int32_t>>{},
+ sized_allocator<bool, std::uint32_t, std::int32_t>{})
+ ->Arg(5140480);
+
+BENCHMARK_CAPTURE(BM_Move_Assignment,
+ vector_bool_uint64_t,
+ std::vector<bool, sized_allocator<bool, std::uint64_t, std::int64_t>>{},
+ sized_allocator<bool, std::uint64_t, std::int64_t>{})
+ ->Arg(5140480);
+
+BENCHMARK_CAPTURE(BM_Move_Assignment,
+ vector_bool_size_t,
+ std::vector<bool, sized_allocator<bool, std::size_t, std::ptrdiff_t>>{},
+ sized_allocator<bool, std::size_t, std::ptrdiff_t>{})
+ ->Arg(5140480);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
index c4866ea4c9b45d..536c74b032ecf9 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp
@@ -7,49 +7,77 @@
//===----------------------------------------------------------------------===//
// <vector>
+// vector<bool>
// vector& operator=(const vector& c);
-#include <vector>
#include <cassert>
-#include "test_macros.h"
-#include "test_allocator.h"
+#include <vector>
+
#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
-TEST_CONSTEXPR_CXX20 bool tests()
-{
- {
- std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
- std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
- l2 = l;
- assert(l2 == l);
- assert(l2.get_allocator() == test_allocator<bool>(3));
- }
- {
- std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
- std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
- l2 = l;
- assert(l2 == l);
- assert(l2.get_allocator() == other_allocator<bool>(5));
- }
-#if TEST_STD_VER >= 11
- {
- std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
- std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
- l2 = l;
- assert(l2 == l);
- assert(l2.get_allocator() == min_allocator<bool>());
- }
-#endif
+TEST_CONSTEXPR_CXX20 void test_copy_assignment(unsigned N) {
+ //
+ // Test with insufficient space where reallocation occurs during assignment
+ //
+ { // pocca = true_type, thus copy-assign the allocator
+ std::vector<bool, other_allocator<bool> > l(N, true, other_allocator<bool>(5));
+ std::vector<bool, other_allocator<bool> > l2(other_allocator<bool>(3));
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == other_allocator<bool>(5));
+ }
+ {
+ std::vector<bool, test_allocator<bool> > l(N + 64, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(10, false, test_allocator<bool>(3));
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == test_allocator<bool>(3));
+ }
+ {
+ std::vector<bool, min_allocator<bool> > l(N + 64, true, min_allocator<bool>());
+ std::vector<bool, min_allocator<bool> > l2(N / 2, false, min_allocator<bool>());
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == min_allocator<bool>());
+ }
+
+ //
+ // Test with sufficient size where no reallocation occurs during assignment
+ //
+ {
+ std::vector<bool, test_allocator<bool> > l(N, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(N + 64, false, test_allocator<bool>(3));
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == test_allocator<bool>(3));
+ }
+ { // pocca = true_type, thus copy-assign the allocator
+ std::vector<bool, other_allocator<bool> > l(N, true, other_allocator<bool>(5));
+ std::vector<bool, other_allocator<bool> > l2(N * 2, false, other_allocator<bool>(3));
+ l2.reserve(5);
+ l2 = l;
+ assert(l2 == l);
+ assert(l2.get_allocator() == other_allocator<bool>(5));
+ }
+}
+
+TEST_CONSTEXPR_CXX20 bool tests() {
+ test_copy_assignment(9);
+ test_copy_assignment(33);
+ test_copy_assignment(65);
+ test_copy_assignment(257);
+ test_copy_assignment(1000);
- return true;
+ return true;
}
-int main(int, char**)
-{
- tests();
-#if TEST_STD_VER > 17
- static_assert(tests());
+int main(int, char**) {
+ tests();
+#if TEST_STD_VER >= 20
+ static_assert(tests());
#endif
- return 0;
+ return 0;
}
diff --git a/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
index 10271efc3f4038..a43399ced94c7c 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp
@@ -9,82 +9,98 @@
// UNSUPPORTED: c++03
// <vector>
+// vector<bool>
// vector& operator=(vector&& c);
-#include <vector>
#include <cassert>
-#include "test_macros.h"
-#include "test_allocator.h"
+#include <vector>
+
#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+TEST_CONSTEXPR_CXX20 void test_move_assignment(unsigned N) {
+ //
+ // Testing for O(1) ownership move
+ //
+ { // Test with pocma = true_type, thus performing an ownership move.
+ std::vector<bool, other_allocator<bool> > l(N, true, other_allocator<bool>(5));
+ std::vector<bool, other_allocator<bool> > lo(N, true, other_allocator<bool>(5));
+ std::vector<bool, other_allocator<bool> > l2(N + 10, false, other_allocator<bool>(42));
+ l2 = std::move(l);
+ assert(l2 == lo);
+ LIBCPP_ASSERT(l.empty()); // After move, source vector is in a vliad but unspecified state. libc++ leaves it empty.
+ assert(l2.get_allocator() == lo.get_allocator());
+ }
+ { // Test with pocma = false_type but equal allocators, thus performing an ownership move.
+ std::vector<bool, test_allocator<bool> > l(N, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > lo(N, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(N + 10, false, test_allocator<bool>(5));
+ l2 = std::move(l);
+ assert(l2 == lo);
+ LIBCPP_ASSERT(l.empty());
+ assert(l2.get_allocator() == lo.get_allocator());
+ }
+ { // Test with pocma = false_type but equal allocators, thus performing an ownership move.
+ std::vector<bool, min_allocator<bool> > l(N, true, min_allocator<bool>{});
+ std::vector<bool, min_allocator<bool> > lo(N, true, min_allocator<bool>{});
+ std::vector<bool, min_allocator<bool> > l2(N + 10, false, min_allocator<bool>{});
+ l2 = std::move(l);
+ assert(l2 == lo);
+ LIBCPP_ASSERT(l.empty());
+ assert(l2.get_allocator() == lo.get_allocator());
+ }
+
+ //
+ // Testing for O(n) element-wise move
+ //
+ { // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
+ // Reallocation occurs during the element-wise move due to empty destination vector.
+ std::vector<bool, test_allocator<bool> > l(N, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > lo(N, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(42));
+ l2 = std::move(l);
+ assert(l2 == lo);
+ LIBCPP_ASSERT(!l.empty());
+ assert(l2.get_allocator() == test_allocator<bool>(42));
+ }
+ { // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
+ // Reallocation occurs during the element-wise move due to insufficient destination space.
+ std::vector<bool, test_allocator<bool> > l(N + 64, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > lo(N + 64, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(10, false, test_allocator<bool>(42));
+ l2 = std::move(l);
+ assert(l2 == lo);
+ LIBCPP_ASSERT(!l.empty());
+ assert(l2.get_allocator() == test_allocator<bool>(42));
+ }
+ { // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
+ // No reallocation occurs since source data fits within destination size.
+ std::vector<bool, test_allocator<bool> > l(N, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > lo(N, true, test_allocator<bool>(5));
+ std::vector<bool, test_allocator<bool> > l2(N * 2, false, test_allocator<bool>(42));
+ l2 = std::move(l);
+ assert(l2 == lo);
+ LIBCPP_ASSERT(!l.empty());
+ assert(l2.get_allocator() == test_allocator<bool>(42));
+ }
+}
-TEST_CONSTEXPR_CXX20 bool tests()
-{
- {
- std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
- std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
- for (int i = 1; i <= 3; ++i)
- {
- l.push_back(i);
- lo.push_back(i);
- }
- std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(5));
- l2 = std::move(l);
- assert(l2 == lo);
- LIBCPP_ASSERT(l.empty());
- assert(l2.get_allocator() == lo.get_allocator());
- }
- {
- std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
- std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
- for (int i = 1; i <= 3; ++i)
- {
- l.push_back(i);
- lo.push_back(i);
- }
- std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(6));
- l2 = std::move(l);
- assert(l2 == lo);
- assert(!l.empty());
- assert(l2.get_allocator() == test_allocator<bool>(6));
- }
- {
- std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
- std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
- for (int i = 1; i <= 3; ++i)
- {
- l.push_back(i);
- lo.push_back(i);
- }
- std::vector<bool, other_allocator<bool> > l2(other_allocator<bool>(6));
- l2 = std::move(l);
- assert(l2 == lo);
- assert(l.empty());
- assert(l2.get_allocator() == lo.get_allocator());
- }
- {
- std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
- std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
- for (int i = 1; i <= 3; ++i)
- {
- l.push_back(i);
- lo.push_back(i);
- }
- std::vector<bool, min_allocator<bool> > l2(min_allocator<bool>{});
- l2 = std::move(l);
- assert(l2 == lo);
- assert(l.empty());
- assert(l2.get_allocator() == lo.get_allocator());
- }
+TEST_CONSTEXPR_CXX20 bool tests() {
+ test_move_assignment(9);
+ test_move_assignment(33);
+ test_move_assignment(65);
+ test_move_assignment(257);
+ test_move_assignment(1000);
- return true;
+ return true;
}
-int main(int, char**)
-{
- tests();
-#if TEST_STD_VER > 17
- static_assert(tests());
+int main(int, char**) {
+ tests();
+#if TEST_STD_VER >= 20
+ static_assert(tests());
#endif
- return 0;
+ return 0;
}
diff --git a/libcxx/test/support/sized_allocator.h b/libcxx/test/support/sized_allocator.h
new file mode 100644
index 00000000000000..8d52f5bf252c74
--- /dev/null
+++ b/libcxx/test/support/sized_allocator.h
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_SUPPORT_SIZED_ALLOCATOR_H
+#define TEST_SUPPORT_SIZED_ALLOCATOR_H
+
+#include <cstddef>
+#include <limits>
+#include <memory>
+#include <new>
+
+#include "test_macros.h"
+
+// Allocator with a provided size_type and difference_type, used to test corner cases
+// like arithmetic on Allocator::size_type in generic code.
+template <typename T, typename Size = std::size_t, typename Difference = std::ptrdiff_t>
+class sized_allocator {
+ template <typename U, typename Sz, typename Diff>
+ friend class sized_allocator;
+
+public:
+ using value_type = T;
+ using size_type = Size;
+ using difference_type = Difference;
+ using propagate_on_container_swap = std::true_type;
+
+ TEST_CONSTEXPR_CXX20 explicit sized_allocator(int d = 0) : data_(d) {}
+
+ template <typename U, typename Sz, typename Diff>
+ TEST_CONSTEXPR_CXX20 sized_allocator(const sized_allocator<U, Sz, Diff>& a) TEST_NOEXCEPT : data_(a.data_) {}
+
+ TEST_CONSTEXPR_CXX20 T* allocate(size_type n) {
+ if (n > max_size())
+ TEST_THROW(std::bad_array_new_length());
+ return std::allocator<T>().allocate(n);
+ }
+
+ TEST_CONSTEXPR_CXX20 void deallocate(T* p, size_type n) TEST_NOEXCEPT { std::allocator<T>().deallocate(p, n); }
+
+ TEST_CONSTEXPR size_type max_size() const TEST_NOEXCEPT {
+ return std::numeric_limits<size_type>::max() / sizeof(value_type);
+ }
+
+private:
+ int data_;
+
+ TEST_CONSTEXPR friend bool operator==(const sized_allocator& a, const sized_allocator& b) {
+ return a.data_ == b.data_;
+ }
+ TEST_CONSTEXPR friend bool operator!=(const sized_allocator& a, const sized_allocator& b) {
+ return a.data_ != b.data_;
+ }
+};
+
+#endif // TEST_SUPPORT_SIZED_ALLOCATOR_H
More information about the libcxx-commits
mailing list