[libcxx-commits] [libcxx] [libc++] Optimize ranges::copy{, _n} for vector<bool>::iterator (PR #121013)
Peng Liu via libcxx-commits
libcxx-commits at lists.llvm.org
Tue Dec 24 15:25:43 PST 2024
https://github.com/winner245 updated https://github.com/llvm/llvm-project/pull/121013
>From 69952374d2b3f45b4d66ea9087ccfaf7c8898a02 Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245 at hotmail.com>
Date: Mon, 23 Dec 2024 08:05:53 -0500
Subject: [PATCH] Optimize ranges::copy{, _n} for vector<bool>::iterator
---
libcxx/docs/ReleaseNotes/20.rst | 3 +
libcxx/include/__algorithm/copy.h | 134 +++++++++++++++++-
libcxx/include/__bit_reference | 131 +----------------
libcxx/include/bitset | 2 +
.../test/benchmarks/algorithms/copy.bench.cpp | 89 ++++++++++++
.../alg.copy/copy.pass.cpp | 34 ++++-
.../alg.copy/copy_n.pass.cpp | 127 ++++++++++-------
.../alg.copy/ranges.copy.pass.cpp | 36 +++++
.../alg.copy/ranges.copy_n.pass.cpp | 53 ++++++-
9 files changed, 424 insertions(+), 185 deletions(-)
create mode 100644 libcxx/test/benchmarks/algorithms/copy.bench.cpp
diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index c8a07fb8b73348..4cb2eff0688176 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -73,6 +73,9 @@ Improvements and New Features
optimized, resulting in a performance improvement of up to 2x for trivial element types (e.g., `std::vector<int>`),
and up to 3.4x for non-trivial element types (e.g., `std::vector<std::vector<int>>`).
+- The ``std::ranges::copy`` and ``std::ranges::copy_n`` algorithms have been optimized for ``std::vector<bool>::iterator``\s,
+ resulting in a performance improvement of up to 2000x.
+
Deprecations and Removals
-------------------------
diff --git a/libcxx/include/__algorithm/copy.h b/libcxx/include/__algorithm/copy.h
index 4f30b2050abbaf..745056fa9ec058 100644
--- a/libcxx/include/__algorithm/copy.h
+++ b/libcxx/include/__algorithm/copy.h
@@ -13,8 +13,10 @@
#include <__algorithm/for_each_segment.h>
#include <__algorithm/min.h>
#include <__config>
+#include <__fwd/bit_reference.h>
#include <__iterator/iterator_traits.h>
#include <__iterator/segmented_iterator.h>
+#include <__memory/pointer_traits.h>
#include <__type_traits/common_type.h>
#include <__type_traits/enable_if.h>
#include <__utility/move.h>
@@ -29,9 +31,129 @@ _LIBCPP_PUSH_MACROS
_LIBCPP_BEGIN_NAMESPACE_STD
+template <class _InputIterator, class _OutputIterator>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result);
+
template <class _InIter, class _Sent, class _OutIter>
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> __copy(_InIter, _Sent, _OutIter);
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned(
+ __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+ using _In = __bit_iterator<_Cp, _IsConst>;
+ using difference_type = typename _In::difference_type;
+ using __storage_type = typename _In::__storage_type;
+
+ const int __bits_per_word = _In::__bits_per_word;
+ difference_type __n = __last - __first;
+ if (__n > 0) {
+ // do first word
+ if (__first.__ctz_ != 0) {
+ unsigned __clz = __bits_per_word - __first.__ctz_;
+ difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
+ __n -= __dn;
+ __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
+ __storage_type __b = *__first.__seg_ & __m;
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b;
+ __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+ __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+ ++__first.__seg_;
+ // __first.__ctz_ = 0;
+ }
+ // __first.__ctz_ == 0;
+ // do middle words
+ __storage_type __nw = __n / __bits_per_word;
+ std::copy(std::__to_address(__first.__seg_),
+ std::__to_address(__first.__seg_ + __nw),
+ std::__to_address(__result.__seg_));
+ __n -= __nw * __bits_per_word;
+ __result.__seg_ += __nw;
+ // do last word
+ if (__n > 0) {
+ __first.__seg_ += __nw;
+ __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+ __storage_type __b = *__first.__seg_ & __m;
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b;
+ __result.__ctz_ = static_cast<unsigned>(__n);
+ }
+ }
+ return __result;
+}
+
+template <class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned(
+ __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
+ using _In = __bit_iterator<_Cp, _IsConst>;
+ using difference_type = typename _In::difference_type;
+ using __storage_type = typename _In::__storage_type;
+
+ const int __bits_per_word = _In::__bits_per_word;
+ difference_type __n = __last - __first;
+ if (__n > 0) {
+ // do first word
+ if (__first.__ctz_ != 0) {
+ unsigned __clz_f = __bits_per_word - __first.__ctz_;
+ difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
+ __n -= __dn;
+ __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+ __storage_type __b = *__first.__seg_ & __m;
+ unsigned __clz_r = __bits_per_word - __result.__ctz_;
+ __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
+ __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
+ *__result.__seg_ &= ~__m;
+ if (__result.__ctz_ > __first.__ctz_)
+ *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_);
+ else
+ *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_);
+ __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
+ __result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
+ __dn -= __ddn;
+ if (__dn > 0) {
+ __m = ~__storage_type(0) >> (__bits_per_word - __dn);
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn);
+ __result.__ctz_ = static_cast<unsigned>(__dn);
+ }
+ ++__first.__seg_;
+ // __first.__ctz_ = 0;
+ }
+ // __first.__ctz_ == 0;
+ // do middle words
+ unsigned __clz_r = __bits_per_word - __result.__ctz_;
+ __storage_type __m = ~__storage_type(0) << __result.__ctz_;
+ for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
+ __storage_type __b = *__first.__seg_;
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b << __result.__ctz_;
+ ++__result.__seg_;
+ *__result.__seg_ &= __m;
+ *__result.__seg_ |= __b >> __clz_r;
+ }
+ // do last word
+ if (__n > 0) {
+ __m = ~__storage_type(0) >> (__bits_per_word - __n);
+ __storage_type __b = *__first.__seg_ & __m;
+ __storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
+ __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b << __result.__ctz_;
+ __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+ __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+ __n -= __dn;
+ if (__n > 0) {
+ __m = ~__storage_type(0) >> (__bits_per_word - __n);
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b >> __dn;
+ __result.__ctz_ = static_cast<unsigned>(__n);
+ }
+ }
+ }
+ return __result;
+}
+
struct __copy_impl {
template <class _InIter, class _Sent, class _OutIter>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
@@ -95,6 +217,16 @@ struct __copy_impl {
}
}
+ template <class _Cp, bool _IsConst>
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
+ operator()(__bit_iterator<_Cp, _IsConst> __first,
+ __bit_iterator<_Cp, _IsConst> __last,
+ __bit_iterator<_Cp, false> __result) {
+ if (__first.__ctz_ == __result.__ctz_)
+ return std::make_pair(__last, std::__copy_aligned(__first, __last, __result));
+ return std::make_pair(__last, std::__copy_unaligned(__first, __last, __result));
+ }
+
// At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
@@ -110,7 +242,7 @@ __copy(_InIter __first, _Sent __last, _OutIter __result) {
}
template <class _InputIterator, class _OutputIterator>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
return std::__copy(__first, __last, __result).second;
}
diff --git a/libcxx/include/__bit_reference b/libcxx/include/__bit_reference
index 9fa24c98d493fd..b51ee1c58dc009 100644
--- a/libcxx/include/__bit_reference
+++ b/libcxx/include/__bit_reference
@@ -10,6 +10,7 @@
#ifndef _LIBCPP___BIT_REFERENCE
#define _LIBCPP___BIT_REFERENCE
+#include <__algorithm/copy.h>
#include <__algorithm/copy_n.h>
#include <__algorithm/min.h>
#include <__bit/countr.h>
@@ -22,6 +23,7 @@
#include <__memory/pointer_traits.h>
#include <__type_traits/conditional.h>
#include <__type_traits/is_constant_evaluated.h>
+#include <__utility/pair.h>
#include <__utility/swap.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -169,130 +171,6 @@ private:
__mask_(__m) {}
};
-// copy
-
-template <class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned(
- __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
- using _In = __bit_iterator<_Cp, _IsConst>;
- using difference_type = typename _In::difference_type;
- using __storage_type = typename _In::__storage_type;
-
- const int __bits_per_word = _In::__bits_per_word;
- difference_type __n = __last - __first;
- if (__n > 0) {
- // do first word
- if (__first.__ctz_ != 0) {
- unsigned __clz = __bits_per_word - __first.__ctz_;
- difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
- __n -= __dn;
- __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
- __storage_type __b = *__first.__seg_ & __m;
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b;
- __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
- __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
- ++__first.__seg_;
- // __first.__ctz_ = 0;
- }
- // __first.__ctz_ == 0;
- // do middle words
- __storage_type __nw = __n / __bits_per_word;
- std::copy_n(std::__to_address(__first.__seg_), __nw, std::__to_address(__result.__seg_));
- __n -= __nw * __bits_per_word;
- __result.__seg_ += __nw;
- // do last word
- if (__n > 0) {
- __first.__seg_ += __nw;
- __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
- __storage_type __b = *__first.__seg_ & __m;
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b;
- __result.__ctz_ = static_cast<unsigned>(__n);
- }
- }
- return __result;
-}
-
-template <class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned(
- __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
- using _In = __bit_iterator<_Cp, _IsConst>;
- using difference_type = typename _In::difference_type;
- using __storage_type = typename _In::__storage_type;
-
- const int __bits_per_word = _In::__bits_per_word;
- difference_type __n = __last - __first;
- if (__n > 0) {
- // do first word
- if (__first.__ctz_ != 0) {
- unsigned __clz_f = __bits_per_word - __first.__ctz_;
- difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
- __n -= __dn;
- __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
- __storage_type __b = *__first.__seg_ & __m;
- unsigned __clz_r = __bits_per_word - __result.__ctz_;
- __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
- __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
- *__result.__seg_ &= ~__m;
- if (__result.__ctz_ > __first.__ctz_)
- *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_);
- else
- *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_);
- __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
- __result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
- __dn -= __ddn;
- if (__dn > 0) {
- __m = ~__storage_type(0) >> (__bits_per_word - __dn);
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn);
- __result.__ctz_ = static_cast<unsigned>(__dn);
- }
- ++__first.__seg_;
- // __first.__ctz_ = 0;
- }
- // __first.__ctz_ == 0;
- // do middle words
- unsigned __clz_r = __bits_per_word - __result.__ctz_;
- __storage_type __m = ~__storage_type(0) << __result.__ctz_;
- for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
- __storage_type __b = *__first.__seg_;
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b << __result.__ctz_;
- ++__result.__seg_;
- *__result.__seg_ &= __m;
- *__result.__seg_ |= __b >> __clz_r;
- }
- // do last word
- if (__n > 0) {
- __m = ~__storage_type(0) >> (__bits_per_word - __n);
- __storage_type __b = *__first.__seg_ & __m;
- __storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
- __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b << __result.__ctz_;
- __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
- __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
- __n -= __dn;
- if (__n > 0) {
- __m = ~__storage_type(0) >> (__bits_per_word - __n);
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b >> __dn;
- __result.__ctz_ = static_cast<unsigned>(__n);
- }
- }
- }
- return __result;
-}
-
-template <class _Cp, bool _IsConst>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false>
-copy(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
- if (__first.__ctz_ == __result.__ctz_)
- return std::__copy_aligned(__first, __last, __result);
- return std::__copy_unaligned(__first, __last, __result);
-}
-
// copy_backward
template <class _Cp, bool _IsConst>
@@ -975,8 +853,9 @@ private:
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_unaligned(
__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
template <class _Dp, bool _IC>
- _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false>
- copy(__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 friend pair<__bit_iterator<_Dp, _IC>, __bit_iterator<_Dp, false> >
+ __copy_impl::operator()(
+ __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
template <class _Dp, bool _IC>
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_aligned(
__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
diff --git a/libcxx/include/bitset b/libcxx/include/bitset
index 8b361824805571..bb09bc5415fb03 100644
--- a/libcxx/include/bitset
+++ b/libcxx/include/bitset
@@ -129,6 +129,8 @@ template <size_t N> struct hash<std::bitset<N>>;
#if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
# include <__cxx03/bitset>
#else
+# include <__algorithm/copy.h>
+# include <__algorithm/copy_backward.h>
# include <__algorithm/count.h>
# include <__algorithm/fill.h>
# include <__algorithm/fill_n.h>
diff --git a/libcxx/test/benchmarks/algorithms/copy.bench.cpp b/libcxx/test/benchmarks/algorithms/copy.bench.cpp
new file mode 100644
index 00000000000000..54006d5a72edea
--- /dev/null
+++ b/libcxx/test/benchmarks/algorithms/copy.bench.cpp
@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <vector>
+
+static void bm_ranges_copy(benchmark::State& state, bool aligned) {
+ auto n = state.range();
+ std::vector<bool> in(n, true);
+ std::vector<bool> out(aligned ? n : n + 8);
+ benchmark::DoNotOptimize(&in);
+ auto dst = aligned ? out.begin() : out.begin() + 4;
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(std::ranges::copy(in, dst));
+ benchmark::DoNotOptimize(&out);
+ }
+}
+
+static void bm_ranges_copy_n(benchmark::State& state, bool aligned) {
+ auto n = state.range();
+ std::vector<bool> in(n, true);
+ std::vector<bool> out(aligned ? n : n + 8);
+ benchmark::DoNotOptimize(&in);
+ auto src = in.begin();
+ auto dst = aligned ? out.begin() : out.begin() + 4;
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(std::ranges::copy_n(src, n, dst));
+ benchmark::DoNotOptimize(&out);
+ }
+}
+
+static void bm_copy(benchmark::State& state, bool aligned) {
+ auto n = state.range();
+ std::vector<bool> in(n, true);
+ std::vector<bool> out(aligned ? n : n + 8);
+ benchmark::DoNotOptimize(&in);
+ auto beg = in.begin();
+ auto end = in.end();
+ auto dst = aligned ? out.begin() : out.begin() + 4;
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(std::copy(beg, end, dst));
+ benchmark::DoNotOptimize(&out);
+ }
+}
+
+static void bm_copy_n(benchmark::State& state, bool aligned) {
+ auto n = state.range();
+ std::vector<bool> in(n, true);
+ std::vector<bool> out(aligned ? n : n + 8);
+ benchmark::DoNotOptimize(&in);
+ auto src = in.begin();
+ auto dst = aligned ? out.begin() : out.begin() + 4;
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(std::copy_n(src, n, dst));
+ benchmark::DoNotOptimize(&out);
+ }
+}
+
+static void bm_ranges_copy_aligned(benchmark::State& state) { bm_ranges_copy(state, true); }
+static void bm_ranges_copy_unaligned(benchmark::State& state) { bm_ranges_copy(state, false); }
+static void bm_ranges_copy_n_aligned(benchmark::State& state) { bm_ranges_copy_n(state, true); }
+static void bm_ranges_copy_n_unaligned(benchmark::State& state) { bm_ranges_copy_n(state, false); }
+
+static void bm_copy_aligned(benchmark::State& state) { bm_copy(state, true); }
+static void bm_copy_unaligned(benchmark::State& state) { bm_copy(state, false); }
+static void bm_copy_n_aligned(benchmark::State& state) { bm_copy_n(state, true); }
+static void bm_copy_n_unaligned(benchmark::State& state) { bm_copy_n(state, false); }
+
+// Test the range version of std::copy for vector<bool>::iterator
+BENCHMARK(bm_ranges_copy_aligned)->Range(8, 1 << 16)->DenseRange(102400, 204800, 4096);
+BENCHMARK(bm_ranges_copy_n_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_ranges_copy_unaligned)->Range(8, 1 << 20);
+BENCHMARK(bm_ranges_copy_n_unaligned)->Range(8, 1 << 20);
+
+// Test the iterator-pair version of std::copy for vector<bool>::iterator
+BENCHMARK(bm_copy_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_copy_n_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_copy_unaligned)->Range(8, 1 << 20);
+BENCHMARK(bm_copy_n_unaligned)->Range(8, 1 << 20);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp
index b5f0a32b986a03..553e4c206ac08a 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp
@@ -14,6 +14,7 @@
#include <algorithm>
#include <cassert>
+#include <vector>
#include "test_macros.h"
#include "test_iterators.h"
@@ -59,6 +60,29 @@ struct TestInIters {
}
};
+template <std::size_t N>
+struct TestBitIter {
+ std::vector<bool> in;
+ TEST_CONSTEXPR_CXX20 TestBitIter() : in(N, false) {
+ for (std::size_t i = 0; i < N; i += 2)
+ in[i] = true;
+ }
+ TEST_CONSTEXPR_CXX20 void operator()() {
+ { // Test copy with aligned bytes
+ std::vector<bool> out(N);
+ std::copy(in.begin(), in.end(), out.begin());
+ assert(in == out);
+ }
+ { // Test copy with unaligned bytes
+ std::vector<bool> out(N + 8);
+ std::copy(in.begin(), in.end(), out.begin() + 4);
+ for (std::size_t i = 0; i < N; ++i)
+ assert(out[i + 4] == in[i]);
+ }
+ }
+ TEST_CONSTEXPR std::size_t size() const { return N; }
+};
+
TEST_CONSTEXPR_CXX20 bool test() {
types::for_each(types::cpp17_input_iterator_list<int*>(), TestInIters());
@@ -78,13 +102,21 @@ TEST_CONSTEXPR_CXX20 bool test() {
assert(std::equal(a, a + 10, expected));
}
+ { // Test vector<bool>::iterator optimization
+ TestBitIter<8>()();
+ TestBitIter<16>()();
+ TestBitIter<32>()();
+ TestBitIter<64>()();
+ TestBitIter<1024>()();
+ }
+
return true;
}
int main(int, char**) {
test();
-#if TEST_STD_VER > 17
+#if TEST_STD_VER >= 20
static_assert(test());
#endif
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp
index b0acc1060101c2..337e0f86a2fe05 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp
@@ -14,6 +14,7 @@
#include <algorithm>
#include <cassert>
+#include <vector>
#include "test_macros.h"
#include "test_iterators.h"
@@ -37,20 +38,18 @@ class Derived : public PaddedBase {
};
template <class InIter, class OutIter>
-TEST_CONSTEXPR_CXX20 void
-test_copy_n()
-{
+TEST_CONSTEXPR_CXX20 void test_copy_n() {
{
const unsigned N = 1000;
- int ia[N] = {};
+ int ia[N] = {};
for (unsigned i = 0; i < N; ++i)
- ia[i] = i;
+ ia[i] = i;
int ib[N] = {0};
- OutIter r = std::copy_n(InIter(ia), UDI(N/2), OutIter(ib));
- assert(base(r) == ib+N/2);
- for (unsigned i = 0; i < N/2; ++i)
- assert(ia[i] == ib[i]);
+ OutIter r = std::copy_n(InIter(ia), UDI(N / 2), OutIter(ib));
+ assert(base(r) == ib + N / 2);
+ for (unsigned i = 0; i < N / 2; ++i)
+ assert(ia[i] == ib[i]);
}
{ // Make sure that padding bits aren't copied
@@ -70,53 +69,81 @@ test_copy_n()
}
}
-TEST_CONSTEXPR_CXX20 bool
-test()
-{
- test_copy_n<cpp17_input_iterator<const int*>, cpp17_output_iterator<int*> >();
- test_copy_n<cpp17_input_iterator<const int*>, cpp17_input_iterator<int*> >();
- test_copy_n<cpp17_input_iterator<const int*>, forward_iterator<int*> >();
- test_copy_n<cpp17_input_iterator<const int*>, bidirectional_iterator<int*> >();
- test_copy_n<cpp17_input_iterator<const int*>, random_access_iterator<int*> >();
- test_copy_n<cpp17_input_iterator<const int*>, int*>();
-
- test_copy_n<forward_iterator<const int*>, cpp17_output_iterator<int*> >();
- test_copy_n<forward_iterator<const int*>, cpp17_input_iterator<int*> >();
- test_copy_n<forward_iterator<const int*>, forward_iterator<int*> >();
- test_copy_n<forward_iterator<const int*>, bidirectional_iterator<int*> >();
- test_copy_n<forward_iterator<const int*>, random_access_iterator<int*> >();
- test_copy_n<forward_iterator<const int*>, int*>();
-
- test_copy_n<bidirectional_iterator<const int*>, cpp17_output_iterator<int*> >();
- test_copy_n<bidirectional_iterator<const int*>, cpp17_input_iterator<int*> >();
- test_copy_n<bidirectional_iterator<const int*>, forward_iterator<int*> >();
- test_copy_n<bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
- test_copy_n<bidirectional_iterator<const int*>, random_access_iterator<int*> >();
- test_copy_n<bidirectional_iterator<const int*>, int*>();
-
- test_copy_n<random_access_iterator<const int*>, cpp17_output_iterator<int*> >();
- test_copy_n<random_access_iterator<const int*>, cpp17_input_iterator<int*> >();
- test_copy_n<random_access_iterator<const int*>, forward_iterator<int*> >();
- test_copy_n<random_access_iterator<const int*>, bidirectional_iterator<int*> >();
- test_copy_n<random_access_iterator<const int*>, random_access_iterator<int*> >();
- test_copy_n<random_access_iterator<const int*>, int*>();
-
- test_copy_n<const int*, cpp17_output_iterator<int*> >();
- test_copy_n<const int*, cpp17_input_iterator<int*> >();
- test_copy_n<const int*, forward_iterator<int*> >();
- test_copy_n<const int*, bidirectional_iterator<int*> >();
- test_copy_n<const int*, random_access_iterator<int*> >();
- test_copy_n<const int*, int*>();
+template <std::size_t N>
+struct TestBitIter {
+ std::vector<bool> in;
+ TEST_CONSTEXPR_CXX20 TestBitIter() : in(N, false) {
+ for (std::size_t i = 0; i < N; i += 2)
+ in[i] = true;
+ }
+ TEST_CONSTEXPR_CXX20 void operator()() {
+ { // Test copy with aligned bytes
+ std::vector<bool> out(N);
+ std::copy_n(in.begin(), N, out.begin());
+ assert(in == out);
+ }
+ { // Test copy with unaligned bytes
+ std::vector<bool> out(N + 8);
+ std::copy_n(in.begin(), N, out.begin() + 4);
+ for (std::size_t i = 0; i < N; ++i)
+ assert(out[i + 4] == in[i]);
+ }
+ }
+ TEST_CONSTEXPR std::size_t size() const { return N; }
+};
+
+TEST_CONSTEXPR_CXX20 bool test() {
+ test_copy_n<cpp17_input_iterator<const int*>, cpp17_output_iterator<int*> >();
+ test_copy_n<cpp17_input_iterator<const int*>, cpp17_input_iterator<int*> >();
+ test_copy_n<cpp17_input_iterator<const int*>, forward_iterator<int*> >();
+ test_copy_n<cpp17_input_iterator<const int*>, bidirectional_iterator<int*> >();
+ test_copy_n<cpp17_input_iterator<const int*>, random_access_iterator<int*> >();
+ test_copy_n<cpp17_input_iterator<const int*>, int*>();
+
+ test_copy_n<forward_iterator<const int*>, cpp17_output_iterator<int*> >();
+ test_copy_n<forward_iterator<const int*>, cpp17_input_iterator<int*> >();
+ test_copy_n<forward_iterator<const int*>, forward_iterator<int*> >();
+ test_copy_n<forward_iterator<const int*>, bidirectional_iterator<int*> >();
+ test_copy_n<forward_iterator<const int*>, random_access_iterator<int*> >();
+ test_copy_n<forward_iterator<const int*>, int*>();
+
+ test_copy_n<bidirectional_iterator<const int*>, cpp17_output_iterator<int*> >();
+ test_copy_n<bidirectional_iterator<const int*>, cpp17_input_iterator<int*> >();
+ test_copy_n<bidirectional_iterator<const int*>, forward_iterator<int*> >();
+ test_copy_n<bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
+ test_copy_n<bidirectional_iterator<const int*>, random_access_iterator<int*> >();
+ test_copy_n<bidirectional_iterator<const int*>, int*>();
+
+ test_copy_n<random_access_iterator<const int*>, cpp17_output_iterator<int*> >();
+ test_copy_n<random_access_iterator<const int*>, cpp17_input_iterator<int*> >();
+ test_copy_n<random_access_iterator<const int*>, forward_iterator<int*> >();
+ test_copy_n<random_access_iterator<const int*>, bidirectional_iterator<int*> >();
+ test_copy_n<random_access_iterator<const int*>, random_access_iterator<int*> >();
+ test_copy_n<random_access_iterator<const int*>, int*>();
+
+ test_copy_n<const int*, cpp17_output_iterator<int*> >();
+ test_copy_n<const int*, cpp17_input_iterator<int*> >();
+ test_copy_n<const int*, forward_iterator<int*> >();
+ test_copy_n<const int*, bidirectional_iterator<int*> >();
+ test_copy_n<const int*, random_access_iterator<int*> >();
+ test_copy_n<const int*, int*>();
+
+ { // Test vector<bool>::iterator optimization
+ TestBitIter<8>()();
+ TestBitIter<16>()();
+ TestBitIter<32>()();
+ TestBitIter<64>()();
+ TestBitIter<1024>()();
+ }
return true;
}
-int main(int, char**)
-{
- test();
+int main(int, char**) {
+ test();
#if TEST_STD_VER > 17
- static_assert(test());
+ static_assert(test());
#endif
return 0;
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp
index 2507e594fe9442..4db2b455b86930 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp
@@ -26,6 +26,7 @@
#include "almost_satisfies_types.h"
#include "test_iterators.h"
+#include "test_macros.h"
#include "type_algorithms.h"
template <class In, class Out = In, class Sent = sentinel_wrapper<In>>
@@ -99,6 +100,31 @@ constexpr void test_iterators() {
}
// clang-format on
+#if TEST_STD_VER >= 23
+template <std::size_t N>
+struct TestBitIter {
+ std::vector<bool> in;
+ TEST_CONSTEXPR_CXX20 TestBitIter() : in(N, false) {
+ for (std::size_t i = 0; i < N; i += 2)
+ in[i] = true;
+ }
+ TEST_CONSTEXPR_CXX20 void operator()() {
+ { // Test copy with aligned bytes
+ std::vector<bool> out(N);
+ std::ranges::copy(in, out.begin());
+ assert(in == out);
+ }
+ { // Test copy with unaligned bytes
+ std::vector<bool> out(N + 8);
+ std::ranges::copy(in, out.begin() + 4);
+ for (std::size_t i = 0; i < N; ++i)
+ assert(out[i + 4] == in[i]);
+ }
+ }
+ TEST_CONSTEXPR std::size_t size() const { return N; }
+};
+#endif
+
constexpr bool test() {
types::for_each(types::forward_iterator_list<int*>{}, []<class Out>() {
test_iterators<cpp20_input_iterator<int*>, Out, sentinel_wrapper<cpp20_input_iterator<int*>>>();
@@ -204,6 +230,16 @@ constexpr bool test() {
}
}
+#if TEST_STD_VER >= 23
+ { // Test vector<bool>::iterator optimization
+ TestBitIter<8>()();
+ TestBitIter<16>()();
+ TestBitIter<32>()();
+ TestBitIter<64>()();
+ TestBitIter<1024>()();
+ }
+#endif
+
return true;
}
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp
index d2a2b7c4888308..760c8c06292bf6 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp
@@ -19,8 +19,10 @@
#include <array>
#include <cassert>
#include <ranges>
+#include <vector>
#include "almost_satisfies_types.h"
+#include "test_macros.h"
#include "test_iterators.h"
template <class In, class Out = In, class Count = std::size_t>
@@ -41,10 +43,10 @@ static_assert(std::is_same_v<std::ranges::copy_result<int, long>, std::ranges::i
template <class In, class Out, class Sent = In>
constexpr void test_iterators() {
{ // simple test
- std::array in {1, 2, 3, 4};
+ std::array in{1, 2, 3, 4};
std::array<int, 4> out;
std::same_as<std::ranges::in_out_result<In, Out>> auto ret =
- std::ranges::copy_n(In(in.data()), in.size(), Out(out.data()));
+ std::ranges::copy_n(In(in.data()), in.size(), Out(out.data()));
assert(in == out);
assert(base(ret.in) == in.data() + in.size());
assert(base(ret.out) == out.data() + out.size());
@@ -70,13 +72,40 @@ constexpr void test_in_iterators() {
template <class Out>
constexpr void test_proxy_in_iterators() {
- test_iterators<ProxyIterator<cpp20_input_iterator<int*>>, Out, sentinel_wrapper<ProxyIterator<cpp20_input_iterator<int*>>>>();
+ test_iterators<ProxyIterator<cpp20_input_iterator<int*>>,
+ Out,
+ sentinel_wrapper<ProxyIterator<cpp20_input_iterator<int*>>>>();
test_iterators<ProxyIterator<forward_iterator<int*>>, Out>();
test_iterators<ProxyIterator<bidirectional_iterator<int*>>, Out>();
test_iterators<ProxyIterator<random_access_iterator<int*>>, Out>();
test_iterators<ProxyIterator<contiguous_iterator<int*>>, Out>();
}
+#if TEST_STD_VER >= 23
+template <std::size_t N>
+struct TestBitIter {
+ std::vector<bool> in;
+ TEST_CONSTEXPR_CXX20 TestBitIter() : in(N, false) {
+ for (std::size_t i = 0; i < N; i += 2)
+ in[i] = true;
+ }
+ TEST_CONSTEXPR_CXX20 void operator()() {
+ { // Test copy with aligned bytes
+ std::vector<bool> out(N);
+ std::ranges::copy_n(in.begin(), N, out.begin());
+ assert(in == out);
+ }
+ { // Test copy with unaligned bytes
+ std::vector<bool> out(N + 8);
+ std::ranges::copy_n(in.begin(), N, out.begin() + 4);
+ for (std::size_t i = 0; i < N; ++i)
+ assert(out[i + 4] == in[i]);
+ }
+ }
+ TEST_CONSTEXPR std::size_t size() const { return N; }
+};
+#endif
+
constexpr bool test() {
test_in_iterators<cpp20_input_iterator<int*>>();
test_in_iterators<forward_iterator<int*>>();
@@ -92,8 +121,8 @@ constexpr bool test() {
{ // check that every element is copied exactly once
struct CopyOnce {
- bool copied = false;
- constexpr CopyOnce() = default;
+ bool copied = false;
+ constexpr CopyOnce() = default;
constexpr CopyOnce(const CopyOnce& other) = delete;
constexpr CopyOnce& operator=(const CopyOnce& other) {
assert(!other.copied);
@@ -101,14 +130,24 @@ constexpr bool test() {
return *this;
}
};
- std::array<CopyOnce, 4> in {};
- std::array<CopyOnce, 4> out {};
+ std::array<CopyOnce, 4> in{};
+ std::array<CopyOnce, 4> out{};
auto ret = std::ranges::copy_n(in.begin(), in.size(), out.begin());
assert(ret.in == in.end());
assert(ret.out == out.end());
assert(std::all_of(out.begin(), out.end(), [](const auto& e) { return e.copied; }));
}
+#if TEST_STD_VER >= 23
+ { // Test vector<bool>::iterator optimization
+ TestBitIter<8>()();
+ TestBitIter<16>()();
+ TestBitIter<32>()();
+ TestBitIter<64>()();
+ TestBitIter<1024>()();
+ }
+#endif
+
return true;
}
More information about the libcxx-commits
mailing list