[libcxx-commits] [libcxx] [libc++][NFC] Use __specialized_algorithm for std::copy __bit_iterator specialization (PR #172270)
Nikolas Klauser via libcxx-commits
libcxx-commits at lists.llvm.org
Sat Dec 20 01:49:54 PST 2025
https://github.com/philnik777 updated https://github.com/llvm/llvm-project/pull/172270
>From 70a8225cc9e074028be7595bcb6bb5c60e84ea95 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser at berlin.de>
Date: Mon, 15 Dec 2025 11:02:02 +0100
Subject: [PATCH] [libc++][NFC] Use __specialized_algorithm for std::copy
__bit_iterator specialization
---
libcxx/include/__algorithm/copy.h | 150 +++---------------
.../__algorithm/specialized_algorithms.h | 1 +
libcxx/include/__bit_reference | 145 +++++++++++++++--
3 files changed, 157 insertions(+), 139 deletions(-)
diff --git a/libcxx/include/__algorithm/copy.h b/libcxx/include/__algorithm/copy.h
index 21fd25ce6fcdc..4caea922dac2d 100644
--- a/libcxx/include/__algorithm/copy.h
+++ b/libcxx/include/__algorithm/copy.h
@@ -12,11 +12,10 @@
#include <__algorithm/copy_move_common.h>
#include <__algorithm/for_each_segment.h>
#include <__algorithm/min.h>
+#include <__algorithm/specialized_algorithms.h>
#include <__config>
-#include <__fwd/bit_reference.h>
#include <__iterator/iterator_traits.h>
#include <__iterator/segmented_iterator.h>
-#include <__memory/pointer_traits.h>
#include <__type_traits/common_type.h>
#include <__type_traits/enable_if.h>
#include <__utility/move.h>
@@ -38,124 +37,14 @@ copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result);
template <class _InIter, class _Sent, class _OutIter>
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> __copy(_InIter, _Sent, _OutIter);
-template <class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned(
- __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
- using _In = __bit_iterator<_Cp, _IsConst>;
- using difference_type = typename _In::difference_type;
- using __storage_type = typename _In::__storage_type;
-
- const int __bits_per_word = _In::__bits_per_word;
- difference_type __n = __last - __first;
- if (__n > 0) {
- // do first word
- if (__first.__ctz_ != 0) {
- unsigned __clz = __bits_per_word - __first.__ctz_;
- difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
- __n -= __dn;
- __storage_type __m = std::__middle_mask<__storage_type>(__clz - __dn, __first.__ctz_);
- __storage_type __b = *__first.__seg_ & __m;
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b;
- __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
- __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
- ++__first.__seg_;
- // __first.__ctz_ = 0;
- }
- // __first.__ctz_ == 0;
- // do middle words
- __storage_type __nw = __n / __bits_per_word;
- std::copy(std::__to_address(__first.__seg_),
- std::__to_address(__first.__seg_ + __nw),
- std::__to_address(__result.__seg_));
- __n -= __nw * __bits_per_word;
- __result.__seg_ += __nw;
- // do last word
- if (__n > 0) {
- __first.__seg_ += __nw;
- __storage_type __m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
- __storage_type __b = *__first.__seg_ & __m;
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b;
- __result.__ctz_ = static_cast<unsigned>(__n);
- }
- }
- return __result;
-}
-
-template <class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned(
- __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
- using _In = __bit_iterator<_Cp, _IsConst>;
- using difference_type = typename _In::difference_type;
- using __storage_type = typename _In::__storage_type;
-
- const int __bits_per_word = _In::__bits_per_word;
- difference_type __n = __last - __first;
- if (__n > 0) {
- // do first word
- if (__first.__ctz_ != 0) {
- unsigned __clz_f = __bits_per_word - __first.__ctz_;
- difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
- __n -= __dn;
- __storage_type __m = std::__middle_mask<__storage_type>(__clz_f - __dn, __first.__ctz_);
- __storage_type __b = *__first.__seg_ & __m;
- unsigned __clz_r = __bits_per_word - __result.__ctz_;
- __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
- __m = std::__middle_mask<__storage_type>(__clz_r - __ddn, __result.__ctz_);
- *__result.__seg_ &= ~__m;
- if (__result.__ctz_ > __first.__ctz_)
- *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_);
- else
- *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_);
- __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
- __result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
- __dn -= __ddn;
- if (__dn > 0) {
- __m = std::__trailing_mask<__storage_type>(__bits_per_word - __dn);
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn);
- __result.__ctz_ = static_cast<unsigned>(__dn);
- }
- ++__first.__seg_;
- // __first.__ctz_ = 0;
- }
- // __first.__ctz_ == 0;
- // do middle words
- unsigned __clz_r = __bits_per_word - __result.__ctz_;
- __storage_type __m = std::__leading_mask<__storage_type>(__result.__ctz_);
- for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
- __storage_type __b = *__first.__seg_;
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b << __result.__ctz_;
- ++__result.__seg_;
- *__result.__seg_ &= __m;
- *__result.__seg_ |= __b >> __clz_r;
- }
- // do last word
- if (__n > 0) {
- __m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
- __storage_type __b = *__first.__seg_ & __m;
- __storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
- __m = std::__middle_mask<__storage_type>(__clz_r - __dn, __result.__ctz_);
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b << __result.__ctz_;
- __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
- __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
- __n -= __dn;
- if (__n > 0) {
- __m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
- *__result.__seg_ &= ~__m;
- *__result.__seg_ |= __b >> __dn;
- __result.__ctz_ = static_cast<unsigned>(__n);
- }
- }
- }
- return __result;
-}
-
struct __copy_impl {
- template <class _InIter, class _Sent, class _OutIter>
+ template <class _InIter,
+ class _Sent,
+ class _OutIter,
+ __enable_if_t<!__specialized_algorithm<_Algorithm::__copy,
+ __iterator_pair<_InIter, _Sent>,
+ __single_iterator<_OutIter> >::__has_algorithm,
+ int> = 0>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
operator()(_InIter __first, _Sent __last, _OutIter __result) const {
while (__first != __last) {
@@ -167,6 +56,19 @@ struct __copy_impl {
return std::make_pair(std::move(__first), std::move(__result));
}
+ template <class _InIter,
+ class _Sent,
+ class _OutIter,
+ __enable_if_t<__specialized_algorithm<_Algorithm::__copy,
+ __iterator_pair<_InIter, _Sent>,
+ __single_iterator<_OutIter> >::__has_algorithm,
+ int> = 0>
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static pair<_InIter, _OutIter>
+ operator()(_InIter __first, _Sent __last, _OutIter __result) {
+ return __specialized_algorithm<_Algorithm::__copy, __iterator_pair<_InIter, _Sent>, __single_iterator<_OutIter> >()(
+ std::move(__first), std::move(__last), std::move(__result));
+ }
+
template <class _InIter, class _OutIter>
struct _CopySegment {
using _Traits _LIBCPP_NODEBUG = __segmented_iterator_traits<_InIter>;
@@ -218,16 +120,6 @@ struct __copy_impl {
}
}
- template <class _Cp, bool _IsConst>
- _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
- operator()(__bit_iterator<_Cp, _IsConst> __first,
- __bit_iterator<_Cp, _IsConst> __last,
- __bit_iterator<_Cp, false> __result) const {
- if (__first.__ctz_ == __result.__ctz_)
- return std::make_pair(__last, std::__copy_aligned(__first, __last, __result));
- return std::make_pair(__last, std::__copy_unaligned(__first, __last, __result));
- }
-
// At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>
diff --git a/libcxx/include/__algorithm/specialized_algorithms.h b/libcxx/include/__algorithm/specialized_algorithms.h
index 1d3bc8723c3f3..7cb94c015f767 100644
--- a/libcxx/include/__algorithm/specialized_algorithms.h
+++ b/libcxx/include/__algorithm/specialized_algorithms.h
@@ -18,6 +18,7 @@
_LIBCPP_BEGIN_NAMESPACE_STD
namespace _Algorithm {
+struct __copy {};
struct __fill_n {};
struct __for_each {};
} // namespace _Algorithm
diff --git a/libcxx/include/__bit_reference b/libcxx/include/__bit_reference
index 00bbc4100ddb9..4f7d5443123a2 100644
--- a/libcxx/include/__bit_reference
+++ b/libcxx/include/__bit_reference
@@ -469,16 +469,6 @@ private:
template <class _Dp>
friend struct __bit_array;
- template <class _Dp, bool _IC>
- _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_aligned(
- __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
- template <class _Dp, bool _IC>
- _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_unaligned(
- __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
- template <class _Dp, bool _IC>
- _LIBCPP_CONSTEXPR_SINCE_CXX20 friend pair<__bit_iterator<_Dp, _IC>, __bit_iterator<_Dp, false> >
- __copy_impl::operator()(
- __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result) const;
template <class _Dp, bool _IC>
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_aligned(
__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
@@ -583,6 +573,141 @@ struct __specialized_algorithm<_Algorithm::__fill_n, __single_iterator<__bit_ite
}
};
+template <class _Cp, bool _IsConst>
+struct __specialized_algorithm<_Algorithm::__copy,
+ __iterator_pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, _IsConst> >,
+ __single_iterator<__bit_iterator<_Cp, false> > > {
+ static const bool __has_algorithm = true;
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static __bit_iterator<_Cp, false>
+ __aligned_impl(__bit_iterator<_Cp, _IsConst> __first,
+ __bit_iterator<_Cp, _IsConst> __last,
+ __bit_iterator<_Cp, false> __result) {
+ using _In = __bit_iterator<_Cp, _IsConst>;
+ using difference_type = typename _In::difference_type;
+ using __storage_type = typename _In::__storage_type;
+
+ const int __bits_per_word = _In::__bits_per_word;
+ difference_type __n = __last - __first;
+ if (__n > 0) {
+ // do first word
+ if (__first.__ctz_ != 0) {
+ unsigned __clz = __bits_per_word - __first.__ctz_;
+ difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
+ __n -= __dn;
+ __storage_type __m = std::__middle_mask<__storage_type>(__clz - __dn, __first.__ctz_);
+ __storage_type __b = *__first.__seg_ & __m;
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b;
+ __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+ __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+ ++__first.__seg_;
+ // __first.__ctz_ = 0;
+ }
+ // __first.__ctz_ == 0;
+ // do middle words
+ __storage_type __nw = __n / __bits_per_word;
+ std::copy(std::__to_address(__first.__seg_),
+ std::__to_address(__first.__seg_ + __nw),
+ std::__to_address(__result.__seg_));
+ __n -= __nw * __bits_per_word;
+ __result.__seg_ += __nw;
+ // do last word
+ if (__n > 0) {
+ __first.__seg_ += __nw;
+ __storage_type __m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
+ __storage_type __b = *__first.__seg_ & __m;
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b;
+ __result.__ctz_ = static_cast<unsigned>(__n);
+ }
+ }
+ return __result;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static __bit_iterator<_Cp, false>
+ __unaligned_impl(__bit_iterator<_Cp, _IsConst> __first,
+ __bit_iterator<_Cp, _IsConst> __last,
+ __bit_iterator<_Cp, false> __result) {
+ using _In = __bit_iterator<_Cp, _IsConst>;
+ using difference_type = typename _In::difference_type;
+ using __storage_type = typename _In::__storage_type;
+
+ const int __bits_per_word = _In::__bits_per_word;
+ difference_type __n = __last - __first;
+ if (__n > 0) {
+ // do first word
+ if (__first.__ctz_ != 0) {
+ unsigned __clz_f = __bits_per_word - __first.__ctz_;
+ difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
+ __n -= __dn;
+ __storage_type __m = std::__middle_mask<__storage_type>(__clz_f - __dn, __first.__ctz_);
+ __storage_type __b = *__first.__seg_ & __m;
+ unsigned __clz_r = __bits_per_word - __result.__ctz_;
+ __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
+ __m = std::__middle_mask<__storage_type>(__clz_r - __ddn, __result.__ctz_);
+ *__result.__seg_ &= ~__m;
+ if (__result.__ctz_ > __first.__ctz_)
+ *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_);
+ else
+ *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_);
+ __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
+ __result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
+ __dn -= __ddn;
+ if (__dn > 0) {
+ __m = std::__trailing_mask<__storage_type>(__bits_per_word - __dn);
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn);
+ __result.__ctz_ = static_cast<unsigned>(__dn);
+ }
+ ++__first.__seg_;
+ // __first.__ctz_ = 0;
+ }
+ // __first.__ctz_ == 0;
+ // do middle words
+ unsigned __clz_r = __bits_per_word - __result.__ctz_;
+ __storage_type __m = std::__leading_mask<__storage_type>(__result.__ctz_);
+ for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
+ __storage_type __b = *__first.__seg_;
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b << __result.__ctz_;
+ ++__result.__seg_;
+ *__result.__seg_ &= __m;
+ *__result.__seg_ |= __b >> __clz_r;
+ }
+ // do last word
+ if (__n > 0) {
+ __m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
+ __storage_type __b = *__first.__seg_ & __m;
+ __storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
+ __m = std::__middle_mask<__storage_type>(__clz_r - __dn, __result.__ctz_);
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b << __result.__ctz_;
+ __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
+ __result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
+ __n -= __dn;
+ if (__n > 0) {
+ __m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
+ *__result.__seg_ &= ~__m;
+ *__result.__seg_ |= __b >> __dn;
+ __result.__ctz_ = static_cast<unsigned>(__n);
+ }
+ }
+ }
+ return __result;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 static pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
+ operator()(__bit_iterator<_Cp, _IsConst> __first,
+ __bit_iterator<_Cp, _IsConst> __last,
+ __bit_iterator<_Cp, false> __result) {
+ if (__first.__ctz_ == __result.__ctz_)
+ return std::make_pair(__last, __aligned_impl(__first, __last, __result));
+ return std::make_pair(__last, __unaligned_impl(__first, __last, __result));
+ }
+};
+
_LIBCPP_END_NAMESPACE_STD
_LIBCPP_POP_MACROS
More information about the libcxx-commits
mailing list