[libcxx-commits] [libcxx] [libc++] Optimize {std, ranges}::distance for segmented iterators (PR #133612)
Peng Liu via libcxx-commits
libcxx-commits at lists.llvm.org
Sun Mar 30 17:04:01 PDT 2025
https://github.com/winner245 updated https://github.com/llvm/llvm-project/pull/133612
>From 5795a5a83a4da9c29062df0786b93ba76c2a0b61 Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245 at hotmail.com>
Date: Sat, 29 Mar 2025 22:29:33 -0400
Subject: [PATCH] Optimize {std,ranges}::distance for segmented iterators
---
libcxx/include/__iterator/distance.h | 68 +++++++++++---
.../benchmarks/iterators/distance.bench.cpp | 71 +++++++++++++++
.../iterator.operations/distance.pass.cpp | 77 +++++++++++-----
.../iterator_sentinel.pass.cpp | 91 +++++++++++++------
4 files changed, 245 insertions(+), 62 deletions(-)
create mode 100644 libcxx/test/benchmarks/iterators/distance.bench.cpp
diff --git a/libcxx/include/__iterator/distance.h b/libcxx/include/__iterator/distance.h
index 1732aa527f64a..8d14299191e30 100644
--- a/libcxx/include/__iterator/distance.h
+++ b/libcxx/include/__iterator/distance.h
@@ -10,41 +10,82 @@
#ifndef _LIBCPP___ITERATOR_DISTANCE_H
#define _LIBCPP___ITERATOR_DISTANCE_H
+#include <__algorithm/for_each_segment.h>
#include <__config>
#include <__iterator/concepts.h>
#include <__iterator/incrementable_traits.h>
#include <__iterator/iterator_traits.h>
+#include <__iterator/segmented_iterator.h>
#include <__ranges/access.h>
#include <__ranges/concepts.h>
#include <__ranges/size.h>
#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
#include <__type_traits/remove_cvref.h>
+#include <__utility/move.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
_LIBCPP_BEGIN_NAMESPACE_STD
-template <class _InputIter>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type
-__distance(_InputIter __first, _InputIter __last, input_iterator_tag) {
- typename iterator_traits<_InputIter>::difference_type __r(0);
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter>
+using __iter_distance_t _LIBCPP_NODEBUG = std::iter_difference_t<_Iter>;
+#else
+template <class _Iter>
+using __iter_distance_t _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::difference_type;
+#endif
+
+template <class _InputIter, class _Sent>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_InputIter>
+__distance(_InputIter __first, _Sent __last) {
+ __iter_distance_t<_InputIter> __r(0);
for (; __first != __last; ++__first)
++__r;
return __r;
}
-template <class _RandIter>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_RandIter>::difference_type
-__distance(_RandIter __first, _RandIter __last, random_access_iterator_tag) {
+template <class _RandIter, __enable_if_t<__has_random_access_iterator_category<_RandIter>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_RandIter>
+__distance(_RandIter __first, _RandIter __last) {
return __last - __first;
}
+template <class _SegmentedIter, class _Difference>
+struct __segment_distance {
+ using _Traits _LIBCPP_NODEBUG = __segmented_iterator_traits<_SegmentedIter>;
+
+ _Difference& __r_;
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __segment_distance(_Difference& __r) : __r_(__r) {}
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+ operator()(typename _Traits::__local_iterator __lfirst, typename _Traits::__local_iterator __llast) {
+ __r_ += std::__distance(__lfirst, __llast);
+ }
+};
+
+template <class _SegmentedIter,
+ __enable_if_t<!__has_random_access_iterator_category<_SegmentedIter>::value &&
+ __is_segmented_iterator<_SegmentedIter>::value,
+ int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_SegmentedIter>
+__distance(_SegmentedIter __first, _SegmentedIter __last) {
+ using __difference_type = __iter_distance_t<_SegmentedIter>;
+ __difference_type __r(0);
+ std::__for_each_segment(__first, __last, std::__segment_distance<_SegmentedIter, __difference_type>(__r));
+ return __r;
+}
+
template <class _InputIter>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_InputIter>
distance(_InputIter __first, _InputIter __last) {
- return std::__distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category());
+ return std::__distance(__first, __last);
}
#if _LIBCPP_STD_VER >= 20
@@ -56,12 +97,7 @@ struct __distance {
template <class _Ip, sentinel_for<_Ip> _Sp>
requires(!sized_sentinel_for<_Sp, _Ip>)
_LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip __first, _Sp __last) const {
- iter_difference_t<_Ip> __n = 0;
- while (__first != __last) {
- ++__first;
- ++__n;
- }
- return __n;
+ return std::__distance(std::move(__first), std::move(__last));
}
template <class _Ip, sized_sentinel_for<decay_t<_Ip>> _Sp>
@@ -92,4 +128,6 @@ inline constexpr auto distance = __distance{};
_LIBCPP_END_NAMESPACE_STD
+_LIBCPP_POP_MACROS
+
#endif // _LIBCPP___ITERATOR_DISTANCE_H
diff --git a/libcxx/test/benchmarks/iterators/distance.bench.cpp b/libcxx/test/benchmarks/iterators/distance.bench.cpp
new file mode 100644
index 0000000000000..db9e6a9201658
--- /dev/null
+++ b/libcxx/test/benchmarks/iterators/distance.bench.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <cstddef>
+#include <deque>
+#include <iterator>
+#include <ranges>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+
+int main(int argc, char** argv) {
+ auto std_distance = [](auto first, auto last) { return std::distance(first, last); };
+
+ // {std,ranges}::distance
+ {
+ auto bm = []<class Container>(std::string name, auto distance, std::size_t seg_size) {
+ benchmark::RegisterBenchmark(
+ name,
+ [distance, seg_size](auto& st) {
+ std::size_t const size = st.range(0);
+ std::size_t const segments = (size + seg_size - 1) / seg_size;
+ Container c(segments);
+ for (std::size_t i = 0, n = size; i < segments; ++i, n -= seg_size) {
+ c[i].resize(std::min(seg_size, n));
+ }
+
+ auto view = c | std::views::join;
+ auto first = view.begin();
+ auto last = view.end();
+
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::DoNotOptimize(c);
+ auto result = distance(first, last);
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(50) // non power-of-two
+ ->Arg(1024)
+ ->Arg(4096)
+ ->Arg(8192)
+ ->Arg(1 << 14)
+ ->Arg(1 << 16)
+ ->Arg(1 << 18)
+ ->Arg(1 << 20);
+ };
+ bm.operator()<std::vector<std::vector<int>>>("std::distance(join_view(vector<vector<int>>))", std_distance, 256);
+ bm.operator()<std::deque<std::deque<int>>>("std::distance(join_view(deque<deque<int>>))", std_distance, 256);
+ bm.operator()<std::vector<std::vector<int>>>(
+ "rng::distance(join_view(vector<vector<int>>)", std::ranges::distance, 256);
+ bm.operator()<std::deque<std::deque<int>>>(
+ "rng::distance(join_view(deque<deque<int>>)", std::ranges::distance, 256);
+
+ // bm.operator()<std::vector<std::vector<int>>>("std::distance(join_view(vector<vector<int>>))", std_distance, 1024);
+ // bm.operator()<std::deque<std::deque<int>>>("std::distance(join_view(deque<deque<int>>))", std_distance, 1024);
+ // bm.operator()<std::vector<std::vector<int>>>("rng::distance(join_view(vector<vector<int>>)", std::ranges::distance, 1024);
+ // bm.operator()<std::deque<std::deque<int>>>("rng::distance(join_view(deque<deque<int>>)", std::ranges::distance, 1024);
+ }
+
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+ benchmark::Shutdown();
+ return 0;
+}
diff --git a/libcxx/test/std/iterators/iterator.primitives/iterator.operations/distance.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/iterator.operations/distance.pass.cpp
index 13caefff92365..bcaeee14cad6d 100644
--- a/libcxx/test/std/iterators/iterator.primitives/iterator.operations/distance.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.primitives/iterator.operations/distance.pass.cpp
@@ -16,38 +16,73 @@
// Iter::difference_type
// distance(Iter first, Iter last); // constexpr in C++17
-#include <iterator>
+#include <array>
#include <cassert>
+#include <deque>
+#include <iterator>
+#include <vector>
#include <type_traits>
#include "test_macros.h"
#include "test_iterators.h"
template <class It>
-TEST_CONSTEXPR_CXX17
-void check_distance(It first, It last, typename std::iterator_traits<It>::difference_type dist)
-{
- typedef typename std::iterator_traits<It>::difference_type Difference;
- static_assert(std::is_same<decltype(std::distance(first, last)), Difference>::value, "");
- assert(std::distance(first, last) == dist);
+TEST_CONSTEXPR_CXX17 void check_distance(It first, It last, typename std::iterator_traits<It>::difference_type dist) {
+ typedef typename std::iterator_traits<It>::difference_type Difference;
+ static_assert(std::is_same<decltype(std::distance(first, last)), Difference>::value, "");
+ assert(std::distance(first, last) == dist);
}
-TEST_CONSTEXPR_CXX17 bool tests()
-{
- const char* s = "1234567890";
- check_distance(cpp17_input_iterator<const char*>(s), cpp17_input_iterator<const char*>(s+10), 10);
- check_distance(forward_iterator<const char*>(s), forward_iterator<const char*>(s+10), 10);
- check_distance(bidirectional_iterator<const char*>(s), bidirectional_iterator<const char*>(s+10), 10);
- check_distance(random_access_iterator<const char*>(s), random_access_iterator<const char*>(s+10), 10);
- check_distance(s, s+10, 10);
- return true;
+#if TEST_STD_VER >= 20
+void test_deque() {
+ using Container = std::deque<std::deque<double>>;
+ Container c;
+ auto view = c | std::views::join;
+ Container::difference_type n = 0;
+ for (std::size_t i = 0; i < 10; ++i) {
+ n += i;
+ c.push_back(Container::value_type(i));
+ }
+ assert(std::distance(view.begin(), view.end()) == n);
+}
+#endif
+
+TEST_CONSTEXPR_CXX17 bool tests() {
+ const char* s = "1234567890";
+ check_distance(cpp17_input_iterator<const char*>(s), cpp17_input_iterator<const char*>(s + 10), 10);
+ check_distance(forward_iterator<const char*>(s), forward_iterator<const char*>(s + 10), 10);
+ check_distance(bidirectional_iterator<const char*>(s), bidirectional_iterator<const char*>(s + 10), 10);
+ check_distance(random_access_iterator<const char*>(s), random_access_iterator<const char*>(s + 10), 10);
+ check_distance(s, s + 10, 10);
+
+#if TEST_STD_VER >= 20
+ {
+ using Container = std::vector<std::vector<int>>;
+ Container c;
+ auto view = c | std::views::join;
+ Container::difference_type n = 0;
+ for (std::size_t i = 0; i < 10; ++i) {
+ n += i;
+ c.push_back(Container::value_type(i));
+ }
+ assert(std::distance(view.begin(), view.end()) == n);
+ }
+ {
+ using Container = std::array<std::array<char, 3>, 10>;
+ Container c;
+ auto view = c | std::views::join;
+ assert(std::distance(view.begin(), view.end()) == 30);
+ }
+ if (!TEST_IS_CONSTANT_EVALUATED)
+ test_deque();
+#endif
+ return true;
}
-int main(int, char**)
-{
- tests();
+int main(int, char**) {
+ tests();
#if TEST_STD_VER >= 17
- static_assert(tests(), "");
+ static_assert(tests(), "");
#endif
- return 0;
+ return 0;
}
diff --git a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.distance/iterator_sentinel.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.distance/iterator_sentinel.pass.cpp
index b4199b73ad76a..a5fcf337387c2 100644
--- a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.distance/iterator_sentinel.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.distance/iterator_sentinel.pass.cpp
@@ -15,18 +15,21 @@
// template<class I, sized_sentinel_for<decay_t<I>> S>
// constexpr iter_difference_t<I> ranges::distance(I&& first, S last); // TODO: update when LWG3664 is resolved
-#include <iterator>
+#include <array>
#include <cassert>
+#include <deque>
+#include <iterator>
+#include <vector>
#include "test_iterators.h"
#include "test_macros.h"
-template<class It, class Sent>
+template <class It, class Sent>
constexpr void test_unsized() {
static_assert(std::sentinel_for<Sent, It> && !std::sized_sentinel_for<Sent, It>);
- int a[3] = {1,2,3};
+ int a[3] = {1, 2, 3};
{
- It first = It(a);
+ It first = It(a);
auto last = Sent(It(a));
assert(std::ranges::distance(first, last) == 0);
assert(std::ranges::distance(It(a), last) == 0);
@@ -36,7 +39,7 @@ constexpr void test_unsized() {
}
{
auto check = [&a]<class ItQual, class SentQual> {
- It first = It(a);
+ It first = It(a);
Sent last = Sent(It(a + 3));
assert(std::ranges::distance(static_cast<ItQual>(first), static_cast<SentQual>(last)) == 3);
};
@@ -61,13 +64,13 @@ constexpr void test_unsized() {
}
}
-template<class It, class Sent>
+template <class It, class Sent>
constexpr void test_sized() {
static_assert(std::sized_sentinel_for<Sent, It>);
- int a[] = {1,2,3};
+ int a[] = {1, 2, 3};
{
auto check = [&a]<class ItQual, class SentQual> {
- It first = It(a + 3);
+ It first = It(a + 3);
Sent last = Sent(It(a));
assert(std::ranges::distance(static_cast<ItQual>(first), static_cast<SentQual>(last)) == -3);
};
@@ -91,7 +94,7 @@ constexpr void test_sized() {
check.template operator()<const It&&, const Sent&&>();
}
{
- It first = It(a);
+ It first = It(a);
auto last = Sent(It(a));
assert(std::ranges::distance(first, last) == 0);
assert(std::ranges::distance(It(a), last) == 0);
@@ -100,7 +103,7 @@ constexpr void test_sized() {
ASSERT_SAME_TYPE(decltype(std::ranges::distance(It(a), Sent(It(a)))), std::iter_difference_t<It>);
}
{
- It first = It(a);
+ It first = It(a);
auto last = Sent(It(a + 3));
assert(std::ranges::distance(first, last) == 3);
assert(std::ranges::distance(It(a), last) == 3);
@@ -110,13 +113,17 @@ constexpr void test_sized() {
}
struct StrideCounter {
- int *it_;
- int *inc_;
- using value_type = int;
+ int* it_;
+ int* inc_;
+ using value_type = int;
using difference_type = int;
explicit StrideCounter();
- constexpr explicit StrideCounter(int *it, int *inc) : it_(it), inc_(inc) {}
- constexpr auto& operator++() { ++it_; *inc_ += 1; return *this; }
+ constexpr explicit StrideCounter(int* it, int* inc) : it_(it), inc_(inc) {}
+ constexpr auto& operator++() {
+ ++it_;
+ *inc_ += 1;
+ return *this;
+ }
StrideCounter operator++(int);
int& operator*() const;
bool operator==(StrideCounter) const;
@@ -125,11 +132,11 @@ static_assert(std::forward_iterator<StrideCounter>);
static_assert(!std::sized_sentinel_for<StrideCounter, StrideCounter>);
struct SizedStrideCounter {
- int *it_;
- int *minus_;
+ int* it_;
+ int* minus_;
using value_type = int;
explicit SizedStrideCounter();
- constexpr explicit SizedStrideCounter(int *it, int *minus) : it_(it), minus_(minus) {}
+ constexpr explicit SizedStrideCounter(int* it, int* minus) : it_(it), minus_(minus) {}
SizedStrideCounter& operator++();
SizedStrideCounter operator++(int);
int& operator*() const;
@@ -147,22 +154,34 @@ constexpr void test_stride_counting() {
int a[] = {1, 2, 3};
int inc = 0;
StrideCounter first(a, &inc);
- StrideCounter last(a+3, nullptr);
+ StrideCounter last(a + 3, nullptr);
std::same_as<int> auto result = std::ranges::distance(first, last);
assert(result == 3);
assert(inc == 3);
}
{
- int a[] = {1, 2, 3};
+ int a[] = {1, 2, 3};
int minus = 0;
SizedStrideCounter first(a, &minus);
- SizedStrideCounter last(a+3, nullptr);
+ SizedStrideCounter last(a + 3, nullptr);
std::same_as<int> auto result = std::ranges::distance(first, last);
assert(result == 3);
assert(minus == 1);
}
}
+void test_deque() {
+ using Container = std::deque<std::deque<double>>;
+ Container c;
+ auto view = c | std::views::join;
+ Container::difference_type n = 0;
+ for (std::size_t i = 0; i < 10; ++i) {
+ n += i;
+ c.push_back(Container::value_type(i));
+ }
+ assert(std::ranges::distance(view.begin(), view.end()) == n);
+}
+
constexpr bool test() {
{
int a[] = {1, 2, 3};
@@ -197,7 +216,7 @@ constexpr bool test() {
test_sized<contiguous_iterator<int*>, contiguous_iterator<int*>>();
{
- using It = cpp20_input_iterator<int*>; // non-copyable, thus not a sentinel for itself
+ using It = cpp20_input_iterator<int*>; // non-copyable, thus not a sentinel for itself
static_assert(!std::is_copy_constructible_v<It>);
static_assert(!std::sentinel_for<It, It>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, It&>);
@@ -206,10 +225,10 @@ constexpr bool test() {
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&&, It&&>);
}
{
- using It = cpp20_input_iterator<int*>; // non-copyable
- using Sent = sentinel_wrapper<It>; // not a sized sentinel
+ using It = cpp20_input_iterator<int*>; // non-copyable
+ using Sent = sentinel_wrapper<It>; // not a sized sentinel
static_assert(std::sentinel_for<Sent, It> && !std::sized_sentinel_for<Sent, It>);
- int a[] = {1,2,3};
+ int a[] = {1, 2, 3};
Sent last = Sent(It(a + 3));
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, Sent&>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, Sent&&>);
@@ -217,7 +236,7 @@ constexpr bool test() {
assert(std::ranges::distance(It(a), Sent(It(a + 3))) == 3);
}
{
- using It = cpp17_input_iterator<int*>; // not a sentinel for itself
+ using It = cpp17_input_iterator<int*>; // not a sentinel for itself
static_assert(!std::sentinel_for<It, It>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, It&>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, It&&>);
@@ -231,6 +250,26 @@ constexpr bool test() {
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), int, int*>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), int*, char*>);
+ {
+ using Container = std::vector<std::vector<int>>;
+ Container c;
+ auto view = c | std::views::join;
+ Container::difference_type n = 0;
+ for (std::size_t i = 0; i < 10; ++i) {
+ n += i;
+ c.push_back(Container::value_type(i));
+ }
+ assert(std::ranges::distance(view.begin(), view.end()) == n);
+ }
+ {
+ using Container = std::array<std::array<char, 3>, 10>;
+ Container c;
+ auto view = c | std::views::join;
+ assert(std::ranges::distance(view.begin(), view.end()) == 30);
+ }
+ if (!TEST_IS_CONSTANT_EVALUATED)
+ test_deque();
+
return true;
}
More information about the libcxx-commits
mailing list