[libcxx-commits] [libcxx] [libc++] Optimize ranges::{for_each, for_each_n} for segmented iterators (PR #132896)
Peng Liu via libcxx-commits
libcxx-commits at lists.llvm.org
Tue Mar 25 00:46:44 PDT 2025
https://github.com/winner245 created https://github.com/llvm/llvm-project/pull/132896
None
>From 49011aa7b5c3e81be7cf3fae5993466bcdc29038 Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245 at hotmail.com>
Date: Tue, 25 Mar 2025 03:44:59 -0400
Subject: [PATCH] Optimize ranges::{for_each, for_each_n} for segmented
iterators
---
libcxx/include/__algorithm/for_each_n.h | 25 +++-
libcxx/include/__algorithm/ranges_for_each.h | 14 ++-
.../include/__algorithm/ranges_for_each_n.h | 14 ++-
.../nonmodifying/for_each_n.bench.cpp | 57 +++++++++
.../alg.foreach/for_each_n.pass.cpp | 119 ++++++++++++------
.../alg.foreach/ranges.for_each.pass.cpp | 46 ++++++-
.../alg.foreach/ranges.for_each_n.pass.cpp | 45 ++++++-
7 files changed, 267 insertions(+), 53 deletions(-)
create mode 100644 libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp
diff --git a/libcxx/include/__algorithm/for_each_n.h b/libcxx/include/__algorithm/for_each_n.h
index fce380b49df3e..3d91124432f56 100644
--- a/libcxx/include/__algorithm/for_each_n.h
+++ b/libcxx/include/__algorithm/for_each_n.h
@@ -10,7 +10,11 @@
#ifndef _LIBCPP___ALGORITHM_FOR_EACH_N_H
#define _LIBCPP___ALGORITHM_FOR_EACH_N_H
+#include <__algorithm/for_each.h>
#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/segmented_iterator.h>
+#include <__type_traits/enable_if.h>
#include <__utility/convert_to_integral.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -21,7 +25,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 17
-template <class _InputIterator, class _Size, class _Function>
+template <class _InputIterator,
+ class _Size,
+ class _Function,
+ __enable_if_t<!__is_segmented_iterator<_InputIterator>::value ||
+ (__has_input_iterator_category<_InputIterator>::value &&
+ !__has_random_access_iterator_category<_InputIterator>::value),
+ int> = 0>
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
@@ -34,6 +44,19 @@ for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
return __first;
}
+template <class _InputIterator,
+ class _Size,
+ class _Function,
+ __enable_if_t<__is_segmented_iterator<_InputIterator>::value &&
+ __has_random_access_iterator_category<_InputIterator>::value,
+ int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
+for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
+ _InputIterator __last = __first + __orig_n;
+ std::for_each(__first, __last, __f);
+ return __last;
+}
+
#endif
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__algorithm/ranges_for_each.h b/libcxx/include/__algorithm/ranges_for_each.h
index de39bc5522753..475f85366188e 100644
--- a/libcxx/include/__algorithm/ranges_for_each.h
+++ b/libcxx/include/__algorithm/ranges_for_each.h
@@ -9,6 +9,7 @@
#ifndef _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
#define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
+#include <__algorithm/for_each.h>
#include <__algorithm/in_fun_result.h>
#include <__config>
#include <__functional/identity.h>
@@ -41,9 +42,16 @@ struct __for_each {
template <class _Iter, class _Sent, class _Proj, class _Func>
_LIBCPP_HIDE_FROM_ABI constexpr static for_each_result<_Iter, _Func>
__for_each_impl(_Iter __first, _Sent __last, _Func& __func, _Proj& __proj) {
- for (; __first != __last; ++__first)
- std::invoke(__func, std::invoke(__proj, *__first));
- return {std::move(__first), std::move(__func)};
+ if constexpr (random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>) {
+ auto __n = __last - __first;
+ auto __end = __first + __n;
+ std::for_each(__first, __end, [&](auto&& __val) { std::invoke(__func, std::invoke(__proj, __val)); });
+ return {std::move(__end), std::move(__func)};
+ } else {
+ for (; __first != __last; ++__first)
+ std::invoke(__func, std::invoke(__proj, *__first));
+ return {std::move(__first), std::move(__func)};
+ }
}
public:
diff --git a/libcxx/include/__algorithm/ranges_for_each_n.h b/libcxx/include/__algorithm/ranges_for_each_n.h
index 603cb723233c8..f3d6e5b85bfe2 100644
--- a/libcxx/include/__algorithm/ranges_for_each_n.h
+++ b/libcxx/include/__algorithm/ranges_for_each_n.h
@@ -10,6 +10,7 @@
#define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
#include <__algorithm/in_fun_result.h>
+#include <__algorithm/ranges_for_each.h>
#include <__config>
#include <__functional/identity.h>
#include <__functional/invoke.h>
@@ -40,11 +41,16 @@ struct __for_each_n {
template <input_iterator _Iter, class _Proj = identity, indirectly_unary_invocable<projected<_Iter, _Proj>> _Func>
_LIBCPP_HIDE_FROM_ABI constexpr for_each_n_result<_Iter, _Func>
operator()(_Iter __first, iter_difference_t<_Iter> __count, _Func __func, _Proj __proj = {}) const {
- while (__count-- > 0) {
- std::invoke(__func, std::invoke(__proj, *__first));
- ++__first;
+ if constexpr (random_access_iterator<_Iter>) {
+ auto __last = __first + __count;
+ return std::ranges::for_each(std::move(__first), std::move(__last), std::move(__func), std::move(__proj));
+ } else {
+ while (__count-- > 0) {
+ std::invoke(__func, std::invoke(__proj, *__first));
+ ++__first;
+ }
+ return {std::move(__first), std::move(__func)};
}
- return {std::move(__first), std::move(__func)};
}
};
diff --git a/libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp b/libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp
new file mode 100644
index 0000000000000..af46371881577
--- /dev/null
+++ b/libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <algorithm>
+#include <cstddef>
+#include <deque>
+#include <list>
+#include <string>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+
+int main(int argc, char** argv) {
+ auto std_for_each_n = [](auto first, auto n, auto f) { return std::for_each_n(first, n, f); };
+
+ // {std,ranges}::for_each_n
+ {
+ auto bm = []<class Container>(std::string name, auto for_each_n) {
+ benchmark::RegisterBenchmark(
+ name,
+ [for_each_n](auto& st) {
+ std::size_t const n = st.range(0);
+ Container c(n, 1);
+ auto first = c.begin();
+
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::DoNotOptimize(c);
+ auto result = for_each_n(first, n, [](int& x) { x = std::clamp(x, 10, 100); });
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(8)
+ ->Arg(32)
+ ->Arg(50) // non power-of-two
+ ->Arg(8192)
+ ->Arg(1 << 20);
+ };
+ bm.operator()<std::vector<int>>("std::for_each_n(vector<int>)", std_for_each_n);
+ bm.operator()<std::deque<int>>("std::for_each_n(deque<int>)", std_for_each_n);
+ bm.operator()<std::list<int>>("std::for_each_n(list<int>)", std_for_each_n);
+ bm.operator()<std::vector<int>>("rng::for_each_n(vector<int>)", std::ranges::for_each_n);
+ bm.operator()<std::deque<int>>("rng::for_each_n(deque<int>)", std::ranges::for_each_n);
+ bm.operator()<std::list<int>>("rng::for_each_n(list<int>)", std::ranges::for_each_n);
+ }
+
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+ benchmark::Shutdown();
+ return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/for_each_n.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/for_each_n.pass.cpp
index 371f6c92f1ed1..b91f640c46bf1 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/for_each_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/for_each_n.pass.cpp
@@ -13,69 +13,112 @@
// constexpr InputIterator // constexpr after C++17
// for_each_n(InputIterator first, Size n, Function f);
-
#include <algorithm>
#include <cassert>
+#include <deque>
#include <functional>
+#include <iterator>
+#include <ranges>
#include "test_macros.h"
#include "test_iterators.h"
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool test_constexpr() {
- int ia[] = {1, 3, 6, 7};
- int expected[] = {3, 5, 8, 9};
- const std::size_t N = 4;
+struct for_each_test {
+ TEST_CONSTEXPR for_each_test(int c) : count(c) {}
+ int count;
+ TEST_CONSTEXPR_CXX14 void operator()(int& i) {
+ ++i;
+ ++count;
+ }
+};
- auto it = std::for_each_n(std::begin(ia), N, [](int &a) { a += 2; });
- return it == (std::begin(ia) + N)
- && std::equal(std::begin(ia), std::end(ia), std::begin(expected))
- ;
- }
-#endif
+struct deque_test {
+ std::deque<int>* d_;
+ int* i_;
+
+ deque_test(std::deque<int>& d, int& i) : d_(&d), i_(&i) {}
-struct for_each_test
-{
- for_each_test(int c) : count(c) {}
- int count;
- void operator()(int& i) {++i; ++count;}
+ void operator()(int& v) {
+ assert(&(*d_)[*i_] == &v);
+ ++*i_;
+ }
};
-int main(int, char**)
-{
+/*TEST_CONSTEXPR_CXX23*/
+void test_segmented_deque_iterator() { // TODO: Mark as TEST_CONSTEXPR_CXX23 once std::deque is constexpr
+ // check that segmented iterators work properly
+ int sizes[] = {0, 1, 2, 1023, 1024, 1025, 2047, 2048, 2049};
+ for (const int size : sizes) {
+ std::deque<int> d(size);
+ int index = 0;
+
+ std::for_each_n(d.begin(), d.size(), deque_test(d, index));
+ }
+}
+
+TEST_CONSTEXPR_CXX20 bool test() {
+ {
typedef cpp17_input_iterator<int*> Iter;
- int ia[] = {0, 1, 2, 3, 4, 5};
- const unsigned s = sizeof(ia)/sizeof(ia[0]);
+ int ia[] = {0, 1, 2, 3, 4, 5};
+ const unsigned s = sizeof(ia) / sizeof(ia[0]);
{
- auto f = for_each_test(0);
- Iter it = std::for_each_n(Iter(ia), 0, std::ref(f));
- assert(it == Iter(ia));
- assert(f.count == 0);
+ auto f = for_each_test(0);
+ Iter it = std::for_each_n(Iter(ia), 0, std::ref(f));
+ assert(it == Iter(ia));
+ assert(f.count == 0);
}
{
- auto f = for_each_test(0);
- Iter it = std::for_each_n(Iter(ia), s, std::ref(f));
+ auto f = for_each_test(0);
+ Iter it = std::for_each_n(Iter(ia), s, std::ref(f));
- assert(it == Iter(ia+s));
- assert(f.count == s);
- for (unsigned i = 0; i < s; ++i)
- assert(ia[i] == static_cast<int>(i+1));
+ assert(it == Iter(ia + s));
+ assert(f.count == s);
+ for (unsigned i = 0; i < s; ++i)
+ assert(ia[i] == static_cast<int>(i + 1));
}
{
- auto f = for_each_test(0);
- Iter it = std::for_each_n(Iter(ia), 1, std::ref(f));
+ auto f = for_each_test(0);
+ Iter it = std::for_each_n(Iter(ia), 1, std::ref(f));
- assert(it == Iter(ia+1));
- assert(f.count == 1);
- for (unsigned i = 0; i < 1; ++i)
- assert(ia[i] == static_cast<int>(i+2));
+ assert(it == Iter(ia + 1));
+ assert(f.count == 1);
+ for (unsigned i = 0; i < 1; ++i)
+ assert(ia[i] == static_cast<int>(i + 2));
}
+ }
+
+#if TEST_STD_VER > 11
+ {
+ int ia[] = {1, 3, 6, 7};
+ int expected[] = {3, 5, 8, 9};
+ const std::size_t N = 4;
+
+ auto it = std::for_each_n(std::begin(ia), N, [](int& a) { a += 2; });
+ assert(it == (std::begin(ia) + N) && std::equal(std::begin(ia), std::end(ia), std::begin(expected)));
+ }
+#endif
+
+ if (!TEST_IS_CONSTANT_EVALUATED) // TODO: Use TEST_STD_AT_LEAST_23_OR_RUNTIME_EVALUATED when std::deque is made constexpr
+ test_segmented_deque_iterator();
+
+#if TEST_STD_VER >= 20
+ { // Make sure that the segmented iterator optimization works during constant evaluation
+ std::vector<std::vector<int>> vec = {{0}, {1, 2}, {3, 4, 5}, {6, 7, 8, 9}, {10}, {11, 12, 13}};
+ auto v = vec | std::views::join;
+ std::for_each_n(v.begin(), std::ranges::distance(v), [i = 0](int& a) mutable { assert(a == i++); });
+ }
+#endif
+
+ return true;
+}
+int main(int, char**) {
+ assert(test());
#if TEST_STD_VER > 17
- static_assert(test_constexpr());
+ static_assert(test());
#endif
return 0;
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each.pass.cpp
index 8b9b6e82cbcb2..2f4bfb9db6dba 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each.pass.cpp
@@ -20,7 +20,10 @@
#include <algorithm>
#include <array>
+#include <cassert>
+#include <deque>
#include <ranges>
+#include <vector>
#include "almost_satisfies_types.h"
#include "test_iterators.h"
@@ -30,7 +33,7 @@ struct Callable {
};
template <class Iter, class Sent = Iter>
-concept HasForEachIt = requires (Iter iter, Sent sent) { std::ranges::for_each(iter, sent, Callable{}); };
+concept HasForEachIt = requires(Iter iter, Sent sent) { std::ranges::for_each(iter, sent, Callable{}); };
static_assert(HasForEachIt<int*>);
static_assert(!HasForEachIt<InputIteratorNotDerivedFrom>);
@@ -47,7 +50,7 @@ static_assert(!HasForEachItFunc<IndirectUnaryPredicateNotPredicate>);
static_assert(!HasForEachItFunc<IndirectUnaryPredicateNotCopyConstructible>);
template <class Range>
-concept HasForEachR = requires (Range range) { std::ranges::for_each(range, Callable{}); };
+concept HasForEachR = requires(Range range) { std::ranges::for_each(range, Callable{}); };
static_assert(HasForEachR<UncheckedRange<int*>>);
static_assert(!HasForEachR<InputRangeNotDerivedFrom>);
@@ -68,7 +71,7 @@ constexpr void test_iterator() {
{ // simple test
{
auto func = [i = 0](int& a) mutable { a += i++; };
- int a[] = {1, 6, 3, 4};
+ int a[] = {1, 6, 3, 4};
std::same_as<std::ranges::for_each_result<Iter, decltype(func)>> decltype(auto) ret =
std::ranges::for_each(Iter(a), Sent(Iter(a + 4)), func);
assert(a[0] == 1);
@@ -81,8 +84,8 @@ constexpr void test_iterator() {
assert(i == 4);
}
{
- auto func = [i = 0](int& a) mutable { a += i++; };
- int a[] = {1, 6, 3, 4};
+ auto func = [i = 0](int& a) mutable { a += i++; };
+ int a[] = {1, 6, 3, 4};
auto range = std::ranges::subrange(Iter(a), Sent(Iter(a + 4)));
std::same_as<std::ranges::for_each_result<Iter, decltype(func)>> decltype(auto) ret =
std::ranges::for_each(range, func);
@@ -110,6 +113,30 @@ constexpr void test_iterator() {
}
}
+struct deque_test {
+ std::deque<int>* d_;
+ int* i_;
+
+ deque_test(std::deque<int>& d, int& i) : d_(&d), i_(&i) {}
+
+ void operator()(int& v) {
+ assert(&(*d_)[*i_] == &v);
+ ++*i_;
+ }
+};
+
+/*TEST_CONSTEXPR_CXX23*/
+void test_segmented_deque_iterator() { // TODO: Mark as TEST_CONSTEXPR_CXX23 once std::deque is constexpr
+ // check that segmented iterators work properly
+ int sizes[] = {0, 1, 2, 1023, 1024, 1025, 2047, 2048, 2049};
+ for (const int size : sizes) {
+ std::deque<int> d(size);
+ int index = 0;
+
+ std::ranges::for_each(d, deque_test(d, index));
+ }
+}
+
constexpr bool test() {
test_iterator<cpp17_input_iterator<int*>, sentinel_wrapper<cpp17_input_iterator<int*>>>();
test_iterator<cpp20_input_iterator<int*>, sentinel_wrapper<cpp20_input_iterator<int*>>>();
@@ -146,6 +173,15 @@ constexpr bool test() {
}
}
+ if (!TEST_IS_CONSTANT_EVALUATED) // TODO: Use TEST_STD_AT_LEAST_23_OR_RUNTIME_EVALUATED when std::deque is made constexpr
+ test_segmented_deque_iterator();
+
+ {
+ std::vector<std::vector<int>> vec = {{0}, {1, 2}, {3, 4, 5}, {6, 7, 8, 9}, {10}, {11, 12, 13}};
+ auto v = vec | std::views::join;
+ std::ranges::for_each(v, [i = 0](int x) mutable { assert(x == 2 * i++); }, [](int x) { return 2 * x; });
+ }
+
return true;
}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each_n.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each_n.pass.cpp
index d4b2d053d08ce..9cf875e40335c 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each_n.pass.cpp
@@ -17,6 +17,10 @@
#include <algorithm>
#include <array>
+#include <cassert>
+#include <deque>
+#include <iterator>
+#include <ranges>
#include <ranges>
#include "almost_satisfies_types.h"
@@ -27,7 +31,7 @@ struct Callable {
};
template <class Iter>
-concept HasForEachN = requires (Iter iter) { std::ranges::for_each_n(iter, 0, Callable{}); };
+concept HasForEachN = requires(Iter iter) { std::ranges::for_each_n(iter, 0, Callable{}); };
static_assert(HasForEachN<int*>);
static_assert(!HasForEachN<InputIteratorNotDerivedFrom>);
@@ -45,7 +49,7 @@ template <class Iter>
constexpr void test_iterator() {
{ // simple test
auto func = [i = 0](int& a) mutable { a += i++; };
- int a[] = {1, 6, 3, 4};
+ int a[] = {1, 6, 3, 4};
std::same_as<std::ranges::for_each_result<Iter, decltype(func)>> auto ret =
std::ranges::for_each_n(Iter(a), 4, func);
assert(a[0] == 1);
@@ -64,6 +68,30 @@ constexpr void test_iterator() {
}
}
+struct deque_test {
+ std::deque<int>* d_;
+ int* i_;
+
+ deque_test(std::deque<int>& d, int& i) : d_(&d), i_(&i) {}
+
+ void operator()(int& v) {
+ assert(&(*d_)[*i_] == &v);
+ ++*i_;
+ }
+};
+
+/*TEST_CONSTEXPR_CXX23*/
+void test_segmented_deque_iterator() { // TODO: Mark as TEST_CONSTEXPR_CXX23 once std::deque is constexpr
+ // check that segmented iterators work properly
+ int sizes[] = {0, 1, 2, 1023, 1024, 1025, 2047, 2048, 2049};
+ for (const int size : sizes) {
+ std::deque<int> d(size);
+ int index = 0;
+
+ std::ranges::for_each_n(d.begin(), d.size(), deque_test(d, index));
+ }
+}
+
constexpr bool test() {
test_iterator<cpp17_input_iterator<int*>>();
test_iterator<cpp20_input_iterator<int*>>();
@@ -89,6 +117,19 @@ constexpr bool test() {
assert(a[2].other == 6);
}
+ if (!TEST_IS_CONSTANT_EVALUATED) // TODO: Use TEST_STD_AT_LEAST_23_OR_RUNTIME_EVALUATED when std::deque is made constexpr
+ test_segmented_deque_iterator();
+
+ {
+ std::vector<std::vector<int>> vec = {{0}, {1, 2}, {3, 4, 5}, {6, 7, 8, 9}, {10}, {11, 12, 13}};
+ auto v = vec | std::views::join;
+ std::ranges::for_each_n(
+ v.begin(),
+ std::ranges::distance(v),
+ [i = 0](int x) mutable { assert(x == 2 * i++); },
+ [](int x) { return 2 * x; });
+ }
+
return true;
}
More information about the libcxx-commits
mailing list