[libcxx-commits] [libcxx] [libc++][pstl] Implementation of parallel std::transform_inclusive_scan (PR #223802)
Michael G. Kazakov via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Sep 23 13:48:46 PDT 2026
https://github.com/mikekazakov updated https://github.com/llvm/llvm-project/pull/223802
>From 5519d9f079c11ac06547e28d44b3e81abb985e8f Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Tue, 15 Sep 2026 20:53:04 +0100
Subject: [PATCH 1/5] Implementation of 3-passes inclusive scan
---
libcxx/include/CMakeLists.txt | 1 +
libcxx/include/__numeric/pstl.h | 57 +++++
libcxx/include/__pstl/backend_fwd.h | 21 ++
libcxx/include/__pstl/backends/default.h | 38 +++
libcxx/include/__pstl/backends/libdispatch.h | 63 +++++
libcxx/include/__pstl/backends/serial.h | 27 +++
libcxx/include/__pstl/backends/std_thread.h | 24 ++
.../cpu_algos/transform_inclusive_scan_init.h | 97 ++++++++
libcxx/include/module.modulemap.in | 3 +
....transform_inclusive_scan_bop_uop.pass.cpp | 159 +++++++++++++
...sform_inclusive_scan_bop_uop_init.pass.cpp | 216 ++++++++++++++++++
11 files changed, 706 insertions(+)
create mode 100644 libcxx/include/__pstl/cpu_algos/transform_inclusive_scan_init.h
create mode 100644 libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/pstl.transform_inclusive_scan_bop_uop.pass.cpp
create mode 100644 libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/pstl.transform_inclusive_scan_bop_uop_init.pass.cpp
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index bc11bda33724b6..93e392dd3a5235 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -709,6 +709,7 @@ set(files
__pstl/cpu_algos/search.h
__pstl/cpu_algos/search_n.h
__pstl/cpu_algos/stable_sort.h
+ __pstl/cpu_algos/transform_inclusive_scan_init.h
__pstl/cpu_algos/transform.h
__pstl/cpu_algos/transform_reduce.h
__pstl/dispatch.h
diff --git a/libcxx/include/__numeric/pstl.h b/libcxx/include/__numeric/pstl.h
index 0b85cfb8c23276..414dedb7e527a9 100644
--- a/libcxx/include/__numeric/pstl.h
+++ b/libcxx/include/__numeric/pstl.h
@@ -82,6 +82,63 @@ reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator _
plus{});
}
+template <class _ExecutionPolicy,
+ class _ForwardIterator1,
+ class _ForwardIterator2,
+ class _BinaryOperation,
+ class _UnaryOperation,
+ class _Tp,
+ class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
+ enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator2 transform_inclusive_scan(
+ _ExecutionPolicy&& __policy,
+ _ForwardIterator1 __first,
+ _ForwardIterator1 __last,
+ _ForwardIterator2 __result,
+ _BinaryOperation __reduce,
+ _UnaryOperation __transform,
+ _Tp __init) {
+ _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_inclusive_scan requires ForwardIterators");
+ _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_inclusive_scan requires ForwardIterators");
+ using _Implementation =
+ __pstl::__dispatch<__pstl::__transform_inclusive_scan_init, __pstl::__current_configuration, _RawPolicy>;
+ return __pstl::__handle_exception<_Implementation>(
+ std::forward<_ExecutionPolicy>(__policy),
+ std::move(__first),
+ std::move(__last),
+ std::move(__result),
+ std::move(__reduce),
+ std::move(__transform),
+ std::move(__init));
+}
+
+template <class _ExecutionPolicy,
+ class _ForwardIterator1,
+ class _ForwardIterator2,
+ class _BinaryOperation,
+ class _UnaryOperation,
+ class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
+ enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator2 transform_inclusive_scan(
+ _ExecutionPolicy&& __policy,
+ _ForwardIterator1 __first,
+ _ForwardIterator1 __last,
+ _ForwardIterator2 __result,
+ _BinaryOperation __reduce,
+ _UnaryOperation __transform) {
+ _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_inclusive_scan requires ForwardIterators");
+ _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_inclusive_scan requires ForwardIterators");
+ using _Implementation =
+ __pstl::__dispatch<__pstl::__transform_inclusive_scan, __pstl::__current_configuration, _RawPolicy>;
+ return __pstl::__handle_exception<_Implementation>(
+ std::forward<_ExecutionPolicy>(__policy),
+ std::move(__first),
+ std::move(__last),
+ std::move(__result),
+ std::move(__reduce),
+ std::move(__transform));
+}
+
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
diff --git a/libcxx/include/__pstl/backend_fwd.h b/libcxx/include/__pstl/backend_fwd.h
index 40a335ae1f2adb..4863db1823592d 100644
--- a/libcxx/include/__pstl/backend_fwd.h
+++ b/libcxx/include/__pstl/backend_fwd.h
@@ -257,6 +257,27 @@ struct __transform_binary;
// _ForwardOutIterator __result,
// _BinaryOperation __op) const noexcept;
+template <class _Backend, class _ExecutionPolicy>
+struct __transform_inclusive_scan;
+// template <class _Policy, class _ForwardIterator1, class _ForwardIterator2,
+// class _BinaryOperation, class _UnaryOperation>
+// optional<_ForwardIterator2>
+// operator()(_Policy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
+// _ForwardIterator2 __result,
+// _BinaryOperation __reduce,
+// _UnaryOperation __transform) const noexcept;
+
+template <class _Backend, class _ExecutionPolicy>
+struct __transform_inclusive_scan_init;
+// template <class _Policy, class _ForwardIterator1, class _ForwardIterator2,
+// class _BinaryOperation, class _UnaryOperation, class _Tp>
+// optional<_ForwardIterator2>
+// operator()(_Policy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
+// _ForwardIterator2 __result,
+// _BinaryOperation __reduce,
+// _UnaryOperation __transform,
+// _Tp __init) const noexcept;
+
template <class _Backend, class _ExecutionPolicy>
struct __replace_copy_if;
// template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _Predicate, class _Tp>
diff --git a/libcxx/include/__pstl/backends/default.h b/libcxx/include/__pstl/backends/default.h
index b5d7045bbb9b85..6e0b23c4d0189b 100644
--- a/libcxx/include/__pstl/backends/default.h
+++ b/libcxx/include/__pstl/backends/default.h
@@ -906,6 +906,44 @@ struct __adjacent_difference<__default_backend_tag, _ExecutionPolicy> {
}
};
+//////////////////////////////////////////////////////////////
+// transform_inclusive_scan_init family
+//////////////////////////////////////////////////////////////
+
+template <class _ExecutionPolicy>
+struct __transform_inclusive_scan<__default_backend_tag, _ExecutionPolicy> {
+ template <class _Policy,
+ class _ForwardIterator1,
+ class _ForwardIterator2,
+ class _BinaryOperation,
+ class _UnaryOperation>
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator2>
+ operator()(_Policy&& __policy,
+ _ForwardIterator1 __first,
+ _ForwardIterator1 __last,
+ _ForwardIterator2 __result,
+ _BinaryOperation __reduce,
+ _UnaryOperation __transform) const noexcept {
+ if (__first != __last) {
+ using _TransformInclusiveScanInit =
+ __dispatch<__transform_inclusive_scan_init, __current_configuration, _ExecutionPolicy>;
+ using _ValueType = __iterator_value_type<_ForwardIterator1>;
+ _ValueType __init = __transform(*__first);
+ *__result++ = __init;
+ if (++__first != __last)
+ return _TransformInclusiveScanInit()(
+ __policy,
+ std::move(__first),
+ std::move(__last),
+ std::move(__result),
+ std::move(__reduce),
+ std::move(__transform),
+ std::move(__init));
+ }
+ return __result;
+ }
+};
+
} // namespace __pstl
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h
index dbd9fa34e1f876..d5cfff3b6829da 100644
--- a/libcxx/include/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__pstl/backends/libdispatch.h
@@ -50,6 +50,7 @@
#include <__pstl/cpu_algos/search_n.h>
#include <__pstl/cpu_algos/stable_sort.h>
#include <__pstl/cpu_algos/transform.h>
+#include <__pstl/cpu_algos/transform_inclusive_scan_init.h>
#include <__pstl/cpu_algos/transform_reduce.h>
#include <__utility/empty.h>
#include <__utility/exception_guard.h>
@@ -270,6 +271,64 @@ struct __cpu_traits<__libdispatch_backend_tag> {
__combiner);
}
+ template <class _RandomAccessIterator1,
+ class _RandomAccessIterator2,
+ class _Value,
+ class _Reduction,
+ class _Accumulation,
+ class _Scan>
+ _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+ __scan(_RandomAccessIterator1 __first,
+ _RandomAccessIterator1 __last,
+ _RandomAccessIterator2 __result,
+ _Value __init,
+ _Reduction __reduction,
+ _Accumulation __accumulation,
+ _Scan __scan) {
+ if (__first == __last)
+ return __empty{};
+
+ auto __partitions = __libdispatch::__partition_chunks(__last - __first);
+
+ auto __destroy = [__count = __partitions.__chunk_count_](_Value* __ptr) {
+ std::destroy_n(__ptr, __count);
+ std::allocator<_Value>().deallocate(__ptr, __count);
+ };
+
+ // TODO: use __uninitialized_buffer
+ // TODO: check for malloc failure
+ // TODO: need one less element - the last chunk is not used in fact
+ unique_ptr<_Value[], decltype(__destroy)> __values(
+ std::allocator<_Value>().allocate(__partitions.__chunk_count_), __destroy);
+
+ // First: compute partial reductions for each chunk in parallel
+ __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) {
+ auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
+ auto __index = __chunk == 0 ? 0
+ : (__chunk * __partitions.__chunk_size_) +
+ (__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
+ std::__construct_at(
+ __values.get() + __chunk, __reduction(__first + __index, __first + __index + __this_chunk_size));
+ });
+
+ // Second: accumulate the partial reductions serially
+ __accumulation(__values.get(), __values.get() + __partitions.__chunk_count_, __init);
+
+ // Third: distribute the accumulated prefix sum to each chunk in a parallel scan
+ __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) {
+ auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
+ auto __index = __chunk == 0 ? 0
+ : (__chunk * __partitions.__chunk_size_) +
+ (__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
+ __scan(__first + __index,
+ __first + __index + __this_chunk_size,
+ __result + __index,
+ std::move(__chunk == 0 ? __init : __values.get()[__chunk - 1]));
+ });
+
+ return __empty{};
+ }
+
template <class _RandomAccessIterator, class _Comp, class _LeafSort>
_LIBCPP_HIDE_FROM_ABI static optional<__empty>
__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) {
@@ -422,6 +481,10 @@ template <class _ExecutionPolicy>
struct __transform_binary<__libdispatch_backend_tag, _ExecutionPolicy>
: __cpu_parallel_transform_binary<__libdispatch_backend_tag, _ExecutionPolicy> {};
+template <class _ExecutionPolicy>
+struct __transform_inclusive_scan_init<__libdispatch_backend_tag, _ExecutionPolicy>
+ : __cpu_parallel_transform_inclusive_scan_init<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
template <class _ExecutionPolicy>
struct __transform_reduce<__libdispatch_backend_tag, _ExecutionPolicy>
: __cpu_parallel_transform_reduce<__libdispatch_backend_tag, _ExecutionPolicy> {};
diff --git a/libcxx/include/__pstl/backends/serial.h b/libcxx/include/__pstl/backends/serial.h
index 85c73b0f15db57..033b68eb32fc23 100644
--- a/libcxx/include/__pstl/backends/serial.h
+++ b/libcxx/include/__pstl/backends/serial.h
@@ -24,6 +24,7 @@
#include <__algorithm/stable_sort.h>
#include <__algorithm/transform.h>
#include <__config>
+#include <__numeric/transform_inclusive_scan.h>
#include <__numeric/transform_reduce.h>
#include <__optional/optional.h>
#include <__pstl/backend_fwd.h>
@@ -239,6 +240,32 @@ struct __transform_binary<__serial_backend_tag, _ExecutionPolicy> {
}
};
+template <class _ExecutionPolicy>
+struct __transform_inclusive_scan_init<__serial_backend_tag, _ExecutionPolicy> {
+ template <class _Policy,
+ class _ForwardIterator1,
+ class _ForwardIterator2,
+ class _BinaryOperation,
+ class _UnaryOperation,
+ class _Tp>
+ _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator2> operator()(
+ _Policy&&,
+ _ForwardIterator1 __first,
+ _ForwardIterator1 __last,
+ _ForwardIterator2 __result,
+ _BinaryOperation __reduce,
+ _UnaryOperation __transform,
+ _Tp __init) const noexcept {
+ return std::transform_inclusive_scan(
+ std::move(__first),
+ std::move(__last),
+ std::move(__result),
+ std::move(__reduce),
+ std::move(__transform),
+ std::move(__init));
+ }
+};
+
template <class _ExecutionPolicy>
struct __transform_reduce<__serial_backend_tag, _ExecutionPolicy> {
template <class _Policy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
diff --git a/libcxx/include/__pstl/backends/std_thread.h b/libcxx/include/__pstl/backends/std_thread.h
index bc7fc020867f8b..85ecc38deb0646 100644
--- a/libcxx/include/__pstl/backends/std_thread.h
+++ b/libcxx/include/__pstl/backends/std_thread.h
@@ -29,6 +29,7 @@
#include <__pstl/cpu_algos/search_n.h>
#include <__pstl/cpu_algos/stable_sort.h>
#include <__pstl/cpu_algos/transform.h>
+#include <__pstl/cpu_algos/transform_inclusive_scan_init.h>
#include <__pstl/cpu_algos/transform_reduce.h>
#include <__utility/empty.h>
#include <__utility/move.h>
@@ -67,6 +68,25 @@ struct __cpu_traits<__std_thread_backend_tag> {
return __reduce(std::move(__first), std::move(__last), std::move(__init));
}
+ template <class _RandomAccessIterator1,
+ class _RandomAccessIterator2,
+ class _Value,
+ class _Reduction,
+ class _Accumulation,
+ class _Scan>
+ _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+ __scan(_RandomAccessIterator1 __first,
+ _RandomAccessIterator1 __last,
+ _RandomAccessIterator2 __result,
+ _Value __init,
+ _Reduction /*__reduction*/,
+ _Accumulation /*__accumulation*/,
+ _Scan __scan) {
+ if (__first != __last)
+ __scan(__first, __last, __result, std::move(__init));
+ return __empty{};
+ }
+
template <class _RandomAccessIterator, class _Compare, class _LeafSort>
_LIBCPP_HIDE_FROM_ABI static optional<__empty>
__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
@@ -153,6 +173,10 @@ template <class _ExecutionPolicy>
struct __transform_binary<__std_thread_backend_tag, _ExecutionPolicy>
: __cpu_parallel_transform_binary<__std_thread_backend_tag, _ExecutionPolicy> {};
+template <class _ExecutionPolicy>
+struct __transform_inclusive_scan_init<__std_thread_backend_tag, _ExecutionPolicy>
+ : __cpu_parallel_transform_inclusive_scan_init<__std_thread_backend_tag, _ExecutionPolicy> {};
+
template <class _ExecutionPolicy>
struct __transform_reduce<__std_thread_backend_tag, _ExecutionPolicy>
: __cpu_parallel_transform_reduce<__std_thread_backend_tag, _ExecutionPolicy> {};
diff --git a/libcxx/include/__pstl/cpu_algos/transform_inclusive_scan_init.h b/libcxx/include/__pstl/cpu_algos/transform_inclusive_scan_init.h
new file mode 100644
index 00000000000000..cba77725a7e74c
--- /dev/null
+++ b/libcxx/include/__pstl/cpu_algos/transform_inclusive_scan_init.h
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_INCLUSIVE_SCAN_INIT_H
+#define _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_INCLUSIVE_SCAN_INIT_H
+
+#include <__config>
+#include <__functional/identity.h>
+#include <__functional/operations.h>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__numeric/inclusive_scan.h>
+#include <__numeric/transform_inclusive_scan.h>
+#include <__numeric/transform_reduce.h>
+#include <__optional/nullopt_t.h>
+#include <__optional/optional.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_transform_inclusive_scan_init {
+ template <class _Policy,
+ class _ForwardIterator1,
+ class _ForwardIterator2,
+ class _BinaryOperation,
+ class _UnaryOperation,
+ class _Tp>
+ _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator2> operator()(
+ _Policy&&,
+ _ForwardIterator1 __first,
+ _ForwardIterator1 __last,
+ _ForwardIterator2 __result,
+ _BinaryOperation __reduce,
+ _UnaryOperation __transform,
+ _Tp __init) const noexcept {
+ if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+ __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
+ __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
+ auto __ret = __cpu_traits<_Backend>::__scan(
+ __first,
+ __last,
+ __result,
+ std::move(__init),
+ [&](auto __chunk_first, auto __chunk_last) {
+ return std::transform_reduce(
+ __chunk_first + 1, __chunk_last, __transform(*__chunk_first), __reduce, __transform);
+ },
+ [&](auto __prefix_first, auto __prefix_last, auto __init_val) {
+ std::inclusive_scan(__prefix_first, __prefix_last, __prefix_first, __reduce, std::move(__init_val));
+ },
+ [&](auto __chunk_first, auto __chunk_last, auto __chunk_result, auto __chunk_init) {
+ std::transform_inclusive_scan(
+ __chunk_first, __chunk_last, __chunk_result, __reduce, __transform, std::move(__chunk_init));
+ });
+ if (!__ret)
+ return nullopt;
+ return __result + (__last - __first);
+ } else {
+ // Non-random access iterators cannot be processed in parallel, fall back to the sequential implementation.
+ return std::transform_inclusive_scan(
+ std::move(__first),
+ std::move(__last),
+ std::move(__result),
+ std::move(__reduce),
+ std::move(__transform),
+ std::move(__init));
+ }
+ }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_INCLUSIVE_SCAN_INIT_H
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 214020750e1afb..b5fbf6c5aecfea 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -2469,6 +2469,9 @@ module std {
header "__pstl/cpu_algos/stable_sort.h"
export std_core.utility_core.empty
}
+ module transform_inclusive_scan_init {
+ header "__pstl/cpu_algos/transform_inclusive_scan_init.h"
+ }
module transform {
header "__pstl/cpu_algos/transform.h"
}
diff --git a/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/pstl.transform_inclusive_scan_bop_uop.pass.cpp b/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/pstl.transform_inclusive_scan_bop_uop.pass.cpp
new file mode 100644
index 00000000000000..c61df348285383
--- /dev/null
+++ b/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/pstl.transform_inclusive_scan_bop_uop.pass.cpp
@@ -0,0 +1,159 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++17
+
+// UNSUPPORTED: libcpp-has-no-incomplete-pstl
+
+// template <class ExecutionPolicy,
+// class ForwardIterator1,
+// class ForwardIterator2,
+// class BinaryOperation,
+// class UnaryOperation,
+// class T>
+// ForwardIterator2 transform_inclusive_scan(ExecutionPolicy&& exec,
+// ForwardIterator1 first,
+// ForwardIterator1 last,
+// ForwardIterator2 result,
+// BinaryOperation reduce,
+// UnaryOperation transform);
+
+#include <algorithm>
+#include <cassert>
+#include <functional>
+#include <iterator>
+#include <limits>
+#include <numeric>
+
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "type_algorithms.h"
+#include "runway_sample.h"
+
+EXECUTION_POLICY_SFINAE_TEST(transform_inclusive_scan);
+
+static_assert(sfinae_test_transform_inclusive_scan<int, int*, int*, int*, int (*)(int, int), int (*)(int)>);
+static_assert(!sfinae_test_transform_inclusive_scan< std::execution::parallel_policy,
+ int*,
+ int*,
+ int*,
+ int (*)(int, int),
+ int (*)(int)>);
+
+struct add_one {
+ template <typename T>
+ constexpr T operator()(T x) const {
+ return x + 1;
+ }
+};
+
+template <typename T>
+constexpr auto triangle(T n) {
+ return n * (n + 1) / 2;
+}
+
+template <class Iter1, class Iter2>
+struct TestInt {
+ template <class ExecutionPolicy>
+ void operator()(ExecutionPolicy&& policy) {
+ { // Check the return type
+ int a[] = {0};
+ auto res = std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::end(a)), Iter2(std::begin(a)), std::plus<int>{}, [](int x) {
+ return x;
+ });
+ static_assert(std::is_same_v<decltype(res), Iter2>);
+ }
+ { // Empty
+ int a[] = {-1};
+ std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::begin(a)), Iter2(std::begin(a)), std::plus<>{}, add_one{});
+ assert(a[0] == -1);
+ }
+ { // Scan a uniform array
+ int a[10];
+ std::fill(std::begin(a), std::end(a), 3);
+ std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::end(a)), Iter2(std::begin(a)), std::plus<>{}, add_one{});
+ for (std::size_t i = 0; i < std::size(a); ++i)
+ assert(a[i] == (static_cast<int>(i) + 1) * 4);
+ }
+ { // Scan an increasing array starting from 0
+ int a[10];
+ std::iota(std::begin(a), std::end(a), 0);
+ std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::end(a)), Iter2(std::begin(a)), std::plus<>{}, add_one{});
+ for (std::size_t i = 0; i < std::size(a); ++i)
+ assert(a[i] == static_cast<int>(triangle(i) + i + 1));
+ }
+ { // Scan an increasing array starting from 1
+ int a[10];
+ std::iota(std::begin(a), std::end(a), 1);
+ std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::end(a)), Iter2(std::begin(a)), std::plus<>{}, add_one{});
+ for (std::size_t i = 0; i < std::size(a); ++i)
+ assert(a[i] == static_cast<int>(triangle(i + 1) + i + 1));
+ }
+ { // Test against pre-computed expected results
+ auto test = [&](auto first, auto last, auto reduce, auto transform, auto exp_first, auto exp_last) {
+ int out[5];
+ // Not in place
+ auto end = std::transform_inclusive_scan(policy, first, last, Iter2(std::begin(out)), reduce, transform);
+ assert(std::equal(Iter2(std::begin(out)), end, exp_first, exp_last));
+ // In place
+ std::copy(first, last, out);
+ end = std::transform_inclusive_scan(
+ policy, Iter2(std::begin(out)), end, Iter2(std::begin(out)), reduce, transform);
+ assert(std::equal(Iter2(std::begin(out)), end, exp_first, exp_last));
+ };
+ const int a[] = {1, 3, 5, 7, 9};
+ const int exp_plus_inc_0[] = {2, 6, 12, 20, 30}; // plus, add_one, init = 0
+ const int exp_mul_inc_0[] = {2, 8, 48, 384, 3840}; // multiplies, add_one, init = 0
+ const int exp_plus_neg_0[] = {-1, -4, -9, -16, -25}; // plus, negate, init = 0
+ const int exp_mul_neg_0[] = {-1, 3, -15, 105, -945}; // multiplies, negate, init = 0
+ static_assert(std::size(a) == std::size(exp_plus_inc_0)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_mul_inc_0)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_plus_neg_0)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_mul_neg_0)); // just to be sure
+ std::plus<> plus;
+ std::multiplies<> mult;
+ std::negate<> negate;
+ for (unsigned int i = 0; i < std::size(a); ++i) {
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), plus, add_one{}, exp_plus_inc_0, exp_plus_inc_0 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), mult, add_one{}, exp_mul_inc_0, exp_mul_inc_0 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), plus, negate, exp_plus_neg_0, exp_plus_neg_0 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), mult, negate, exp_mul_neg_0, exp_mul_neg_0 + i);
+ }
+ }
+ { // Constuct iota from std::plus<> and add_one
+ int ia[1073];
+ int ib[1073];
+ std::fill(std::begin(ia), std::end(ia), 0);
+ runway_sample(std::size(ia) + 1, [&](size_t i) {
+ Iter2 r = std::transform_inclusive_scan(
+ policy, Iter1(std::begin(ia)), Iter1(std::begin(ia) + i), Iter2(std::begin(ib)), std::plus<>{}, add_one{});
+ assert(r == Iter2(std::begin(ib) + i));
+ for (size_t j = 0; j < i; ++j) {
+ assert(ib[j] == static_cast<int>(j) + 1);
+ }
+ std::fill(std::begin(ib), std::begin(ib) + i, -1);
+ });
+ }
+ }
+};
+
+int main(int, char**) {
+ types::for_each(types::forward_iterator_list<const int*>{}, types::apply_type_identity{[](auto v) {
+ using Iter = typename decltype(v)::type;
+ types::for_each(
+ types::forward_iterator_list<int*>{},
+ TestIteratorWithPolicies<types::partial_instantiation<TestInt, Iter>::template apply>{});
+ }});
+ return 0;
+}
diff --git a/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/pstl.transform_inclusive_scan_bop_uop_init.pass.cpp b/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/pstl.transform_inclusive_scan_bop_uop_init.pass.cpp
new file mode 100644
index 00000000000000..a993816ac91c97
--- /dev/null
+++ b/libcxx/test/std/numerics/numeric.ops/transform.inclusive.scan/pstl.transform_inclusive_scan_bop_uop_init.pass.cpp
@@ -0,0 +1,216 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++17
+
+// UNSUPPORTED: libcpp-has-no-incomplete-pstl
+
+// template <class ExecutionPolicy,
+// class ForwardIterator1,
+// class ForwardIterator2,
+// class BinaryOperation,
+// class UnaryOperation,
+// class T>
+// ForwardIterator2 transform_inclusive_scan(ExecutionPolicy&& exec,
+// ForwardIterator1 first,
+// ForwardIterator1 last,
+// ForwardIterator2 result,
+// BinaryOperation reduce,
+// UnaryOperation transform,
+// T init);
+
+#include <algorithm>
+#include <cassert>
+#include <functional>
+#include <iterator>
+#include <limits>
+#include <numeric>
+
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "type_algorithms.h"
+#include "runway_sample.h"
+
+EXECUTION_POLICY_SFINAE_TEST(transform_inclusive_scan);
+
+static_assert(sfinae_test_transform_inclusive_scan<int, int*, int*, int*, int (*)(int, int), int (*)(int), int>);
+static_assert(!sfinae_test_transform_inclusive_scan<
+ std::execution::parallel_policy,
+ int*,
+ int*,
+ int*,
+ int (*)(int, int),
+ int (*)(int),
+ int>);
+
+struct add_one {
+ template <typename T>
+ constexpr T operator()(T x) const {
+ return x + 1;
+ }
+};
+
+template <typename T>
+constexpr auto triangle(T n) {
+ return n * (n + 1) / 2;
+}
+
+template <class Iter1, class Iter2>
+struct TestInt {
+ template <class ExecutionPolicy>
+ void operator()(ExecutionPolicy&& policy) {
+ { // Check the return type
+ int a[] = {0};
+ auto res = std::transform_inclusive_scan(
+ policy,
+ Iter1(std::begin(a)),
+ Iter1(std::end(a)),
+ Iter2(std::begin(a)),
+ std::plus<int>{},
+ [](int x) { return x; },
+ 0);
+ static_assert(std::is_same_v<decltype(res), Iter2>);
+ }
+ { // Empty
+ int a[] = {-1};
+ std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::begin(a)), Iter2(std::begin(a)), std::plus<>{}, add_one{}, 10);
+ assert(a[0] == -1);
+ }
+ { // Scan a uniform array
+ int a[10];
+ std::fill(std::begin(a), std::end(a), 3);
+ std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::end(a)), Iter2(std::begin(a)), std::plus<>{}, add_one{}, 50);
+ for (std::size_t i = 0; i < std::size(a); ++i)
+ assert(a[i] == 50 + (static_cast<int>(i) + 1) * 4);
+ }
+ { // Scan an increasing array starting from 0
+ int a[10];
+ std::iota(std::begin(a), std::end(a), 0);
+ std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::end(a)), Iter2(std::begin(a)), std::plus<>{}, add_one{}, 30);
+ for (std::size_t i = 0; i < std::size(a); ++i)
+ assert(a[i] == static_cast<int>(30 + triangle(i) + i + 1));
+ }
+ { // Scan an increasing array starting from 1
+ int a[10];
+ std::iota(std::begin(a), std::end(a), 1);
+ std::transform_inclusive_scan(
+ policy, Iter1(std::begin(a)), Iter1(std::end(a)), Iter2(std::begin(a)), std::plus<>{}, add_one{}, 40);
+ for (std::size_t i = 0; i < std::size(a); ++i)
+ assert(a[i] == static_cast<int>(40 + triangle(i + 1) + i + 1));
+ }
+ { // Test against pre-computed expected results
+ auto test = [&](auto first, auto last, auto reduce, auto transform, auto init, auto exp_first, auto exp_last) {
+ int out[5];
+ // Not in place
+ auto end = std::transform_inclusive_scan(policy, first, last, Iter2(std::begin(out)), reduce, transform, init);
+ assert(std::equal(Iter2(std::begin(out)), end, exp_first, exp_last));
+ // In place
+ std::copy(first, last, out);
+ end = std::transform_inclusive_scan(
+ policy, Iter2(std::begin(out)), end, Iter2(std::begin(out)), reduce, transform, init);
+ assert(std::equal(Iter2(std::begin(out)), end, exp_first, exp_last));
+ };
+ const int a[] = {1, 3, 5, 7, 9};
+ const int exp_plus_inc_0[] = {2, 6, 12, 20, 30}; // plus, add_one, init = 0
+ const int exp_mul_inc_0[] = {0, 0, 0, 0, 0}; // multiplies, add_one, init = 0
+ const int exp_plus_neg_0[] = {-1, -4, -9, -16, -25}; // plus, negate, init = 0
+ const int exp_mul_neg_0[] = {0, 0, 0, 0, 0}; // multiplies, negate, init = 0
+ const int exp_plus_inc_2[] = {4, 8, 14, 22, 32}; // plus, add_one, init = 2
+ const int exp_mul_inc_2[] = {4, 16, 96, 768, 7680}; // multiplies, add_one, init = 2
+ const int exp_plus_neg_2[] = {1, -2, -7, -14, -23}; // plus, negate, init = 2
+ const int exp_mul_neg_2[] = {-2, 6, -30, 210, -1890}; // multiplies, negate, init = 2
+ static_assert(std::size(a) == std::size(exp_plus_inc_0)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_mul_inc_0)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_plus_neg_0)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_mul_neg_0)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_plus_inc_2)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_mul_inc_2)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_plus_neg_2)); // just to be sure
+ static_assert(std::size(a) == std::size(exp_mul_neg_2)); // just to be sure
+ std::plus<> plus;
+ std::multiplies<> mult;
+ std::negate<> negate;
+ for (unsigned int i = 0; i < std::size(a); ++i) {
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), plus, add_one{}, 0, exp_plus_inc_0, exp_plus_inc_0 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), mult, add_one{}, 0, exp_mul_inc_0, exp_mul_inc_0 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), plus, negate, 0, exp_plus_neg_0, exp_plus_neg_0 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), mult, negate, 0, exp_mul_neg_0, exp_mul_neg_0 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), plus, add_one{}, 2, exp_plus_inc_2, exp_plus_inc_2 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), mult, add_one{}, 2, exp_mul_inc_2, exp_mul_inc_2 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), plus, negate, 2, exp_plus_neg_2, exp_plus_neg_2 + i);
+ test(Iter1(std::begin(a)), Iter1(std::begin(a) + i), mult, negate, 2, exp_mul_neg_2, exp_mul_neg_2 + i);
+ }
+ }
+ { // Constuct iota from std::plus<> and add_one
+ int ia[1073];
+ int ib[1073];
+ std::fill(std::begin(ia), std::end(ia), 0);
+ runway_sample(std::size(ia) + 1, [&](size_t i) {
+ Iter2 r = std::transform_inclusive_scan(
+ policy,
+ Iter1(std::begin(ia)),
+ Iter1(std::begin(ia) + i),
+ Iter2(std::begin(ib)),
+ std::plus<>{},
+ add_one{},
+ 1);
+ assert(r == Iter2(std::begin(ib) + i));
+ for (size_t j = 0; j < i; ++j) {
+ assert(ib[j] == static_cast<int>(j) + 2);
+ }
+ std::fill(std::begin(ib), std::begin(ib) + i, -1);
+ });
+ }
+ }
+};
+
+template <class Iter1, class Iter2>
+struct TestUnsignedCharToSize_t {
+ template <class ExecutionPolicy>
+ void operator()(ExecutionPolicy&& policy) {
+ // Make sure that the calculations are done using the init typedef
+ unsigned char v[10];
+ std::iota(std::begin(v), std::end(v), static_cast<unsigned char>(1));
+ std::size_t res[10];
+ std::transform_inclusive_scan(
+ policy,
+ Iter1(std::begin(v)),
+ Iter1(std::end(v)),
+ Iter2(std::begin(res)),
+ std::multiplies<>(),
+ add_one{},
+ std::size_t{1});
+ std::size_t j = 2;
+ assert(res[0] == 2);
+ for (std::size_t i = 1; i < std::size(res); ++i) {
+ j *= i + 2;
+ assert(res[i] == j);
+ }
+ }
+};
+
+int main(int, char**) {
+ types::for_each(types::forward_iterator_list<const int*>{}, types::apply_type_identity{[](auto v) {
+ using Iter = typename decltype(v)::type;
+ types::for_each(
+ types::forward_iterator_list<int*>{},
+ TestIteratorWithPolicies<types::partial_instantiation<TestInt, Iter>::template apply>{});
+ }});
+ types::for_each(
+ types::forward_iterator_list<const unsigned char*>{}, types::apply_type_identity{[](auto v) {
+ using Iter = typename decltype(v)::type;
+ types::for_each(
+ types::forward_iterator_list<std::size_t*>{},
+ TestIteratorWithPolicies<types::partial_instantiation<TestUnsignedCharToSize_t, Iter>::template apply>{});
+ }});
+ return 0;
+}
>From b7e8f83cfd9a2b2c12cae1018a42586c46afd596 Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Wed, 16 Sep 2026 15:44:07 +0100
Subject: [PATCH 2/5] Correct file order
---
libcxx/include/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 93e392dd3a5235..a02822ef95ca5b 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -709,8 +709,8 @@ set(files
__pstl/cpu_algos/search.h
__pstl/cpu_algos/search_n.h
__pstl/cpu_algos/stable_sort.h
- __pstl/cpu_algos/transform_inclusive_scan_init.h
__pstl/cpu_algos/transform.h
+ __pstl/cpu_algos/transform_inclusive_scan_init.h
__pstl/cpu_algos/transform_reduce.h
__pstl/dispatch.h
__pstl/handle_exception.h
>From 8815dcc2d1b9f545d8055a8fde0434dbb822a029 Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Wed, 23 Sep 2026 21:17:56 +0100
Subject: [PATCH 3/5] More tests
---
.../pstl.iterator-requirements.verify.cpp | 33 +++++++++++++++++++
.../pstl.exception_handling.pass.cpp | 17 ++++++++++
2 files changed, 50 insertions(+)
diff --git a/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp b/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp
index f12d5bdcb31218..6099b0066b97a5 100644
--- a/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp
+++ b/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp
@@ -313,6 +313,39 @@ void f(non_forward_iterator non_fwd,
pol, non_fwd, non_fwd, val, func, func); // expected-error@*:* {{static assertion failed: transform_reduce}}
}
+ {
+ (void)std::transform_inclusive_scan(
+ pol,
+ non_fwd,
+ non_fwd,
+ it,
+ func,
+ func); // expected-error@*:* {{static assertion failed: transform_inclusive_scan}}
+ (void)std::transform_inclusive_scan(
+ pol,
+ it,
+ it,
+ non_fwd,
+ func,
+ func); // expected-error@*:* {{static assertion failed: transform_inclusive_scan}}
+ (void)std::transform_inclusive_scan(
+ pol,
+ non_fwd,
+ non_fwd,
+ it,
+ func,
+ func,
+ val); // expected-error@*:* {{static assertion failed: transform_inclusive_scan}}
+ (void)std::transform_inclusive_scan(
+ pol,
+ it,
+ it,
+ non_fwd,
+ func,
+ func,
+ val); // expected-error@*:* {{static assertion failed: transform_inclusive_scan}}
+ }
+
{
(void)std::adjacent_difference(
pol, it, it, non_fwd); // expected-error@*:* {{static assertion failed: adjacent_difference}}
diff --git a/libcxx/test/std/algorithms/pstl.exception_handling.pass.cpp b/libcxx/test/std/algorithms/pstl.exception_handling.pass.cpp
index b85cc984d969d9..0fd16ff853bf7f 100644
--- a/libcxx/test/std/algorithms/pstl.exception_handling.pass.cpp
+++ b/libcxx/test/std/algorithms/pstl.exception_handling.pass.cpp
@@ -490,6 +490,23 @@ int main(int, char**) {
});
}
+ {
+ auto unary = maybe_throw(tokens[5], [](int x) -> int { return x * 2; });
+ auto binary = maybe_throw(tokens[5], [](int x, int y) -> int { return x * y; });
+
+ // transform_inclusive_scan(first, last, dest, unary_op)
+ assert_non_throwing([=, &policy] {
+ (void)std::transform_inclusive_scan(
+ policy, std::move(first1), std::move(last1), std::move(dest), binary, unary);
+ });
+
+ // transform_inclusive_scan(first, last, dest, unary_op, init)
+ assert_non_throwing([=, &policy] {
+ (void)std::transform_inclusive_scan(
+ policy, std::move(first1), std::move(last1), std::move(dest), binary, unary, init);
+ });
+ }
+
{
auto reduction = maybe_throw(tokens[5], [](int x, int y) -> int { return x + y; });
auto transform_unary = maybe_throw(tokens[6], [](int x) -> int { return x * 2; });
>From aa5f914e97f0e391d961ac661051308059d9180d Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Wed, 23 Sep 2026 21:18:42 +0100
Subject: [PATCH 4/5] Benchmark
---
.../pstl.transform_inclusive_scan.bench.cpp | 83 +++++++++++++++++++
1 file changed, 83 insertions(+)
create mode 100644 libcxx/test/benchmarks/numeric/pstl.transform_inclusive_scan.bench.cpp
diff --git a/libcxx/test/benchmarks/numeric/pstl.transform_inclusive_scan.bench.cpp b/libcxx/test/benchmarks/numeric/pstl.transform_inclusive_scan.bench.cpp
new file mode 100644
index 00000000000000..8399f24706217c
--- /dev/null
+++ b/libcxx/test/benchmarks/numeric/pstl.transform_inclusive_scan.bench.cpp
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++17
+
+// UNSUPPORTED: libcpp-has-no-incomplete-pstl
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cmath>
+#include <numeric>
+#include <string>
+#include <type_traits>
+#include <vector>
+#include <execution>
+#include <utility>
+
+#include <benchmark/benchmark.h>
+
+int main(int argc, char** argv) {
+ // A transformation/reduction that does just a bit more than a no-op
+ struct minimal {
+ double operator()(double x) const { return -x; }
+ double operator()(double x, double y) const { return x + y; }
+ };
+
+ // A transformation/reduction that does miniscule work per element
+ struct cheap {
+ double operator()(double x) const { return -x + 0.25; }
+ double operator()(double x, double y) const { return x / 2. * 2. / 2. * 2. + y / 2. * 2. / 2. * 2.; }
+ };
+
+ // A transformation/reduction that does significant work per element
+ struct expensive {
+ double operator()(double x) const { return (-x + 0.25) * (x + 10.); }
+ double operator()(double x, double y) const {
+ return x / 2. * 2. / 3. * 3. / 4. * 4. / 5. * 5. + y / 2. * 2. / 2. * 2. / 4. * 4. / 5. * 5.;
+ }
+ };
+
+ auto bm = [](std::string name, auto&& policy, auto&& func) {
+ benchmark::RegisterBenchmark(
+ name,
+ [&policy, func](auto& st) {
+ std::size_t size = st.range(0);
+ std::vector<double> c(size), r(size);
+ std::iota(c.begin(), c.end(), 1.);
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::ClobberMemory();
+ std::transform_inclusive_scan(policy, c.begin(), c.end(), r.begin(), func, func, 0.);
+ benchmark::DoNotOptimize(c);
+ benchmark::DoNotOptimize(r);
+ }
+ })
+ ->Arg(1 << 6) // 64
+ ->Arg(1 << 16) // 65'536
+ ->Arg(1 << 26) // 67'108'864
+ ->UseRealTime();
+ };
+#if defined(TEST_PSTL_ENABLE_SEQ_BASELINES)
+ bm("std::transform_inclusive_scan(std::execution::seq, vector<double>) (minimal)", std::execution::seq, minimal{});
+ bm("std::transform_inclusive_scan(std::execution::seq, vector<double>) (cheap)", std::execution::seq, cheap{});
+ bm("std::transform_inclusive_scan(std::execution::seq, vector<double>) (expensive)",
+ std::execution::seq,
+ expensive{});
+#endif
+ bm("std::transform_inclusive_scan(std::execution::par, vector<double>) (minimal)", std::execution::par, minimal{});
+ bm("std::transform_inclusive_scan(std::execution::par, vector<double>) (cheap)", std::execution::par, cheap{});
+ bm("std::transform_inclusive_scan(std::execution::par, vector<double>) (expensive)",
+ std::execution::par,
+ expensive{});
+
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+ benchmark::Shutdown();
+ return 0;
+}
>From f85c94fe61ed4f13fe7efdc0c1a4747a1a81e906 Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Wed, 23 Sep 2026 21:48:25 +0100
Subject: [PATCH 5/5] Single-pass parallel scan
---
libcxx/include/CMakeLists.txt | 1 +
libcxx/include/__pstl/backends/libdispatch.h | 60 ++-----
libcxx/include/__pstl/backends/std_thread.h | 22 +--
.../cpu_algos/transform_inclusive_scan_init.h | 99 +++++++++--
libcxx/include/__pstl/decoupled_lookback.h | 161 ++++++++++++++++++
libcxx/include/module.modulemap.in | 1 +
6 files changed, 268 insertions(+), 76 deletions(-)
create mode 100644 libcxx/include/__pstl/decoupled_lookback.h
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index a02822ef95ca5b..9615d2196bfde5 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -712,6 +712,7 @@ set(files
__pstl/cpu_algos/transform.h
__pstl/cpu_algos/transform_inclusive_scan_init.h
__pstl/cpu_algos/transform_reduce.h
+ __pstl/decoupled_lookback.h
__pstl/dispatch.h
__pstl/handle_exception.h
__random/bernoulli_distribution.h
diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h
index d5cfff3b6829da..f515659054ca0e 100644
--- a/libcxx/include/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__pstl/backends/libdispatch.h
@@ -52,6 +52,7 @@
#include <__pstl/cpu_algos/transform.h>
#include <__pstl/cpu_algos/transform_inclusive_scan_init.h>
#include <__pstl/cpu_algos/transform_reduce.h>
+#include <__pstl/decoupled_lookback.h>
#include <__utility/empty.h>
#include <__utility/exception_guard.h>
#include <__utility/move.h>
@@ -271,59 +272,28 @@ struct __cpu_traits<__libdispatch_backend_tag> {
__combiner);
}
- template <class _RandomAccessIterator1,
- class _RandomAccessIterator2,
- class _Value,
- class _Reduction,
- class _Accumulation,
- class _Scan>
+ template <class _Value, class _RandomAccessIterator1, class _PartitionScan>
_LIBCPP_HIDE_FROM_ABI static optional<__empty>
- __scan(_RandomAccessIterator1 __first,
- _RandomAccessIterator1 __last,
- _RandomAccessIterator2 __result,
- _Value __init,
- _Reduction __reduction,
- _Accumulation __accumulation,
- _Scan __scan) {
+ __lookback_scan(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _PartitionScan __scan) {
if (__first == __last)
- return __empty{};
-
- auto __partitions = __libdispatch::__partition_chunks(__last - __first);
+ return __empty{}; // nothing to do
- auto __destroy = [__count = __partitions.__chunk_count_](_Value* __ptr) {
- std::destroy_n(__ptr, __count);
- std::allocator<_Value>().deallocate(__ptr, __count);
- };
-
- // TODO: use __uninitialized_buffer
- // TODO: check for malloc failure
- // TODO: need one less element - the last chunk is not used in fact
- unique_ptr<_Value[], decltype(__destroy)> __values(
- std::allocator<_Value>().allocate(__partitions.__chunk_count_), __destroy);
-
- // First: compute partial reductions for each chunk in parallel
- __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) {
- auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
- auto __index = __chunk == 0 ? 0
- : (__chunk * __partitions.__chunk_size_) +
- (__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
- std::__construct_at(
- __values.get() + __chunk, __reduction(__first + __index, __first + __index + __this_chunk_size));
- });
-
- // Second: accumulate the partial reductions serially
- __accumulation(__values.get(), __values.get() + __partitions.__chunk_count_, __init);
+ // Partition the input and allocate the lookback storage
+ __libdispatch::__chunk_partitions __partitions = __libdispatch::__partition_chunks(__last - __first);
+ __decoupled_lookback<_Value> __lookback{static_cast<size_t>(__partitions.__chunk_count_ - 1)};
+ if (__partitions.__chunk_count_ > 1 && __lookback.__size() == 0) {
+ return nullopt; // failed to allocate the lookback storage
+ }
- // Third: distribute the accumulated prefix sum to each chunk in a parallel scan
- __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) {
+ // Run the single-pass scan with decoupled lookback
+ atomic<size_t> __next_chunk{0};
+ __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t /*__apply_chunk_out_of_order*/) {
+ size_t __chunk = __next_chunk.fetch_add(1, std::memory_order_relaxed);
auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
auto __index = __chunk == 0 ? 0
: (__chunk * __partitions.__chunk_size_) +
(__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
- __scan(__first + __index,
- __first + __index + __this_chunk_size,
- __result + __index,
- std::move(__chunk == 0 ? __init : __values.get()[__chunk - 1]));
+ __scan(__first + __index, __first + __index + __this_chunk_size, __chunk, __lookback);
});
return __empty{};
diff --git a/libcxx/include/__pstl/backends/std_thread.h b/libcxx/include/__pstl/backends/std_thread.h
index 85ecc38deb0646..de0d9a519a526b 100644
--- a/libcxx/include/__pstl/backends/std_thread.h
+++ b/libcxx/include/__pstl/backends/std_thread.h
@@ -31,6 +31,7 @@
#include <__pstl/cpu_algos/transform.h>
#include <__pstl/cpu_algos/transform_inclusive_scan_init.h>
#include <__pstl/cpu_algos/transform_reduce.h>
+#include <__pstl/decoupled_lookback.h>
#include <__utility/empty.h>
#include <__utility/move.h>
@@ -68,22 +69,13 @@ struct __cpu_traits<__std_thread_backend_tag> {
return __reduce(std::move(__first), std::move(__last), std::move(__init));
}
- template <class _RandomAccessIterator1,
- class _RandomAccessIterator2,
- class _Value,
- class _Reduction,
- class _Accumulation,
- class _Scan>
+ template <class _Value, class _RandomAccessIterator, class _PartitionScan>
_LIBCPP_HIDE_FROM_ABI static optional<__empty>
- __scan(_RandomAccessIterator1 __first,
- _RandomAccessIterator1 __last,
- _RandomAccessIterator2 __result,
- _Value __init,
- _Reduction /*__reduction*/,
- _Accumulation /*__accumulation*/,
- _Scan __scan) {
- if (__first != __last)
- __scan(__first, __last, __result, std::move(__init));
+ __lookback_scan(_RandomAccessIterator __first, _RandomAccessIterator __last, _PartitionScan __scan) {
+ if (__first == __last)
+ return __empty{}; // nothing to do
+ __decoupled_lookback<_Value> __lookback{0};
+ __scan(__first, __last, 0, __lookback);
return __empty{};
}
diff --git a/libcxx/include/__pstl/cpu_algos/transform_inclusive_scan_init.h b/libcxx/include/__pstl/cpu_algos/transform_inclusive_scan_init.h
index cba77725a7e74c..91e398b9721fa1 100644
--- a/libcxx/include/__pstl/cpu_algos/transform_inclusive_scan_init.h
+++ b/libcxx/include/__pstl/cpu_algos/transform_inclusive_scan_init.h
@@ -14,13 +14,13 @@
#include <__functional/operations.h>
#include <__iterator/concepts.h>
#include <__iterator/iterator_traits.h>
-#include <__numeric/inclusive_scan.h>
#include <__numeric/transform_inclusive_scan.h>
#include <__numeric/transform_reduce.h>
#include <__optional/nullopt_t.h>
#include <__optional/optional.h>
#include <__pstl/backend_fwd.h>
#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__pstl/decoupled_lookback.h>
#include <__type_traits/is_execution_policy.h>
#include <__utility/move.h>
@@ -55,22 +55,89 @@ struct __cpu_parallel_transform_inclusive_scan_init {
if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
__has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
__has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
- auto __ret = __cpu_traits<_Backend>::__scan(
- __first,
- __last,
- __result,
- std::move(__init),
- [&](auto __chunk_first, auto __chunk_last) {
- return std::transform_reduce(
- __chunk_first + 1, __chunk_last, __transform(*__chunk_first), __reduce, __transform);
- },
- [&](auto __prefix_first, auto __prefix_last, auto __init_val) {
- std::inclusive_scan(__prefix_first, __prefix_last, __prefix_first, __reduce, std::move(__init_val));
- },
- [&](auto __chunk_first, auto __chunk_last, auto __chunk_result, auto __chunk_init) {
+ auto __scan = [&](_ForwardIterator1 __chunk_first,
+ _ForwardIterator1 __chunk_last,
+ size_t __chunk_index,
+ __decoupled_lookback<_Tp>& __lookback) {
+ // Derive the destination iterator for this chunk.
+ _ForwardIterator2 __chunk_result = __result + (__chunk_first - __first);
+
+ if (__chunk_index == 0) {
+ // Handling of the first chunk.
+ // It is special because it doesn't have a previous partition and must immediately push its inclusive prefix
+ // once the local reduction is complete.
+
+ if (__chunk_index < __lookback.__size()) {
+ // Calculate and publish the inclusive prefix only if storage for this partition exists.
+ // For the edge case when the entire workset consists of a single partition, the lookback will be empty.
+ __decoupled_lookback_partition< _Tp >& __partition = __lookback.__partition(__chunk_index);
+ __partition.__construct_inclusive_prefix(
+ std::transform_reduce(__chunk_first, __chunk_last, __init, __reduce, __transform));
+ }
+
+ // Perform the scanning into the destination.
+ std::transform_inclusive_scan(
+ __chunk_first, __chunk_last, __chunk_result, __reduce, __transform, std::move(__init));
+
+ } else if (__chunk_index < __lookback.__size()) {
+ // General case: a chunk in the middle.
+
+ // Compute and publish the local aggregate.
+ _Tp __aggregate = std::transform_reduce(
+ __chunk_first + 1, __chunk_last, __transform(*__chunk_first), __reduce, __transform);
+ __decoupled_lookback_partition< _Tp >& __partition = __lookback.__partition(__chunk_index);
+ __partition.__construct_aggregate(__aggregate);
+
+ // Depending on the state of the previous chunk, or
+ size_t __prev_chunk_index = __chunk_index - 1;
+ if (__decoupled_lookback_partition< _Tp >& __prev_partition = __lookback.__partition(__prev_chunk_index);
+ __prev_partition.__acquire_available_status() & __decoupled_lookback_status_prefix_available) {
+ // Use the existing inclusive prefix directly (avoid copies)
+ const _Tp& __exclusive_prefix = __prev_partition.__inclusive_prefix();
+ // Calculate and publish the inclusive prefix for the current chunk.
+ __partition.__construct_inclusive_prefix(__reduce(__exclusive_prefix, std::move(__aggregate)));
+ // Perform the scanning into the destination.
+ std::transform_inclusive_scan(
+ __chunk_first, __chunk_last, __chunk_result, __reduce, __transform, __exclusive_prefix);
+ } else {
+ // Calculate the exclusive prefix starting with the aggregate of the previous chunk.
+ _Tp __exclusive_prefix = __lookback.__calculate_exclusive_prefix(__prev_chunk_index, __reduce);
+ // Calculate and publish the inclusive prefix for the current chunk.
+ __partition.__construct_inclusive_prefix(__reduce(__exclusive_prefix, std::move(__aggregate)));
+ // Perform the scanning into the destination.
+ std::transform_inclusive_scan(
+ __chunk_first, __chunk_last, __chunk_result, __reduce, __transform, std::move(__exclusive_prefix));
+ }
+
+ } else {
+ // Handling of the last chunk (which is also not the first one).
+ // It doesn't have its lookback partition, so get the exclusive prefix from the previous chunk and perform
+ // scanning without publishing the inclusive prefix.
+
+ size_t __prev_chunk_index = __chunk_index - 1;
+ if (__decoupled_lookback_partition<_Tp>& __prev_partition = __lookback.__partition(__prev_chunk_index);
+ __prev_partition.__acquire_available_status() & __decoupled_lookback_status_prefix_available) {
+ // Perform the scanning using the existing inclusive prefix of the previous chunk.
+ std::transform_inclusive_scan(
+ __chunk_first,
+ __chunk_last,
+ __chunk_result,
+ __reduce,
+ __transform,
+ __prev_partition.__inclusive_prefix());
+ } else {
+ // Calculate the exclusive prefix for the previous chunk and perform the scanning.
std::transform_inclusive_scan(
- __chunk_first, __chunk_last, __chunk_result, __reduce, __transform, std::move(__chunk_init));
- });
+ __chunk_first,
+ __chunk_last,
+ __chunk_result,
+ __reduce,
+ __transform,
+ __lookback.__calculate_exclusive_prefix(__prev_chunk_index, __reduce));
+ }
+ }
+ };
+ auto __ret = __cpu_traits<_Backend>::template __lookback_scan<_Tp>(__first, __last, __scan);
if (!__ret)
return nullopt;
return __result + (__last - __first);
diff --git a/libcxx/include/__pstl/decoupled_lookback.h b/libcxx/include/__pstl/decoupled_lookback.h
new file mode 100644
index 00000000000000..91699c56841c6e
--- /dev/null
+++ b/libcxx/include/__pstl/decoupled_lookback.h
@@ -0,0 +1,161 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_DECOUPLED_LOOKBACK_H
+#define _LIBCPP___PSTL_DECOUPLED_LOOKBACK_H
+
+#include <__atomic/atomic.h>
+#include <__atomic/atomic_sync.h>
+#include <__config>
+#include <__memory/construct_at.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+// Based on the paper "Single-pass Parallel Prefix Scan with Decoupled Look-back" (Merrill, Garland, 2016).
+
+inline constexpr unsigned char __decoupled_lookback_status_invalid = 0;
+inline constexpr unsigned char __decoupled_lookback_status_aggregate_available = 1 << 0;
+inline constexpr unsigned char __decoupled_lookback_status_prefix_available = 1 << 1;
+
+template <typename _Tp>
+struct _LIBCPP_HIDE_FROM_ABI __decoupled_lookback_partition {
+ __decoupled_lookback_partition() = default;
+ __decoupled_lookback_partition(const __decoupled_lookback_partition&) = delete;
+ __decoupled_lookback_partition& operator=(const __decoupled_lookback_partition&) = delete;
+
+ // Destructor: destroy the aggregate and inclusive prefix if they have been constructed.
+ ~__decoupled_lookback_partition() {
+ unsigned char __flag = __status_flag.load(std::memory_order_relaxed);
+ if (__flag & __decoupled_lookback_status_aggregate_available) {
+ __aggregate().~_Tp();
+ }
+ if (__flag & __decoupled_lookback_status_prefix_available) {
+ __inclusive_prefix().~_Tp();
+ }
+ }
+
+ // Acquire the current status of the partition.
+ // If no value is published is available yet, wait until one becomes available.
+ _LIBCPP_HIDE_FROM_ABI unsigned char __acquire_available_status() {
+ unsigned char __current_status = __status_flag.load(std::memory_order_acquire);
+ if (__current_status != __decoupled_lookback_status_invalid)
+ return __current_status;
+ std::__atomic_wait(__status_flag, __decoupled_lookback_status_invalid, std::memory_order_acquire);
+ return __status_flag.load(std::memory_order_acquire);
+ }
+
+ // Construct the aggregate value of the partition and publish this change.
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI void __construct_aggregate(_Args&&... __args) {
+ std::__construct_at(&reinterpret_cast<_Tp&>(__aggregate_storage), std::forward<_Args>(__args)...);
+ __status_flag.store(__status_flag.load(std::memory_order_relaxed) | __decoupled_lookback_status_aggregate_available,
+ std::memory_order_release);
+ std::__atomic_notify_all(__status_flag);
+ }
+
+ // Access the aggregate value of the partition.
+ // Precondition: __acquire_available_status() & __decoupled_lookback_status_aggregate_available
+ _LIBCPP_HIDE_FROM_ABI const _Tp& __aggregate() const { return reinterpret_cast<const _Tp&>(__aggregate_storage); }
+
+ // Construct the inclusive prefix of the partition and publish this change.
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI void __construct_inclusive_prefix(_Args&&... __args) {
+ std::__construct_at(&reinterpret_cast<_Tp&>(__inclusive_prefix_storage), std::forward<_Args>(__args)...);
+ __status_flag.store(__status_flag.load(std::memory_order_relaxed) | __decoupled_lookback_status_prefix_available,
+ std::memory_order_release);
+ std::__atomic_notify_all(__status_flag);
+ }
+
+ // Access the inclusive prefix of the partition.
+ // Precondition: __acquire_available_status() & __decoupled_lookback_status_prefix_available
+ _LIBCPP_HIDE_FROM_ABI const _Tp& __inclusive_prefix() const {
+ return reinterpret_cast<const _Tp&>(__inclusive_prefix_storage);
+ }
+
+private:
+ // Atomic/waitable flag indicating the status of the partition.
+ std::atomic<unsigned char> __status_flag{__decoupled_lookback_status_invalid};
+
+ // Storage for the aggregate reduced value of the partition.
+ alignas(_Tp) unsigned char __aggregate_storage[sizeof(_Tp)];
+
+ // Storage for the inclusive prefix of the partition.
+ alignas(_Tp) unsigned char __inclusive_prefix_storage[sizeof(_Tp)];
+};
+
+template <typename _Tp>
+class _LIBCPP_HIDE_FROM_ABI __decoupled_lookback {
+public:
+ // Allocates the memory for the lookback partitions.
+ // Does not throw, instead sets the size to 0 if allocation fails.
+ _LIBCPP_HIDE_FROM_ABI explicit __decoupled_lookback(size_t __size) {
+ __partitions_ = __size > 0 ? new (std::nothrow) __decoupled_lookback_partition<_Tp>[__size] : nullptr;
+ __partitions_count_ = __partitions_ ? __size : 0;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI ~__decoupled_lookback() { delete[] __partitions_; }
+
+ __decoupled_lookback(const __decoupled_lookback&) = delete;
+
+ __decoupled_lookback& operator=(const __decoupled_lookback&) = delete;
+
+ _LIBCPP_HIDE_FROM_ABI size_t __size() const { return __partitions_count_; }
+
+ _LIBCPP_HIDE_FROM_ABI __decoupled_lookback_partition<_Tp>* __partitions() const { return __partitions_; }
+
+ _LIBCPP_HIDE_FROM_ABI __decoupled_lookback_partition<_Tp>& __partition(size_t __index) const {
+ return __partitions_[__index];
+ }
+
+ // Calculate an exclusive prefix starting at the given partition.
+ // Performs reductions of aggregate values until it arrives at a partition with an available prefix.
+ // Precondition: __partition(__index).status is __decoupled_lookback_status_aggregate_available.
+ template <class _BinaryOperation>
+ _LIBCPP_HIDE_FROM_ABI _Tp __calculate_exclusive_prefix(size_t __index, _BinaryOperation __reduce) {
+ // Start at the given partition
+ _Tp __prefix = __partitions_[__index].__aggregate();
+ while (true) {
+ // Move to the previous partition and check its status.
+ --__index;
+ __decoupled_lookback_partition< _Tp >& __prev_partition = __partitions_[__index];
+ unsigned char __status = __prev_partition.__acquire_available_status();
+ if (__status & __decoupled_lookback_status_prefix_available) {
+ // Found a partition with an available prefix - can perform a final reduction and terminate.
+ __prefix = __reduce(__prev_partition.__inclusive_prefix(), std::move(__prefix));
+ break;
+ } else /* if(__status & __decoupled_lookback_status_aggregate_available) */ {
+ // Reduce with another aggregate and continue moving left.
+ __prefix = __reduce(__prev_partition.__aggregate(), std::move(__prefix));
+ }
+ }
+ return __prefix;
+ }
+
+private:
+ __decoupled_lookback_partition<_Tp>* __partitions_;
+ size_t __partitions_count_;
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_DECOUPLED_LOOKBACK_H
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index b5fbf6c5aecfea..c967ddb8cbe1e8 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -2479,6 +2479,7 @@ module std {
header "__pstl/cpu_algos/transform_reduce.h"
}
}
+ module decoupled_lookback { header "__pstl/decoupled_lookback.h" }
module dispatch { header "__pstl/dispatch.h" }
module handle_exception { header "__pstl/handle_exception.h" }
}
More information about the libcxx-commits
mailing list