[libcxx-commits] [libcxx] [libc++][pstl] Implementation of parallel std::copy_if (PR #229385)
Michael G. Kazakov via libcxx-commits
libcxx-commits at lists.llvm.org
Wed Oct 7 03:25:27 PDT 2026
https://github.com/mikekazakov updated https://github.com/llvm/llvm-project/pull/229385
>From 53267331f4c71a15da86fd84d7dc45cd3613e6da Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Tue, 6 Oct 2026 12:34:01 +0100
Subject: [PATCH 1/5] Implementation of copy_if based on __lookback_scan
---
libcxx/include/CMakeLists.txt | 2 +
libcxx/include/__algorithm/pstl.h | 27 +++
libcxx/include/__pstl/backend_fwd.h | 8 +
libcxx/include/__pstl/backends/libdispatch.h | 69 ++++++
libcxx/include/__pstl/backends/std_thread.h | 28 +++
libcxx/include/__pstl/cpu_algos/copy_if.h | 201 ++++++++++++++++++
libcxx/include/__pstl/decoupled_lookback.h | 161 ++++++++++++++
libcxx/include/module.modulemap.in | 3 +
.../alg.copy/pstl.copy_if.pass.cpp | 158 ++++++++++++++
9 files changed, 657 insertions(+)
create mode 100644 libcxx/include/__pstl/cpu_algos/copy_if.h
create mode 100644 libcxx/include/__pstl/decoupled_lookback.h
create mode 100644 libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/pstl.copy_if.pass.cpp
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 1ee2081561fd97..198de5be233b55 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -697,6 +697,7 @@ set(files
__pstl/backends/serial.h
__pstl/backends/std_thread.h
__pstl/cpu_algos/any_of.h
+ __pstl/cpu_algos/copy_if.h
__pstl/cpu_algos/cpu_traits.h
__pstl/cpu_algos/fill.h
__pstl/cpu_algos/find_end.h
@@ -715,6 +716,7 @@ set(files
__pstl/cpu_algos/transform.h
__pstl/cpu_algos/transform_reduce.h
__pstl/cpu_algos/uninitialized_algorithms.h
+ __pstl/decoupled_lookback.h
__pstl/dispatch.h
__pstl/handle_exception.h
__random/bernoulli_distribution.h
diff --git a/libcxx/include/__algorithm/pstl.h b/libcxx/include/__algorithm/pstl.h
index 492dd3f3c5d5a7..39927e73584295 100644
--- a/libcxx/include/__algorithm/pstl.h
+++ b/libcxx/include/__algorithm/pstl.h
@@ -92,6 +92,33 @@ copy(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __l
std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__result));
}
+template <class _ExecutionPolicy,
+ class _ForwardIterator,
+ class _ForwardOutIterator,
+ class _UnaryPred,
+ class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
+ enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator
+copy_if(_ExecutionPolicy&& __policy,
+ _ForwardIterator __first,
+ _ForwardIterator __last,
+ _ForwardOutIterator __result,
+ _UnaryPred __pred) {
+ _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
+ _ForwardIterator, "copy_if(first, last, result) requires [first, last) to be ForwardIterators");
+ _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
+ _ForwardOutIterator, "copy_if(first, last, result) requires result to be a ForwardIterator");
+ _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(
+ _ForwardOutIterator, decltype(*__first), "copy_if(first, last, result) requires result to be an OutputIterator");
+ using _Implementation = __pstl::__dispatch<__pstl::__copy_if, __pstl::__current_configuration, _RawPolicy>;
+ return __pstl::__handle_exception<_Implementation>(
+ std::forward<_ExecutionPolicy>(__policy),
+ std::move(__first),
+ std::move(__last),
+ std::move(__result),
+ std::move(__pred));
+}
+
template <class _ExecutionPolicy,
class _ForwardIterator,
class _ForwardOutIterator,
diff --git a/libcxx/include/__pstl/backend_fwd.h b/libcxx/include/__pstl/backend_fwd.h
index a5b83f977b7098..64963dcd3bf36b 100644
--- a/libcxx/include/__pstl/backend_fwd.h
+++ b/libcxx/include/__pstl/backend_fwd.h
@@ -296,6 +296,14 @@ struct __copy;
// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
// _ForwardOutIterator __out_it) const noexcept;
+template <class _Backend, class _ExecutionPolicy>
+struct __copy_if;
+// template <class _Policy, class _ForwardIterator, class _ForwardOutIterator, class _Predicate>
+// optional<_ForwardOutIterator>
+// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last,
+// _ForwardOutIterator __out_it,
+// _Predicate __pred) const noexcept;
+
template <class _Backend, class _ExecutionPolicy>
struct __copy_n;
// template <class _Policy, class _ForwardIterator, class _Size, class _ForwardOutIterator>
diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h
index 09ea15d411dd1c..8b62e6ffebf075 100644
--- a/libcxx/include/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__pstl/backends/libdispatch.h
@@ -36,6 +36,7 @@
#include <__optional/optional.h>
#include <__pstl/backend_fwd.h>
#include <__pstl/cpu_algos/any_of.h>
+#include <__pstl/cpu_algos/copy_if.h>
#include <__pstl/cpu_algos/cpu_traits.h>
#include <__pstl/cpu_algos/fill.h>
#include <__pstl/cpu_algos/find_end.h>
@@ -54,6 +55,8 @@
#include <__pstl/cpu_algos/transform.h>
#include <__pstl/cpu_algos/transform_reduce.h>
#include <__pstl/cpu_algos/uninitialized_algorithms.h>
+#include <__pstl/decoupled_lookback.h>
+#include <__thread/thread.h>
#include <__utility/empty.h>
#include <__utility/exception_guard.h>
#include <__utility/move.h>
@@ -273,6 +276,68 @@ struct __cpu_traits<__libdispatch_backend_tag> {
__combiner);
}
+ template <class _Value,
+ class _RandomAccessIterator,
+ class _WorkerPrologue,
+ class _ScanHead,
+ class _ScanMiddle,
+ class _ScanTail,
+ class _WorkerEpilogue>
+ _LIBCPP_HIDE_FROM_ABI static optional<__empty> __lookback_scan(
+ _RandomAccessIterator __first,
+ _RandomAccessIterator __last,
+ _WorkerPrologue __worker_prologue,
+ _ScanHead __scan_head,
+ _ScanMiddle __scan_middle,
+ _ScanTail __scan_tail,
+ _WorkerEpilogue __worker_epilogue) {
+ if (__first == __last)
+ return __empty{}; // nothing to do
+
+ // Partition the input and allocate the lookback storage
+ __libdispatch::__chunk_partitions __partitions = __libdispatch::__partition_chunks(__last - __first);
+ __decoupled_lookback<_Value> __lookback{static_cast<size_t>(__partitions.__chunk_count_ - 1)};
+ if (__partitions.__chunk_count_ > 1 && __lookback.__size() == 0) {
+ return nullopt; // failed to allocate the lookback storage
+ }
+
+ size_t __max_workers_count = thread::hardware_concurrency();
+
+ // Run the single-pass scan with decoupled lookback
+ atomic<size_t> __next_chunk{0};
+ __libdispatch::__dispatch_apply(__max_workers_count, [&](size_t /*__worker_id*/) {
+ auto __worker_ctx = __worker_prologue(
+ static_cast<size_t>(std::max(__partitions.__first_chunk_size_, __partitions.__chunk_size_)));
+
+ size_t __chunk;
+ while ((__chunk = __next_chunk.fetch_add(1, std::memory_order_relaxed)) <
+ static_cast<size_t>(__partitions.__chunk_count_)) {
+ auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
+ auto __index = __chunk == 0 ? 0
+ : (__chunk * __partitions.__chunk_size_) +
+ (__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
+ if (__chunk == 0) {
+ __scan_head(__worker_ctx,
+ __first + __index,
+ __first + __index + __this_chunk_size,
+ __partitions.__chunk_count_ > 1 ? &__lookback.__partition(0) : nullptr);
+ } else if (__chunk < __lookback.__size()) {
+ __scan_middle(
+ __worker_ctx, __first + __index, __first + __index + __this_chunk_size, &__lookback.__partition(__chunk));
+ } else {
+ __scan_tail(__worker_ctx,
+ __first + __index,
+ __first + __index + __this_chunk_size,
+ &__lookback.__partition(__chunk - 1) + 1);
+ }
+
+ __worker_epilogue(std::move(__worker_ctx));
+ }
+ });
+
+ return __empty{};
+ }
+
template <class _RandomAccessIterator, class _Comp, class _LeafSort>
_LIBCPP_HIDE_FROM_ABI static optional<__empty>
__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) {
@@ -369,6 +434,10 @@ struct __cpu_traits<__libdispatch_backend_tag> {
};
// Mandatory implementations of the computational basis
+template <class _ExecutionPolicy>
+struct __copy_if<__libdispatch_backend_tag, _ExecutionPolicy>
+ : __cpu_parallel_copy_if<__libdispatch_backend_tag, _ExecutionPolicy> {};
+
template <class _ExecutionPolicy>
struct __find_end<__libdispatch_backend_tag, _ExecutionPolicy>
: __cpu_parallel_find_end<__libdispatch_backend_tag, _ExecutionPolicy> {};
diff --git a/libcxx/include/__pstl/backends/std_thread.h b/libcxx/include/__pstl/backends/std_thread.h
index e8b7e2fbd9458d..1f79287c5a7409 100644
--- a/libcxx/include/__pstl/backends/std_thread.h
+++ b/libcxx/include/__pstl/backends/std_thread.h
@@ -14,6 +14,7 @@
#include <__optional/optional.h>
#include <__pstl/backend_fwd.h>
#include <__pstl/cpu_algos/any_of.h>
+#include <__pstl/cpu_algos/copy_if.h>
#include <__pstl/cpu_algos/cpu_traits.h>
#include <__pstl/cpu_algos/fill.h>
#include <__pstl/cpu_algos/find_end.h>
@@ -69,6 +70,29 @@ struct __cpu_traits<__std_thread_backend_tag> {
return __reduce(std::move(__first), std::move(__last), std::move(__init));
}
+ template <class _Value,
+ class _RandomAccessIterator,
+ class _WorkerPrologue,
+ class _ScanHead,
+ class _ScanMiddle,
+ class _ScanTail,
+ class _WorkerEpilogue>
+ _LIBCPP_HIDE_FROM_ABI static optional<__empty> __lookback_scan(
+ _RandomAccessIterator __first,
+ _RandomAccessIterator __last,
+ _WorkerPrologue __worker_prologue,
+ _ScanHead __scan_head,
+ _ScanMiddle /*__scan_middle*/,
+ _ScanTail /*__scan_tail*/,
+ _WorkerEpilogue __worker_epilogue) {
+ if (__first == __last)
+ return __empty{}; // nothing to do
+ auto __ctx = __worker_prologue(static_cast<size_t>(__last - __first));
+ __scan_head(__ctx, __first, __last, nullptr);
+ __worker_epilogue(std::move(__ctx));
+ return __empty{};
+ }
+
template <class _RandomAccessIterator, class _Compare, class _LeafSort>
_LIBCPP_HIDE_FROM_ABI static optional<__empty>
__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
@@ -99,6 +123,10 @@ struct __cpu_traits<__std_thread_backend_tag> {
};
// Mandatory implementations of the computational basis
+template <class _ExecutionPolicy>
+struct __copy_if<__std_thread_backend_tag, _ExecutionPolicy>
+ : __cpu_parallel_copy_if<__std_thread_backend_tag, _ExecutionPolicy> {};
+
template <class _ExecutionPolicy>
struct __find_end<__std_thread_backend_tag, _ExecutionPolicy>
: __cpu_parallel_find_end<__std_thread_backend_tag, _ExecutionPolicy> {};
diff --git a/libcxx/include/__pstl/cpu_algos/copy_if.h b/libcxx/include/__pstl/cpu_algos/copy_if.h
new file mode 100644
index 00000000000000..a5fdc4576fca48
--- /dev/null
+++ b/libcxx/include/__pstl/cpu_algos/copy_if.h
@@ -0,0 +1,201 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_COPY_IF_H
+#define _LIBCPP___PSTL_CPU_ALGOS_COPY_IF_H
+
+#include <__algorithm/copy_if.h>
+#include <__algorithm/fill.h>
+#include <__config>
+#include <__functional/operations.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/unique_ptr.h>
+#include <__new/global_new_delete.h>
+#include <__new/nothrow_t.h>
+#include <__optional/optional.h>
+#include <__pstl/backend_fwd.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
+#include <__pstl/cpu_algos/find_if.h>
+#include <__pstl/decoupled_lookback.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/move.h>
+#include <climits> // for CHAR_BIT
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+struct _LIBCPP_HIDE_FROM_ABI __dynamic_bitset {
+ size_t __size;
+ std::unique_ptr<size_t[]> __bits;
+
+ _LIBCPP_HIDE_FROM_ABI __dynamic_bitset(size_t __max_chunk_size)
+ : __size((__max_chunk_size + CHAR_BIT * sizeof(size_t) - 1) / (CHAR_BIT * sizeof(size_t))),
+ __bits(new (std::nothrow) size_t[__size]) {}
+ _LIBCPP_HIDE_FROM_ABI __dynamic_bitset(__dynamic_bitset&&) noexcept = default;
+ _LIBCPP_HIDE_FROM_ABI __dynamic_bitset& operator=(__dynamic_bitset&&) noexcept = default;
+ _LIBCPP_HIDE_FROM_ABI bool __test(size_t __pos) const {
+ size_t __word_index = __pos / (CHAR_BIT * sizeof(size_t));
+ size_t __bit_index = __pos % (CHAR_BIT * sizeof(size_t));
+ return (__bits[__word_index] & (size_t(1) << __bit_index)) != 0;
+ }
+ _LIBCPP_HIDE_FROM_ABI void __set(size_t __pos) {
+ size_t __word_index = __pos / (CHAR_BIT * sizeof(size_t));
+ size_t __bit_index = __pos % (CHAR_BIT * sizeof(size_t));
+ __bits[__word_index] |= (size_t(1) << __bit_index);
+ }
+ _LIBCPP_HIDE_FROM_ABI void __reset(size_t __pos) {
+ size_t __word_index = __pos / (CHAR_BIT * sizeof(size_t));
+ size_t __bit_index = __pos % (CHAR_BIT * sizeof(size_t));
+ __bits[__word_index] &= ~(size_t(1) << __bit_index);
+ }
+ _LIBCPP_HIDE_FROM_ABI void __reset() {
+ for (size_t i = 0; i < __size; ++i) {
+ __bits[i] = 0;
+ }
+ }
+};
+
+template <class _ForwardIterator, class _Pred>
+_LIBCPP_HIDE_FROM_ABI size_t __count_and_cache_predicate_results(
+ _ForwardIterator __first, _ForwardIterator __last, const _Pred& __pred, __dynamic_bitset& __bitset) {
+ size_t __occupied_count = 0;
+ size_t __local_idx = 0;
+ for (auto __it = __first; __it != __last; ++__it) {
+ if (__pred(*__it)) {
+ __bitset.__set(__local_idx);
+ ++__occupied_count;
+ }
+ ++__local_idx;
+ }
+ return __occupied_count;
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator2 __copy_if_flag_set(
+ _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result, __dynamic_bitset& __bitset) {
+ size_t __local_idx = 0;
+ for (auto __it = __first; __it != __last; ++__it) {
+ if (__bitset.__test(__local_idx)) {
+ *__result = *__it;
+ ++__result;
+ }
+ ++__local_idx;
+ }
+ return __result;
+}
+
+template <class _Backend, class _RawExecutionPolicy>
+struct __cpu_parallel_copy_if {
+ template <class _Policy, class _ForwardIterator1, class _ForwardIterator2, class _Pred>
+ _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator2>
+ operator()(_Policy&&, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result, _Pred __pred)
+ const noexcept {
+ if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
+ __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
+ __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
+ auto __prologue = [](size_t __max_chunk_size) { return __dynamic_bitset(__max_chunk_size); };
+ auto __epilogue = [](__dynamic_bitset&&) {};
+
+ _ForwardIterator2 __out_iter = __result;
+
+ // TODO: support the case when the worker context was not allocated
+
+ auto __scan_head =
+ [&](__dynamic_bitset& __worker_ctx,
+ _ForwardIterator1 __chunk_first,
+ _ForwardIterator1 __chunk_last,
+ __decoupled_lookback_partition<size_t>* __optional_lookback_partition) {
+ __worker_ctx.__reset();
+
+ size_t __occupied_count =
+ __count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
+
+ if (__optional_lookback_partition != nullptr) {
+ __optional_lookback_partition->__construct_inclusive_prefix(__occupied_count);
+ }
+
+ _ForwardIterator2 __chunk_result = __copy_if_flag_set(__chunk_first, __chunk_last, __result, __worker_ctx);
+
+ if (__optional_lookback_partition == nullptr) {
+ __out_iter = __chunk_result; // TODO: explain
+ }
+ };
+
+ auto __scan_middle =
+ [&](__dynamic_bitset& __worker_ctx,
+ _ForwardIterator1 __chunk_first,
+ _ForwardIterator1 __chunk_last,
+ __decoupled_lookback_partition<size_t>* __lookback_partition) {
+ __worker_ctx.__reset();
+
+ size_t __occupied_count =
+ __count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
+
+ __lookback_partition->__construct_aggregate(__occupied_count);
+ __decoupled_lookback_partition<size_t>* __prev_partition = __lookback_partition - 1;
+ size_t __exclusive_prefix =
+ (__prev_partition->__acquire_available_status() & __decoupled_lookback_status_prefix_available)
+ ? __prev_partition->__inclusive_prefix()
+ : __calculate_inclusive_prefix_of_partition(__prev_partition, plus<>{});
+
+ __lookback_partition->__construct_inclusive_prefix(__exclusive_prefix + __occupied_count);
+
+ __copy_if_flag_set(__chunk_first, __chunk_last, __result + __exclusive_prefix, __worker_ctx);
+ };
+
+ auto __scan_tail =
+ [&](__dynamic_bitset& __worker_ctx,
+ _ForwardIterator1 __chunk_first,
+ _ForwardIterator1 __chunk_last,
+ __decoupled_lookback_partition<size_t>* __nonexistent_lookback_partition) {
+ __worker_ctx.__reset();
+
+ __count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
+
+ __decoupled_lookback_partition< size_t >* __prev_partition = __nonexistent_lookback_partition - 1;
+
+ size_t __exclusive_prefix =
+ (__prev_partition->__acquire_available_status() & __decoupled_lookback_status_prefix_available)
+ ? __prev_partition->__inclusive_prefix()
+ : __calculate_inclusive_prefix_of_partition(__prev_partition, plus<>{});
+
+ _ForwardIterator2 __chunk_result =
+ __copy_if_flag_set(__chunk_first, __chunk_last, __result + __exclusive_prefix, __worker_ctx);
+
+ __out_iter = __chunk_result;
+ };
+
+ auto __ret = __cpu_traits<_Backend>::template __lookback_scan<size_t>(
+ __first, __last, __prologue, __scan_head, __scan_middle, __scan_tail, __epilogue);
+ if (!__ret)
+ return nullopt;
+
+ return __out_iter;
+ } else {
+ return std::copy_if(std::move(__first), std::move(__last), std::move(__result), std::move(__pred));
+ }
+ }
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_COPY_IF_H
diff --git a/libcxx/include/__pstl/decoupled_lookback.h b/libcxx/include/__pstl/decoupled_lookback.h
new file mode 100644
index 00000000000000..5759cba18ec967
--- /dev/null
+++ b/libcxx/include/__pstl/decoupled_lookback.h
@@ -0,0 +1,161 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_DECOUPLED_LOOKBACK_H
+#define _LIBCPP___PSTL_DECOUPLED_LOOKBACK_H
+
+#include <__atomic/atomic.h>
+#include <__atomic/atomic_sync.h>
+#include <__config>
+#include <__memory/construct_at.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+// Based on the paper "Single-pass Parallel Prefix Scan with Decoupled Look-back" (Merrill, Garland, 2016).
+
+inline constexpr unsigned char __decoupled_lookback_status_invalid = 0;
+inline constexpr unsigned char __decoupled_lookback_status_aggregate_available = 1 << 0;
+inline constexpr unsigned char __decoupled_lookback_status_prefix_available = 1 << 1;
+
+template <typename _Tp>
+struct _LIBCPP_HIDE_FROM_ABI __decoupled_lookback_partition {
+ __decoupled_lookback_partition() = default;
+ __decoupled_lookback_partition(const __decoupled_lookback_partition&) = delete;
+ __decoupled_lookback_partition& operator=(const __decoupled_lookback_partition&) = delete;
+
+ // Destructor: destroy the aggregate and inclusive prefix if they have been constructed.
+ ~__decoupled_lookback_partition() {
+ unsigned char __flag = __status_flag.load(std::memory_order_relaxed);
+ if (__flag & __decoupled_lookback_status_aggregate_available) {
+ __aggregate().~_Tp();
+ }
+ if (__flag & __decoupled_lookback_status_prefix_available) {
+ __inclusive_prefix().~_Tp();
+ }
+ }
+
+ // Acquire the current status of the partition.
+ // If no value is published is available yet, wait until one becomes available.
+ _LIBCPP_HIDE_FROM_ABI unsigned char __acquire_available_status() {
+ unsigned char __current_status = __status_flag.load(std::memory_order_acquire);
+ if (__current_status != __decoupled_lookback_status_invalid)
+ return __current_status;
+ std::__atomic_wait(__status_flag, __decoupled_lookback_status_invalid, std::memory_order_acquire);
+ return __status_flag.load(std::memory_order_acquire);
+ }
+
+ // Construct the aggregate value of the partition and publish this change.
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI void __construct_aggregate(_Args&&... __args) {
+ std::__construct_at(&reinterpret_cast<_Tp&>(__aggregate_storage), std::forward<_Args>(__args)...);
+ __status_flag.store(__status_flag.load(std::memory_order_relaxed) | __decoupled_lookback_status_aggregate_available,
+ std::memory_order_release);
+ std::__atomic_notify_all(__status_flag);
+ }
+
+ // Access the aggregate value of the partition.
+ // Precondition: __acquire_available_status() & __decoupled_lookback_status_aggregate_available
+ _LIBCPP_HIDE_FROM_ABI const _Tp& __aggregate() const { return reinterpret_cast<const _Tp&>(__aggregate_storage); }
+
+ // Construct the inclusive prefix of the partition and publish this change.
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI void __construct_inclusive_prefix(_Args&&... __args) {
+ std::__construct_at(&reinterpret_cast<_Tp&>(__inclusive_prefix_storage), std::forward<_Args>(__args)...);
+ __status_flag.store(__status_flag.load(std::memory_order_relaxed) | __decoupled_lookback_status_prefix_available,
+ std::memory_order_release);
+ std::__atomic_notify_all(__status_flag);
+ }
+
+ // Access the inclusive prefix of the partition.
+ // Precondition: __acquire_available_status() & __decoupled_lookback_status_prefix_available
+ _LIBCPP_HIDE_FROM_ABI const _Tp& __inclusive_prefix() const {
+ return reinterpret_cast<const _Tp&>(__inclusive_prefix_storage);
+ }
+
+private:
+ // Atomic/waitable flag indicating the status of the partition.
+ std::atomic<unsigned char> __status_flag{__decoupled_lookback_status_invalid};
+
+ // Storage for the aggregate reduced value of the partition.
+ alignas(_Tp) unsigned char __aggregate_storage[sizeof(_Tp)];
+
+ // Storage for the inclusive prefix of the partition.
+ alignas(_Tp) unsigned char __inclusive_prefix_storage[sizeof(_Tp)];
+};
+
+// Calculate an inclusive prefix of a given partition that currently has only an aggregate.
+// Performs reductions of aggregate values until it arrives at a partition with an available prefix.
+// Precondition: __partition->__acquire_available_status() == __decoupled_lookback_status_aggregate_available.
+template <typename _Tp, class _BinaryOperation>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__calculate_inclusive_prefix_of_partition(__decoupled_lookback_partition<_Tp>* __partition, _BinaryOperation __reduce) {
+ // Start at the given partition
+ _Tp __prefix = __partition->__aggregate();
+ while (true) {
+ // Move to the previous partition and check its status.
+ --__partition;
+ unsigned char __status = __partition->__acquire_available_status();
+ if (__status & __decoupled_lookback_status_prefix_available) {
+ // Found a partition with an available prefix - can perform a final reduction and terminate.
+ __prefix = __reduce(__partition->__inclusive_prefix(), std::move(__prefix));
+ break;
+ } else /* if(__status & __decoupled_lookback_status_aggregate_available) */ {
+ // Reduce with another aggregate and continue moving left.
+ __prefix = __reduce(__partition->__aggregate(), std::move(__prefix));
+ }
+ }
+ return __prefix;
+}
+
+template <typename _Tp>
+class _LIBCPP_HIDE_FROM_ABI __decoupled_lookback {
+public:
+ // Allocates the memory for the lookback partitions.
+ // Does not throw, instead sets the size to 0 if allocation fails.
+ _LIBCPP_HIDE_FROM_ABI explicit __decoupled_lookback(size_t __size) {
+ __partitions_ = __size > 0 ? new (std::nothrow) __decoupled_lookback_partition<_Tp>[__size] : nullptr;
+ __partitions_count_ = __partitions_ ? __size : 0;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI ~__decoupled_lookback() { delete[] __partitions_; }
+
+ __decoupled_lookback(const __decoupled_lookback&) = delete;
+
+ __decoupled_lookback& operator=(const __decoupled_lookback&) = delete;
+
+ _LIBCPP_HIDE_FROM_ABI size_t __size() const { return __partitions_count_; }
+
+ _LIBCPP_HIDE_FROM_ABI __decoupled_lookback_partition<_Tp>* __partitions() const { return __partitions_; }
+
+ _LIBCPP_HIDE_FROM_ABI __decoupled_lookback_partition<_Tp>& __partition(size_t __index) const {
+ return __partitions_[__index];
+ }
+
+private:
+ __decoupled_lookback_partition<_Tp>* __partitions_;
+ size_t __partitions_count_;
+};
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_DECOUPLED_LOOKBACK_H
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 28ae9ee8de46a3..5bbc6f4718a9ea 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -2426,6 +2426,9 @@ module std {
module any_of {
header "__pstl/cpu_algos/any_of.h"
}
+ module copy_if {
+ header "__pstl/cpu_algos/copy_if.h"
+ }
module cpu_traits {
header "__pstl/cpu_algos/cpu_traits.h"
}
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/pstl.copy_if.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/pstl.copy_if.pass.cpp
new file mode 100644
index 00000000000000..607569089d589e
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/pstl.copy_if.pass.cpp
@@ -0,0 +1,158 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++17
+
+// UNSUPPORTED: libcpp-has-no-incomplete-pstl
+
+// template <class ExecutionPolicy,
+// class ForwardIterator1,
+// class ForwardIterator2,
+// class UnaryPredicate>
+// ForwardIterator2 copy_if(ExecutionPolicy&& exec,
+// ForwardIterator1 first,
+// ForwardIterator1 last,
+// ForwardIterator2 result,
+// UnaryPredicate pred);
+
+#include <algorithm>
+#include <cassert>
+#include <functional>
+#include <iterator>
+#include <limits>
+#include <numeric>
+
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "type_algorithms.h"
+#include "runway_sample.h"
+
+EXECUTION_POLICY_SFINAE_TEST(copy_if);
+
+static_assert(sfinae_test_copy_if<int, int*, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_copy_if<std::execution::parallel_policy, int*, int*, int*, bool (*)(int)>);
+
+template <class Iter1, class Iter2>
+struct TestInt {
+ template <class ExecutionPolicy>
+ void operator()(ExecutionPolicy&& policy) {
+ auto always_true = [](int) { return true; };
+ auto always_false = [](int) { return false; };
+ { // Check the return types
+ int in[] = {0};
+ int out[] = {-1};
+ auto res =
+ std::copy_if(policy, Iter1(std::begin(in)), Iter1(std::begin(in)), Iter2(std::begin(out)), always_true);
+ static_assert(std::is_same_v<decltype(res), Iter2>);
+ }
+ { // Empty range
+ int in[] = {0};
+ int out[] = {-1};
+ auto res =
+ std::copy_if(policy, Iter1(std::begin(in)), Iter1(std::begin(in)), Iter2(std::begin(out)), always_true);
+ assert(res == Iter2(std::begin(out)));
+ }
+ { // Single, true
+ int in[] = {0};
+ int out[] = {-1};
+ auto res = std::copy_if(policy, Iter1(std::begin(in)), Iter1(std::end(in)), Iter2(std::begin(out)), always_true);
+ assert(res == Iter2(std::begin(out) + 1));
+ assert(out[0] == 0);
+ }
+ { // Single, false
+ int in[] = {0};
+ int out[] = {-1};
+ auto res = std::copy_if(policy, Iter1(std::begin(in)), Iter1(std::end(in)), Iter2(std::begin(out)), always_false);
+ assert(res == Iter2(std::begin(out)));
+ assert(out[0] == -1);
+ }
+ {
+ // Two, true, false
+ int in[] = {0, 1};
+ int out[] = {-1, -1};
+ auto res = std::copy_if(policy, Iter1(std::begin(in)), Iter1(std::end(in)), Iter2(std::begin(out)), [](int x) {
+ return x == 0;
+ });
+ assert(res == Iter2(std::begin(out) + 1));
+ assert(out[0] == 0);
+ assert(out[1] == -1);
+ }
+ {
+ // Two, false, true
+ int in[] = {0, 1};
+ int out[] = {-1, -1};
+ auto res = std::copy_if(policy, Iter1(std::begin(in)), Iter1(std::end(in)), Iter2(std::begin(out)), [](int x) {
+ return x != 0;
+ });
+ assert(res == Iter2(std::begin(out) + 1));
+ assert(out[0] == 1);
+ assert(out[1] == -1);
+ }
+ { // iotaed input sampled at different lengths, pred: always true
+ int in[1073];
+ int out[1073];
+ std::iota(std::begin(in), std::end(in), 0);
+ runway_sample(std::size(in) + 1, [&](size_t n) {
+ std::fill(std::begin(out), std::end(out), -1);
+ auto res =
+ std::copy_if(policy, Iter1(std::begin(in)), Iter1(std::begin(in) + n), Iter2(std::begin(out)), always_true);
+ assert(res == Iter2(std::begin(out) + n));
+ for (size_t i = 0; i < std::size(out); ++i) {
+ assert(out[i] == (i < n ? in[i] : -1));
+ }
+ });
+ }
+ { // iotaed input sampled at different lengths, pred: always false
+ int in[1073];
+ int out[1073];
+ std::iota(std::begin(in), std::end(in), 0);
+ runway_sample(std::size(in) + 1, [&](size_t n) {
+ std::fill(std::begin(out), std::end(out), -1);
+ auto res = std::copy_if(
+ policy, Iter1(std::begin(in)), Iter1(std::begin(in) + n), Iter2(std::begin(out)), always_false);
+ assert(res == Iter2(std::begin(out)));
+ for (size_t i = 0; i < std::size(out); ++i) {
+ assert(out[i] == -1);
+ }
+ });
+ }
+ { // iotaed input sampled at different lengths, pred: x % 3 == 0
+ int in[1073];
+ int out[1073];
+ std::iota(std::begin(in), std::end(in), 0);
+ runway_sample(std::size(in) + 1, [&](size_t n) {
+ std::fill(std::begin(out), std::end(out), -1);
+ auto res =
+ std::copy_if(policy, Iter1(std::begin(in)), Iter1(std::begin(in) + n), Iter2(std::begin(out)), [](int x) {
+ return x % 3 == 0;
+ });
+ assert(res == Iter2(std::begin(out) + (n + 2) / 3));
+ auto out_count = std::distance(Iter2(std::begin(out)), res);
+ for (size_t i = 0; i < std::size(out); ++i) {
+ if (i < static_cast<size_t>(out_count)) {
+ assert(out[i] % 3 == 0);
+ assert(out[i] == (i > 0 ? out[i - 1] + 3 : 0));
+ } else {
+ assert(out[i] == -1);
+ }
+ }
+ });
+ }
+ }
+};
+
+int main(int, char**) {
+ types::for_each(types::forward_iterator_list<const int*>{}, types::apply_type_identity{[](auto v) {
+ using Iter = typename decltype(v)::type;
+ types::for_each(
+ types::forward_iterator_list<int*>{},
+ TestIteratorWithPolicies<types::partial_instantiation<TestInt, Iter>::template apply>{});
+ }});
+ return 0;
+}
>From c6b78525c9e6437738ca82b12862261969eea28b Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Tue, 6 Oct 2026 16:57:51 +0100
Subject: [PATCH 2/5] Require C++20 for now
---
libcxx/include/__pstl/backends/libdispatch.h | 7 ++++
libcxx/include/__pstl/cpu_algos/copy_if.h | 39 ++++++++++---------
libcxx/include/__pstl/decoupled_lookback.h | 39 ++++++++++---------
.../alg.copy/pstl.copy_if.pass.cpp | 3 +-
4 files changed, 49 insertions(+), 39 deletions(-)
diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h
index 8b62e6ffebf075..46ce57013f2971 100644
--- a/libcxx/include/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__pstl/backends/libdispatch.h
@@ -276,6 +276,8 @@ struct __cpu_traits<__libdispatch_backend_tag> {
__combiner);
}
+# if _LIBCPP_STD_VER >= 20 // TODO: remove once https://github.com/llvm/llvm-project/pull/224356 is merged
+
template <class _Value,
class _RandomAccessIterator,
class _WorkerPrologue,
@@ -338,6 +340,8 @@ struct __cpu_traits<__libdispatch_backend_tag> {
return __empty{};
}
+# endif // _LIBCPP_STD_VER >= 20
+
template <class _RandomAccessIterator, class _Comp, class _LeafSort>
_LIBCPP_HIDE_FROM_ABI static optional<__empty>
__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) {
@@ -434,9 +438,12 @@ struct __cpu_traits<__libdispatch_backend_tag> {
};
// Mandatory implementations of the computational basis
+
+# if _LIBCPP_STD_VER >= 20 // TODO: remove once https://github.com/llvm/llvm-project/pull/224356 is merged
template <class _ExecutionPolicy>
struct __copy_if<__libdispatch_backend_tag, _ExecutionPolicy>
: __cpu_parallel_copy_if<__libdispatch_backend_tag, _ExecutionPolicy> {};
+# endif // _LIBCPP_STD_VER >= 20
template <class _ExecutionPolicy>
struct __find_end<__libdispatch_backend_tag, _ExecutionPolicy>
diff --git a/libcxx/include/__pstl/cpu_algos/copy_if.h b/libcxx/include/__pstl/cpu_algos/copy_if.h
index a5fdc4576fca48..1ce4b7534e24c2 100644
--- a/libcxx/include/__pstl/cpu_algos/copy_if.h
+++ b/libcxx/include/__pstl/cpu_algos/copy_if.h
@@ -33,38 +33,38 @@
_LIBCPP_PUSH_MACROS
#include <__undef_macros>
-#if _LIBCPP_STD_VER >= 17
+#if _LIBCPP_STD_VER >= 20 // TODO: should be 17 once https://github.com/llvm/llvm-project/pull/224356 is merged
_LIBCPP_BEGIN_NAMESPACE_STD
namespace __pstl {
struct _LIBCPP_HIDE_FROM_ABI __dynamic_bitset {
- size_t __size;
- std::unique_ptr<size_t[]> __bits;
+ size_t __size_;
+ std::unique_ptr<size_t[]> __bits_;
_LIBCPP_HIDE_FROM_ABI __dynamic_bitset(size_t __max_chunk_size)
- : __size((__max_chunk_size + CHAR_BIT * sizeof(size_t) - 1) / (CHAR_BIT * sizeof(size_t))),
- __bits(new (std::nothrow) size_t[__size]) {}
+ : __size_((__max_chunk_size + CHAR_BIT * sizeof(size_t) - 1) / (CHAR_BIT * sizeof(size_t))),
+ __bits_(new (std::nothrow) size_t[__size_]) {}
_LIBCPP_HIDE_FROM_ABI __dynamic_bitset(__dynamic_bitset&&) noexcept = default;
_LIBCPP_HIDE_FROM_ABI __dynamic_bitset& operator=(__dynamic_bitset&&) noexcept = default;
_LIBCPP_HIDE_FROM_ABI bool __test(size_t __pos) const {
size_t __word_index = __pos / (CHAR_BIT * sizeof(size_t));
size_t __bit_index = __pos % (CHAR_BIT * sizeof(size_t));
- return (__bits[__word_index] & (size_t(1) << __bit_index)) != 0;
+ return (__bits_[__word_index] & (size_t(1) << __bit_index)) != 0;
}
_LIBCPP_HIDE_FROM_ABI void __set(size_t __pos) {
size_t __word_index = __pos / (CHAR_BIT * sizeof(size_t));
size_t __bit_index = __pos % (CHAR_BIT * sizeof(size_t));
- __bits[__word_index] |= (size_t(1) << __bit_index);
+ __bits_[__word_index] |= (size_t(1) << __bit_index);
}
_LIBCPP_HIDE_FROM_ABI void __reset(size_t __pos) {
size_t __word_index = __pos / (CHAR_BIT * sizeof(size_t));
size_t __bit_index = __pos % (CHAR_BIT * sizeof(size_t));
- __bits[__word_index] &= ~(size_t(1) << __bit_index);
+ __bits_[__word_index] &= ~(size_t(1) << __bit_index);
}
_LIBCPP_HIDE_FROM_ABI void __reset() {
- for (size_t i = 0; i < __size; ++i) {
- __bits[i] = 0;
+ for (size_t i = 0; i < __size_; ++i) {
+ __bits_[i] = 0;
}
}
};
@@ -122,13 +122,14 @@ struct __cpu_parallel_copy_if {
__worker_ctx.__reset();
size_t __occupied_count =
- __count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
+ __pstl::__count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
if (__optional_lookback_partition != nullptr) {
__optional_lookback_partition->__construct_inclusive_prefix(__occupied_count);
}
- _ForwardIterator2 __chunk_result = __copy_if_flag_set(__chunk_first, __chunk_last, __result, __worker_ctx);
+ _ForwardIterator2 __chunk_result =
+ __pstl::__copy_if_flag_set(__chunk_first, __chunk_last, __result, __worker_ctx);
if (__optional_lookback_partition == nullptr) {
__out_iter = __chunk_result; // TODO: explain
@@ -143,18 +144,18 @@ struct __cpu_parallel_copy_if {
__worker_ctx.__reset();
size_t __occupied_count =
- __count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
+ __pstl::__count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
__lookback_partition->__construct_aggregate(__occupied_count);
__decoupled_lookback_partition<size_t>* __prev_partition = __lookback_partition - 1;
size_t __exclusive_prefix =
(__prev_partition->__acquire_available_status() & __decoupled_lookback_status_prefix_available)
? __prev_partition->__inclusive_prefix()
- : __calculate_inclusive_prefix_of_partition(__prev_partition, plus<>{});
+ : __pstl::__calculate_inclusive_prefix_of_partition(__prev_partition, plus<>{});
__lookback_partition->__construct_inclusive_prefix(__exclusive_prefix + __occupied_count);
- __copy_if_flag_set(__chunk_first, __chunk_last, __result + __exclusive_prefix, __worker_ctx);
+ __pstl::__copy_if_flag_set(__chunk_first, __chunk_last, __result + __exclusive_prefix, __worker_ctx);
};
auto __scan_tail =
@@ -164,17 +165,17 @@ struct __cpu_parallel_copy_if {
__decoupled_lookback_partition<size_t>* __nonexistent_lookback_partition) {
__worker_ctx.__reset();
- __count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
+ __pstl::__count_and_cache_predicate_results(__chunk_first, __chunk_last, __pred, __worker_ctx);
__decoupled_lookback_partition< size_t >* __prev_partition = __nonexistent_lookback_partition - 1;
size_t __exclusive_prefix =
(__prev_partition->__acquire_available_status() & __decoupled_lookback_status_prefix_available)
? __prev_partition->__inclusive_prefix()
- : __calculate_inclusive_prefix_of_partition(__prev_partition, plus<>{});
+ : __pstl::__calculate_inclusive_prefix_of_partition(__prev_partition, plus<>{});
_ForwardIterator2 __chunk_result =
- __copy_if_flag_set(__chunk_first, __chunk_last, __result + __exclusive_prefix, __worker_ctx);
+ __pstl::__copy_if_flag_set(__chunk_first, __chunk_last, __result + __exclusive_prefix, __worker_ctx);
__out_iter = __chunk_result;
};
@@ -194,7 +195,7 @@ struct __cpu_parallel_copy_if {
} // namespace __pstl
_LIBCPP_END_NAMESPACE_STD
-#endif // _LIBCPP_STD_VER >= 17
+#endif // _LIBCPP_STD_VER >= 20
_LIBCPP_POP_MACROS
diff --git a/libcxx/include/__pstl/decoupled_lookback.h b/libcxx/include/__pstl/decoupled_lookback.h
index 5759cba18ec967..a25c73b3cfca7d 100644
--- a/libcxx/include/__pstl/decoupled_lookback.h
+++ b/libcxx/include/__pstl/decoupled_lookback.h
@@ -21,7 +21,7 @@
_LIBCPP_PUSH_MACROS
#include <__undef_macros>
-#if _LIBCPP_STD_VER >= 17
+#if _LIBCPP_STD_VER >= 20 // TODO: should be 17 once https://github.com/llvm/llvm-project/pull/224356 is merged
_LIBCPP_BEGIN_NAMESPACE_STD
namespace __pstl {
@@ -40,7 +40,7 @@ struct _LIBCPP_HIDE_FROM_ABI __decoupled_lookback_partition {
// Destructor: destroy the aggregate and inclusive prefix if they have been constructed.
~__decoupled_lookback_partition() {
- unsigned char __flag = __status_flag.load(std::memory_order_relaxed);
+ unsigned char __flag = __status_flag_.load(std::memory_order_relaxed);
if (__flag & __decoupled_lookback_status_aggregate_available) {
__aggregate().~_Tp();
}
@@ -52,50 +52,51 @@ struct _LIBCPP_HIDE_FROM_ABI __decoupled_lookback_partition {
// Acquire the current status of the partition.
// If no value is published is available yet, wait until one becomes available.
_LIBCPP_HIDE_FROM_ABI unsigned char __acquire_available_status() {
- unsigned char __current_status = __status_flag.load(std::memory_order_acquire);
+ unsigned char __current_status = __status_flag_.load(std::memory_order_acquire);
if (__current_status != __decoupled_lookback_status_invalid)
return __current_status;
- std::__atomic_wait(__status_flag, __decoupled_lookback_status_invalid, std::memory_order_acquire);
- return __status_flag.load(std::memory_order_acquire);
+ std::__atomic_wait(__status_flag_, __decoupled_lookback_status_invalid, std::memory_order_acquire);
+ return __status_flag_.load(std::memory_order_acquire);
}
// Construct the aggregate value of the partition and publish this change.
template <class... _Args>
_LIBCPP_HIDE_FROM_ABI void __construct_aggregate(_Args&&... __args) {
- std::__construct_at(&reinterpret_cast<_Tp&>(__aggregate_storage), std::forward<_Args>(__args)...);
- __status_flag.store(__status_flag.load(std::memory_order_relaxed) | __decoupled_lookback_status_aggregate_available,
- std::memory_order_release);
- std::__atomic_notify_all(__status_flag);
+ std::__construct_at(&reinterpret_cast<_Tp&>(__aggregate_storage_), std::forward<_Args>(__args)...);
+ __status_flag_.store(
+ __status_flag_.load(std::memory_order_relaxed) | __decoupled_lookback_status_aggregate_available,
+ std::memory_order_release);
+ std::__atomic_notify_all(__status_flag_);
}
// Access the aggregate value of the partition.
// Precondition: __acquire_available_status() & __decoupled_lookback_status_aggregate_available
- _LIBCPP_HIDE_FROM_ABI const _Tp& __aggregate() const { return reinterpret_cast<const _Tp&>(__aggregate_storage); }
+ _LIBCPP_HIDE_FROM_ABI const _Tp& __aggregate() const { return reinterpret_cast<const _Tp&>(__aggregate_storage_); }
// Construct the inclusive prefix of the partition and publish this change.
template <class... _Args>
_LIBCPP_HIDE_FROM_ABI void __construct_inclusive_prefix(_Args&&... __args) {
- std::__construct_at(&reinterpret_cast<_Tp&>(__inclusive_prefix_storage), std::forward<_Args>(__args)...);
- __status_flag.store(__status_flag.load(std::memory_order_relaxed) | __decoupled_lookback_status_prefix_available,
- std::memory_order_release);
- std::__atomic_notify_all(__status_flag);
+ std::__construct_at(&reinterpret_cast<_Tp&>(__inclusive_prefix_storage_), std::forward<_Args>(__args)...);
+ __status_flag_.store(__status_flag_.load(std::memory_order_relaxed) | __decoupled_lookback_status_prefix_available,
+ std::memory_order_release);
+ std::__atomic_notify_all(__status_flag_);
}
// Access the inclusive prefix of the partition.
// Precondition: __acquire_available_status() & __decoupled_lookback_status_prefix_available
_LIBCPP_HIDE_FROM_ABI const _Tp& __inclusive_prefix() const {
- return reinterpret_cast<const _Tp&>(__inclusive_prefix_storage);
+ return reinterpret_cast<const _Tp&>(__inclusive_prefix_storage_);
}
private:
// Atomic/waitable flag indicating the status of the partition.
- std::atomic<unsigned char> __status_flag{__decoupled_lookback_status_invalid};
+ std::atomic<unsigned char> __status_flag_{__decoupled_lookback_status_invalid};
// Storage for the aggregate reduced value of the partition.
- alignas(_Tp) unsigned char __aggregate_storage[sizeof(_Tp)];
+ alignas(_Tp) unsigned char __aggregate_storage_[sizeof(_Tp)];
// Storage for the inclusive prefix of the partition.
- alignas(_Tp) unsigned char __inclusive_prefix_storage[sizeof(_Tp)];
+ alignas(_Tp) unsigned char __inclusive_prefix_storage_[sizeof(_Tp)];
};
// Calculate an inclusive prefix of a given partition that currently has only an aggregate.
@@ -154,7 +155,7 @@ class _LIBCPP_HIDE_FROM_ABI __decoupled_lookback {
} // namespace __pstl
_LIBCPP_END_NAMESPACE_STD
-#endif // _LIBCPP_STD_VER >= 17
+#endif // _LIBCPP_STD_VER >= 20
_LIBCPP_POP_MACROS
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/pstl.copy_if.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/pstl.copy_if.pass.cpp
index 607569089d589e..1d4f83b63cbd6f 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/pstl.copy_if.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/pstl.copy_if.pass.cpp
@@ -6,7 +6,8 @@
//
//===----------------------------------------------------------------------===//
-// REQUIRES: std-at-least-c++17
+// TODO: should be 17 one https://github.com/llvm/llvm-project/pull/224356 is merged
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: libcpp-has-no-incomplete-pstl
>From 78d1075ebb4818a14af92f7edecbb100602a9d33 Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Tue, 6 Oct 2026 17:18:39 +0100
Subject: [PATCH 3/5] Sync, more gating
---
libcxx/include/__pstl/backends/std_thread.h | 2 ++
libcxx/include/__pstl/decoupled_lookback.h | 7 +++++++
2 files changed, 9 insertions(+)
diff --git a/libcxx/include/__pstl/backends/std_thread.h b/libcxx/include/__pstl/backends/std_thread.h
index 1f79287c5a7409..cc8096bc41c494 100644
--- a/libcxx/include/__pstl/backends/std_thread.h
+++ b/libcxx/include/__pstl/backends/std_thread.h
@@ -123,9 +123,11 @@ struct __cpu_traits<__std_thread_backend_tag> {
};
// Mandatory implementations of the computational basis
+# if _LIBCPP_STD_VER >= 20 // TODO: remove once https://github.com/llvm/llvm-project/pull/224356 is merged
template <class _ExecutionPolicy>
struct __copy_if<__std_thread_backend_tag, _ExecutionPolicy>
: __cpu_parallel_copy_if<__std_thread_backend_tag, _ExecutionPolicy> {};
+# endif // _LIBCPP_STD_VER >= 20
template <class _ExecutionPolicy>
struct __find_end<__std_thread_backend_tag, _ExecutionPolicy>
diff --git a/libcxx/include/__pstl/decoupled_lookback.h b/libcxx/include/__pstl/decoupled_lookback.h
index a25c73b3cfca7d..6eb041d69b5472 100644
--- a/libcxx/include/__pstl/decoupled_lookback.h
+++ b/libcxx/include/__pstl/decoupled_lookback.h
@@ -11,8 +11,15 @@
#include <__atomic/atomic.h>
#include <__atomic/atomic_sync.h>
+#include <__atomic/memory_order.h>
#include <__config>
+#include <__cstddef/size_t.h>
#include <__memory/construct_at.h>
+#include <__new/global_new_delete.h>
+#include <__new/nothrow_t.h>
+#include <__utility/empty.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
>From c0bbd3768bfd2e9f1633ab0f7b4f6ac6a374b8f0 Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Tue, 6 Oct 2026 17:35:45 +0100
Subject: [PATCH 4/5] Missing entry in modulemap
---
libcxx/include/module.modulemap.in | 1 +
1 file changed, 1 insertion(+)
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 5bbc6f4718a9ea..f39e8839656542 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -2487,6 +2487,7 @@ module std {
header "__pstl/cpu_algos/uninitialized_algorithms.h"
}
}
+ module decoupled_lookback { header "__pstl/decoupled_lookback.h" }
module dispatch { header "__pstl/dispatch.h" }
module handle_exception { header "__pstl/handle_exception.h" }
}
>From 48518fe1b61d0996afec81ba08bc20947c8b67bf Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Wed, 7 Oct 2026 11:25:01 +0100
Subject: [PATCH 5/5] Missing includes
---
libcxx/include/__pstl/backends/libdispatch.h | 1 +
libcxx/include/__pstl/cpu_algos/copy_if.h | 7 +++++--
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h
index 46ce57013f2971..dfa162668d602c 100644
--- a/libcxx/include/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__pstl/backends/libdispatch.h
@@ -20,6 +20,7 @@
#include <__algorithm/move.h>
#include <__algorithm/upper_bound.h>
#include <__atomic/atomic.h>
+#include <__atomic/memory_order.h>
#include <__config>
#include <__cstddef/ptrdiff_t.h>
#include <__cstddef/size_t.h>
diff --git a/libcxx/include/__pstl/cpu_algos/copy_if.h b/libcxx/include/__pstl/cpu_algos/copy_if.h
index 1ce4b7534e24c2..4688a41114dab3 100644
--- a/libcxx/include/__pstl/cpu_algos/copy_if.h
+++ b/libcxx/include/__pstl/cpu_algos/copy_if.h
@@ -12,11 +12,14 @@
#include <__algorithm/copy_if.h>
#include <__algorithm/fill.h>
#include <__config>
+#include <__cstddef/size_t.h>
#include <__functional/operations.h>
+#include <__iterator/concepts.h>
#include <__iterator/iterator_traits.h>
#include <__memory/unique_ptr.h>
#include <__new/global_new_delete.h>
#include <__new/nothrow_t.h>
+#include <__optional/nullopt_t.h>
#include <__optional/optional.h>
#include <__pstl/backend_fwd.h>
#include <__pstl/cpu_algos/cpu_traits.h>
@@ -63,8 +66,8 @@ struct _LIBCPP_HIDE_FROM_ABI __dynamic_bitset {
__bits_[__word_index] &= ~(size_t(1) << __bit_index);
}
_LIBCPP_HIDE_FROM_ABI void __reset() {
- for (size_t i = 0; i < __size_; ++i) {
- __bits_[i] = 0;
+ for (size_t __i = 0; __i < __size_; ++__i) {
+ __bits_[__i] = 0;
}
}
};
More information about the libcxx-commits
mailing list