[libcxx-commits] [libcxx] [libc++] atomic timed wait (PR #172214)
via libcxx-commits
libcxx-commits at lists.llvm.org
Sun Dec 14 07:11:59 PST 2025
https://github.com/huixie90 created https://github.com/llvm/llvm-project/pull/172214
None
>From a8a1f9b11e9413f13dca85433622fb714112ed5c Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 14 Dec 2025 15:11:31 +0000
Subject: [PATCH] [libc++] atomic timed wait
---
libcxx/include/CMakeLists.txt | 2 +
libcxx/include/__atomic/atomic.h | 1 +
libcxx/include/__atomic/atomic_flag.h | 1 +
libcxx/include/__atomic/atomic_ref.h | 1 +
libcxx/include/__atomic/atomic_sync.h | 87 +----------
libcxx/include/__atomic/atomic_sync_timed.h | 143 ++++++++++++++++++
.../include/__atomic/atomic_waitable_traits.h | 100 ++++++++++++
libcxx/include/__thread/poll_with_backoff.h | 32 +++-
.../include/__thread/timed_backoff_policy.h | 5 +-
libcxx/include/atomic | 1 +
libcxx/include/module.modulemap.in | 2 +
libcxx/include/semaphore | 7 +-
libcxx/src/atomic.cpp | 87 ++++++++---
13 files changed, 359 insertions(+), 110 deletions(-)
create mode 100644 libcxx/include/__atomic/atomic_sync_timed.h
create mode 100644 libcxx/include/__atomic/atomic_waitable_traits.h
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 53ca1f7d3ded2..17c231ca2b5b4 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -214,6 +214,8 @@ set(files
__atomic/atomic_lock_free.h
__atomic/atomic_ref.h
__atomic/atomic_sync.h
+ __atomic/atomic_sync_timed.h
+ __atomic/atomic_waitable_traits.h
__atomic/check_memory_order.h
__atomic/contention_t.h
__atomic/fence.h
diff --git a/libcxx/include/__atomic/atomic.h b/libcxx/include/__atomic/atomic.h
index 554c111d695f2..fbd17eff75d36 100644
--- a/libcxx/include/__atomic/atomic.h
+++ b/libcxx/include/__atomic/atomic.h
@@ -10,6 +10,7 @@
#define _LIBCPP___ATOMIC_ATOMIC_H
#include <__atomic/atomic_sync.h>
+#include <__atomic/atomic_waitable_traits.h>
#include <__atomic/check_memory_order.h>
#include <__atomic/floating_point_helper.h>
#include <__atomic/is_always_lock_free.h>
diff --git a/libcxx/include/__atomic/atomic_flag.h b/libcxx/include/__atomic/atomic_flag.h
index 321a6283ba7ad..f78301256820e 100644
--- a/libcxx/include/__atomic/atomic_flag.h
+++ b/libcxx/include/__atomic/atomic_flag.h
@@ -10,6 +10,7 @@
#define _LIBCPP___ATOMIC_ATOMIC_FLAG_H
#include <__atomic/atomic_sync.h>
+#include <__atomic/atomic_waitable_traits.h>
#include <__atomic/contention_t.h>
#include <__atomic/memory_order.h>
#include <__atomic/support.h>
diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 9a36aaa3b84fe..b00685f7ce74c 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -19,6 +19,7 @@
#include <__assert>
#include <__atomic/atomic_sync.h>
+#include <__atomic/atomic_waitable_traits.h>
#include <__atomic/check_memory_order.h>
#include <__atomic/floating_point_helper.h>
#include <__atomic/memory_order.h>
diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h
index d0119ab5d35ec..e27816e27d801 100644
--- a/libcxx/include/__atomic/atomic_sync.h
+++ b/libcxx/include/__atomic/atomic_sync.h
@@ -9,6 +9,7 @@
#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H
#define _LIBCPP___ATOMIC_ATOMIC_SYNC_H
+#include <__atomic/atomic_waitable_traits.h>
#include <__atomic/contention_t.h>
#include <__atomic/memory_order.h>
#include <__atomic/to_gcc_order.h>
@@ -18,10 +19,8 @@
#include <__thread/poll_with_backoff.h>
#include <__type_traits/conjunction.h>
#include <__type_traits/decay.h>
-#include <__type_traits/has_unique_object_representation.h>
#include <__type_traits/invoke.h>
#include <__type_traits/is_same.h>
-#include <__type_traits/is_trivially_copyable.h>
#include <__type_traits/void_t.h>
#include <__utility/declval.h>
#include <cstring>
@@ -32,34 +31,6 @@
_LIBCPP_BEGIN_NAMESPACE_STD
-// The customisation points to enable the following functions:
-// - __atomic_wait
-// - __atomic_wait_unless
-// - __atomic_notify_one
-// - __atomic_notify_all
-// Note that std::atomic<T>::wait was back-ported to C++03
-// The below implementations look ugly to support C++03
-template <class _Tp, class = void>
-struct __atomic_waitable_traits {
- using __value_type _LIBCPP_NODEBUG = void;
-
- template <class _AtomicWaitable>
- static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;
-
- template <class _AtomicWaitable>
- static void __atomic_contention_address(_AtomicWaitable&&) = delete;
-};
-
-template <class _Tp, class = void>
-struct __atomic_waitable : false_type {};
-
-template <class _Tp>
-struct __atomic_waitable< _Tp,
- __void_t<decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(
- std::declval<const _Tp&>(), std::declval<memory_order>())),
- decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(
- std::declval<const _Tp&>()))> > : true_type {};
-
#if _LIBCPP_STD_VER >= 20
# if _LIBCPP_HAS_THREADS
@@ -108,48 +79,6 @@ _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one
template <std::size_t _Size>
_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(const void*) _NOEXCEPT;
-# ifdef __linux__
-# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4)
-# elif defined(__APPLE__)
-# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) \
- _APPLY(4) \
- _APPLY(8)
-# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
-# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
-# elif defined(_WIN32)
-# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
-# else
-# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t))
-# endif // __linux__
-
-// concepts defines the types are supported natively by the platform's wait
-
-# if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
-
-_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl(size_t __size) {
- switch (__size) {
-# define _LIBCPP_MAKE_CASE(n) \
- case n: \
- return true;
- _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE)
- default:
- return false;
-# undef _LIBCPP_MAKE_CASE
- };
-}
-
-template <class _Tp>
-concept __has_native_atomic_wait =
- has_unique_object_representations_v<_Tp> && is_trivially_copyable_v<_Tp> &&
- __has_native_atomic_wait_impl(sizeof(_Tp));
-
-# else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
-
-template <class _Tp>
-concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>;
-
-# endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
-
# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
template <class _AtomicWaitable, class _Poll>
@@ -161,7 +90,7 @@ struct __atomic_wait_backoff_impl {
using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type;
- _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
+ _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
if (__elapsed > chrono::microseconds(4)) {
auto __contention_address = const_cast<const void*>(
static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_)));
@@ -169,18 +98,18 @@ struct __atomic_wait_backoff_impl {
if constexpr (__has_native_atomic_wait<__value_type>) {
auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
if (__poll_(__atomic_value))
- return true;
+ return __backoff_results::__poll_success;
std::__atomic_wait_native<sizeof(__value_type)>(__contention_address, std::addressof(__atomic_value));
} else {
__cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address);
auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
if (__poll_(__atomic_value))
- return true;
+ return __backoff_results::__poll_success;
std::__atomic_wait_global_table(__contention_address, __monitor_val);
}
} else {
} // poll
- return false;
+ return __backoff_results::__continue_poll;
}
};
@@ -215,16 +144,16 @@ struct __atomic_wait_backoff_impl {
return __poll_(__current_val);
}
- _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
+ _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
if (__elapsed > chrono::microseconds(4)) {
auto __contention_address = __waitable_traits::__atomic_contention_address(__a_);
__cxx_contention_t __monitor_val;
if (__update_monitor_val_and_poll(__contention_address, __monitor_val))
- return true;
+ return __backoff_results::__poll_success;
std::__libcpp_atomic_wait(__contention_address, __monitor_val);
} else {
} // poll
- return false;
+ return __backoff_results::__continue_poll;
}
};
diff --git a/libcxx/include/__atomic/atomic_sync_timed.h b/libcxx/include/__atomic/atomic_sync_timed.h
new file mode 100644
index 0000000000000..ca7502b64c304
--- /dev/null
+++ b/libcxx/include/__atomic/atomic_sync_timed.h
@@ -0,0 +1,143 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H
+#define _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H
+
+#include <__atomic/atomic_waitable_traits.h>
+#include <__atomic/contention_t.h>
+#include <__atomic/memory_order.h>
+#include <__atomic/to_gcc_order.h>
+#include <__chrono/duration.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__thread/poll_with_backoff.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/has_unique_object_representation.h>
+#include <__type_traits/invoke.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <cstdint>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+# if _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+_LIBCPP_AVAILABILITY_NEW_SYNC
+ _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __address) _NOEXCEPT;
+
+// wait on the global contention state to be changed from the given value for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout(
+ void const* __address, __cxx_contention_t __monitor_value, uint64_t __timeout_ns) _NOEXCEPT;
+
+// wait on the address directly with the native platform wait
+template <std::size_t _Size>
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) _NOEXCEPT;
+
+template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
+struct __atomic_wait_timed_backoff_impl {
+ const _AtomicWaitable& __a_;
+ _Poll __poll_;
+ memory_order __order_;
+ chrono::duration<_Rep, _Period> __rel_time_;
+
+ using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+ using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type;
+
+ _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
+ if (__elapsed > chrono::microseconds(4)) {
+ auto __contention_address = const_cast<const void*>(
+ static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_)));
+
+ uint64_t __timeout_ns =
+ static_cast<uint64_t>((chrono::duration_cast<chrono::nanoseconds>(__rel_time_) - __elapsed).count());
+
+ if constexpr (__has_native_atomic_wait<__value_type>) {
+ auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
+ if (__poll_(__atomic_value))
+ return __backoff_results::__poll_success;
+ std::__atomic_wait_native_with_timeout<sizeof(__value_type)>(
+ __contention_address, std::addressof(__atomic_value), __timeout_ns);
+ } else {
+ __cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address);
+ auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
+ if (__poll_(__atomic_value))
+ return __backoff_results::__poll_success;
+ std::__atomic_wait_global_table_with_timeout(__contention_address, __monitor_val, __timeout_ns);
+ }
+ } else {
+ } // poll
+ return __backoff_results::__continue_poll;
+ }
+};
+
+// The semantics of this function are similar to `atomic`'s
+// `.wait(T old, std::memory_order order)`, but instead of having a hardcoded
+// predicate (is the loaded value unequal to `old`?), the predicate function is
+// specified as an argument. The loaded value is given as an in-out argument to
+// the predicate. If the predicate function returns `true`,
+// `__atomic_wait_unless` will return. If the predicate function returns
+// `false`, it must set the argument to its current understanding of the atomic
+// value. The predicate function must not return `false` spuriously.
+template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
+_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout(
+ const _AtomicWaitable& __a,
+ memory_order __order,
+ _Poll&& __poll,
+ chrono::duration<_Rep, _Period> const& __rel_time) {
+ static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+ __atomic_wait_timed_backoff_impl<_AtomicWaitable, __decay_t<_Poll>, _Rep, _Period> __backoff_fn = {
+ __a, __poll, __order, __rel_time};
+ auto __poll_result = std::__libcpp_thread_poll_with_backoff(
+ /* poll */
+ [&]() {
+ auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
+ return __poll(__current_val);
+ },
+ /* backoff */ __backoff_fn,
+ __rel_time);
+
+ return __poll_result == __poll_with_backoff_results::__poll_success;
+}
+
+# else // _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
+_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout(
+ const _AtomicWaitable& __a,
+ memory_order __order,
+ _Poll&& __poll,
+ chrono::duration<_Rep, _Period> const& __rel_time) {
+ auto __res = std::__libcpp_thread_poll_with_backoff(
+ /* poll */
+ [&]() {
+ auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
+ return __poll(__current_val);
+ },
+ /* backoff */ __libcpp_timed_backoff_policy(),
+ __rel_time);
+ return __res == __poll_with_backoff_results::__poll_success;
+}
+
+# endif // _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+#endif // C++20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H
diff --git a/libcxx/include/__atomic/atomic_waitable_traits.h b/libcxx/include/__atomic/atomic_waitable_traits.h
new file mode 100644
index 0000000000000..6363ab7df5beb
--- /dev/null
+++ b/libcxx/include/__atomic/atomic_waitable_traits.h
@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H
+#define _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H
+
+#include <__atomic/contention_t.h>
+#include <__atomic/memory_order.h>
+#include <__config>
+#include <__type_traits/decay.h>
+#include <__type_traits/has_unique_object_representation.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// The customisation points to enable the following functions:
+// - __atomic_wait
+// - __atomic_wait_unless
+// - __atomic_notify_one
+// - __atomic_notify_all
+// Note that std::atomic<T>::wait was back-ported to C++03
+// The below implementations look ugly to support C++03
+template <class _Tp, class = void>
+struct __atomic_waitable_traits {
+ using __value_type _LIBCPP_NODEBUG = void;
+
+ template <class _AtomicWaitable>
+ static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;
+
+ template <class _AtomicWaitable>
+ static void __atomic_contention_address(_AtomicWaitable&&) = delete;
+};
+
+template <class _Tp, class = void>
+struct __atomic_waitable : false_type {};
+
+template <class _Tp>
+struct __atomic_waitable< _Tp,
+ __void_t<decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(
+ std::declval<const _Tp&>(), std::declval<memory_order>())),
+ decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(
+ std::declval<const _Tp&>()))> > : true_type {};
+
+#ifdef __linux__
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4)
+#elif defined(__APPLE__)
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) \
+ _APPLY(4) \
+ _APPLY(8)
+#elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
+#elif defined(_WIN32)
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
+#else
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t))
+#endif // __linux__
+
+// concepts defines the types are supported natively by the platform's wait
+
+#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
+
+_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl(size_t __size) {
+ switch (__size) {
+# define _LIBCPP_MAKE_CASE(n) \
+ case n: \
+ return true;
+ _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE)
+ default:
+ return false;
+# undef _LIBCPP_MAKE_CASE
+ };
+}
+
+template <class _Tp>
+concept __has_native_atomic_wait =
+ has_unique_object_representations_v<_Tp> && is_trivially_copyable_v<_Tp> &&
+ __has_native_atomic_wait_impl(sizeof(_Tp));
+
+#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+template <class _Tp>
+concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>;
+
+#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H
diff --git a/libcxx/include/__thread/poll_with_backoff.h b/libcxx/include/__thread/poll_with_backoff.h
index b42b1285c13c8..94f075c37e193 100644
--- a/libcxx/include/__thread/poll_with_backoff.h
+++ b/libcxx/include/__thread/poll_with_backoff.h
@@ -13,6 +13,7 @@
#include <__chrono/duration.h>
#include <__chrono/high_resolution_clock.h>
#include <__config>
+#include <cstdint>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -33,22 +34,39 @@ static _LIBCPP_CONSTEXPR const int __libcpp_polling_count = 64;
//
// - __max_elapsed is the maximum duration to try polling for. If the maximum duration is exceeded,
// the polling loop will return false to report a timeout.
+
+enum class __poll_with_backoff_results : uint8_t {
+ __poll_success,
+ __timeout,
+ __backoff_failure,
+};
+
+enum class __backoff_results : uint8_t {
+ __poll_success,
+ __timeout,
+ __backoff_failure,
+ __continue_poll,
+};
+
template <class _Poll, class _Backoff>
-_LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_poll_with_backoff(
+_LIBCPP_HIDE_FROM_ABI __poll_with_backoff_results __libcpp_thread_poll_with_backoff(
_Poll&& __poll, _Backoff&& __backoff, chrono::nanoseconds __max_elapsed = chrono::nanoseconds::zero()) {
auto const __start = chrono::high_resolution_clock::now();
for (int __count = 0;;) {
if (__poll())
- return true; // __poll completion means success
+ return __poll_with_backoff_results::__poll_success;
if (__count < __libcpp_polling_count) {
__count += 1;
continue;
}
chrono::nanoseconds const __elapsed = chrono::high_resolution_clock::now() - __start;
if (__max_elapsed != chrono::nanoseconds::zero() && __max_elapsed < __elapsed)
- return false; // timeout failure
- if (__backoff(__elapsed))
- return false; // __backoff completion means failure
+ return __poll_with_backoff_results::__timeout;
+ if (auto __backoff_res = __backoff(__elapsed); __backoff_res == __backoff_results::__continue_poll) {
+ continue;
+ } else {
+ return static_cast<__poll_with_backoff_results>(__backoff_res);
+ }
}
}
@@ -59,7 +77,9 @@ _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_poll_with_backoff(
// so this should most likely only be used on single-threaded systems where there
// are no other threads to compete with.
struct __spinning_backoff_policy {
- _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(chrono::nanoseconds const&) const { return false; }
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __backoff_results operator()(chrono::nanoseconds const&) const {
+ return __backoff_results::__continue_poll;
+ }
};
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__thread/timed_backoff_policy.h b/libcxx/include/__thread/timed_backoff_policy.h
index 35a72eb61f77e..43d047c18c64e 100644
--- a/libcxx/include/__thread/timed_backoff_policy.h
+++ b/libcxx/include/__thread/timed_backoff_policy.h
@@ -10,6 +10,7 @@
#ifndef _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
#define _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
+#include "__thread/poll_with_backoff.h"
#include <__config>
#if _LIBCPP_HAS_THREADS
@@ -24,7 +25,7 @@
_LIBCPP_BEGIN_NAMESPACE_STD
struct __libcpp_timed_backoff_policy {
- _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
+ _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
if (__elapsed > chrono::milliseconds(128))
__libcpp_thread_sleep_for(chrono::milliseconds(8));
else if (__elapsed > chrono::microseconds(64))
@@ -33,7 +34,7 @@ struct __libcpp_timed_backoff_policy {
__libcpp_thread_yield();
else {
} // poll
- return false;
+ return __backoff_results::__continue_poll;
}
};
diff --git a/libcxx/include/atomic b/libcxx/include/atomic
index 75af5de33ca4c..eff826d0ddd49 100644
--- a/libcxx/include/atomic
+++ b/libcxx/include/atomic
@@ -608,6 +608,7 @@ template <class T>
# include <__atomic/atomic_init.h>
# include <__atomic/atomic_lock_free.h>
# include <__atomic/atomic_sync.h>
+# include <__atomic/atomic_sync_timed.h>
# include <__atomic/check_memory_order.h>
# include <__atomic/contention_t.h>
# include <__atomic/fence.h>
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 04be734e37384..9e8c230128ef5 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -869,6 +869,8 @@ module std [system] {
module atomic_lock_free { header "__atomic/atomic_lock_free.h" }
module atomic_ref { header "__atomic/atomic_ref.h" }
module atomic_sync { header "__atomic/atomic_sync.h" }
+ module atomic_sync_tim { header "__atomic/atomic_sync_timed.h" }
+ module atomic_waitable_traits { header "__atomic/atomic_waitable_traits.h" }
module atomic {
header "__atomic/atomic.h"
export std.atomic.atomic_base // most of std::atomic methods are defined there
diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore
index 1f19d50e32af7..f93f8243bb15a 100644
--- a/libcxx/include/semaphore
+++ b/libcxx/include/semaphore
@@ -55,6 +55,7 @@ using binary_semaphore = counting_semaphore<1>; // since C++20
# include <__assert>
# include <__atomic/atomic.h>
# include <__atomic/atomic_sync.h>
+# include <__atomic/atomic_sync_timed.h>
# include <__atomic/memory_order.h>
# include <__chrono/time_point.h>
# include <__cstddef/ptrdiff_t.h>
@@ -107,8 +108,10 @@ public:
_LIBCPP_HIDE_FROM_ABI bool try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) {
if (__rel_time == chrono::duration<_Rep, _Period>::zero())
return try_acquire();
- auto const __poll_fn = [this]() { return try_acquire(); };
- return std::__libcpp_thread_poll_with_backoff(__poll_fn, __libcpp_timed_backoff_policy(), __rel_time);
+
+ return std::__atomic_wait_unless_with_timeout(__a_, memory_order_relaxed, [this](ptrdiff_t& __old) {
+ return __try_acquire_impl(__old);
+ }, __rel_time);
}
_LIBCPP_HIDE_FROM_ABI bool try_acquire() {
auto __old = __a_.load(memory_order_relaxed);
diff --git a/libcxx/src/atomic.cpp b/libcxx/src/atomic.cpp
index 2b1ebfa177e1c..a8bfb2c880c3b 100644
--- a/libcxx/src/atomic.cpp
+++ b/libcxx/src/atomic.cpp
@@ -10,6 +10,7 @@
#include <atomic>
#include <climits>
#include <cstddef>
+#include <cstdint>
#include <cstring>
#include <functional>
#include <new>
@@ -63,11 +64,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#ifdef __linux__
template <std::size_t _Size>
-static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) {
static_assert(_Size == 4, "Can only wait on 4 bytes value");
char buffer[_Size];
std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
- static constexpr timespec __timeout = {2, 0};
+ static constexpr timespec __default_timeout = {2, 0};
+ timespec __timeout;
+ if (__timeout_ns == 0) {
+ __timeout = __default_timeout;
+ } else {
+ __timeout.tv_sec = __timeout_ns / 1'000'000'000;
+ __timeout.tv_nsec = __timeout_ns % 1'000'000'000;
+ }
_LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, *reinterpret_cast<__cxx_contention_t const*>(&buffer), &__timeout, 0, 0);
}
@@ -89,14 +97,17 @@ extern "C" int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value)
# define ULF_WAKE_ALL 0x00000100
template <std::size_t _Size>
-static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) {
static_assert(_Size == 8 || _Size == 4, "Can only wait on 8 bytes or 4 bytes value");
char buffer[_Size];
std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
+ auto __timeout_us = __timeout_ns == 0 ? 0 : static_cast<uint32_t>(__timeout_ns / 1000);
if constexpr (_Size == 4)
- __ulock_wait(UL_COMPARE_AND_WAIT, const_cast<void*>(__ptr), *reinterpret_cast<uint32_t const*>(&buffer), 0);
+ __ulock_wait(
+ UL_COMPARE_AND_WAIT, const_cast<void*>(__ptr), *reinterpret_cast<uint32_t const*>(&buffer), __timeout_us);
else
- __ulock_wait(UL_COMPARE_AND_WAIT64, const_cast<void*>(__ptr), *reinterpret_cast<uint64_t const*>(&buffer), 0);
+ __ulock_wait(
+ UL_COMPARE_AND_WAIT64, const_cast<void*>(__ptr), *reinterpret_cast<uint64_t const*>(&buffer), __timeout_us);
}
template <std::size_t _Size>
@@ -117,11 +128,20 @@ static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
*/
template <std::size_t _Size>
-static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) {
static_assert(_Size == 8, "Can only wait on 8 bytes value");
- char buffer[_Size];
- std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
- _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, *reinterpret_cast<__cxx_contention_t*>(&buffer), nullptr, nullptr);
+ if (__timeout_ns == 0) {
+ char buffer[_Size];
+ std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
+ _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, *reinterpret_cast<__cxx_contention_t*>(&buffer), nullptr, nullptr);
+ } else {
+ // TODO the doc says it supports timeout but does not say how to use it
+ // https://man.freebsd.org/cgi/man.cgi?query=_umtx_op
+ __libcpp_thread_poll_with_backoff(
+ [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
+ __libcpp_timed_backoff_policy(),
+ std::chrono::nanoseconds(__timeout_ns));
+ }
}
template <std::size_t _Size>
@@ -157,17 +177,21 @@ static void* win32_get_synch_api_function(const char* function_name) {
}
template <std::size_t _Size>
-static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) {
static_assert(_Size == 8, "Can only wait on 8 bytes value");
// WaitOnAddress was added in Windows 8 (build 9200)
static auto wait_on_address =
reinterpret_cast<BOOL(WINAPI*)(void*, PVOID, SIZE_T, DWORD)>(win32_get_synch_api_function("WaitOnAddress"));
if (wait_on_address != nullptr) {
- wait_on_address(const_cast<void*>(__ptr), const_cast<void*>(__val), _Size, INFINITE);
+ wait_on_address(const_cast<void*>(__ptr),
+ const_cast<void*>(__val),
+ _Size,
+ __timeout_ns == 0 ? INFINITE : static_cast<DWORD>(__timeout_ns / 1'000'000));
} else {
__libcpp_thread_poll_with_backoff(
[=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
- __libcpp_timed_backoff_policy());
+ __libcpp_timed_backoff_policy(),
+ std::chrono::nanoseconds(__timeout_ns));
}
}
@@ -202,10 +226,11 @@ static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
// Baseline is just a timed backoff
template <std::size_t _Size>
-static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) {
__libcpp_thread_poll_with_backoff(
[=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
- __libcpp_timed_backoff_policy());
+ __libcpp_timed_backoff_policy(),
+ std::chrono::nanoseconds(__timeout_ns));
}
template <std::size_t _Size>
@@ -229,14 +254,16 @@ __contention_notify(__cxx_atomic_contention_t* __waiter_count, void const* __add
}
template <std::size_t _Size>
-static void
-__contention_wait(__cxx_atomic_contention_t* __waiter_count, void const* __address_to_wait, void const* __old_value) {
+static void __contention_wait(__cxx_atomic_contention_t* __waiter_count,
+ void const* __address_to_wait,
+ void const* __old_value,
+ uint64_t __timeout_ns) {
__cxx_atomic_fetch_add(__waiter_count, __cxx_contention_t(1), memory_order_relaxed);
// https://llvm.org/PR109290
// There are no platform guarantees of a memory barrier in the platform wait implementation
__cxx_atomic_thread_fence(memory_order_seq_cst);
// We sleep as long as the monitored value hasn't changed.
- __platform_wait_on_address<_Size>(__address_to_wait, __old_value);
+ __platform_wait_on_address<_Size>(__address_to_wait, __old_value, __timeout_ns);
__cxx_atomic_fetch_sub(__waiter_count, __cxx_contention_t(1), memory_order_release);
}
@@ -305,7 +332,14 @@ _LIBCPP_EXPORTED_FROM_ABI void
__atomic_wait_global_table(void const* __location, __cxx_contention_t __old_value) noexcept {
auto const __entry = __get_global_contention_state(__location);
__contention_wait<sizeof(__cxx_atomic_contention_t)>(
- &__entry->__waiter_count, &__entry->__platform_state, &__old_value);
+ &__entry->__waiter_count, &__entry->__platform_state, &__old_value, 0);
+}
+
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout(
+ void const* __location, __cxx_contention_t __old_value, uint64_t __timeout_ns) _NOEXCEPT {
+ auto const __entry = __get_global_contention_state(__location);
+ __contention_wait<sizeof(__cxx_atomic_contention_t)>(
+ &__entry->__waiter_count, &__entry->__platform_state, &__old_value, __timeout_ns);
}
_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const* __location) noexcept {
@@ -320,7 +354,13 @@ _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const* __lo
template <std::size_t _Size>
_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native(void const* __address, void const* __old_value) noexcept {
- __contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value);
+ __contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value, 0);
+}
+
+template <std::size_t _Size>
+_LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) noexcept {
+ __contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value, __timeout_ns);
}
template <std::size_t _Size>
@@ -341,6 +381,8 @@ _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(void const* __location
# define _INSTANTIATE(_SIZE) \
template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native<_SIZE>(void const*, void const*) noexcept; \
+ template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<_SIZE>( \
+ void const*, void const*, uint64_t) noexcept; \
template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native<_SIZE>(void const*) noexcept; \
template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native<_SIZE>(void const*) noexcept;
@@ -353,6 +395,9 @@ _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_INSTANTIATE)
template _LIBCPP_EXPORTED_FROM_ABI void
__atomic_wait_native<sizeof(__cxx_contention_t)>(void const* __address, void const* __old_value) noexcept;
+template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<sizeof(__cxx_contention_t)>(
+ void const* __address, void const* __old_value, uint64_t) noexcept;
+
template _LIBCPP_EXPORTED_FROM_ABI void
__atomic_notify_one_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
@@ -382,7 +427,7 @@ _LIBCPP_EXPORTED_FROM_ABI void
__libcpp_atomic_wait(void const volatile* __location, __cxx_contention_t __old_value) noexcept {
auto const __entry = __get_global_contention_state(const_cast<void const*>(__location));
__contention_wait<sizeof(__cxx_atomic_contention_t)>(
- &__entry->__waiter_count, &__entry->__platform_state, &__old_value);
+ &__entry->__waiter_count, &__entry->__platform_state, &__old_value, 0);
}
_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile* __location) noexcept {
@@ -401,7 +446,7 @@ _LIBCPP_EXPORTED_FROM_ABI void
__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile* __location, __cxx_contention_t __old_value) noexcept {
auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
__contention_wait<sizeof(__cxx_atomic_contention_t)>(
- __get_native_waiter_count(__location_cast), __location_cast, &__old_value);
+ __get_native_waiter_count(__location_cast), __location_cast, &__old_value, 0);
}
// this function is even unused in the old ABI
More information about the libcxx-commits
mailing list