[libcxx-commits] [libcxx] [libc++] P0718R2: Implementation of std::atomic<shared_ptr<T>> and std::atomic<weak_ptr<T>> (PR #194215)
Vladislav Semykin via libcxx-commits
libcxx-commits at lists.llvm.org
Mon Aug 31 08:25:58 PDT 2026
https://github.com/ViNN280801 updated https://github.com/llvm/llvm-project/pull/194215
>From 368b9e95212e00b93e291c7a6bd02d27c6f1ce42 Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Sun, 26 Apr 2026 10:03:26 +0300
Subject: [PATCH 01/46] [libcxx] Implementation of non lock-free atomic
shared_ptr
Signed-off-by: ViNN280801 <vladislav.semykin at gmail.com>
---
libcxx/include/CMakeLists.txt | 2 +
libcxx/include/__atomic/atomic_sync_lite.h | 45 ++
libcxx/include/__memory/atomic_shared_ptr.h | 491 ++++++++++++++++++
libcxx/include/__memory/shared_ptr.h | 22 +-
libcxx/include/module.modulemap.in | 5 +
.../atomic_shared_ptr_aliasing.pass.cpp | 71 +++
.../atomic_shared_ptr_class.pass.cpp | 70 +++
.../atomic_shared_ptr_memory_order.verify.cpp | 58 +++
.../atomic_shared_ptr_nullptr.pass.cpp | 54 ++
.../atomic_shared_ptr_refcount.pass.cpp | 105 ++++
.../atomic_shared_ptr_stress.pass.cpp | 107 ++++
.../atomic_weak_ptr_class.pass.cpp | 91 ++++
12 files changed, 1119 insertions(+), 2 deletions(-)
create mode 100644 libcxx/include/__atomic/atomic_sync_lite.h
create mode 100644 libcxx/include/__memory/atomic_shared_ptr.h
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_aliasing.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_class.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_weak_ptr_class.pass.cpp
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 5e9040a62dd53..48968128e47bd 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -216,6 +216,7 @@ set(files
__atomic/atomic_lock_free.h
__atomic/atomic_ref.h
__atomic/atomic_sync.h
+ __atomic/atomic_sync_lite.h
__atomic/atomic_sync_timed.h
__atomic/atomic_waitable_traits.h
__atomic/check_memory_order.h
@@ -596,6 +597,7 @@ set(files
__memory/allocator_traits.h
__memory/array_cookie.h
__memory/assume_aligned.h
+ __memory/atomic_shared_ptr.h
__memory/auto_ptr.h
__memory/compressed_pair.h
__memory/concepts.h
diff --git a/libcxx/include/__atomic/atomic_sync_lite.h b/libcxx/include/__atomic/atomic_sync_lite.h
new file mode 100644
index 0000000000000..a0433e16b506a
--- /dev/null
+++ b/libcxx/include/__atomic/atomic_sync_lite.h
@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_LITE_H
+#define _LIBCPP___ATOMIC_ATOMIC_SYNC_LITE_H
+
+#include <__atomic/contention_t.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
+
+# if !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+// Old dylib interface kept for backwards compatibility.
+_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
+_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT;
+_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*) _NOEXCEPT;
+_LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t) _NOEXCEPT;
+# endif // !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+// New dylib interface.
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
+__atomic_monitor_global(void const* __address) _NOEXCEPT;
+
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_global_table(void const* __address, __cxx_contention_t __monitor_value) _NOEXCEPT;
+
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const*) _NOEXCEPT;
+
+#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_LITE_H
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
new file mode 100644
index 0000000000000..000b56e08bae6
--- /dev/null
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -0,0 +1,491 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_H
+#define _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_H
+
+#include <__atomic/atomic_sync_lite.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/memory_order.h>
+#include <__atomic/support.h>
+#include <__config>
+#include <__cstddef/nullptr_t.h>
+#include <__memory/shared_count.h>
+#include <__utility/move.h>
+
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+#if defined(__SANITIZE_THREAD__) || (defined(__has_feature) && __has_feature(thread_sanitizer))
+# if __has_include(<sanitizer/tsan_interface.h>)
+# include <sanitizer/tsan_interface.h>
+# define _LIBCPP_ATOMIC_SHARED_PTR_TSAN 1
+# endif
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+// TSAN annotations model the lock-bit protocol on __ctrl_.
+#if defined(_LIBCPP_ATOMIC_SHARED_PTR_TSAN)
+# define _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(addr) \
+ ::__tsan_mutex_pre_lock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
+# define _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(addr) \
+ ::__tsan_mutex_post_lock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0, 0)
+# define _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(addr) \
+ ::__tsan_mutex_pre_unlock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
+# define _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(addr) \
+ ::__tsan_mutex_post_unlock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
+#else
+# define _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(addr) ((void)(addr))
+# define _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(addr) ((void)(addr))
+# define _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(addr) ((void)(addr))
+# define _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(addr) ((void)(addr))
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+
+template <class _Tp>
+class shared_ptr;
+
+template <class _Tp>
+class weak_ptr;
+
+template <class _Tp>
+struct atomic;
+
+// Split state into pointer word and control word.
+// The control word stores control-block pointer plus lock/notify bits.
+struct __atomic_smart_ptr_storage {
+ static constexpr uintptr_t __lock_bit_ = uintptr_t{1};
+ static constexpr uintptr_t __notify_bit_ = uintptr_t{2};
+ static constexpr uintptr_t __ptr_mask_ = ~(__lock_bit_ | __notify_bit_);
+
+ _LIBCPP_HIDE_FROM_ABI static uintptr_t __encode(__shared_weak_count* __ctrl, uintptr_t __bits) _NOEXCEPT {
+ return (reinterpret_cast<uintptr_t>(__ctrl) & __ptr_mask_) | (__bits & ~__ptr_mask_);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI static __shared_weak_count* __decode(uintptr_t __word) _NOEXCEPT {
+ return reinterpret_cast<__shared_weak_count*>(__word & __ptr_mask_);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI static bool __has_lock(uintptr_t __word) _NOEXCEPT { return (__word & __lock_bit_) != 0; }
+ _LIBCPP_HIDE_FROM_ABI static bool __has_notify(uintptr_t __word) _NOEXCEPT { return (__word & __notify_bit_) != 0; }
+};
+
+_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_one(const void* __address) _NOEXCEPT {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ std::__atomic_notify_one_global_table(__address);
+# else
+ std::__cxx_atomic_notify_one(reinterpret_cast<void const volatile*>(__address));
+# endif
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_all(const void* __address) _NOEXCEPT {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ std::__atomic_notify_all_global_table(__address);
+# else
+ std::__cxx_atomic_notify_all(reinterpret_cast<void const volatile*>(__address));
+# endif
+}
+
+template <class _Poll>
+_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_wait_on_address(const void* __address, _Poll&& __poll) _NOEXCEPT {
+ while (!__poll()) {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ auto __monitor_value = std::__atomic_monitor_global(__address);
+ if (__poll())
+ return;
+ std::__atomic_wait_global_table(__address, __monitor_value);
+# else
+ void const volatile* __volatile_address = reinterpret_cast<void const volatile*>(__address);
+ auto __monitor_value = std::__libcpp_atomic_monitor(__volatile_address);
+ if (__poll())
+ return;
+ std::__libcpp_atomic_wait(__volatile_address, __monitor_value);
+# endif
+ }
+}
+
+template <class _Element>
+struct __atomic_smart_ptr_fields {
+ mutable __cxx_atomic_impl<_Element*> __ptr_;
+ mutable __cxx_atomic_impl<uintptr_t> __ctrl_;
+
+ _LIBCPP_HIDE_FROM_ABI __atomic_smart_ptr_fields(_Element* __p, __shared_weak_count* __c) _NOEXCEPT
+ : __ptr_(__p),
+ __ctrl_(__atomic_smart_ptr_storage::__encode(__c, 0)) {}
+
+ _LIBCPP_HIDE_FROM_ABI const void* __ctrl_address() const _NOEXCEPT {
+ return static_cast<const void*>(__builtin_addressof(__ctrl_));
+ }
+
+ // Acquire lock bit on __ctrl_. Contended path sets notify bit and waits.
+ _LIBCPP_HIDE_FROM_ABI void __lock() const _NOEXCEPT {
+ _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(&__ctrl_);
+ uintptr_t __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
+ for (;;) {
+ if (!__atomic_smart_ptr_storage::__has_lock(__expected)) {
+ uintptr_t __desired = __expected | __atomic_smart_ptr_storage::__lock_bit_;
+ if (std::__cxx_atomic_compare_exchange_weak(
+ __builtin_addressof(__ctrl_),
+ __builtin_addressof(__expected),
+ __desired,
+ memory_order_acquire,
+ memory_order_relaxed)) {
+ _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(&__ctrl_);
+ return;
+ }
+ continue;
+ }
+
+ uintptr_t __with_notify = __expected | __atomic_smart_ptr_storage::__notify_bit_;
+ if (!__atomic_smart_ptr_storage::__has_notify(__expected)) {
+ if (!std::__cxx_atomic_compare_exchange_weak(
+ __builtin_addressof(__ctrl_),
+ __builtin_addressof(__expected),
+ __with_notify,
+ memory_order_relaxed,
+ memory_order_relaxed))
+ continue;
+ __expected = __with_notify;
+ }
+
+ std::__atomic_smart_ptr_wait_on_address(__ctrl_address(), [&] {
+ __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
+ return !__atomic_smart_ptr_storage::__has_lock(__expected);
+ });
+ }
+ }
+
+ // Publish new control pointer, clear bits, and notify waiters if needed.
+ _LIBCPP_HIDE_FROM_ABI void __unlock(__shared_weak_count* __ctrl_to_publish) const _NOEXCEPT {
+ _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(&__ctrl_);
+ uintptr_t __new_word = __atomic_smart_ptr_storage::__encode(__ctrl_to_publish, 0);
+ uintptr_t __previous = std::__cxx_atomic_exchange(__builtin_addressof(__ctrl_), __new_word, memory_order_release);
+ if (__atomic_smart_ptr_storage::__has_notify(__previous))
+ std::__atomic_smart_ptr_notify_all(__ctrl_address());
+ _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(&__ctrl_);
+ }
+};
+
+// [util.smartptr.atomic.shared]: same stored pointer and same ownership, or both empty.
+template <class _Element>
+_LIBCPP_HIDE_FROM_ABI inline bool __atomic_smart_ptr_equivalent(
+ _Element* __ptr,
+ __shared_weak_count* __ctrl,
+ _Element* __expected_ptr,
+ __shared_weak_count* __expected_ctrl) _NOEXCEPT {
+ if (__ctrl == nullptr && __expected_ctrl == nullptr)
+ return true;
+ return __ptr == __expected_ptr && __ctrl == __expected_ctrl;
+}
+
+template <class _Tp>
+struct atomic<shared_ptr<_Tp>> {
+ using value_type = shared_ptr<_Tp>;
+
+ static constexpr bool is_always_lock_free = false;
+
+ _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT : __fields_(nullptr, nullptr) {}
+ _LIBCPP_HIDE_FROM_ABI constexpr atomic(nullptr_t) _NOEXCEPT : __fields_(nullptr, nullptr) {}
+ _LIBCPP_HIDE_FROM_ABI atomic(shared_ptr<_Tp> __desired) _NOEXCEPT : __fields_(__desired.__ptr_, __desired.__cntrl_) {
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ }
+
+ atomic(const atomic&) = delete;
+ atomic& operator=(const atomic&) = delete;
+
+ _LIBCPP_HIDE_FROM_ABI ~atomic() {
+ if (auto* __c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed)))
+ __c->__release_shared();
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { return false; }
+
+ _LIBCPP_HIDE_FROM_ABI void operator=(shared_ptr<_Tp> __desired) _NOEXCEPT { store(std::move(__desired)); }
+ _LIBCPP_HIDE_FROM_ABI void operator=(nullptr_t) _NOEXCEPT { store(nullptr); }
+ _LIBCPP_HIDE_FROM_ABI operator shared_ptr<_Tp>() const _NOEXCEPT { return load(); }
+
+ _LIBCPP_HIDE_FROM_ABI void store(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT
+ _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+ (void)__m;
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ __fields_.__lock();
+ __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+
+ if (__old_c)
+ __old_c->__release_shared();
+ }
+
+ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+ _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+ (void)__m;
+ __fields_.__lock();
+ _Tp* __ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ if (__c)
+ __c->__add_shared();
+ __fields_.__unlock(__c);
+ return shared_ptr<_Tp>::__create_with_control_block(__ptr, __c);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
+ exchange(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ (void)__m;
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ __fields_.__lock();
+ _Tp* __old_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+
+ return shared_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_c);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) _NOEXCEPT
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ (void)__success;
+ (void)__failure;
+ __fields_.__lock();
+ _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+
+ if (__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+ if (__cur_c)
+ __cur_c->__release_shared();
+ return true;
+ }
+
+ if (__cur_c)
+ __cur_c->__add_shared();
+ __fields_.__unlock(__cur_c);
+ __expected = shared_ptr<_Tp>::__create_with_control_block(__cur_ptr, __cur_c);
+ return false;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) _NOEXCEPT
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ return compare_exchange_strong(__expected, std::move(__desired), __success, __failure);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ // Wait until the stored value is not equivalent to __old.
+ // __ctrl_ is the wait address; pointer changes are published with control updates.
+ _LIBCPP_HIDE_FROM_ABI void wait(shared_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+ _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
+ _Tp* __old_ptr = __old.__ptr_;
+ __shared_weak_count* __old_c = __old.__cntrl_;
+
+ std::__atomic_smart_ptr_wait_on_address(__fields_.__ctrl_address(), [&] {
+ uintptr_t __word = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), __m);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(__word);
+ if (__cur_c != __old_c)
+ return true;
+ _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), __m);
+ return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ });
+ }
+
+ _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
+ _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
+
+private:
+ __atomic_smart_ptr_fields<_Tp> __fields_;
+};
+
+template <class _Tp>
+struct atomic<weak_ptr<_Tp>> {
+ using value_type = weak_ptr<_Tp>;
+
+ static constexpr bool is_always_lock_free = false;
+
+ _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT : __fields_(nullptr, nullptr) {}
+ _LIBCPP_HIDE_FROM_ABI atomic(weak_ptr<_Tp> __desired) _NOEXCEPT : __fields_(__desired.__ptr_, __desired.__cntrl_) {
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ }
+
+ atomic(const atomic&) = delete;
+ atomic& operator=(const atomic&) = delete;
+
+ _LIBCPP_HIDE_FROM_ABI ~atomic() {
+ if (auto* __c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed)))
+ __c->__release_weak();
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { return false; }
+
+ _LIBCPP_HIDE_FROM_ABI void operator=(weak_ptr<_Tp> __desired) _NOEXCEPT { store(std::move(__desired)); }
+ _LIBCPP_HIDE_FROM_ABI operator weak_ptr<_Tp>() const _NOEXCEPT { return load(); }
+
+ _LIBCPP_HIDE_FROM_ABI void store(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT
+ _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+ (void)__m;
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ __fields_.__lock();
+ __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+
+ if (__old_c)
+ __old_c->__release_weak();
+ }
+
+ _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+ _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+ (void)__m;
+ __fields_.__lock();
+ _Tp* __ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ if (__c)
+ __c->__add_weak();
+ __fields_.__unlock(__c);
+ return weak_ptr<_Tp>::__create_with_control_block(__ptr, __c);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp>
+ exchange(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ (void)__m;
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ __fields_.__lock();
+ _Tp* __old_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+
+ return weak_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_c);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) _NOEXCEPT
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ (void)__success;
+ (void)__failure;
+ __fields_.__lock();
+ _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+
+ if (__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+ if (__cur_c)
+ __cur_c->__release_weak();
+ return true;
+ }
+
+ if (__cur_c)
+ __cur_c->__add_weak();
+ __fields_.__unlock(__cur_c);
+ __expected = weak_ptr<_Tp>::__create_with_control_block(__cur_ptr, __cur_c);
+ return false;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) _NOEXCEPT
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ return compare_exchange_strong(__expected, std::move(__desired), __success, __failure);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ _LIBCPP_HIDE_FROM_ABI void wait(weak_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+ _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
+ _Tp* __old_ptr = __old.__ptr_;
+ __shared_weak_count* __old_c = __old.__cntrl_;
+
+ std::__atomic_smart_ptr_wait_on_address(__fields_.__ctrl_address(), [&] {
+ uintptr_t __word = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), __m);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(__word);
+ if (__cur_c != __old_c)
+ return true;
+ _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), __m);
+ return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ });
+ }
+
+ _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
+ _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
+
+private:
+ __atomic_smart_ptr_fields<_Tp> __fields_;
+};
+
+#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_H
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 66442c8868005..45a82abebbb15 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -606,6 +606,9 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI shared_ptr {
return __r;
}
+ template <class>
+ friend struct atomic;
+
private:
template <class _Yp, bool = is_function<_Yp>::value>
struct __shared_ptr_default_allocator {
@@ -1078,8 +1081,8 @@ template <class _Tp, class _Up>
#endif
template <class _Tp, class _Up>
-[[__nodiscard__]] inline
- _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
+[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
+dynamic_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
typedef typename shared_ptr<_Tp>::element_type _ET;
_ET* __p = dynamic_cast<_ET*>(__r.get());
return __p ? shared_ptr<_Tp>(__r, __p) : shared_ptr<_Tp>();
@@ -1251,6 +1254,17 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI weak_ptr {
friend class weak_ptr;
template <class _Up>
friend class shared_ptr;
+
+ template <class>
+ friend struct atomic;
+
+ template <class _Yp, class _CntrlBlk>
+ _LIBCPP_HIDE_FROM_ABI static weak_ptr<_Tp> __create_with_control_block(_Yp* __p, _CntrlBlk* __cntrl) _NOEXCEPT {
+ weak_ptr<_Tp> __r;
+ __r.__ptr_ = __p;
+ __r.__cntrl_ = __cntrl;
+ return __r;
+ }
};
#if _LIBCPP_STD_VER >= 17
@@ -1472,6 +1486,10 @@ inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
_LIBCPP_END_NAMESPACE_STD
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+# include <__memory/atomic_shared_ptr.h>
+#endif
+
_LIBCPP_POP_MACROS
#endif // _LIBCPP___MEMORY_SHARED_PTR_H
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 51a2b77a9ba45..704af8af9a5e2 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -874,6 +874,7 @@ module std [system] {
module atomic_lock_free { header "__atomic/atomic_lock_free.h" }
module atomic_ref { header "__atomic/atomic_ref.h" }
module atomic_sync { header "__atomic/atomic_sync.h" }
+ module atomic_sync_lite { header "__atomic/atomic_sync_lite.h" }
module atomic_sync_timed { header "__atomic/atomic_sync_timed.h" }
module atomic_waitable_traits { header "__atomic/atomic_waitable_traits.h" }
module atomic {
@@ -1688,6 +1689,10 @@ module std [system] {
header "__memory/shared_ptr.h"
export std.functional.hash
}
+ module atomic_shared_ptr {
+ header "__memory/atomic_shared_ptr.h"
+ export std.memory.shared_ptr
+ }
module swap_allocator { header "__memory/swap_allocator.h" }
module temp_value { header "__memory/temp_value.h" }
module temporary_buffer {
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_aliasing.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_aliasing.pass.cpp
new file mode 100644
index 0000000000000..8c70c371475bd
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_aliasing.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-threads
+
+// [util.smartptr.atomic.shared] equivalence checks with aliasing shared_ptr.
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+struct Pair {
+ int a;
+ int b;
+};
+
+int main(int, char**) {
+ auto owner = std::make_shared<Pair>(Pair{10, 20});
+
+ std::shared_ptr<int> view_a(owner, &owner->a);
+ std::shared_ptr<int> view_b(owner, &owner->b);
+ assert(view_a.use_count() == view_b.use_count());
+ assert(view_a.get() != view_b.get());
+
+ std::atomic<std::shared_ptr<int>> atom(view_a);
+
+ // Same control block, different stored pointer -> CAS must fail.
+ {
+ std::shared_ptr<int> expected = view_b;
+ auto fresh = std::make_shared<int>(99);
+ bool ok = atom.compare_exchange_strong(expected, fresh);
+ assert(!ok);
+ assert(expected.get() == view_a.get());
+ }
+
+ // Same stored pointer with the matching control block -> CAS must succeed.
+ {
+ std::shared_ptr<int> expected = view_a;
+ bool ok = atom.compare_exchange_strong(expected, view_b);
+ assert(ok);
+ auto got = atom.load();
+ assert(got.get() == view_b.get());
+ }
+
+ // Both empty are equivalent.
+ {
+ std::atomic<std::shared_ptr<int>> empty_atom;
+ std::shared_ptr<int> expected;
+ bool ok = empty_atom.compare_exchange_strong(expected, view_a);
+ assert(ok);
+ auto got = empty_atom.load();
+ assert(got.get() == view_a.get());
+ }
+
+ // Empty vs non-empty are not equivalent.
+ {
+ std::atomic<std::shared_ptr<int>> non_empty(view_a);
+ std::shared_ptr<int> expected;
+ bool ok = non_empty.compare_exchange_strong(expected, view_b);
+ assert(!ok);
+ assert(expected.get() == view_a.get());
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_class.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_class.pass.cpp
new file mode 100644
index 0000000000000..6eb3ab65fc2e1
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_class.pass.cpp
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-threads
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <thread>
+
+int main(int, char**) {
+ std::atomic<std::shared_ptr<int>> a;
+
+ auto p1 = std::make_shared<int>(1);
+ auto p2 = std::make_shared<int>(2);
+
+ a.store(p1);
+ {
+ auto got = a.load();
+ assert(got && *got == 1);
+ }
+
+ {
+ auto old = a.exchange(p2);
+ assert(old && *old == 1);
+ auto got = a.load();
+ assert(got && *got == 2);
+ }
+
+ {
+ auto expected = p2;
+ bool ok = a.compare_exchange_strong(expected, p1);
+ assert(ok);
+ auto got = a.load();
+ assert(got && *got == 1);
+ }
+
+ {
+ auto expected = p2;
+ bool ok = a.compare_exchange_strong(expected, p2);
+ assert(!ok);
+ assert(expected && *expected == 1);
+ }
+
+#if __cpp_lib_atomic_wait >= 201907L
+ {
+ std::atomic<bool> started{false};
+ std::thread t([&] {
+ auto old = a.load();
+ started.store(true, std::memory_order_release);
+ a.wait(old);
+ });
+
+ while (!started.load(std::memory_order_acquire)) {
+ }
+
+ a.store(p2);
+ a.notify_all();
+ t.join();
+ }
+#endif
+
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
new file mode 100644
index 0000000000000..047c2785cab6e
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: diagnose-if-support
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-threads
+
+#include <atomic>
+#include <memory>
+
+void test() {
+ std::atomic<std::shared_ptr<int>> atom;
+ std::shared_ptr<int> expected;
+ std::shared_ptr<int> desired;
+
+ atom.store(desired,
+ std::memory_order_consume); // expected-warning {{memory order argument to atomic operation is invalid}}
+ atom.store(desired,
+ std::memory_order_acquire); // expected-warning {{memory order argument to atomic operation is invalid}}
+ atom.store(desired,
+ std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+
+ (void)atom.load(
+ std::memory_order_release); // expected-warning {{memory order argument to atomic operation is invalid}}
+ (void)atom.load(
+ std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+
+ atom.wait(expected,
+ std::memory_order_release); // expected-warning {{memory order argument to atomic operation is invalid}}
+ atom.wait(expected,
+ std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+
+ atom.compare_exchange_strong(
+ expected,
+ desired,
+ std::memory_order_seq_cst,
+ std::memory_order_release); // expected-warning {{memory order argument to atomic operation is invalid}}
+ atom.compare_exchange_strong(
+ expected,
+ desired,
+ std::memory_order_seq_cst,
+ std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+ atom.compare_exchange_weak(
+ expected,
+ desired,
+ std::memory_order_seq_cst,
+ std::memory_order_release); // expected-warning {{memory order argument to atomic operation is invalid}}
+ atom.compare_exchange_weak(
+ expected,
+ desired,
+ std::memory_order_seq_cst,
+ std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
new file mode 100644
index 0000000000000..3662c491f72ee
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-threads
+
+// LWG 3661 and LWG 3893 coverage.
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <type_traits>
+
+int main(int, char**) {
+ static_assert(std::is_constructible_v<std::atomic<std::shared_ptr<int>>, std::nullptr_t>);
+ static_assert(std::is_constructible_v<std::atomic<std::weak_ptr<int>>>);
+ static_assert(std::is_assignable_v<std::atomic<std::shared_ptr<int>>&, std::nullptr_t>);
+ static_assert(std::is_assignable_v<std::atomic<std::shared_ptr<int>>&, std::shared_ptr<int>>);
+
+ {
+ std::atomic<std::shared_ptr<int>> a(nullptr);
+ auto loaded = a.load();
+ assert(!loaded);
+ }
+
+ {
+ auto p = std::make_shared<int>(7);
+ std::atomic<std::shared_ptr<int>> a(p);
+ a = nullptr;
+ auto loaded = a.load();
+ assert(!loaded);
+ }
+
+ {
+ auto p = std::make_shared<int>(13);
+ std::atomic<std::shared_ptr<int>> a;
+ a = p;
+ auto loaded = a.load();
+ assert(loaded && *loaded == 13);
+ }
+
+ {
+ std::atomic<std::shared_ptr<int>> a;
+ std::shared_ptr<int> got = a;
+ assert(!got);
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
new file mode 100644
index 0000000000000..197506cd5b0ac
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-threads
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+namespace {
+struct Tracker {
+ static int alive;
+ Tracker() { ++alive; }
+ ~Tracker() { --alive; }
+};
+int Tracker::alive = 0;
+
+struct ReentrantDeleter {
+ std::atomic<std::shared_ptr<int>>* watch;
+ void operator()(int* p) const {
+ if (watch) {
+ auto current = watch->load();
+ (void)current;
+ }
+ delete p;
+ }
+};
+} // namespace
+
+int main(int, char**) {
+ // store releases the previous value.
+ {
+ std::atomic<std::shared_ptr<Tracker>> a;
+ a.store(std::make_shared<Tracker>());
+ assert(Tracker::alive == 1);
+ a.store(std::make_shared<Tracker>());
+ assert(Tracker::alive == 1);
+ a.store(nullptr);
+ assert(Tracker::alive == 0);
+ }
+
+ // exchange returns old ownership and preserves refcounts.
+ {
+ auto first = std::make_shared<Tracker>();
+ std::atomic<std::shared_ptr<Tracker>> a(first);
+ assert(first.use_count() == 2);
+ auto second = std::make_shared<Tracker>();
+ auto old = a.exchange(second);
+ assert(old.get() == first.get());
+ assert(first.use_count() == 2);
+ assert(second.use_count() == 2);
+ assert(Tracker::alive == 2);
+ }
+
+ // compare_exchange success/failure paths preserve ownership.
+ {
+ auto p1 = std::make_shared<Tracker>();
+ auto p2 = std::make_shared<Tracker>();
+ std::atomic<std::shared_ptr<Tracker>> a(p1);
+ {
+ std::shared_ptr<Tracker> expected = p2;
+ bool ok = a.compare_exchange_strong(expected, p1);
+ assert(!ok);
+ assert(expected.get() == p1.get());
+ }
+ {
+ std::shared_ptr<Tracker> expected = p1;
+ bool ok = a.compare_exchange_strong(expected, p2);
+ assert(ok);
+ }
+ assert(p1.use_count() == 1);
+ assert(p2.use_count() == 2);
+ }
+ assert(Tracker::alive == 0);
+
+ // atomic destruction releases held shared ownership.
+ {
+ auto p = std::make_shared<Tracker>();
+ {
+ std::atomic<std::shared_ptr<Tracker>> a(p);
+ assert(p.use_count() == 2);
+ }
+ assert(p.use_count() == 1);
+ }
+ assert(Tracker::alive == 0);
+
+ // Deleter must run after unlock; reentrant load must succeed.
+ {
+ std::atomic<std::shared_ptr<int>> a;
+ {
+ ReentrantDeleter d{&a};
+ std::shared_ptr<int> p(new int(7), d);
+ a.store(std::move(p));
+ }
+ a.store(std::shared_ptr<int>{});
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
new file mode 100644
index 0000000000000..e02d74931fce6
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
@@ -0,0 +1,107 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-threads
+
+#include <array>
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <thread>
+#include <vector>
+
+namespace {
+constexpr int kCandidateCount = 4;
+constexpr int kWriters = 4;
+constexpr int kReaders = 4;
+constexpr int kIterations = 4000;
+
+template <class Atomic, class Make>
+void run(Atomic& a, Make make_value, const std::array<int, kCandidateCount>& expected) {
+ std::atomic<bool> stop{false};
+ std::vector<std::thread> threads;
+
+ for (int w = 0; w < kWriters; ++w) {
+ threads.emplace_back([&, w] {
+ for (int i = 0; i < kIterations; ++i) {
+ a.store(make_value(expected[(w + i) % kCandidateCount]));
+ }
+ });
+ }
+
+ for (int r = 0; r < kReaders; ++r) {
+ threads.emplace_back([&, r] {
+ while (!stop.load(std::memory_order_relaxed)) {
+ auto val = a.load();
+ std::shared_ptr<int> sp;
+ if constexpr (requires { val.lock(); }) {
+ sp = val.lock();
+ } else {
+ sp = val;
+ }
+ if (sp) {
+ int observed = *sp;
+ bool ok = false;
+ for (int candidate : expected) {
+ if (candidate == observed) {
+ ok = true;
+ break;
+ }
+ }
+ assert(ok);
+ }
+ (void)r;
+ }
+ });
+ }
+
+ for (int w = 0; w < kWriters; ++w) {
+ threads[w].join();
+ }
+ stop.store(true, std::memory_order_relaxed);
+ for (int r = 0; r < kReaders; ++r) {
+ threads[kWriters + r].join();
+ }
+}
+} // namespace
+
+int main(int, char**) {
+ const std::array<int, kCandidateCount> expected = {1, 2, 3, 4};
+ std::array<std::shared_ptr<int>, kCandidateCount> pool;
+ for (int i = 0; i < kCandidateCount; ++i) {
+ pool[i] = std::make_shared<int>(expected[i]);
+ }
+
+ {
+ std::atomic<std::shared_ptr<int>> a(pool[0]);
+ auto make_shared_value = [&](int v) {
+ for (int i = 0; i < kCandidateCount; ++i) {
+ if (expected[i] == v)
+ return pool[i];
+ }
+ return pool[0];
+ };
+ run(a, make_shared_value, expected);
+ }
+
+ {
+ std::atomic<std::weak_ptr<int>> a;
+ a.store(std::weak_ptr<int>(pool[0]));
+ auto make_weak_value = [&](int v) -> std::weak_ptr<int> {
+ for (int i = 0; i < kCandidateCount; ++i) {
+ if (expected[i] == v)
+ return std::weak_ptr<int>(pool[i]);
+ }
+ return std::weak_ptr<int>(pool[0]);
+ };
+ run(a, make_weak_value, expected);
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_weak_ptr_class.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_weak_ptr_class.pass.cpp
new file mode 100644
index 0000000000000..2b0c68234d3f0
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_weak_ptr_class.pass.cpp
@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-threads
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <thread>
+
+int main(int, char**) {
+ std::atomic<std::weak_ptr<int>> a;
+
+ auto sp1 = std::make_shared<int>(1);
+ auto sp2 = std::make_shared<int>(2);
+ std::weak_ptr<int> wp1 = sp1;
+ std::weak_ptr<int> wp2 = sp2;
+
+ a.store(wp1);
+ {
+ std::weak_ptr<int> got = a.load();
+ auto locked = got.lock();
+ assert(locked && *locked == 1);
+ }
+
+ {
+ std::weak_ptr<int> old = a.exchange(wp2);
+ auto locked = old.lock();
+ assert(locked && *locked == 1);
+ std::weak_ptr<int> got = a.load();
+ auto locked2 = got.lock();
+ assert(locked2 && *locked2 == 2);
+ }
+
+ {
+ std::weak_ptr<int> expected = wp2;
+ bool ok = a.compare_exchange_strong(expected, wp1);
+ assert(ok);
+ std::weak_ptr<int> got = a.load();
+ auto locked = got.lock();
+ assert(locked && *locked == 1);
+ }
+
+ {
+ std::weak_ptr<int> expected = wp2;
+ bool ok = a.compare_exchange_strong(expected, wp2);
+ assert(!ok);
+ auto locked = expected.lock();
+ assert(locked && *locked == 1);
+ }
+
+ // Expired weak pointer remains representable and loadable.
+ {
+ auto sp3 = std::make_shared<int>(3);
+ std::weak_ptr<int> wp3 = sp3;
+ a.store(wp3);
+ sp3.reset();
+ std::weak_ptr<int> got = a.load();
+ assert(got.expired());
+ }
+
+#if __cpp_lib_atomic_wait >= 201907L
+ {
+ auto sp_for_wait = std::make_shared<int>(42);
+ std::weak_ptr<int> wp_for_wait = sp_for_wait;
+ a.store(wp_for_wait);
+
+ std::atomic<bool> started{false};
+ std::thread t([&] {
+ std::weak_ptr<int> old = a.load();
+ started.store(true, std::memory_order_release);
+ a.wait(old);
+ });
+
+ while (!started.load(std::memory_order_acquire)) {
+ }
+
+ a.store(wp1);
+ a.notify_all();
+ t.join();
+ }
+#endif
+
+ return 0;
+}
>From 5d0484bdbd2e33ae90a03bcf2187d6da3ea7c343 Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Sun, 26 Apr 2026 15:51:41 +0300
Subject: [PATCH 02/46] [libcxx] Updated docs; replaced old-style _NOEXCEPT
with noexcept; added necessary headers for arm platforms
Signed-off-by: ViNN280801 <vladislav.semykin at gmail.com>
---
libcxx/docs/FeatureTestMacroTable.rst | 2 +-
libcxx/docs/Status/Cxx20Papers.csv | 2 +-
libcxx/docs/Status/Cxx23Issues.csv | 2 +-
libcxx/docs/Status/Cxx2cIssues.csv | 2 +-
libcxx/include/__memory/atomic_shared_ptr.h | 91 +++++++++----------
libcxx/include/version | 2 +-
.../atomic.version.compile.pass.cpp | 48 +++-------
.../version.version.compile.pass.cpp | 48 +++-------
.../atomic_shared_ptr_aliasing.pass.cpp | 2 +-
.../atomic_shared_ptr_class.pass.cpp | 2 +-
.../atomic_shared_ptr_memory_order.verify.cpp | 2 +-
.../atomic_shared_ptr_nullptr.pass.cpp | 3 +-
.../atomic_shared_ptr_refcount.pass.cpp | 3 +-
.../atomic_shared_ptr_stress.pass.cpp | 2 +-
.../atomic_weak_ptr_class.pass.cpp | 2 +-
.../generate_feature_test_macro_components.py | 1 -
16 files changed, 89 insertions(+), 125 deletions(-)
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index ae48eaed1f46b..902e9df4e1df2 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -180,7 +180,7 @@ Status
---------------------------------------------------------- -----------------
``__cpp_lib_atomic_ref`` ``201806L``
---------------------------------------------------------- -----------------
- ``__cpp_lib_atomic_shared_ptr`` *unimplemented*
+ ``__cpp_lib_atomic_shared_ptr`` ``201711L``
---------------------------------------------------------- -----------------
``__cpp_lib_atomic_value_initialization`` ``201911L``
---------------------------------------------------------- -----------------
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index 7fbccde688009..3b1dc43a7fd5b 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -12,7 +12,7 @@
"`P0600R1 <https://wg21.link/P0600R1>`__","nodiscard in the Library","2017-11 (Albuquerque)","|Complete|","16","`#104010 <https://github.com/llvm/llvm-project/issues/104010>`__",""
"`P0616R0 <https://wg21.link/P0616R0>`__","de-pessimize legacy <numeric> algorithms with std::move","2017-11 (Albuquerque)","|Complete|","12","`#104011 <https://github.com/llvm/llvm-project/issues/104011>`__",""
"`P0653R2 <https://wg21.link/P0653R2>`__","Utility to convert a pointer to a raw pointer","2017-11 (Albuquerque)","|Complete|","6","`#104012 <https://github.com/llvm/llvm-project/issues/104012>`__",""
-"`P0718R2 <https://wg21.link/P0718R2>`__","Atomic shared_ptr","2017-11 (Albuquerque)","","","`#99980 <https://github.com/llvm/llvm-project/issues/99980>`__",""
+"`P0718R2 <https://wg21.link/P0718R2>`__","Atomic shared_ptr","2017-11 (Albuquerque)","|Complete|","22","`#99980 <https://github.com/llvm/llvm-project/issues/99980>`__",""
"`P0767R1 <https://wg21.link/P0767R1>`__","Deprecate POD","2017-11 (Albuquerque)","|Complete|","21","`#104013 <https://github.com/llvm/llvm-project/issues/104013>`__","It was previously erroneously marked as complete in LLVM 7."
"`P0768R1 <https://wg21.link/P0768R1>`__","Library Support for the Spaceship (Comparison) Operator","2017-11 (Albuquerque)","|Complete|","","`#104014 <https://github.com/llvm/llvm-project/issues/104014>`__",""
"`P0777R1 <https://wg21.link/P0777R1>`__","Treating Unnecessary ``decay``\ ","2017-11 (Albuquerque)","|Complete|","7","`#104016 <https://github.com/llvm/llvm-project/issues/104016>`__",""
diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index 0a2f0d59484de..7788cd285d997 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -157,7 +157,7 @@
"`LWG3654 <https://wg21.link/LWG3654>`__","``basic_format_context::arg(size_t)`` should be ``noexcept`` ","2022-02 (Virtual)","|Complete|","15","`#104959 <https://github.com/llvm/llvm-project/issues/104959>`__",""
"`LWG3657 <https://wg21.link/LWG3657>`__","``std::hash<std::filesystem::path>`` is not enabled","2022-02 (Virtual)","|Complete|","17","`#104960 <https://github.com/llvm/llvm-project/issues/104960>`__",""
"`LWG3660 <https://wg21.link/LWG3660>`__","``iterator_traits<common_iterator>::pointer`` should conform to ยง[iterator.traits]","2022-02 (Virtual)","|Complete|","14","`#104961 <https://github.com/llvm/llvm-project/issues/104961>`__",""
-"`LWG3661 <https://wg21.link/LWG3661>`__","``constinit atomic<shared_ptr<T>> a(nullptr);`` should work","2022-02 (Virtual)","","","`#104962 <https://github.com/llvm/llvm-project/issues/104962>`__",""
+"`LWG3661 <https://wg21.link/LWG3661>`__","``constinit atomic<shared_ptr<T>> a(nullptr);`` should work","2022-02 (Virtual)","|Complete|","22","`#104962 <https://github.com/llvm/llvm-project/issues/104962>`__",""
"","","","","","",""
"`LWG3564 <https://wg21.link/LWG3564>`__","``transform_view::iterator<true>::value_type`` and ``iterator_category`` should use ``const F&``","2022-07 (Virtual)","|Complete|","20","`#104963 <https://github.com/llvm/llvm-project/issues/104963>`__",""
"`LWG3617 <https://wg21.link/LWG3617>`__","``function``/``packaged_task`` deduction guides and deducing ``this``","2022-07 (Virtual)","","","`#104965 <https://github.com/llvm/llvm-project/issues/104965>`__",""
diff --git a/libcxx/docs/Status/Cxx2cIssues.csv b/libcxx/docs/Status/Cxx2cIssues.csv
index e329e14f70ab3..0a868e89eba6b 100644
--- a/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/libcxx/docs/Status/Cxx2cIssues.csv
@@ -3,7 +3,7 @@
"`LWG3884 <https://wg21.link/LWG3884>`__","``flat_foo`` is missing allocator-extended copy/move constructors","2023-06 (Varna)","|Complete|","21","`#105269 <https://github.com/llvm/llvm-project/issues/105269>`__",""
"`LWG3885 <https://wg21.link/LWG3885>`__","``op`` should be in [zombie.names]","2023-06 (Varna)","|Nothing To Do|","","`#105270 <https://github.com/llvm/llvm-project/issues/105270>`__",""
"`LWG3887 <https://wg21.link/LWG3887>`__","Version macro for ``allocate_at_least``","2023-06 (Varna)","|Complete|","19","`#105271 <https://github.com/llvm/llvm-project/issues/105271>`__",""
-"`LWG3893 <https://wg21.link/LWG3893>`__","LWG 3661 broke ``atomic<shared_ptr<T>> a; a = nullptr;``","2023-06 (Varna)","","","`#105273 <https://github.com/llvm/llvm-project/issues/105273>`__",""
+"`LWG3893 <https://wg21.link/LWG3893>`__","LWG 3661 broke ``atomic<shared_ptr<T>> a; a = nullptr;``","2023-06 (Varna)","|Complete|","22","`#105273 <https://github.com/llvm/llvm-project/issues/105273>`__",""
"`LWG3894 <https://wg21.link/LWG3894>`__","``generator::promise_type::yield_value(ranges::elements_of<Rng, Alloc>)`` should not be ``noexcept``","2023-06 (Varna)","","","`#105274 <https://github.com/llvm/llvm-project/issues/105274>`__",""
"`LWG3903 <https://wg21.link/LWG3903>`__","span destructor is redundantly noexcept","2023-06 (Varna)","|Complete|","7","`#105275 <https://github.com/llvm/llvm-project/issues/105275>`__",""
"`LWG3904 <https://wg21.link/LWG3904>`__","``lazy_split_view::outer-iterator``'s const-converting constructor isn't setting ``trailing_empty_``","2023-06 (Varna)","","","`#105276 <https://github.com/llvm/llvm-project/issues/105276>`__",""
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index 000b56e08bae6..a4875b9687f45 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -72,19 +72,19 @@ struct __atomic_smart_ptr_storage {
static constexpr uintptr_t __notify_bit_ = uintptr_t{2};
static constexpr uintptr_t __ptr_mask_ = ~(__lock_bit_ | __notify_bit_);
- _LIBCPP_HIDE_FROM_ABI static uintptr_t __encode(__shared_weak_count* __ctrl, uintptr_t __bits) _NOEXCEPT {
+ _LIBCPP_HIDE_FROM_ABI static uintptr_t __encode(__shared_weak_count* __ctrl, uintptr_t __bits) noexcept {
return (reinterpret_cast<uintptr_t>(__ctrl) & __ptr_mask_) | (__bits & ~__ptr_mask_);
}
- _LIBCPP_HIDE_FROM_ABI static __shared_weak_count* __decode(uintptr_t __word) _NOEXCEPT {
+ _LIBCPP_HIDE_FROM_ABI static __shared_weak_count* __decode(uintptr_t __word) noexcept {
return reinterpret_cast<__shared_weak_count*>(__word & __ptr_mask_);
}
- _LIBCPP_HIDE_FROM_ABI static bool __has_lock(uintptr_t __word) _NOEXCEPT { return (__word & __lock_bit_) != 0; }
- _LIBCPP_HIDE_FROM_ABI static bool __has_notify(uintptr_t __word) _NOEXCEPT { return (__word & __notify_bit_) != 0; }
+ _LIBCPP_HIDE_FROM_ABI static bool __has_lock(uintptr_t __word) noexcept { return (__word & __lock_bit_) != 0; }
+ _LIBCPP_HIDE_FROM_ABI static bool __has_notify(uintptr_t __word) noexcept { return (__word & __notify_bit_) != 0; }
};
-_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_one(const void* __address) _NOEXCEPT {
+_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_one(const void* __address) noexcept {
# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
std::__atomic_notify_one_global_table(__address);
# else
@@ -92,7 +92,7 @@ _LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_one(const void* __ad
# endif
}
-_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_all(const void* __address) _NOEXCEPT {
+_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_all(const void* __address) noexcept {
# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
std::__atomic_notify_all_global_table(__address);
# else
@@ -101,7 +101,7 @@ _LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_all(const void* __ad
}
template <class _Poll>
-_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_wait_on_address(const void* __address, _Poll&& __poll) _NOEXCEPT {
+_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_wait_on_address(const void* __address, _Poll&& __poll) noexcept {
while (!__poll()) {
# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
auto __monitor_value = std::__atomic_monitor_global(__address);
@@ -123,16 +123,15 @@ struct __atomic_smart_ptr_fields {
mutable __cxx_atomic_impl<_Element*> __ptr_;
mutable __cxx_atomic_impl<uintptr_t> __ctrl_;
- _LIBCPP_HIDE_FROM_ABI __atomic_smart_ptr_fields(_Element* __p, __shared_weak_count* __c) _NOEXCEPT
- : __ptr_(__p),
- __ctrl_(__atomic_smart_ptr_storage::__encode(__c, 0)) {}
+ _LIBCPP_HIDE_FROM_ABI __atomic_smart_ptr_fields(_Element* __p, __shared_weak_count* __c) noexcept
+ : __ptr_(__p), __ctrl_(__atomic_smart_ptr_storage::__encode(__c, 0)) {}
- _LIBCPP_HIDE_FROM_ABI const void* __ctrl_address() const _NOEXCEPT {
+ _LIBCPP_HIDE_FROM_ABI const void* __ctrl_address() const noexcept {
return static_cast<const void*>(__builtin_addressof(__ctrl_));
}
// Acquire lock bit on __ctrl_. Contended path sets notify bit and waits.
- _LIBCPP_HIDE_FROM_ABI void __lock() const _NOEXCEPT {
+ _LIBCPP_HIDE_FROM_ABI void __lock() const noexcept {
_LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(&__ctrl_);
uintptr_t __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
for (;;) {
@@ -170,7 +169,7 @@ struct __atomic_smart_ptr_fields {
}
// Publish new control pointer, clear bits, and notify waiters if needed.
- _LIBCPP_HIDE_FROM_ABI void __unlock(__shared_weak_count* __ctrl_to_publish) const _NOEXCEPT {
+ _LIBCPP_HIDE_FROM_ABI void __unlock(__shared_weak_count* __ctrl_to_publish) const noexcept {
_LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(&__ctrl_);
uintptr_t __new_word = __atomic_smart_ptr_storage::__encode(__ctrl_to_publish, 0);
uintptr_t __previous = std::__cxx_atomic_exchange(__builtin_addressof(__ctrl_), __new_word, memory_order_release);
@@ -186,7 +185,7 @@ _LIBCPP_HIDE_FROM_ABI inline bool __atomic_smart_ptr_equivalent(
_Element* __ptr,
__shared_weak_count* __ctrl,
_Element* __expected_ptr,
- __shared_weak_count* __expected_ctrl) _NOEXCEPT {
+ __shared_weak_count* __expected_ctrl) noexcept {
if (__ctrl == nullptr && __expected_ctrl == nullptr)
return true;
return __ptr == __expected_ptr && __ctrl == __expected_ctrl;
@@ -198,9 +197,9 @@ struct atomic<shared_ptr<_Tp>> {
static constexpr bool is_always_lock_free = false;
- _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT : __fields_(nullptr, nullptr) {}
- _LIBCPP_HIDE_FROM_ABI constexpr atomic(nullptr_t) _NOEXCEPT : __fields_(nullptr, nullptr) {}
- _LIBCPP_HIDE_FROM_ABI atomic(shared_ptr<_Tp> __desired) _NOEXCEPT : __fields_(__desired.__ptr_, __desired.__cntrl_) {
+ _LIBCPP_HIDE_FROM_ABI atomic() noexcept : __fields_(nullptr, nullptr) {}
+ _LIBCPP_HIDE_FROM_ABI constexpr atomic(nullptr_t) noexcept : __fields_(nullptr, nullptr) {}
+ _LIBCPP_HIDE_FROM_ABI atomic(shared_ptr<_Tp> __desired) noexcept : __fields_(__desired.__ptr_, __desired.__cntrl_) {
__desired.__ptr_ = nullptr;
__desired.__cntrl_ = nullptr;
}
@@ -214,13 +213,13 @@ struct atomic<shared_ptr<_Tp>> {
__c->__release_shared();
}
- _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { return false; }
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return false; }
- _LIBCPP_HIDE_FROM_ABI void operator=(shared_ptr<_Tp> __desired) _NOEXCEPT { store(std::move(__desired)); }
- _LIBCPP_HIDE_FROM_ABI void operator=(nullptr_t) _NOEXCEPT { store(nullptr); }
- _LIBCPP_HIDE_FROM_ABI operator shared_ptr<_Tp>() const _NOEXCEPT { return load(); }
+ _LIBCPP_HIDE_FROM_ABI void operator=(shared_ptr<_Tp> __desired) noexcept { store(std::move(__desired)); }
+ _LIBCPP_HIDE_FROM_ABI void operator=(nullptr_t) noexcept { store(nullptr); }
+ _LIBCPP_HIDE_FROM_ABI operator shared_ptr<_Tp>() const noexcept { return load(); }
- _LIBCPP_HIDE_FROM_ABI void store(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT
+ _LIBCPP_HIDE_FROM_ABI void store(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept
_LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
(void)__m;
_Tp* __desired_ptr = __desired.__ptr_;
@@ -238,7 +237,7 @@ struct atomic<shared_ptr<_Tp>> {
__old_c->__release_shared();
}
- _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
(void)__m;
__fields_.__lock();
@@ -252,7 +251,7 @@ struct atomic<shared_ptr<_Tp>> {
}
_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
- exchange(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ exchange(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
(void)__m;
_Tp* __desired_ptr = __desired.__ptr_;
__shared_weak_count* __desired_c = __desired.__cntrl_;
@@ -270,7 +269,7 @@ struct atomic<shared_ptr<_Tp>> {
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
- shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) _NOEXCEPT
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
(void)__success;
(void)__failure;
@@ -300,24 +299,24 @@ struct atomic<shared_ptr<_Tp>> {
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
- shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
- shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) _NOEXCEPT
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
return compare_exchange_strong(__expected, std::move(__desired), __success, __failure);
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
- shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
}
// Wait until the stored value is not equivalent to __old.
// __ctrl_ is the wait address; pointer changes are published with control updates.
- _LIBCPP_HIDE_FROM_ABI void wait(shared_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+ _LIBCPP_HIDE_FROM_ABI void wait(shared_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const noexcept
_LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
_Tp* __old_ptr = __old.__ptr_;
__shared_weak_count* __old_c = __old.__cntrl_;
@@ -332,8 +331,8 @@ struct atomic<shared_ptr<_Tp>> {
});
}
- _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
- _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
+ _LIBCPP_HIDE_FROM_ABI void notify_one() noexcept { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
+ _LIBCPP_HIDE_FROM_ABI void notify_all() noexcept { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
private:
__atomic_smart_ptr_fields<_Tp> __fields_;
@@ -345,8 +344,8 @@ struct atomic<weak_ptr<_Tp>> {
static constexpr bool is_always_lock_free = false;
- _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT : __fields_(nullptr, nullptr) {}
- _LIBCPP_HIDE_FROM_ABI atomic(weak_ptr<_Tp> __desired) _NOEXCEPT : __fields_(__desired.__ptr_, __desired.__cntrl_) {
+ _LIBCPP_HIDE_FROM_ABI atomic() noexcept : __fields_(nullptr, nullptr) {}
+ _LIBCPP_HIDE_FROM_ABI atomic(weak_ptr<_Tp> __desired) noexcept : __fields_(__desired.__ptr_, __desired.__cntrl_) {
__desired.__ptr_ = nullptr;
__desired.__cntrl_ = nullptr;
}
@@ -360,12 +359,12 @@ struct atomic<weak_ptr<_Tp>> {
__c->__release_weak();
}
- _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { return false; }
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return false; }
- _LIBCPP_HIDE_FROM_ABI void operator=(weak_ptr<_Tp> __desired) _NOEXCEPT { store(std::move(__desired)); }
- _LIBCPP_HIDE_FROM_ABI operator weak_ptr<_Tp>() const _NOEXCEPT { return load(); }
+ _LIBCPP_HIDE_FROM_ABI void operator=(weak_ptr<_Tp> __desired) noexcept { store(std::move(__desired)); }
+ _LIBCPP_HIDE_FROM_ABI operator weak_ptr<_Tp>() const noexcept { return load(); }
- _LIBCPP_HIDE_FROM_ABI void store(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT
+ _LIBCPP_HIDE_FROM_ABI void store(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept
_LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
(void)__m;
_Tp* __desired_ptr = __desired.__ptr_;
@@ -383,7 +382,7 @@ struct atomic<weak_ptr<_Tp>> {
__old_c->__release_weak();
}
- _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+ _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
(void)__m;
__fields_.__lock();
@@ -397,7 +396,7 @@ struct atomic<weak_ptr<_Tp>> {
}
_LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp>
- exchange(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ exchange(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
(void)__m;
_Tp* __desired_ptr = __desired.__ptr_;
__shared_weak_count* __desired_c = __desired.__cntrl_;
@@ -415,7 +414,7 @@ struct atomic<weak_ptr<_Tp>> {
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
- weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) _NOEXCEPT
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
(void)__success;
(void)__failure;
@@ -445,22 +444,22 @@ struct atomic<weak_ptr<_Tp>> {
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
- weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
- weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) _NOEXCEPT
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
return compare_exchange_strong(__expected, std::move(__desired), __success, __failure);
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
- weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
}
- _LIBCPP_HIDE_FROM_ABI void wait(weak_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
+ _LIBCPP_HIDE_FROM_ABI void wait(weak_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const noexcept
_LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
_Tp* __old_ptr = __old.__ptr_;
__shared_weak_count* __old_c = __old.__cntrl_;
@@ -475,8 +474,8 @@ struct atomic<weak_ptr<_Tp>> {
});
}
- _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
- _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
+ _LIBCPP_HIDE_FROM_ABI void notify_one() noexcept { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
+ _LIBCPP_HIDE_FROM_ABI void notify_all() noexcept { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
private:
__atomic_smart_ptr_fields<_Tp> __fields_;
diff --git a/libcxx/include/version b/libcxx/include/version
index 1c683b67e5700..ab11cf1fbcca7 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -398,7 +398,7 @@ __cpp_lib_void_t 201411L <type_traits>
# define __cpp_lib_atomic_float 201711L
# define __cpp_lib_atomic_lock_free_type_aliases 201907L
# define __cpp_lib_atomic_ref 201806L
-// # define __cpp_lib_atomic_shared_ptr 201711L
+# define __cpp_lib_atomic_shared_ptr 201711L
# define __cpp_lib_atomic_value_initialization 201911L
# define __cpp_lib_atomic_wait 201907L
# if _LIBCPP_HAS_THREADS
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/atomic.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/atomic.version.compile.pass.cpp
index e62416257a696..599d3654ffc92 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/atomic.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/atomic.version.compile.pass.cpp
@@ -188,17 +188,11 @@
# error "__cpp_lib_atomic_ref should have the value 201806L in c++20"
# endif
-# if !defined(_LIBCPP_VERSION)
-# ifndef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should be defined in c++20"
-# endif
-# if __cpp_lib_atomic_shared_ptr != 201711L
-# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++20"
-# endif
-# else
-# ifdef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should not be defined because it is unimplemented in libc++!"
-# endif
+# ifndef __cpp_lib_atomic_shared_ptr
+# error "__cpp_lib_atomic_shared_ptr should be defined in c++20"
+# endif
+# if __cpp_lib_atomic_shared_ptr != 201711L
+# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++20"
# endif
# ifndef __cpp_lib_atomic_value_initialization
@@ -269,17 +263,11 @@
# error "__cpp_lib_atomic_ref should have the value 201806L in c++23"
# endif
-# if !defined(_LIBCPP_VERSION)
-# ifndef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should be defined in c++23"
-# endif
-# if __cpp_lib_atomic_shared_ptr != 201711L
-# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++23"
-# endif
-# else
-# ifdef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should not be defined because it is unimplemented in libc++!"
-# endif
+# ifndef __cpp_lib_atomic_shared_ptr
+# error "__cpp_lib_atomic_shared_ptr should be defined in c++23"
+# endif
+# if __cpp_lib_atomic_shared_ptr != 201711L
+# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++23"
# endif
# ifndef __cpp_lib_atomic_value_initialization
@@ -359,17 +347,11 @@
# error "__cpp_lib_atomic_ref should have the value 202603L in c++26"
# endif
-# if !defined(_LIBCPP_VERSION)
-# ifndef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should be defined in c++26"
-# endif
-# if __cpp_lib_atomic_shared_ptr != 201711L
-# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++26"
-# endif
-# else
-# ifdef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should not be defined because it is unimplemented in libc++!"
-# endif
+# ifndef __cpp_lib_atomic_shared_ptr
+# error "__cpp_lib_atomic_shared_ptr should be defined in c++26"
+# endif
+# if __cpp_lib_atomic_shared_ptr != 201711L
+# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++26"
# endif
# ifndef __cpp_lib_atomic_value_initialization
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index dfee4b6d458db..203b03eb16ca8 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -3242,17 +3242,11 @@
# error "__cpp_lib_atomic_ref should have the value 201806L in c++20"
# endif
-# if !defined(_LIBCPP_VERSION)
-# ifndef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should be defined in c++20"
-# endif
-# if __cpp_lib_atomic_shared_ptr != 201711L
-# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++20"
-# endif
-# else
-# ifdef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should not be defined because it is unimplemented in libc++!"
-# endif
+# ifndef __cpp_lib_atomic_shared_ptr
+# error "__cpp_lib_atomic_shared_ptr should be defined in c++20"
+# endif
+# if __cpp_lib_atomic_shared_ptr != 201711L
+# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++20"
# endif
# ifndef __cpp_lib_atomic_value_initialization
@@ -4711,17 +4705,11 @@
# error "__cpp_lib_atomic_ref should have the value 201806L in c++23"
# endif
-# if !defined(_LIBCPP_VERSION)
-# ifndef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should be defined in c++23"
-# endif
-# if __cpp_lib_atomic_shared_ptr != 201711L
-# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++23"
-# endif
-# else
-# ifdef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should not be defined because it is unimplemented in libc++!"
-# endif
+# ifndef __cpp_lib_atomic_shared_ptr
+# error "__cpp_lib_atomic_shared_ptr should be defined in c++23"
+# endif
+# if __cpp_lib_atomic_shared_ptr != 201711L
+# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++23"
# endif
# ifndef __cpp_lib_atomic_value_initialization
@@ -6414,17 +6402,11 @@
# error "__cpp_lib_atomic_ref should have the value 202603L in c++26"
# endif
-# if !defined(_LIBCPP_VERSION)
-# ifndef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should be defined in c++26"
-# endif
-# if __cpp_lib_atomic_shared_ptr != 201711L
-# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++26"
-# endif
-# else
-# ifdef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should not be defined because it is unimplemented in libc++!"
-# endif
+# ifndef __cpp_lib_atomic_shared_ptr
+# error "__cpp_lib_atomic_shared_ptr should be defined in c++26"
+# endif
+# if __cpp_lib_atomic_shared_ptr != 201711L
+# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++26"
# endif
# ifndef __cpp_lib_atomic_value_initialization
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_aliasing.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_aliasing.pass.cpp
index 8c70c371475bd..33bfe02376f0c 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_aliasing.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_aliasing.pass.cpp
@@ -6,7 +6,7 @@
//
//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14, c++17
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: no-threads
// [util.smartptr.atomic.shared] equivalence checks with aliasing shared_ptr.
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_class.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_class.pass.cpp
index 6eb3ab65fc2e1..68156b9cafb96 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_class.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_class.pass.cpp
@@ -6,7 +6,7 @@
//
//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14, c++17
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: no-threads
#include <atomic>
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
index 047c2785cab6e..7b92e1b23dfcd 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
// REQUIRES: diagnose-if-support
-// UNSUPPORTED: c++03, c++11, c++14, c++17
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: no-threads
#include <atomic>
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
index 3662c491f72ee..3889e86717948 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
@@ -6,13 +6,14 @@
//
//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14, c++17
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: no-threads
// LWG 3661 and LWG 3893 coverage.
#include <atomic>
#include <cassert>
+#include <cstddef> // Armv7, Armv8 (-fno-exceptions) require for std::nullptr_t
#include <memory>
#include <type_traits>
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
index 197506cd5b0ac..7957fb7a7b787 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
@@ -6,12 +6,13 @@
//
//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14, c++17
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: no-threads
#include <atomic>
#include <cassert>
#include <memory>
+#include <utility> // Armv7, Armv8 require for std::move
namespace {
struct Tracker {
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
index e02d74931fce6..86b56bfa920bc 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
@@ -6,7 +6,7 @@
//
//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14, c++17
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: no-threads
#include <array>
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_weak_ptr_class.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_weak_ptr_class.pass.cpp
index 2b0c68234d3f0..9c8f0355d46d4 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_weak_ptr_class.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_weak_ptr_class.pass.cpp
@@ -6,7 +6,7 @@
//
//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14, c++17
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: no-threads
#include <atomic>
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index f81eb0e0e8060..119bfb7e540f2 100644
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -205,7 +205,6 @@ def add_version_header(tc):
"name": "__cpp_lib_atomic_shared_ptr",
"values": {"c++20": 201711},
"headers": ["atomic"],
- "unimplemented": True,
},
{
"name": "__cpp_lib_atomic_value_initialization",
>From 064b6f6c428afbeddbfaa840b5f20da0c1a9bd11 Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Mon, 27 Apr 2026 10:51:48 +0300
Subject: [PATCH 03/46] [libcxx] Added nodiscard tests for the
atomic<shared_ptr>::load|::is_lock_free methods; additionaly marked 'load()'
as nodiscard
Signed-off-by: ViNN280801 <vladislav.semykin at gmail.com>
---
libcxx/include/__memory/atomic_shared_ptr.h | 4 ++--
.../test/libcxx/atomics/nodiscard.verify.cpp | 21 +++++++++++++++++++
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index a4875b9687f45..9b8b5f291bbfb 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -237,7 +237,7 @@ struct atomic<shared_ptr<_Tp>> {
__old_c->__release_shared();
}
- _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
(void)__m;
__fields_.__lock();
@@ -382,7 +382,7 @@ struct atomic<weak_ptr<_Tp>> {
__old_c->__release_weak();
}
- _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
(void)__m;
__fields_.__lock();
diff --git a/libcxx/test/libcxx/atomics/nodiscard.verify.cpp b/libcxx/test/libcxx/atomics/nodiscard.verify.cpp
index c1c2b95ecf89f..642d85ae67723 100644
--- a/libcxx/test/libcxx/atomics/nodiscard.verify.cpp
+++ b/libcxx/test/libcxx/atomics/nodiscard.verify.cpp
@@ -11,6 +11,7 @@
// Check that functions are marked [[nodiscard]]
#include <atomic>
+#include <memory>
#include "test_macros.h"
@@ -31,6 +32,26 @@ void test() {
// expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
atRef.load();
}
+
+ {
+ std::atomic<std::shared_ptr<int>> asp;
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ asp.is_lock_free();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ asp.load();
+ }
+
+ {
+ std::atomic<std::weak_ptr<int>> awp;
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ awp.is_lock_free();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ awp.load();
+ }
#endif
{
>From d7c61f1dcaece8609d533b8fb65f913a8c5f97b8 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 28 Apr 2026 08:39:57 +0300
Subject: [PATCH 04/46] Apply suggestion from @H-G-Hristov
Co-authored-by: Hristo Hristov <hghristov.rmm at gmail.com>
---
libcxx/docs/Status/Cxx20Papers.csv | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index 3b1dc43a7fd5b..c57723fa7a309 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -12,7 +12,7 @@
"`P0600R1 <https://wg21.link/P0600R1>`__","nodiscard in the Library","2017-11 (Albuquerque)","|Complete|","16","`#104010 <https://github.com/llvm/llvm-project/issues/104010>`__",""
"`P0616R0 <https://wg21.link/P0616R0>`__","de-pessimize legacy <numeric> algorithms with std::move","2017-11 (Albuquerque)","|Complete|","12","`#104011 <https://github.com/llvm/llvm-project/issues/104011>`__",""
"`P0653R2 <https://wg21.link/P0653R2>`__","Utility to convert a pointer to a raw pointer","2017-11 (Albuquerque)","|Complete|","6","`#104012 <https://github.com/llvm/llvm-project/issues/104012>`__",""
-"`P0718R2 <https://wg21.link/P0718R2>`__","Atomic shared_ptr","2017-11 (Albuquerque)","|Complete|","22","`#99980 <https://github.com/llvm/llvm-project/issues/99980>`__",""
+"`P0718R2 <https://wg21.link/P0718R2>`__","Atomic shared_ptr","2017-11 (Albuquerque)","|Complete|","23","`#99980 <https://github.com/llvm/llvm-project/issues/99980>`__",""
"`P0767R1 <https://wg21.link/P0767R1>`__","Deprecate POD","2017-11 (Albuquerque)","|Complete|","21","`#104013 <https://github.com/llvm/llvm-project/issues/104013>`__","It was previously erroneously marked as complete in LLVM 7."
"`P0768R1 <https://wg21.link/P0768R1>`__","Library Support for the Spaceship (Comparison) Operator","2017-11 (Albuquerque)","|Complete|","","`#104014 <https://github.com/llvm/llvm-project/issues/104014>`__",""
"`P0777R1 <https://wg21.link/P0777R1>`__","Treating Unnecessary ``decay``\ ","2017-11 (Albuquerque)","|Complete|","7","`#104016 <https://github.com/llvm/llvm-project/issues/104016>`__",""
>From 40c38a7949c62023fc80c58f43708b1b01162742 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 28 Apr 2026 08:41:14 +0300
Subject: [PATCH 05/46] Update libcxx/docs/Status/Cxx2cIssues.csv
Co-authored-by: Hristo Hristov <hghristov.rmm at gmail.com>
---
libcxx/docs/Status/Cxx2cIssues.csv | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/Status/Cxx2cIssues.csv b/libcxx/docs/Status/Cxx2cIssues.csv
index 0a868e89eba6b..f14240167e729 100644
--- a/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/libcxx/docs/Status/Cxx2cIssues.csv
@@ -3,7 +3,7 @@
"`LWG3884 <https://wg21.link/LWG3884>`__","``flat_foo`` is missing allocator-extended copy/move constructors","2023-06 (Varna)","|Complete|","21","`#105269 <https://github.com/llvm/llvm-project/issues/105269>`__",""
"`LWG3885 <https://wg21.link/LWG3885>`__","``op`` should be in [zombie.names]","2023-06 (Varna)","|Nothing To Do|","","`#105270 <https://github.com/llvm/llvm-project/issues/105270>`__",""
"`LWG3887 <https://wg21.link/LWG3887>`__","Version macro for ``allocate_at_least``","2023-06 (Varna)","|Complete|","19","`#105271 <https://github.com/llvm/llvm-project/issues/105271>`__",""
-"`LWG3893 <https://wg21.link/LWG3893>`__","LWG 3661 broke ``atomic<shared_ptr<T>> a; a = nullptr;``","2023-06 (Varna)","|Complete|","22","`#105273 <https://github.com/llvm/llvm-project/issues/105273>`__",""
+"`LWG3893 <https://wg21.link/LWG3893>`__","LWG 3661 broke ``atomic<shared_ptr<T>> a; a = nullptr;``","2023-06 (Varna)","|Complete|","23","`#105273 <https://github.com/llvm/llvm-project/issues/105273>`__",""
"`LWG3894 <https://wg21.link/LWG3894>`__","``generator::promise_type::yield_value(ranges::elements_of<Rng, Alloc>)`` should not be ``noexcept``","2023-06 (Varna)","","","`#105274 <https://github.com/llvm/llvm-project/issues/105274>`__",""
"`LWG3903 <https://wg21.link/LWG3903>`__","span destructor is redundantly noexcept","2023-06 (Varna)","|Complete|","7","`#105275 <https://github.com/llvm/llvm-project/issues/105275>`__",""
"`LWG3904 <https://wg21.link/LWG3904>`__","``lazy_split_view::outer-iterator``'s const-converting constructor isn't setting ``trailing_empty_``","2023-06 (Varna)","","","`#105276 <https://github.com/llvm/llvm-project/issues/105276>`__",""
>From 4df1f1637e3cf26ca5c02e9f51437ec3463d33c5 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 28 Apr 2026 08:41:32 +0300
Subject: [PATCH 06/46] Update libcxx/docs/Status/Cxx23Issues.csv
Co-authored-by: Hristo Hristov <hghristov.rmm at gmail.com>
---
libcxx/docs/Status/Cxx23Issues.csv | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index 7788cd285d997..676968340d420 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -157,7 +157,7 @@
"`LWG3654 <https://wg21.link/LWG3654>`__","``basic_format_context::arg(size_t)`` should be ``noexcept`` ","2022-02 (Virtual)","|Complete|","15","`#104959 <https://github.com/llvm/llvm-project/issues/104959>`__",""
"`LWG3657 <https://wg21.link/LWG3657>`__","``std::hash<std::filesystem::path>`` is not enabled","2022-02 (Virtual)","|Complete|","17","`#104960 <https://github.com/llvm/llvm-project/issues/104960>`__",""
"`LWG3660 <https://wg21.link/LWG3660>`__","``iterator_traits<common_iterator>::pointer`` should conform to ยง[iterator.traits]","2022-02 (Virtual)","|Complete|","14","`#104961 <https://github.com/llvm/llvm-project/issues/104961>`__",""
-"`LWG3661 <https://wg21.link/LWG3661>`__","``constinit atomic<shared_ptr<T>> a(nullptr);`` should work","2022-02 (Virtual)","|Complete|","22","`#104962 <https://github.com/llvm/llvm-project/issues/104962>`__",""
+"`LWG3661 <https://wg21.link/LWG3661>`__","``constinit atomic<shared_ptr<T>> a(nullptr);`` should work","2022-02 (Virtual)","|Complete|","23","`#104962 <https://github.com/llvm/llvm-project/issues/104962>`__",""
"","","","","","",""
"`LWG3564 <https://wg21.link/LWG3564>`__","``transform_view::iterator<true>::value_type`` and ``iterator_category`` should use ``const F&``","2022-07 (Virtual)","|Complete|","20","`#104963 <https://github.com/llvm/llvm-project/issues/104963>`__",""
"`LWG3617 <https://wg21.link/LWG3617>`__","``function``/``packaged_task`` deduction guides and deducing ``this``","2022-07 (Virtual)","","","`#104965 <https://github.com/llvm/llvm-project/issues/104965>`__",""
>From 235aee6c1e3254a8fff9cf4f9df0d7dbf54775b3 Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Tue, 28 Apr 2026 20:56:39 +0300
Subject: [PATCH 07/46] [libcxx] Fix addressof and poll callable in
atomic<shared_ptr>
Use __builtin_addressof instead of & in TSAN macro call sites to avoid
potentially overloaded operator&. Take the poll callable by const lvalue
reference in __atomic_smart_ptr_wait_on_address since it is called
multiple times and does not need to be forwarded.
---
libcxx/include/__memory/atomic_shared_ptr.h | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index 9b8b5f291bbfb..1baa0060d3425 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -101,7 +101,8 @@ _LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_all(const void* __ad
}
template <class _Poll>
-_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_wait_on_address(const void* __address, _Poll&& __poll) noexcept {
+_LIBCPP_HIDE_FROM_ABI inline void
+__atomic_smart_ptr_wait_on_address(const void* __address, const _Poll& __poll) noexcept {
while (!__poll()) {
# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
auto __monitor_value = std::__atomic_monitor_global(__address);
@@ -132,7 +133,7 @@ struct __atomic_smart_ptr_fields {
// Acquire lock bit on __ctrl_. Contended path sets notify bit and waits.
_LIBCPP_HIDE_FROM_ABI void __lock() const noexcept {
- _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(&__ctrl_);
+ _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(__builtin_addressof(__ctrl_));
uintptr_t __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
for (;;) {
if (!__atomic_smart_ptr_storage::__has_lock(__expected)) {
@@ -143,7 +144,7 @@ struct __atomic_smart_ptr_fields {
__desired,
memory_order_acquire,
memory_order_relaxed)) {
- _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(&__ctrl_);
+ _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(__builtin_addressof(__ctrl_));
return;
}
continue;
@@ -170,12 +171,12 @@ struct __atomic_smart_ptr_fields {
// Publish new control pointer, clear bits, and notify waiters if needed.
_LIBCPP_HIDE_FROM_ABI void __unlock(__shared_weak_count* __ctrl_to_publish) const noexcept {
- _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(&__ctrl_);
+ _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(__builtin_addressof(__ctrl_));
uintptr_t __new_word = __atomic_smart_ptr_storage::__encode(__ctrl_to_publish, 0);
uintptr_t __previous = std::__cxx_atomic_exchange(__builtin_addressof(__ctrl_), __new_word, memory_order_release);
if (__atomic_smart_ptr_storage::__has_notify(__previous))
std::__atomic_smart_ptr_notify_all(__ctrl_address());
- _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(&__ctrl_);
+ _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(__builtin_addressof(__ctrl_));
}
};
>From 45bd0d80342cbb0711df815fb6e615d6f61cd27b Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Tue, 28 Apr 2026 21:26:04 +0300
Subject: [PATCH 08/46] [libcxx] Written 2 simple benchmarks
Signed-off-by: ViNN280801 <vladislav.semykin at gmail.com>
---
.../benchmarks/atomic_shared_ptr.bench.cpp | 34 +++++++++++++++++++
1 file changed, 34 insertions(+)
create mode 100644 libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
diff --git a/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp b/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
new file mode 100644
index 0000000000000..47239b5c9a613
--- /dev/null
+++ b/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+
+#include <atomic>
+#include <memory>
+
+#include "benchmark/benchmark.h"
+
+static void BM_AtomicSharedPtrLoadUncontended(benchmark::State& st) {
+ std::atomic<std::shared_ptr<int>> atom(std::make_shared<int>(42));
+ while (st.KeepRunning()) {
+ auto snap = atom.load();
+ benchmark::DoNotOptimize(snap);
+ }
+}
+BENCHMARK(BM_AtomicSharedPtrLoadUncontended);
+
+static void BM_AtomicSharedPtrStoreUncontended(benchmark::State& st) {
+ std::atomic<std::shared_ptr<int>> atom;
+ auto keep = std::make_shared<int>(7);
+ while (st.KeepRunning()) {
+ atom.store(keep);
+ }
+}
+BENCHMARK(BM_AtomicSharedPtrStoreUncontended);
+
+BENCHMARK_MAIN();
>From b0d0a5a3b7a922c45220a0b7d323afdb79916a61 Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Fri, 1 May 2026 15:37:16 +0300
Subject: [PATCH 09/46] [libcxx] Add per-method tests for atomic<shared_ptr<T>>
and atomic<weak_ptr<T>>; update <memory> synopsis
Signed-off-by: ViNN280801 <vladislav.semykin at gmail.com>
---
libcxx/include/memory | 8 ++
.../test/libcxx/atomics/nodiscard.verify.cpp | 21 ----
.../utilities/memory/nodiscard.verify.cpp | 22 ++++
.../atomic/atomic_smart_ptr_test_types.h | 102 ++++++++++++++++++
.../atomic/is_lock_free.compile.pass.cpp | 50 +++++++++
.../atomic/shared/assign.pass.cpp | 40 +++++++
.../shared/compare_exchange_strong.pass.cpp | 62 +++++++++++
.../shared/compare_exchange_weak.pass.cpp | 64 +++++++++++
.../atomic/shared/convert.pass.cpp | 41 +++++++
.../atomic/shared/exchange.pass.cpp | 45 ++++++++
.../is_always_lock_free.compile.pass.cpp | 40 +++++++
.../atomic/shared/load.compile.pass.cpp | 44 ++++++++
.../atomic/shared/notify_all.pass.cpp | 67 ++++++++++++
.../atomic/shared/notify_one.pass.cpp | 54 ++++++++++
.../atomic/shared/store.pass.cpp | 49 +++++++++
.../atomic/shared/types.compile.pass.cpp | 66 ++++++++++++
.../util.smartptr/atomic/shared/wait.pass.cpp | 58 ++++++++++
.../util.smartptr/atomic/weak/assign.pass.cpp | 50 +++++++++
.../weak/compare_exchange_strong.pass.cpp | 67 ++++++++++++
.../weak/compare_exchange_weak.pass.cpp | 69 ++++++++++++
.../atomic/weak/convert.pass.cpp | 44 ++++++++
.../atomic/weak/exchange.pass.cpp | 57 ++++++++++
.../weak/is_always_lock_free.compile.pass.cpp | 40 +++++++
.../atomic/weak/is_lock_free.compile.pass.cpp | 34 ++++++
.../atomic/weak/load.compile.pass.cpp | 44 ++++++++
.../atomic/weak/notify_all.pass.cpp | 69 ++++++++++++
.../atomic/weak/notify_one.pass.cpp | 57 ++++++++++
.../util.smartptr/atomic/weak/store.pass.cpp | 50 +++++++++
.../atomic/weak/types.compile.pass.cpp | 61 +++++++++++
.../util.smartptr/atomic/weak/wait.pass.cpp | 64 +++++++++++
30 files changed, 1518 insertions(+), 21 deletions(-)
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.compile.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/assign.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_weak.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.compile.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.compile.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_all.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_one.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/store.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/wait.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/assign.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_weak.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.compile.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.compile.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.compile.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_all.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_one.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/store.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp
create mode 100644 libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/wait.pass.cpp
diff --git a/libcxx/include/memory b/libcxx/include/memory
index ca880c83a544d..177a7efbb4e10 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -861,6 +861,14 @@ template<class T>
atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
shared_ptr<T> w, memory_order success,
memory_order failure);
+
+// [util.smartptr.atomic.shared], partial specialization for shared_ptr
+template <class T>
+struct atomic<shared_ptr<T>>;
+// [util.smartptr.atomic.weak], partial specialization for weak_ptr
+template <class T>
+struct atomic<weak_ptr<T>>;
+
// Hash support
template <class T> struct hash;
template <class T, class D> struct hash<unique_ptr<T, D> >;
diff --git a/libcxx/test/libcxx/atomics/nodiscard.verify.cpp b/libcxx/test/libcxx/atomics/nodiscard.verify.cpp
index 642d85ae67723..c1c2b95ecf89f 100644
--- a/libcxx/test/libcxx/atomics/nodiscard.verify.cpp
+++ b/libcxx/test/libcxx/atomics/nodiscard.verify.cpp
@@ -11,7 +11,6 @@
// Check that functions are marked [[nodiscard]]
#include <atomic>
-#include <memory>
#include "test_macros.h"
@@ -32,26 +31,6 @@ void test() {
// expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
atRef.load();
}
-
- {
- std::atomic<std::shared_ptr<int>> asp;
-
- // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
- asp.is_lock_free();
-
- // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
- asp.load();
- }
-
- {
- std::atomic<std::weak_ptr<int>> awp;
-
- // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
- awp.is_lock_free();
-
- // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
- awp.load();
- }
#endif
{
diff --git a/libcxx/test/libcxx/utilities/memory/nodiscard.verify.cpp b/libcxx/test/libcxx/utilities/memory/nodiscard.verify.cpp
index 6bddeceb9951a..5533e9f741983 100644
--- a/libcxx/test/libcxx/utilities/memory/nodiscard.verify.cpp
+++ b/libcxx/test/libcxx/utilities/memory/nodiscard.verify.cpp
@@ -124,4 +124,26 @@ void test() {
it.base(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
}
#endif
+
+#if TEST_STD_VER >= 20 && !defined(TEST_HAS_NO_THREADS)
+ {
+ std::atomic<std::shared_ptr<int>> asp;
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ asp.is_lock_free();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ asp.load();
+ }
+
+ {
+ std::atomic<std::weak_ptr<int>> awp;
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ awp.is_lock_free();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ awp.load();
+ }
+#endif
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
new file mode 100644
index 0000000000000..309cc11bb5026
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
@@ -0,0 +1,102 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Shared fixtures for libc++ tests under util.smartptr/atomic/{shared,weak}/:
+// heterogeneous value types (built-in, standard library, and user-defined).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBCXX_TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
+#define LIBCXX_TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+namespace libcxx_atomic_smart_ptr_test {
+
+// --- User-defined and semi-random scalar-like types ---------------------------------
+
+struct TrackedPod {
+ std::uint32_t gen{};
+ std::int64_t salt{};
+ friend constexpr bool operator==(TrackedPod lhs, TrackedPod rhs) noexcept {
+ return lhs.gen == rhs.gen && lhs.salt == rhs.salt;
+ }
+};
+
+class Handle {
+ double coeff_{};
+
+public:
+ explicit Handle(double c = 0.0) : coeff_(c) {}
+ double coeff() const noexcept { return coeff_; }
+ friend bool operator==(Handle const& lhs, Handle const& rhs) noexcept { return lhs.coeff_ == rhs.coeff_; }
+};
+
+enum class Flag : std::uint16_t { Off = 0, Stale = 4099, On = 60000 };
+
+// --- Distinct shared states for compare/exchange style tests ------------------------
+
+template <class T>
+struct SpValues;
+
+template <>
+struct SpValues<int> {
+ static std::shared_ptr<int> state_a() { return std::make_shared<int>(-90210); }
+ static std::shared_ptr<int> state_b() { return std::make_shared<int>(404); }
+ static std::shared_ptr<int> state_c() { return std::make_shared<int>(7331); }
+};
+
+template <>
+struct SpValues<double> {
+ static std::shared_ptr<double> state_a() { return std::make_shared<double>(1.4142135623730951); }
+ static std::shared_ptr<double> state_b() { return std::make_shared<double>(2.7182818284590452); }
+ static std::shared_ptr<double> state_c() { return std::make_shared<double>(3.1415926535897932); }
+};
+
+template <>
+struct SpValues<std::string> {
+ static std::shared_ptr<std::string> state_a() { return std::make_shared<std::string>("kappa"); }
+ static std::shared_ptr<std::string> state_b() { return std::make_shared<std::string>("lambda"); }
+ static std::shared_ptr<std::string> state_c() { return std::make_shared<std::string>("mu"); }
+};
+
+template <>
+struct SpValues<TrackedPod> {
+ static std::shared_ptr<TrackedPod> state_a() { return std::make_shared<TrackedPod>(TrackedPod{3u, -77L}); }
+ static std::shared_ptr<TrackedPod> state_b() { return std::make_shared<TrackedPod>(TrackedPod{101u, 1L << 20}); }
+ static std::shared_ptr<TrackedPod> state_c() { return std::make_shared<TrackedPod>(TrackedPod{255u, -1L}); }
+};
+
+template <>
+struct SpValues<Handle> {
+ static std::shared_ptr<Handle> state_a() { return std::make_shared<Handle>(0.125); }
+ static std::shared_ptr<Handle> state_b() { return std::make_shared<Handle>(-4096.5); }
+ static std::shared_ptr<Handle> state_c() { return std::make_shared<Handle>(8192.25); }
+};
+
+template <>
+struct SpValues<Flag> {
+ static std::shared_ptr<Flag> state_a() { return std::make_shared<Flag>(Flag::Off); }
+ static std::shared_ptr<Flag> state_b() { return std::make_shared<Flag>(Flag::Stale); }
+ static std::shared_ptr<Flag> state_c() { return std::make_shared<Flag>(Flag::On); }
+};
+
+} // namespace libcxx_atomic_smart_ptr_test
+
+// Instantiate the same runtime coverage for every value type.
+#define LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(M) \
+ M(int) \
+ M(double) \
+ M(std::string) \
+ M(libcxx_atomic_smart_ptr_test::TrackedPod) \
+ M(libcxx_atomic_smart_ptr_test::Handle) \
+ M(libcxx_atomic_smart_ptr_test::Flag)
+
+#endif // LIBCXX_TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.compile.pass.cpp
new file mode 100644
index 0000000000000..71464f598f5eb
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.compile.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// [util.smartptr.atomic.shared]
+// bool is_lock_free() const noexcept;
+
+// [util.smartptr.atomic.weak]
+// bool is_lock_free() const noexcept;
+
+#include <atomic>
+#include <memory>
+
+#include "atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void check(const std::atomic<std::shared_ptr<T>>& asp) noexcept {
+ static_assert(!std::atomic<std::shared_ptr<T>>::is_always_lock_free);
+ std::same_as<bool> decltype(auto) lf = asp.is_lock_free();
+ (void)lf;
+ ASSERT_SAME_TYPE(decltype(asp.is_lock_free()), bool);
+ ASSERT_NOEXCEPT(asp.is_lock_free());
+}
+
+template <class T>
+void check(const std::atomic<std::weak_ptr<T>>& awp) noexcept {
+ static_assert(!std::atomic<std::weak_ptr<T>>::is_always_lock_free);
+ std::same_as<bool> decltype(auto) lf = awp.is_lock_free();
+ (void)lf;
+ ASSERT_SAME_TYPE(decltype(awp.is_lock_free()), bool);
+ ASSERT_NOEXCEPT(awp.is_lock_free());
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_IS_LOCK_FREE(T) check<T>(std::atomic<std::shared_ptr<T>>());
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_IS_LOCK_FREE)
+#undef LIBCXX_ATOMIC_SP_RUN_IS_LOCK_FREE
+#define LIBCXX_ATOMIC_SP_RUN_W_ILF(T) check<T>(std::atomic<std::weak_ptr<T>>());
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_ILF)
+#undef LIBCXX_ATOMIC_SP_RUN_W_ILF
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/assign.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/assign.pass.cpp
new file mode 100644
index 0000000000000..3a39c30cb7413
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/assign.pass.cpp
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void operator=(shared_ptr<T>) noexcept;
+// void operator=(nullptr_t) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_assign() {
+ std::atomic<std::shared_ptr<T>> a;
+ auto p = libcxx_atomic_smart_ptr_test::SpValues<T>::state_a();
+ a = std::shared_ptr<T>(p);
+ assert(a.load().get() == p.get());
+ assert(*a.load() == *p);
+ a = nullptr;
+ assert(!a.load());
+ ASSERT_NOEXCEPT(a = nullptr);
+ ASSERT_NOEXCEPT(a = std::shared_ptr<T>(p));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_ASSIGN(T) test_assign<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_ASSIGN)
+#undef LIBCXX_ATOMIC_SP_RUN_ASSIGN
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
new file mode 100644
index 0000000000000..5ceca3efad536
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// bool compare_exchange_strong(shared_ptr<T>&, shared_ptr<T>, memory_order, memory_order) noexcept;
+// bool compare_exchange_strong(shared_ptr<T>&, shared_ptr<T>, memory_order = memory_order_seq_cst) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_compare_exchange_strong() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto p1 = SpValues<T>::state_a();
+ auto p2 = SpValues<T>::state_b();
+ std::atomic<std::shared_ptr<T>> a((std::shared_ptr<T>(p1)));
+
+ {
+ std::shared_ptr<T> expected = p1;
+ std::same_as<bool> decltype(auto) ok = a.compare_exchange_strong(expected, std::shared_ptr<T>(p2));
+ assert(ok);
+ assert(expected.get() == p1.get());
+ assert(*a.load() == *p2);
+ ASSERT_NOEXCEPT(a.compare_exchange_strong(expected, std::shared_ptr<T>(p2)));
+ }
+
+ {
+ std::shared_ptr<T> expected = p1;
+ bool ok =
+ a.compare_exchange_strong(expected, std::make_shared<T>(*SpValues<T>::state_c()), std::memory_order_seq_cst);
+ assert(!ok);
+ assert(*expected == *p2);
+ assert(*a.load() == *p2);
+ }
+
+ {
+ std::shared_ptr<T> expected = p2;
+ const std::shared_ptr<T> next = SpValues<T>::state_c();
+ bool ok = a.compare_exchange_strong(
+ expected, std::shared_ptr<T>(next), std::memory_order_release, std::memory_order_relaxed);
+ assert(ok);
+ assert(*a.load() == *next);
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_CXS(T) test_compare_exchange_strong<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_CXS)
+#undef LIBCXX_ATOMIC_SP_RUN_CXS
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_weak.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_weak.pass.cpp
new file mode 100644
index 0000000000000..713d04b7f5c47
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_weak.pass.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// bool compare_exchange_weak(shared_ptr<T>&, shared_ptr<T>, memory_order, memory_order) noexcept;
+// bool compare_exchange_weak(shared_ptr<T>&, shared_ptr<T>, memory_order = memory_order_seq_cst) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_compare_exchange_weak() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto p1 = SpValues<T>::state_a();
+ auto p2 = SpValues<T>::state_b();
+ std::atomic<std::shared_ptr<T>> a((std::shared_ptr<T>(p1)));
+
+ {
+ std::shared_ptr<T> expected = p1;
+ bool ok = false;
+ while (!ok)
+ ok = a.compare_exchange_weak(expected, std::shared_ptr<T>(p2));
+ assert(ok);
+ assert(*a.load() == *p2);
+ ASSERT_NOEXCEPT(a.compare_exchange_weak(expected, std::shared_ptr<T>(p2)));
+ }
+
+ {
+ std::shared_ptr<T> expected = p1;
+ bool ok =
+ a.compare_exchange_weak(expected, std::make_shared<T>(*SpValues<T>::state_c()), std::memory_order_seq_cst);
+ assert(!ok);
+ assert(*expected == *p2);
+ }
+
+ {
+ std::shared_ptr<T> expected = p2;
+ const std::shared_ptr<T> next = SpValues<T>::state_c();
+ bool ok = false;
+ while (!ok)
+ ok = a.compare_exchange_weak(
+ expected, std::shared_ptr<T>(next), std::memory_order_release, std::memory_order_relaxed);
+ assert(ok);
+ assert(*a.load() == *next);
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_CXW(T) test_compare_exchange_weak<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_CXW)
+#undef LIBCXX_ATOMIC_SP_RUN_CXW
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
new file mode 100644
index 0000000000000..4c219ae175d18
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// operator shared_ptr<T>() const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_convert() {
+ auto p = libcxx_atomic_smart_ptr_test::SpValues<T>::state_a();
+ const std::atomic<std::shared_ptr<T>> a((std::shared_ptr<T>(p)));
+
+ std::shared_ptr<T> s = a;
+ assert(s.get() == p.get());
+ assert(*s == *p);
+
+ std::same_as<std::shared_ptr<T>> decltype(auto) s2 = static_cast<std::shared_ptr<T>>(a);
+ assert(s2.get() == p.get());
+
+ ASSERT_NOEXCEPT(static_cast<std::shared_ptr<T>>(a));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_CONVERT(T) test_convert<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_CONVERT)
+#undef LIBCXX_ATOMIC_SP_RUN_CONVERT
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
new file mode 100644
index 0000000000000..fa957d8ff7ebe
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// shared_ptr<T> exchange(shared_ptr<T> desired, memory_order order = memory_order::seq_cst) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_exchange() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto p1 = SpValues<T>::state_a();
+ auto p2 = SpValues<T>::state_b();
+ std::atomic<std::shared_ptr<T>> a((std::shared_ptr<T>(p1)));
+
+ std::same_as<std::shared_ptr<T>> decltype(auto) out = a.exchange(std::shared_ptr<T>(p2));
+ assert(*out == *p1);
+ assert(out.get() == p1.get());
+ assert(*a.load() == *p2);
+
+ out = a.exchange(nullptr, std::memory_order_seq_cst);
+ assert(*out == *p2);
+ assert(!a.load());
+
+ ASSERT_NOEXCEPT(a.exchange(nullptr));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_EXCHANGE(T) test_exchange<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_EXCHANGE)
+#undef LIBCXX_ATOMIC_SP_RUN_EXCHANGE
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.compile.pass.cpp
new file mode 100644
index 0000000000000..b7ff230832d09
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.compile.pass.cpp
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// static constexpr bool is_always_lock_free;
+// bool is_lock_free() const noexcept;
+
+#include <atomic>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void check() {
+ using A = std::atomic<std::shared_ptr<T>>;
+
+ static_assert(std::same_as<decltype(A::is_always_lock_free), const bool>);
+ static_assert(!static_cast<bool>(A::is_always_lock_free));
+
+ const A asp;
+ std::same_as<bool> decltype(auto) lf = asp.is_lock_free();
+ (void)lf;
+ ASSERT_NOEXCEPT(A::is_always_lock_free);
+ ASSERT_NOEXCEPT(asp.is_lock_free());
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_IS_ALWAYS_LF(T) check<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_IS_ALWAYS_LF)
+#undef LIBCXX_ATOMIC_SP_RUN_IS_ALWAYS_LF
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.compile.pass.cpp
new file mode 100644
index 0000000000000..48bcb81ab6cce
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.compile.pass.cpp
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// shared_ptr<T> load(memory_order order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void check(const std::atomic<std::shared_ptr<T>>& asp) {
+ std::same_as<std::shared_ptr<T>> decltype(auto) no_arg = asp.load();
+ ASSERT_SAME_TYPE(decltype(asp.load()), std::shared_ptr<T>);
+ ASSERT_NOEXCEPT(asp.load());
+
+ std::same_as<std::shared_ptr<T>> decltype(auto) with_order = asp.load(std::memory_order_seq_cst);
+ ASSERT_SAME_TYPE(decltype(asp.load(std::memory_order_acquire)), std::shared_ptr<T>);
+ ASSERT_NOEXCEPT(asp.load(std::memory_order_seq_cst));
+ static_cast<void>(no_arg);
+ static_cast<void>(with_order);
+
+ {
+ const std::atomic<std::shared_ptr<T>> const_a;
+ static_assert(noexcept(const_a.load()));
+ ASSERT_SAME_TYPE(decltype(const_a.load()), std::shared_ptr<T>);
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_LOAD(T) check<T>(std::atomic<std::shared_ptr<T>>());
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_LOAD)
+#undef LIBCXX_ATOMIC_SP_RUN_LOAD
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_all.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_all.pass.cpp
new file mode 100644
index 0000000000000..5d4c9b7714b23
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_all.pass.cpp
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void notify_all() noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <thread>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_notify_all() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ std::atomic<std::shared_ptr<T>> a;
+
+ ASSERT_NOEXCEPT(a.notify_all());
+
+#if __cpp_lib_atomic_wait >= 201907L
+ auto p1 = SpValues<T>::state_a();
+ a.store(std::shared_ptr<T>(p1));
+
+ std::atomic<int> phase{0};
+ std::thread t1([&] {
+ std::shared_ptr<T> old = a.load();
+ phase.fetch_add(1, std::memory_order_acq_rel);
+ while (phase.load(std::memory_order_acquire) < 2) {
+ std::this_thread::yield();
+ }
+ a.wait(old);
+ });
+ std::thread t2([&] {
+ std::shared_ptr<T> old = a.load();
+ phase.fetch_add(1, std::memory_order_acq_rel);
+ while (phase.load(std::memory_order_acquire) < 2) {
+ std::this_thread::yield();
+ }
+ a.wait(old);
+ });
+
+ while (phase.load(std::memory_order_acquire) < 2) {
+ std::this_thread::yield();
+ }
+
+ a.store(std::make_shared<T>(*SpValues<T>::state_b()));
+ a.notify_all();
+ t1.join();
+ t2.join();
+#endif
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_NOTIFY_ALL(T) test_notify_all<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_NOTIFY_ALL)
+#undef LIBCXX_ATOMIC_SP_RUN_NOTIFY_ALL
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_one.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_one.pass.cpp
new file mode 100644
index 0000000000000..b178fb13d1a0f
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_one.pass.cpp
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void notify_one() noexcept;
+
+#include <atomic>
+#include <memory>
+#include <thread>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_notify_one() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ std::atomic<std::shared_ptr<T>> a;
+
+ ASSERT_NOEXCEPT(a.notify_one());
+
+#if __cpp_lib_atomic_wait >= 201907L
+ auto p1 = SpValues<T>::state_a();
+ a.store(std::shared_ptr<T>(p1));
+
+ std::atomic<bool> started{false};
+ std::thread t([&] {
+ std::shared_ptr<T> old = a.load();
+ started.store(true, std::memory_order_release);
+ a.wait(old);
+ });
+
+ while (!started.load(std::memory_order_acquire)) {
+ std::this_thread::yield();
+ }
+
+ a.store(std::make_shared<T>(*SpValues<T>::state_b()));
+ a.notify_one();
+ t.join();
+#endif
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_NOTIFY_ONE(T) test_notify_one<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_NOTIFY_ONE)
+#undef LIBCXX_ATOMIC_SP_RUN_NOTIFY_ONE
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/store.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/store.pass.cpp
new file mode 100644
index 0000000000000..9ab808cbd0fae
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/store.pass.cpp
@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void store(shared_ptr<T> desired, memory_order order = memory_order::seq_cst) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_store() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ std::atomic<std::shared_ptr<T>> a;
+
+ auto p = SpValues<T>::state_a();
+ a.store(std::shared_ptr<T>(p));
+ assert(a.load().get() == p.get());
+ assert(*a.load() == *p);
+
+ a.store(std::make_shared<T>(*SpValues<T>::state_b()));
+ assert(*a.load() == *SpValues<T>::state_b());
+
+ a.store(nullptr, std::memory_order_relaxed);
+ assert(!a.load());
+
+ ASSERT_NOEXCEPT(a.store(nullptr));
+ {
+ std::shared_ptr<T> desired = std::make_shared<T>(*SpValues<T>::state_c());
+ ASSERT_NOEXCEPT(a.store(std::move(desired), std::memory_order_seq_cst));
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_STORE(T) test_store<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_STORE)
+#undef LIBCXX_ATOMIC_SP_RUN_STORE
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp
new file mode 100644
index 0000000000000..39a49c810f9a4
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// using value_type = shared_ptr<T>;
+// static constexpr bool is_always_lock_free = false;
+// atomic() noexcept;
+// constexpr atomic(nullptr_t) noexcept;
+// atomic(shared_ptr<T>) noexcept;
+// atomic(const atomic&) = delete;
+// atomic& operator=(const atomic&) = delete;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <type_traits>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void traits() {
+ using A = std::atomic<std::shared_ptr<T>>;
+
+ static_assert(std::same_as<typename A::value_type, std::shared_ptr<T>>);
+ static_assert(!A::is_always_lock_free);
+
+ static_assert(!std::is_copy_constructible_v<A>);
+ static_assert(!std::is_copy_assignable_v<A>);
+
+ static_assert(std::is_nothrow_default_constructible_v<A>);
+ static_assert(std::is_nothrow_constructible_v<A, std::nullptr_t>);
+
+ {
+ A a;
+ assert(!a.load());
+ }
+
+ {
+ auto p = libcxx_atomic_smart_ptr_test::SpValues<T>::state_a();
+ A a((std::shared_ptr<T>(p)));
+ assert(a.load().get() == p.get());
+ assert(*a.load() == *p);
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_TRAITS(T) traits<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_TRAITS)
+#undef LIBCXX_ATOMIC_SP_RUN_TRAITS
+
+ {
+ static_assert(std::is_nothrow_constructible_v<std::atomic<std::shared_ptr<int>>, std::nullptr_t>);
+ std::atomic<std::shared_ptr<int>> a(nullptr);
+ assert(!a.load());
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/wait.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/wait.pass.cpp
new file mode 100644
index 0000000000000..fef98a7008c27
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/wait.pass.cpp
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void wait(shared_ptr<T> old, memory_order order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <thread>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_wait() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ std::atomic<std::shared_ptr<T>> a;
+
+#if __cpp_lib_atomic_wait >= 201907L
+ auto p1 = SpValues<T>::state_a();
+ a.store(std::shared_ptr<T>(p1));
+
+ std::atomic<bool> started{false};
+ std::thread t([&] {
+ std::shared_ptr<T> old = a.load();
+ started.store(true, std::memory_order_release);
+ a.wait(old);
+ });
+
+ while (!started.load(std::memory_order_acquire)) {
+ std::this_thread::yield();
+ }
+
+ auto pWake = std::make_shared<T>(*SpValues<T>::state_c());
+ a.store(std::move(pWake));
+ a.notify_all();
+ t.join();
+
+ assert(*a.load() == *SpValues<T>::state_c());
+#endif
+
+ ASSERT_NOEXCEPT(a.wait(std::shared_ptr<T>(), std::memory_order_seq_cst));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_WAIT(T) test_wait<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_WAIT)
+#undef LIBCXX_ATOMIC_SP_RUN_WAIT
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/assign.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/assign.pass.cpp
new file mode 100644
index 0000000000000..efe373d1d8036
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/assign.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void operator=(weak_ptr<T>) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_assign_weak() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto sp1 = SpValues<T>::state_a();
+ auto sp2 = SpValues<T>::state_b();
+ std::weak_ptr<T> wp1 = sp1;
+ std::weak_ptr<T> wp2 = sp2;
+
+ std::atomic<std::weak_ptr<T>> a;
+ a = std::weak_ptr<T>(wp1);
+ {
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp1);
+ }
+
+ a = std::weak_ptr<T>(wp2);
+ {
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp2);
+ }
+
+ ASSERT_NOEXCEPT(a = std::weak_ptr<T>(wp1));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_ASSIGN(T) test_assign_weak<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_ASSIGN)
+#undef LIBCXX_ATOMIC_SP_RUN_W_ASSIGN
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
new file mode 100644
index 0000000000000..61047d17d264b
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// bool compare_exchange_strong(weak_ptr<T>&, weak_ptr<T>, memory_order, memory_order) noexcept;
+// bool compare_exchange_strong(weak_ptr<T>&, weak_ptr<T>, memory_order = memory_order_seq_cst) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_compare_exchange_strong_weakptr() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto sp1 = SpValues<T>::state_a();
+ auto sp2 = SpValues<T>::state_b();
+ std::weak_ptr<T> wp1 = sp1;
+ std::weak_ptr<T> wp2 = sp2;
+
+ std::atomic<std::weak_ptr<T>> a((std::weak_ptr<T>(wp1)));
+
+ {
+ std::weak_ptr<T> expected = wp1;
+ std::same_as<bool> decltype(auto) ok = a.compare_exchange_strong(expected, std::weak_ptr<T>(wp2));
+ assert(ok);
+ {
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp2);
+ }
+ ASSERT_NOEXCEPT(a.compare_exchange_strong(expected, std::weak_ptr<T>(wp2)));
+ }
+
+ {
+ std::weak_ptr<T> expected = wp1;
+ bool ok = a.compare_exchange_strong(expected, std::weak_ptr<T>(wp1), std::memory_order_seq_cst);
+ assert(!ok);
+ auto locked = expected.lock();
+ assert(locked && *locked == *sp2);
+ }
+
+ {
+ std::weak_ptr<T> expected = wp2;
+ auto sp3 = SpValues<T>::state_c();
+ bool ok = a.compare_exchange_strong(
+ expected, std::weak_ptr<T>(sp3), std::memory_order_release, std::memory_order_relaxed);
+ assert(ok);
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp3);
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_CXS(T) test_compare_exchange_strong_weakptr<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_CXS)
+#undef LIBCXX_ATOMIC_SP_RUN_W_CXS
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_weak.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_weak.pass.cpp
new file mode 100644
index 0000000000000..f44450b46476c
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_weak.pass.cpp
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// bool compare_exchange_weak(weak_ptr<T>&, weak_ptr<T>, memory_order, memory_order) noexcept;
+// bool compare_exchange_weak(weak_ptr<T>&, weak_ptr<T>, memory_order = memory_order_seq_cst) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_compare_exchange_weak_weakptr() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto sp1 = SpValues<T>::state_a();
+ auto sp2 = SpValues<T>::state_b();
+ std::weak_ptr<T> wp1 = sp1;
+ std::weak_ptr<T> wp2 = sp2;
+
+ std::atomic<std::weak_ptr<T>> a((std::weak_ptr<T>(wp1)));
+
+ {
+ std::weak_ptr<T> expected = wp1;
+ bool ok = false;
+ while (!ok)
+ ok = a.compare_exchange_weak(expected, std::weak_ptr<T>(wp2));
+ assert(ok);
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp2);
+ ASSERT_NOEXCEPT(a.compare_exchange_weak(expected, std::weak_ptr<T>(wp2)));
+ }
+
+ {
+ std::weak_ptr<T> expected = wp1;
+ bool ok = a.compare_exchange_weak(expected, std::weak_ptr<T>(wp1), std::memory_order_seq_cst);
+ assert(!ok);
+ auto locked = expected.lock();
+ assert(locked && *locked == *sp2);
+ }
+
+ {
+ std::weak_ptr<T> expected = wp2;
+ auto sp3 = SpValues<T>::state_c();
+ bool ok = false;
+ while (!ok)
+ ok = a.compare_exchange_weak(
+ expected, std::weak_ptr<T>(sp3), std::memory_order_release, std::memory_order_relaxed);
+ assert(ok);
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp3);
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_CXW(T) test_compare_exchange_weak_weakptr<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_CXW)
+#undef LIBCXX_ATOMIC_SP_RUN_W_CXW
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
new file mode 100644
index 0000000000000..c74d5f4925641
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// operator weak_ptr<T>() const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_convert_weak() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto sp = SpValues<T>::state_a();
+ std::weak_ptr<T> wp = sp;
+ const std::atomic<std::weak_ptr<T>> a((std::weak_ptr<T>(wp)));
+
+ std::weak_ptr<T> w = a;
+ auto locked = w.lock();
+ assert(locked && *locked == *sp);
+
+ std::same_as<std::weak_ptr<T>> decltype(auto) w2 = static_cast<std::weak_ptr<T>>(a);
+ auto locked2 = w2.lock();
+ assert(locked2 && *locked2 == *sp);
+
+ ASSERT_NOEXCEPT(static_cast<std::weak_ptr<T>>(a));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_CONVERT(T) test_convert_weak<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_CONVERT)
+#undef LIBCXX_ATOMIC_SP_RUN_W_CONVERT
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
new file mode 100644
index 0000000000000..2c777efaabb5b
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// weak_ptr<T> exchange(weak_ptr<T> desired, memory_order order = memory_order::seq_cst) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_exchange_weak() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto sp1 = SpValues<T>::state_a();
+ auto sp2 = SpValues<T>::state_b();
+ std::weak_ptr<T> wp1 = sp1;
+ std::weak_ptr<T> wp2 = sp2;
+
+ std::atomic<std::weak_ptr<T>> a((std::weak_ptr<T>(wp1)));
+
+ std::same_as<std::weak_ptr<T>> decltype(auto) out = a.exchange(std::weak_ptr<T>(wp2));
+ {
+ auto locked = out.lock();
+ assert(locked && *locked == *sp1);
+ }
+ {
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp2);
+ }
+
+ std::weak_ptr<T> empty;
+ out = a.exchange(std::weak_ptr<T>(empty), std::memory_order_seq_cst);
+ {
+ auto locked = out.lock();
+ assert(locked && *locked == *sp2);
+ }
+ assert(a.load().expired());
+
+ ASSERT_NOEXCEPT(a.exchange(std::weak_ptr<T>(wp1)));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_EXCHANGE(T) test_exchange_weak<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_EXCHANGE)
+#undef LIBCXX_ATOMIC_SP_RUN_W_EXCHANGE
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.compile.pass.cpp
new file mode 100644
index 0000000000000..ceb3dbce26c29
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.compile.pass.cpp
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// static constexpr bool is_always_lock_free;
+// bool is_lock_free() const noexcept;
+
+#include <atomic>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void check() {
+ using A = std::atomic<std::weak_ptr<T>>;
+
+ static_assert(std::same_as<decltype(A::is_always_lock_free), const bool>);
+ static_assert(A::is_always_lock_free == false);
+
+ const A awp;
+ std::same_as<bool> decltype(auto) lf = awp.is_lock_free();
+ (void)lf;
+ ASSERT_NOEXCEPT(A::is_always_lock_free);
+ ASSERT_NOEXCEPT(awp.is_lock_free());
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_IALF(T) check<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_IALF)
+#undef LIBCXX_ATOMIC_SP_RUN_W_IALF
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.compile.pass.cpp
new file mode 100644
index 0000000000000..55421abb2026b
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.compile.pass.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// bool is_lock_free() const noexcept;
+
+#include <atomic>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void check(const std::atomic<std::weak_ptr<T>>& awp) noexcept {
+ static_assert(!std::atomic<std::weak_ptr<T>>::is_always_lock_free);
+ std::same_as<bool> decltype(auto) lf = awp.is_lock_free();
+ (void)lf;
+ ASSERT_SAME_TYPE(decltype(awp.is_lock_free()), bool);
+ ASSERT_NOEXCEPT(awp.is_lock_free());
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_ILF(T) check<T>(std::atomic<std::weak_ptr<T>>());
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_ILF)
+#undef LIBCXX_ATOMIC_SP_RUN_W_ILF
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.compile.pass.cpp
new file mode 100644
index 0000000000000..b372c94cd1597
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.compile.pass.cpp
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// weak_ptr<T> load(memory_order order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void check(const std::atomic<std::weak_ptr<T>>& awp) {
+ std::same_as<std::weak_ptr<T>> decltype(auto) no_arg = awp.load();
+ ASSERT_SAME_TYPE(decltype(awp.load()), std::weak_ptr<T>);
+ ASSERT_NOEXCEPT(awp.load());
+
+ std::same_as<std::weak_ptr<T>> decltype(auto) with_order = awp.load(std::memory_order_seq_cst);
+ ASSERT_SAME_TYPE(decltype(awp.load(std::memory_order_acquire)), std::weak_ptr<T>);
+ ASSERT_NOEXCEPT(awp.load(std::memory_order_seq_cst));
+ static_cast<void>(no_arg);
+ static_cast<void>(with_order);
+
+ {
+ const std::atomic<std::weak_ptr<T>> const_a;
+ static_assert(noexcept(const_a.load()));
+ ASSERT_SAME_TYPE(decltype(const_a.load()), std::weak_ptr<T>);
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_LOAD(T) check<T>(std::atomic<std::weak_ptr<T>>());
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_LOAD)
+#undef LIBCXX_ATOMIC_SP_RUN_W_LOAD
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_all.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_all.pass.cpp
new file mode 100644
index 0000000000000..3211c651143f3
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_all.pass.cpp
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void notify_all() noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <thread>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_notify_all_weak() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ std::atomic<std::weak_ptr<T>> a;
+
+ ASSERT_NOEXCEPT(a.notify_all());
+
+#if __cpp_lib_atomic_wait >= 201907L
+ auto sp1 = SpValues<T>::state_a();
+ std::weak_ptr<T> wp1 = sp1;
+ a.store(std::weak_ptr<T>(wp1));
+
+ std::atomic<int> phase{0};
+ std::thread t1([&] {
+ std::weak_ptr<T> old = a.load();
+ phase.fetch_add(1, std::memory_order_acq_rel);
+ while (phase.load(std::memory_order_acquire) < 2) {
+ std::this_thread::yield();
+ }
+ a.wait(old);
+ });
+ std::thread t2([&] {
+ std::weak_ptr<T> old = a.load();
+ phase.fetch_add(1, std::memory_order_acq_rel);
+ while (phase.load(std::memory_order_acquire) < 2) {
+ std::this_thread::yield();
+ }
+ a.wait(old);
+ });
+
+ while (phase.load(std::memory_order_acquire) < 2) {
+ std::this_thread::yield();
+ }
+
+ auto sp2 = SpValues<T>::state_b();
+ a.store(std::weak_ptr<T>(sp2));
+ a.notify_all();
+ t1.join();
+ t2.join();
+#endif
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_NA(T) test_notify_all_weak<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_NA)
+#undef LIBCXX_ATOMIC_SP_RUN_W_NA
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_one.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_one.pass.cpp
new file mode 100644
index 0000000000000..440aab1d3b5e7
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_one.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void notify_one() noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <thread>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_notify_one_weak() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ std::atomic<std::weak_ptr<T>> a;
+
+ ASSERT_NOEXCEPT(a.notify_one());
+
+#if __cpp_lib_atomic_wait >= 201907L
+ auto sp1 = SpValues<T>::state_a();
+ std::weak_ptr<T> wp1 = sp1;
+ a.store(std::weak_ptr<T>(wp1));
+
+ std::atomic<bool> started{false};
+ std::thread t([&] {
+ std::weak_ptr<T> old = a.load();
+ started.store(true, std::memory_order_release);
+ a.wait(old);
+ });
+
+ while (!started.load(std::memory_order_acquire)) {
+ std::this_thread::yield();
+ }
+
+ auto spB = SpValues<T>::state_b();
+ a.store(std::weak_ptr<T>(spB));
+ a.notify_one();
+ t.join();
+#endif
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_N1(T) test_notify_one_weak<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_N1)
+#undef LIBCXX_ATOMIC_SP_RUN_W_N1
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/store.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/store.pass.cpp
new file mode 100644
index 0000000000000..35baa907c1437
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/store.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void store(weak_ptr<T> desired, memory_order order = memory_order::seq_cst) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_store_weak() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ auto sp1 = SpValues<T>::state_a();
+ auto sp2 = SpValues<T>::state_b();
+ std::weak_ptr<T> wp1 = sp1;
+ std::weak_ptr<T> wp2 = sp2;
+
+ std::atomic<std::weak_ptr<T>> a;
+ a.store(std::weak_ptr<T>(wp1));
+ {
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp1);
+ }
+
+ a.store(std::weak_ptr<T>(wp2), std::memory_order_relaxed);
+ {
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp2);
+ }
+
+ ASSERT_NOEXCEPT(a.store(std::weak_ptr<T>(wp1)));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_STORE(T) test_store_weak<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_STORE)
+#undef LIBCXX_ATOMIC_SP_RUN_W_STORE
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp
new file mode 100644
index 0000000000000..a130c9920d99c
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// Note: atomic<weak_ptr<T>> intentionally has no constexpr atomic(nullptr_t) constructor,
+// unlike atomic<shared_ptr<T>>. See [util.smartptr.atomic.weak].
+
+// using value_type = weak_ptr<T>;
+// static constexpr bool is_always_lock_free = false;
+// atomic() noexcept;
+// atomic(weak_ptr<T>) noexcept;
+// atomic(const atomic&) = delete;
+// atomic& operator=(const atomic&) = delete;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <type_traits>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void traits() {
+ using A = std::atomic<std::weak_ptr<T>>;
+
+ static_assert(std::same_as<typename A::value_type, std::weak_ptr<T>>);
+ static_assert(!A::is_always_lock_free);
+
+ static_assert(!std::is_copy_constructible_v<A>);
+ static_assert(!std::is_copy_assignable_v<A>);
+
+ static_assert(std::is_nothrow_default_constructible_v<A>);
+
+ {
+ A a;
+ assert(a.load().expired());
+ }
+
+ {
+ auto sp = libcxx_atomic_smart_ptr_test::SpValues<T>::state_a();
+ std::weak_ptr<T> wp = sp;
+ A a((std::weak_ptr<T>(wp)));
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp);
+ }
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_TRAITS(T) traits<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_TRAITS)
+#undef LIBCXX_ATOMIC_SP_RUN_W_TRAITS
+ return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/wait.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/wait.pass.cpp
new file mode 100644
index 0000000000000..f840c4ec00d32
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/wait.pass.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+
+// void wait(weak_ptr<T> old, memory_order order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+#include <thread>
+
+#include "../atomic_smart_ptr_test_types.h"
+#include "test_macros.h"
+
+template <class T>
+void test_wait_weak() {
+ using libcxx_atomic_smart_ptr_test::SpValues;
+ std::atomic<std::weak_ptr<T>> a;
+
+#if __cpp_lib_atomic_wait >= 201907L
+ auto sp_for_wait = SpValues<T>::state_a();
+ std::weak_ptr<T> wp_for_wait = sp_for_wait;
+ a.store(std::weak_ptr<T>(wp_for_wait));
+
+ auto sp1 = SpValues<T>::state_b();
+ std::weak_ptr<T> wp1 = sp1;
+
+ std::atomic<bool> started{false};
+ std::thread t([&] {
+ std::weak_ptr<T> old = a.load();
+ started.store(true, std::memory_order_release);
+ a.wait(old);
+ });
+
+ while (!started.load(std::memory_order_acquire)) {
+ std::this_thread::yield();
+ }
+
+ a.store(std::weak_ptr<T>(wp1));
+ a.notify_all();
+ t.join();
+
+ {
+ auto locked = a.load().lock();
+ assert(locked && *locked == *sp1);
+ }
+#endif
+
+ ASSERT_NOEXCEPT(a.wait(std::weak_ptr<T>(), std::memory_order_seq_cst));
+}
+
+int main(int, char**) {
+#define LIBCXX_ATOMIC_SP_RUN_W_WAIT(T) test_wait_weak<T>();
+ LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_WAIT)
+#undef LIBCXX_ATOMIC_SP_RUN_W_WAIT
+ return 0;
+}
>From 3be5c0f2ca555cf522a234a9cffcccbcc3231330 Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Sat, 2 May 2026 14:33:43 +0300
Subject: [PATCH 10/46] [libcxx][NFC] Yielding another thread in stress test to
decrease the execution time; fixed issue with same_as on Armv7 and Armv8 (and
with -fno-exceptions); applied formatting changes suggested by GH bot
Signed-off-by: ViNN280801 <vladislav.semykin at gmail.com>
---
.../atomic/atomic_smart_ptr_test_types.h | 12 ++++++------
.../atomic/shared/types.compile.pass.cpp | 1 +
.../util.smartptr/atomic/weak/types.compile.pass.cpp | 1 +
.../atomic_shared_ptr_stress.pass.cpp | 1 +
4 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
index 309cc11bb5026..4ea5a52472891 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
@@ -91,12 +91,12 @@ struct SpValues<Flag> {
} // namespace libcxx_atomic_smart_ptr_test
// Instantiate the same runtime coverage for every value type.
-#define LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(M) \
- M(int) \
- M(double) \
- M(std::string) \
- M(libcxx_atomic_smart_ptr_test::TrackedPod) \
- M(libcxx_atomic_smart_ptr_test::Handle) \
+#define LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(M) \
+ M(int) \
+ M(double) \
+ M(std::string) \
+ M(libcxx_atomic_smart_ptr_test::TrackedPod) \
+ M(libcxx_atomic_smart_ptr_test::Handle) \
M(libcxx_atomic_smart_ptr_test::Flag)
#endif // LIBCXX_TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp
index 39a49c810f9a4..bc8ee81abe2b1 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp
@@ -19,6 +19,7 @@
#include <atomic>
#include <cassert>
+#include <concepts>
#include <memory>
#include <type_traits>
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp
index a130c9920d99c..40e0f4d5e38d6 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp
@@ -21,6 +21,7 @@
#include <atomic>
#include <cassert>
+#include <concepts>
#include <memory>
#include <type_traits>
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
index 86b56bfa920bc..da73a98b44526 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_stress.pass.cpp
@@ -56,6 +56,7 @@ void run(Atomic& a, Make make_value, const std::array<int, kCandidateCount>& exp
}
assert(ok);
}
+ std::this_thread::yield();
(void)r;
}
});
>From 861960381cadf6a3edc57b4a89dc7bb7fa048812 Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Sat, 2 May 2026 16:31:27 +0300
Subject: [PATCH 11/46] [libcxx][NFC] Applied review feedback to
atomic<shared_ptr>/<weak_ptr> tests
Signed-off-by: ViNN280801 <vladislav.semykin at gmail.com>
---
.../atomic/atomic_smart_ptr_test_types.h | 29 +++++++++----------
...compile.pass.cpp => is_lock_free.pass.cpp} | 24 ++++++++-------
.../atomic/shared/assign.pass.cpp | 15 ++++++----
.../shared/compare_exchange_strong.pass.cpp | 12 ++++----
.../shared/compare_exchange_weak.pass.cpp | 12 ++++----
.../atomic/shared/convert.pass.cpp | 13 +++++----
.../atomic/shared/exchange.pass.cpp | 12 ++++----
....pass.cpp => is_always_lock_free.pass.cpp} | 13 +++++----
.../{load.compile.pass.cpp => load.pass.cpp} | 18 +++++++-----
.../atomic/shared/notify_all.pass.cpp | 12 ++++----
.../atomic/shared/notify_one.pass.cpp | 12 ++++----
.../atomic/shared/store.pass.cpp | 14 +++++----
...{types.compile.pass.cpp => types.pass.cpp} | 11 ++++---
.../util.smartptr/atomic/shared/wait.pass.cpp | 12 ++++----
.../util.smartptr/atomic/weak/assign.pass.cpp | 12 ++++----
.../weak/compare_exchange_strong.pass.cpp | 12 ++++----
.../weak/compare_exchange_weak.pass.cpp | 12 ++++----
.../atomic/weak/convert.pass.cpp | 12 ++++----
.../atomic/weak/exchange.pass.cpp | 12 ++++----
....pass.cpp => is_always_lock_free.pass.cpp} | 13 +++++----
...compile.pass.cpp => is_lock_free.pass.cpp} | 12 ++++----
.../{load.compile.pass.cpp => load.pass.cpp} | 18 +++++++-----
.../atomic/weak/notify_all.pass.cpp | 12 ++++----
.../atomic/weak/notify_one.pass.cpp | 12 ++++----
.../util.smartptr/atomic/weak/store.pass.cpp | 12 ++++----
...{types.compile.pass.cpp => types.pass.cpp} | 11 ++++---
.../util.smartptr/atomic/weak/wait.pass.cpp | 12 ++++----
27 files changed, 215 insertions(+), 156 deletions(-)
rename libcxx/test/std/utilities/memory/util.smartptr/atomic/{is_lock_free.compile.pass.cpp => is_lock_free.pass.cpp} (66%)
rename libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/{is_always_lock_free.compile.pass.cpp => is_always_lock_free.pass.cpp} (77%)
rename libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/{load.compile.pass.cpp => load.pass.cpp} (68%)
rename libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/{types.compile.pass.cpp => types.pass.cpp} (88%)
rename libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/{is_always_lock_free.compile.pass.cpp => is_always_lock_free.pass.cpp} (78%)
rename libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/{is_lock_free.compile.pass.cpp => is_lock_free.pass.cpp} (75%)
rename libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/{load.compile.pass.cpp => load.pass.cpp} (68%)
rename libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/{types.compile.pass.cpp => types.pass.cpp} (86%)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
index 4ea5a52472891..b8a235e8e1416 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
@@ -11,15 +11,13 @@
//
//===----------------------------------------------------------------------===//
-#ifndef LIBCXX_TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
-#define LIBCXX_TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
+#ifndef TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
+#define TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
#include <cstdint>
#include <memory>
#include <string>
-namespace libcxx_atomic_smart_ptr_test {
-
// --- User-defined and semi-random scalar-like types ---------------------------------
struct TrackedPod {
@@ -88,15 +86,16 @@ struct SpValues<Flag> {
static std::shared_ptr<Flag> state_c() { return std::make_shared<Flag>(Flag::On); }
};
-} // namespace libcxx_atomic_smart_ptr_test
-
-// Instantiate the same runtime coverage for every value type.
-#define LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(M) \
- M(int) \
- M(double) \
- M(std::string) \
- M(libcxx_atomic_smart_ptr_test::TrackedPod) \
- M(libcxx_atomic_smart_ptr_test::Handle) \
- M(libcxx_atomic_smart_ptr_test::Flag)
+struct ForEachSmartPtrType {
+ template <template <class> class Fn>
+ void operator()() const {
+ Fn<int>{}();
+ Fn<double>{}();
+ Fn<std::string>{}();
+ Fn<TrackedPod>{}();
+ Fn<Handle>{}();
+ Fn<Flag>{}();
+ }
+};
-#endif // LIBCXX_TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
+#endif // TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
similarity index 66%
rename from libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.compile.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
index 71464f598f5eb..574a89fababb3 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
@@ -26,8 +26,7 @@ void check(const std::atomic<std::shared_ptr<T>>& asp) noexcept {
static_assert(!std::atomic<std::shared_ptr<T>>::is_always_lock_free);
std::same_as<bool> decltype(auto) lf = asp.is_lock_free();
(void)lf;
- ASSERT_SAME_TYPE(decltype(asp.is_lock_free()), bool);
- ASSERT_NOEXCEPT(asp.is_lock_free());
+ static_assert(noexcept(asp.is_lock_free()));
}
template <class T>
@@ -35,16 +34,21 @@ void check(const std::atomic<std::weak_ptr<T>>& awp) noexcept {
static_assert(!std::atomic<std::weak_ptr<T>>::is_always_lock_free);
std::same_as<bool> decltype(auto) lf = awp.is_lock_free();
(void)lf;
- ASSERT_SAME_TYPE(decltype(awp.is_lock_free()), bool);
- ASSERT_NOEXCEPT(awp.is_lock_free());
+ static_assert(noexcept(awp.is_lock_free()));
}
+template <class T>
+struct TestIsLockFreeShared {
+ void operator()() const { check<T>(std::atomic<std::shared_ptr<T>>()); }
+};
+
+template <class T>
+struct TestIsLockFreeWeak {
+ void operator()() const { check<T>(std::atomic<std::weak_ptr<T>>()); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_IS_LOCK_FREE(T) check<T>(std::atomic<std::shared_ptr<T>>());
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_IS_LOCK_FREE)
-#undef LIBCXX_ATOMIC_SP_RUN_IS_LOCK_FREE
-#define LIBCXX_ATOMIC_SP_RUN_W_ILF(T) check<T>(std::atomic<std::weak_ptr<T>>());
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_ILF)
-#undef LIBCXX_ATOMIC_SP_RUN_W_ILF
+ ForEachSmartPtrType{}.template operator()<TestIsLockFreeShared>();
+ ForEachSmartPtrType{}.template operator()<TestIsLockFreeWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/assign.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/assign.pass.cpp
index 3a39c30cb7413..c3ad8ec3904c5 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/assign.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/assign.pass.cpp
@@ -22,19 +22,22 @@
template <class T>
void test_assign() {
std::atomic<std::shared_ptr<T>> a;
- auto p = libcxx_atomic_smart_ptr_test::SpValues<T>::state_a();
+ auto p = SpValues<T>::state_a();
a = std::shared_ptr<T>(p);
assert(a.load().get() == p.get());
assert(*a.load() == *p);
a = nullptr;
assert(!a.load());
- ASSERT_NOEXCEPT(a = nullptr);
- ASSERT_NOEXCEPT(a = std::shared_ptr<T>(p));
+ static_assert(noexcept(a = nullptr));
+ static_assert(noexcept(a = std::shared_ptr<T>(p)));
}
+template <class T>
+struct TestAssign {
+ void operator()() const { test_assign<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_ASSIGN(T) test_assign<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_ASSIGN)
-#undef LIBCXX_ATOMIC_SP_RUN_ASSIGN
+ ForEachSmartPtrType{}.template operator()<TestAssign>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
index 5ceca3efad536..bb10b75746c2d 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
@@ -21,7 +21,6 @@
template <class T>
void test_compare_exchange_strong() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto p1 = SpValues<T>::state_a();
auto p2 = SpValues<T>::state_b();
std::atomic<std::shared_ptr<T>> a((std::shared_ptr<T>(p1)));
@@ -32,7 +31,7 @@ void test_compare_exchange_strong() {
assert(ok);
assert(expected.get() == p1.get());
assert(*a.load() == *p2);
- ASSERT_NOEXCEPT(a.compare_exchange_strong(expected, std::shared_ptr<T>(p2)));
+ static_assert(noexcept(a.compare_exchange_strong(expected, std::shared_ptr<T>(p2))));
}
{
@@ -54,9 +53,12 @@ void test_compare_exchange_strong() {
}
}
+template <class T>
+struct TestCompareExchangeStrong {
+ void operator()() const { test_compare_exchange_strong<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_CXS(T) test_compare_exchange_strong<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_CXS)
-#undef LIBCXX_ATOMIC_SP_RUN_CXS
+ ForEachSmartPtrType{}.template operator()<TestCompareExchangeStrong>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_weak.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_weak.pass.cpp
index 713d04b7f5c47..73de9af913df8 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_weak.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_weak.pass.cpp
@@ -21,7 +21,6 @@
template <class T>
void test_compare_exchange_weak() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto p1 = SpValues<T>::state_a();
auto p2 = SpValues<T>::state_b();
std::atomic<std::shared_ptr<T>> a((std::shared_ptr<T>(p1)));
@@ -33,7 +32,7 @@ void test_compare_exchange_weak() {
ok = a.compare_exchange_weak(expected, std::shared_ptr<T>(p2));
assert(ok);
assert(*a.load() == *p2);
- ASSERT_NOEXCEPT(a.compare_exchange_weak(expected, std::shared_ptr<T>(p2)));
+ static_assert(noexcept(a.compare_exchange_weak(expected, std::shared_ptr<T>(p2))));
}
{
@@ -56,9 +55,12 @@ void test_compare_exchange_weak() {
}
}
+template <class T>
+struct TestCompareExchangeWeak {
+ void operator()() const { test_compare_exchange_weak<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_CXW(T) test_compare_exchange_weak<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_CXW)
-#undef LIBCXX_ATOMIC_SP_RUN_CXW
+ ForEachSmartPtrType{}.template operator()<TestCompareExchangeWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
index 4c219ae175d18..85de5e07b7058 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
@@ -20,7 +20,7 @@
template <class T>
void test_convert() {
- auto p = libcxx_atomic_smart_ptr_test::SpValues<T>::state_a();
+ auto p = SpValues<T>::state_a();
const std::atomic<std::shared_ptr<T>> a((std::shared_ptr<T>(p)));
std::shared_ptr<T> s = a;
@@ -30,12 +30,15 @@ void test_convert() {
std::same_as<std::shared_ptr<T>> decltype(auto) s2 = static_cast<std::shared_ptr<T>>(a);
assert(s2.get() == p.get());
- ASSERT_NOEXCEPT(static_cast<std::shared_ptr<T>>(a));
+ static_assert(noexcept(static_cast<std::shared_ptr<T>>(a)));
}
+template <class T>
+struct TestConvert {
+ void operator()() const { test_convert<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_CONVERT(T) test_convert<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_CONVERT)
-#undef LIBCXX_ATOMIC_SP_RUN_CONVERT
+ ForEachSmartPtrType{}.template operator()<TestConvert>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
index fa957d8ff7ebe..5c5339389ac7b 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
@@ -20,7 +20,6 @@
template <class T>
void test_exchange() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto p1 = SpValues<T>::state_a();
auto p2 = SpValues<T>::state_b();
std::atomic<std::shared_ptr<T>> a((std::shared_ptr<T>(p1)));
@@ -34,12 +33,15 @@ void test_exchange() {
assert(*out == *p2);
assert(!a.load());
- ASSERT_NOEXCEPT(a.exchange(nullptr));
+ static_assert(noexcept(a.exchange(nullptr)));
}
+template <class T>
+struct TestExchange {
+ void operator()() const { test_exchange<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_EXCHANGE(T) test_exchange<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_EXCHANGE)
-#undef LIBCXX_ATOMIC_SP_RUN_EXCHANGE
+ ForEachSmartPtrType{}.template operator()<TestExchange>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
similarity index 77%
rename from libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.compile.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
index b7ff230832d09..879fd1f204ceb 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
@@ -28,13 +28,16 @@ void check() {
const A asp;
std::same_as<bool> decltype(auto) lf = asp.is_lock_free();
(void)lf;
- ASSERT_NOEXCEPT(A::is_always_lock_free);
- ASSERT_NOEXCEPT(asp.is_lock_free());
+ static_assert(noexcept(A::is_always_lock_free));
+ static_assert(noexcept(asp.is_lock_free()));
}
+template <class T>
+struct TestIsAlwaysLockFreeShared {
+ void operator()() const { check<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_IS_ALWAYS_LF(T) check<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_IS_ALWAYS_LF)
-#undef LIBCXX_ATOMIC_SP_RUN_IS_ALWAYS_LF
+ ForEachSmartPtrType{}.template operator()<TestIsAlwaysLockFreeShared>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
similarity index 68%
rename from libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.compile.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
index 48bcb81ab6cce..f6271923d3db8 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
@@ -20,25 +20,27 @@
template <class T>
void check(const std::atomic<std::shared_ptr<T>>& asp) {
std::same_as<std::shared_ptr<T>> decltype(auto) no_arg = asp.load();
- ASSERT_SAME_TYPE(decltype(asp.load()), std::shared_ptr<T>);
- ASSERT_NOEXCEPT(asp.load());
+ static_assert(noexcept(asp.load()));
std::same_as<std::shared_ptr<T>> decltype(auto) with_order = asp.load(std::memory_order_seq_cst);
- ASSERT_SAME_TYPE(decltype(asp.load(std::memory_order_acquire)), std::shared_ptr<T>);
- ASSERT_NOEXCEPT(asp.load(std::memory_order_seq_cst));
+ static_assert(noexcept(asp.load(std::memory_order_seq_cst)));
static_cast<void>(no_arg);
static_cast<void>(with_order);
{
const std::atomic<std::shared_ptr<T>> const_a;
static_assert(noexcept(const_a.load()));
- ASSERT_SAME_TYPE(decltype(const_a.load()), std::shared_ptr<T>);
+ std::same_as<std::shared_ptr<T>> decltype(auto) loaded = const_a.load();
+ (void)loaded;
}
}
+template <class T>
+struct TestLoadShared {
+ void operator()() const { check<T>(std::atomic<std::shared_ptr<T>>()); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_LOAD(T) check<T>(std::atomic<std::shared_ptr<T>>());
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_LOAD)
-#undef LIBCXX_ATOMIC_SP_RUN_LOAD
+ ForEachSmartPtrType{}.template operator()<TestLoadShared>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_all.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_all.pass.cpp
index 5d4c9b7714b23..6c9cef0eceb5a 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_all.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_all.pass.cpp
@@ -21,10 +21,9 @@
template <class T>
void test_notify_all() {
- using libcxx_atomic_smart_ptr_test::SpValues;
std::atomic<std::shared_ptr<T>> a;
- ASSERT_NOEXCEPT(a.notify_all());
+ static_assert(noexcept(a.notify_all()));
#if __cpp_lib_atomic_wait >= 201907L
auto p1 = SpValues<T>::state_a();
@@ -59,9 +58,12 @@ void test_notify_all() {
#endif
}
+template <class T>
+struct TestNotifyAll {
+ void operator()() const { test_notify_all<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_NOTIFY_ALL(T) test_notify_all<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_NOTIFY_ALL)
-#undef LIBCXX_ATOMIC_SP_RUN_NOTIFY_ALL
+ ForEachSmartPtrType{}.template operator()<TestNotifyAll>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_one.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_one.pass.cpp
index b178fb13d1a0f..8d9d3a3ecdcc8 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_one.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/notify_one.pass.cpp
@@ -20,10 +20,9 @@
template <class T>
void test_notify_one() {
- using libcxx_atomic_smart_ptr_test::SpValues;
std::atomic<std::shared_ptr<T>> a;
- ASSERT_NOEXCEPT(a.notify_one());
+ static_assert(noexcept(a.notify_one()));
#if __cpp_lib_atomic_wait >= 201907L
auto p1 = SpValues<T>::state_a();
@@ -46,9 +45,12 @@ void test_notify_one() {
#endif
}
+template <class T>
+struct TestNotifyOne {
+ void operator()() const { test_notify_one<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_NOTIFY_ONE(T) test_notify_one<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_NOTIFY_ONE)
-#undef LIBCXX_ATOMIC_SP_RUN_NOTIFY_ONE
+ ForEachSmartPtrType{}.template operator()<TestNotifyOne>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/store.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/store.pass.cpp
index 9ab808cbd0fae..51f1df0f2a2d9 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/store.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/store.pass.cpp
@@ -20,7 +20,6 @@
template <class T>
void test_store() {
- using libcxx_atomic_smart_ptr_test::SpValues;
std::atomic<std::shared_ptr<T>> a;
auto p = SpValues<T>::state_a();
@@ -34,16 +33,19 @@ void test_store() {
a.store(nullptr, std::memory_order_relaxed);
assert(!a.load());
- ASSERT_NOEXCEPT(a.store(nullptr));
+ static_assert(noexcept(a.store(nullptr)));
{
std::shared_ptr<T> desired = std::make_shared<T>(*SpValues<T>::state_c());
- ASSERT_NOEXCEPT(a.store(std::move(desired), std::memory_order_seq_cst));
+ static_assert(noexcept(a.store(std::move(desired), std::memory_order_seq_cst)));
}
}
+template <class T>
+struct TestStore {
+ void operator()() const { test_store<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_STORE(T) test_store<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_STORE)
-#undef LIBCXX_ATOMIC_SP_RUN_STORE
+ ForEachSmartPtrType{}.template operator()<TestStore>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.pass.cpp
similarity index 88%
rename from libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.pass.cpp
index bc8ee81abe2b1..563c383d76177 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/types.pass.cpp
@@ -45,17 +45,20 @@ void traits() {
}
{
- auto p = libcxx_atomic_smart_ptr_test::SpValues<T>::state_a();
+ auto p = SpValues<T>::state_a();
A a((std::shared_ptr<T>(p)));
assert(a.load().get() == p.get());
assert(*a.load() == *p);
}
}
+template <class T>
+struct TestTraitsShared {
+ void operator()() const { traits<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_TRAITS(T) traits<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_TRAITS)
-#undef LIBCXX_ATOMIC_SP_RUN_TRAITS
+ ForEachSmartPtrType{}.template operator()<TestTraitsShared>();
{
static_assert(std::is_nothrow_constructible_v<std::atomic<std::shared_ptr<int>>, std::nullptr_t>);
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/wait.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/wait.pass.cpp
index fef98a7008c27..fc9600688e039 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/wait.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/wait.pass.cpp
@@ -21,7 +21,6 @@
template <class T>
void test_wait() {
- using libcxx_atomic_smart_ptr_test::SpValues;
std::atomic<std::shared_ptr<T>> a;
#if __cpp_lib_atomic_wait >= 201907L
@@ -47,12 +46,15 @@ void test_wait() {
assert(*a.load() == *SpValues<T>::state_c());
#endif
- ASSERT_NOEXCEPT(a.wait(std::shared_ptr<T>(), std::memory_order_seq_cst));
+ static_assert(noexcept(a.wait(std::shared_ptr<T>(), std::memory_order_seq_cst)));
}
+template <class T>
+struct TestWait {
+ void operator()() const { test_wait<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_WAIT(T) test_wait<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_WAIT)
-#undef LIBCXX_ATOMIC_SP_RUN_WAIT
+ ForEachSmartPtrType{}.template operator()<TestWait>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/assign.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/assign.pass.cpp
index efe373d1d8036..21d182ba32de4 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/assign.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/assign.pass.cpp
@@ -20,7 +20,6 @@
template <class T>
void test_assign_weak() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto sp1 = SpValues<T>::state_a();
auto sp2 = SpValues<T>::state_b();
std::weak_ptr<T> wp1 = sp1;
@@ -39,12 +38,15 @@ void test_assign_weak() {
assert(locked && *locked == *sp2);
}
- ASSERT_NOEXCEPT(a = std::weak_ptr<T>(wp1));
+ static_assert(noexcept(a = std::weak_ptr<T>(wp1)));
}
+template <class T>
+struct TestAssignWeak {
+ void operator()() const { test_assign_weak<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_ASSIGN(T) test_assign_weak<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_ASSIGN)
-#undef LIBCXX_ATOMIC_SP_RUN_W_ASSIGN
+ ForEachSmartPtrType{}.template operator()<TestAssignWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
index 61047d17d264b..9d225ae0ba3f5 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
@@ -21,7 +21,6 @@
template <class T>
void test_compare_exchange_strong_weakptr() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto sp1 = SpValues<T>::state_a();
auto sp2 = SpValues<T>::state_b();
std::weak_ptr<T> wp1 = sp1;
@@ -37,7 +36,7 @@ void test_compare_exchange_strong_weakptr() {
auto locked = a.load().lock();
assert(locked && *locked == *sp2);
}
- ASSERT_NOEXCEPT(a.compare_exchange_strong(expected, std::weak_ptr<T>(wp2)));
+ static_assert(noexcept(a.compare_exchange_strong(expected, std::weak_ptr<T>(wp2))));
}
{
@@ -59,9 +58,12 @@ void test_compare_exchange_strong_weakptr() {
}
}
+template <class T>
+struct TestCompareExchangeStrongWeak {
+ void operator()() const { test_compare_exchange_strong_weakptr<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_CXS(T) test_compare_exchange_strong_weakptr<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_CXS)
-#undef LIBCXX_ATOMIC_SP_RUN_W_CXS
+ ForEachSmartPtrType{}.template operator()<TestCompareExchangeStrongWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_weak.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_weak.pass.cpp
index f44450b46476c..028b40fd13f50 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_weak.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_weak.pass.cpp
@@ -21,7 +21,6 @@
template <class T>
void test_compare_exchange_weak_weakptr() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto sp1 = SpValues<T>::state_a();
auto sp2 = SpValues<T>::state_b();
std::weak_ptr<T> wp1 = sp1;
@@ -37,7 +36,7 @@ void test_compare_exchange_weak_weakptr() {
assert(ok);
auto locked = a.load().lock();
assert(locked && *locked == *sp2);
- ASSERT_NOEXCEPT(a.compare_exchange_weak(expected, std::weak_ptr<T>(wp2)));
+ static_assert(noexcept(a.compare_exchange_weak(expected, std::weak_ptr<T>(wp2))));
}
{
@@ -61,9 +60,12 @@ void test_compare_exchange_weak_weakptr() {
}
}
+template <class T>
+struct TestCompareExchangeWeakWeak {
+ void operator()() const { test_compare_exchange_weak_weakptr<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_CXW(T) test_compare_exchange_weak_weakptr<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_CXW)
-#undef LIBCXX_ATOMIC_SP_RUN_W_CXW
+ ForEachSmartPtrType{}.template operator()<TestCompareExchangeWeakWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
index c74d5f4925641..110491bca6b09 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
@@ -20,7 +20,6 @@
template <class T>
void test_convert_weak() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto sp = SpValues<T>::state_a();
std::weak_ptr<T> wp = sp;
const std::atomic<std::weak_ptr<T>> a((std::weak_ptr<T>(wp)));
@@ -33,12 +32,15 @@ void test_convert_weak() {
auto locked2 = w2.lock();
assert(locked2 && *locked2 == *sp);
- ASSERT_NOEXCEPT(static_cast<std::weak_ptr<T>>(a));
+ static_assert(noexcept(static_cast<std::weak_ptr<T>>(a)));
}
+template <class T>
+struct TestConvertWeak {
+ void operator()() const { test_convert_weak<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_CONVERT(T) test_convert_weak<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_CONVERT)
-#undef LIBCXX_ATOMIC_SP_RUN_W_CONVERT
+ ForEachSmartPtrType{}.template operator()<TestConvertWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
index 2c777efaabb5b..de4677fc1c166 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
@@ -20,7 +20,6 @@
template <class T>
void test_exchange_weak() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto sp1 = SpValues<T>::state_a();
auto sp2 = SpValues<T>::state_b();
std::weak_ptr<T> wp1 = sp1;
@@ -46,12 +45,15 @@ void test_exchange_weak() {
}
assert(a.load().expired());
- ASSERT_NOEXCEPT(a.exchange(std::weak_ptr<T>(wp1)));
+ static_assert(noexcept(a.exchange(std::weak_ptr<T>(wp1))));
}
+template <class T>
+struct TestExchangeWeak {
+ void operator()() const { test_exchange_weak<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_EXCHANGE(T) test_exchange_weak<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_EXCHANGE)
-#undef LIBCXX_ATOMIC_SP_RUN_W_EXCHANGE
+ ForEachSmartPtrType{}.template operator()<TestExchangeWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
similarity index 78%
rename from libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.compile.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
index ceb3dbce26c29..2f8a679f21fab 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
@@ -28,13 +28,16 @@ void check() {
const A awp;
std::same_as<bool> decltype(auto) lf = awp.is_lock_free();
(void)lf;
- ASSERT_NOEXCEPT(A::is_always_lock_free);
- ASSERT_NOEXCEPT(awp.is_lock_free());
+ static_assert(noexcept(A::is_always_lock_free));
+ static_assert(noexcept(awp.is_lock_free()));
}
+template <class T>
+struct TestIsAlwaysLockFreeWeak {
+ void operator()() const { check<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_IALF(T) check<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_IALF)
-#undef LIBCXX_ATOMIC_SP_RUN_W_IALF
+ ForEachSmartPtrType{}.template operator()<TestIsAlwaysLockFreeWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
similarity index 75%
rename from libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.compile.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
index 55421abb2026b..15c9b81f34e80 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
@@ -22,13 +22,15 @@ void check(const std::atomic<std::weak_ptr<T>>& awp) noexcept {
static_assert(!std::atomic<std::weak_ptr<T>>::is_always_lock_free);
std::same_as<bool> decltype(auto) lf = awp.is_lock_free();
(void)lf;
- ASSERT_SAME_TYPE(decltype(awp.is_lock_free()), bool);
- ASSERT_NOEXCEPT(awp.is_lock_free());
+ static_assert(noexcept(awp.is_lock_free()));
}
+template <class T>
+struct TestIsLockFreeWeak {
+ void operator()() const { check<T>(std::atomic<std::weak_ptr<T>>()); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_ILF(T) check<T>(std::atomic<std::weak_ptr<T>>());
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_ILF)
-#undef LIBCXX_ATOMIC_SP_RUN_W_ILF
+ ForEachSmartPtrType{}.template operator()<TestIsLockFreeWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.pass.cpp
similarity index 68%
rename from libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.compile.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.pass.cpp
index b372c94cd1597..84206c5832636 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.pass.cpp
@@ -20,25 +20,27 @@
template <class T>
void check(const std::atomic<std::weak_ptr<T>>& awp) {
std::same_as<std::weak_ptr<T>> decltype(auto) no_arg = awp.load();
- ASSERT_SAME_TYPE(decltype(awp.load()), std::weak_ptr<T>);
- ASSERT_NOEXCEPT(awp.load());
+ static_assert(noexcept(awp.load()));
std::same_as<std::weak_ptr<T>> decltype(auto) with_order = awp.load(std::memory_order_seq_cst);
- ASSERT_SAME_TYPE(decltype(awp.load(std::memory_order_acquire)), std::weak_ptr<T>);
- ASSERT_NOEXCEPT(awp.load(std::memory_order_seq_cst));
+ static_assert(noexcept(awp.load(std::memory_order_seq_cst)));
static_cast<void>(no_arg);
static_cast<void>(with_order);
{
const std::atomic<std::weak_ptr<T>> const_a;
static_assert(noexcept(const_a.load()));
- ASSERT_SAME_TYPE(decltype(const_a.load()), std::weak_ptr<T>);
+ std::same_as<std::weak_ptr<T>> decltype(auto) loaded = const_a.load();
+ (void)loaded;
}
}
+template <class T>
+struct TestLoadWeak {
+ void operator()() const { check<T>(std::atomic<std::weak_ptr<T>>()); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_LOAD(T) check<T>(std::atomic<std::weak_ptr<T>>());
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_LOAD)
-#undef LIBCXX_ATOMIC_SP_RUN_W_LOAD
+ ForEachSmartPtrType{}.template operator()<TestLoadWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_all.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_all.pass.cpp
index 3211c651143f3..29e7f78b275db 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_all.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_all.pass.cpp
@@ -21,10 +21,9 @@
template <class T>
void test_notify_all_weak() {
- using libcxx_atomic_smart_ptr_test::SpValues;
std::atomic<std::weak_ptr<T>> a;
- ASSERT_NOEXCEPT(a.notify_all());
+ static_assert(noexcept(a.notify_all()));
#if __cpp_lib_atomic_wait >= 201907L
auto sp1 = SpValues<T>::state_a();
@@ -61,9 +60,12 @@ void test_notify_all_weak() {
#endif
}
+template <class T>
+struct TestNotifyAllWeak {
+ void operator()() const { test_notify_all_weak<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_NA(T) test_notify_all_weak<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_NA)
-#undef LIBCXX_ATOMIC_SP_RUN_W_NA
+ ForEachSmartPtrType{}.template operator()<TestNotifyAllWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_one.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_one.pass.cpp
index 440aab1d3b5e7..58fa25e9dda7f 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_one.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/notify_one.pass.cpp
@@ -21,10 +21,9 @@
template <class T>
void test_notify_one_weak() {
- using libcxx_atomic_smart_ptr_test::SpValues;
std::atomic<std::weak_ptr<T>> a;
- ASSERT_NOEXCEPT(a.notify_one());
+ static_assert(noexcept(a.notify_one()));
#if __cpp_lib_atomic_wait >= 201907L
auto sp1 = SpValues<T>::state_a();
@@ -49,9 +48,12 @@ void test_notify_one_weak() {
#endif
}
+template <class T>
+struct TestNotifyOneWeak {
+ void operator()() const { test_notify_one_weak<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_N1(T) test_notify_one_weak<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_N1)
-#undef LIBCXX_ATOMIC_SP_RUN_W_N1
+ ForEachSmartPtrType{}.template operator()<TestNotifyOneWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/store.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/store.pass.cpp
index 35baa907c1437..1f50b44b84116 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/store.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/store.pass.cpp
@@ -20,7 +20,6 @@
template <class T>
void test_store_weak() {
- using libcxx_atomic_smart_ptr_test::SpValues;
auto sp1 = SpValues<T>::state_a();
auto sp2 = SpValues<T>::state_b();
std::weak_ptr<T> wp1 = sp1;
@@ -39,12 +38,15 @@ void test_store_weak() {
assert(locked && *locked == *sp2);
}
- ASSERT_NOEXCEPT(a.store(std::weak_ptr<T>(wp1)));
+ static_assert(noexcept(a.store(std::weak_ptr<T>(wp1))));
}
+template <class T>
+struct TestStoreWeak {
+ void operator()() const { test_store_weak<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_STORE(T) test_store_weak<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_STORE)
-#undef LIBCXX_ATOMIC_SP_RUN_W_STORE
+ ForEachSmartPtrType{}.template operator()<TestStoreWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.pass.cpp
similarity index 86%
rename from libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.pass.cpp
index 40e0f4d5e38d6..4c1a60ea2e78d 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.compile.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/types.pass.cpp
@@ -46,7 +46,7 @@ void traits() {
}
{
- auto sp = libcxx_atomic_smart_ptr_test::SpValues<T>::state_a();
+ auto sp = SpValues<T>::state_a();
std::weak_ptr<T> wp = sp;
A a((std::weak_ptr<T>(wp)));
auto locked = a.load().lock();
@@ -54,9 +54,12 @@ void traits() {
}
}
+template <class T>
+struct TestTraitsWeak {
+ void operator()() const { traits<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_TRAITS(T) traits<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_TRAITS)
-#undef LIBCXX_ATOMIC_SP_RUN_W_TRAITS
+ ForEachSmartPtrType{}.template operator()<TestTraitsWeak>();
return 0;
}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/wait.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/wait.pass.cpp
index f840c4ec00d32..20b6474121f07 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/wait.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/wait.pass.cpp
@@ -21,7 +21,6 @@
template <class T>
void test_wait_weak() {
- using libcxx_atomic_smart_ptr_test::SpValues;
std::atomic<std::weak_ptr<T>> a;
#if __cpp_lib_atomic_wait >= 201907L
@@ -53,12 +52,15 @@ void test_wait_weak() {
}
#endif
- ASSERT_NOEXCEPT(a.wait(std::weak_ptr<T>(), std::memory_order_seq_cst));
+ static_assert(noexcept(a.wait(std::weak_ptr<T>(), std::memory_order_seq_cst)));
}
+template <class T>
+struct TestWaitWeak {
+ void operator()() const { test_wait_weak<T>(); }
+};
+
int main(int, char**) {
-#define LIBCXX_ATOMIC_SP_RUN_W_WAIT(T) test_wait_weak<T>();
- LIBCXX_ATOMIC_SP_FOR_ALL_RUNTIME_TYPES(LIBCXX_ATOMIC_SP_RUN_W_WAIT)
-#undef LIBCXX_ATOMIC_SP_RUN_W_WAIT
+ ForEachSmartPtrType{}.template operator()<TestWaitWeak>();
return 0;
}
>From 9f1af1e17f8d0b41192c63c7806462a0bb964392 Mon Sep 17 00:00:00 2001
From: ViNN280801 <vladislav.semykin at gmail.com>
Date: Sat, 2 May 2026 21:21:38 +0300
Subject: [PATCH 12/46] [libcxx] Included <concepts> for Armv7 and Armv8
(including -fno-exceptions); TODO: figure out why stress test on AIX (32 and
64 bit) is timeouting
Signed-off-by: ViNN280801 <vladislav.semykin at gmail.com>
---
.../utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp | 1 +
.../util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp | 1 +
.../memory/util.smartptr/atomic/shared/convert.pass.cpp | 1 +
.../memory/util.smartptr/atomic/shared/exchange.pass.cpp | 1 +
.../util.smartptr/atomic/shared/is_always_lock_free.pass.cpp | 1 +
.../utilities/memory/util.smartptr/atomic/shared/load.pass.cpp | 1 +
.../util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp | 1 +
.../utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp | 1 +
.../utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp | 1 +
.../util.smartptr/atomic/weak/is_always_lock_free.pass.cpp | 1 +
.../memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp | 1 +
.../std/utilities/memory/util.smartptr/atomic/weak/load.pass.cpp | 1 +
12 files changed, 12 insertions(+)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
index 574a89fababb3..16ccf9bac493b 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
@@ -16,6 +16,7 @@
// bool is_lock_free() const noexcept;
#include <atomic>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
index bb10b75746c2d..a94a9563839c8 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
@@ -14,6 +14,7 @@
#include <atomic>
#include <cassert>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
index 85de5e07b7058..7c5ddcea6c8c7 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
@@ -13,6 +13,7 @@
#include <atomic>
#include <cassert>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
index 5c5339389ac7b..39fa057141d92 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
@@ -13,6 +13,7 @@
#include <atomic>
#include <cassert>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
index 879fd1f204ceb..ac685fbf8bfc1 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
@@ -13,6 +13,7 @@
// bool is_lock_free() const noexcept;
#include <atomic>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
index f6271923d3db8..2942cfe6c1017 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
@@ -12,6 +12,7 @@
// shared_ptr<T> load(memory_order order = memory_order::seq_cst) const noexcept;
#include <atomic>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
index 9d225ae0ba3f5..8f282baef9bb2 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
@@ -14,6 +14,7 @@
#include <atomic>
#include <cassert>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
index 110491bca6b09..97d46023e4e37 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
@@ -13,6 +13,7 @@
#include <atomic>
#include <cassert>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
index de4677fc1c166..d553dbc34390a 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
@@ -13,6 +13,7 @@
#include <atomic>
#include <cassert>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
index 2f8a679f21fab..31b7fe877a92c 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
@@ -13,6 +13,7 @@
// bool is_lock_free() const noexcept;
#include <atomic>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
index 15c9b81f34e80..f65b24d55449d 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
@@ -12,6 +12,7 @@
// bool is_lock_free() const noexcept;
#include <atomic>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.pass.cpp
index 84206c5832636..73fb38efdf9ac 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/load.pass.cpp
@@ -12,6 +12,7 @@
// weak_ptr<T> load(memory_order order = memory_order::seq_cst) const noexcept;
#include <atomic>
+#include <concepts> // needed on Armv7/Armv8 with -fmodules
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From 7817a3b7bdaae473fd490a82c9c1a3294b871a90 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Thu, 28 May 2026 22:13:59 +0300
Subject: [PATCH 13/46] [libc++] The '__to_failure_order' function has been
moved to a separate file to avoid CE on gcc 15 and make it reusable; Used
macro '_LIBCPP_{BEGIN|END}_EXPLICIT_ABI_ANNOTATIONS' from #193045 to avoid CE
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/include/CMakeLists.txt | 1 +
libcxx/include/__atomic/atomic_sync_lite.h | 4 +++
libcxx/include/__atomic/support/c11.h | 8 +-----
libcxx/include/__atomic/to_failure_order.h | 30 +++++++++++++++++++++
libcxx/include/__memory/atomic_shared_ptr.h | 1 +
libcxx/include/__memory/shared_ptr.h | 8 +++---
libcxx/include/module.modulemap.in | 1 +
7 files changed, 42 insertions(+), 11 deletions(-)
create mode 100644 libcxx/include/__atomic/to_failure_order.h
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 48968128e47bd..0c7278c30330e 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -229,6 +229,7 @@ set(files
__atomic/support.h
__atomic/support/c11.h
__atomic/support/gcc.h
+ __atomic/to_failure_order.h
__atomic/to_gcc_order.h
__bit/bit_cast.h
__bit/bit_ceil.h
diff --git a/libcxx/include/__atomic/atomic_sync_lite.h b/libcxx/include/__atomic/atomic_sync_lite.h
index a0433e16b506a..8354d12bdea7a 100644
--- a/libcxx/include/__atomic/atomic_sync_lite.h
+++ b/libcxx/include/__atomic/atomic_sync_lite.h
@@ -20,6 +20,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
+_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
+
# if !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
// Old dylib interface kept for backwards compatibility.
_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
@@ -38,6 +40,8 @@ __atomic_wait_global_table(void const* __address, __cxx_contention_t __monitor_v
_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const*) _NOEXCEPT;
_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const*) _NOEXCEPT;
+_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
+
#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__atomic/support/c11.h b/libcxx/include/__atomic/support/c11.h
index 1ad299882a12a..84db7accc8c73 100644
--- a/libcxx/include/__atomic/support/c11.h
+++ b/libcxx/include/__atomic/support/c11.h
@@ -10,6 +10,7 @@
#define _LIBCPP___ATOMIC_SUPPORT_C11_H
#include <__atomic/memory_order.h>
+#include <__atomic/to_failure_order.h>
#include <__config>
#include <__cstddef/ptrdiff_t.h>
#include <__memory/addressof.h>
@@ -110,13 +111,6 @@ __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_orde
std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
}
-_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
- // Avoid switch statement to make this a constexpr.
- return __order == memory_order_release
- ? memory_order_relaxed
- : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
-}
-
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
__cxx_atomic_base_impl<_Tp> volatile* __a,
diff --git a/libcxx/include/__atomic/to_failure_order.h b/libcxx/include/__atomic/to_failure_order.h
new file mode 100644
index 0000000000000..7e7017ab5d8ac
--- /dev/null
+++ b/libcxx/include/__atomic/to_failure_order.h
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_TO_FAILURE_ORDER_H
+#define _LIBCPP___ATOMIC_TO_FAILURE_ORDER_H
+
+#include <__atomic/memory_order.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
+ // Avoid switch statement to make this a constexpr.
+ return __order == memory_order_release
+ ? memory_order_relaxed
+ : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_TO_FAILURE_ORDER_H
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index 1baa0060d3425..156885d9a688d 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -14,6 +14,7 @@
#include <__atomic/check_memory_order.h>
#include <__atomic/memory_order.h>
#include <__atomic/support.h>
+#include <__atomic/to_failure_order.h>
#include <__config>
#include <__cstddef/nullptr_t.h>
#include <__memory/shared_count.h>
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 45a82abebbb15..7466a2ceb620a 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -733,8 +733,8 @@ struct __unbounded_array_control_block<_Tp[], _Alloc> : __shared_weak_count {
// for the whole allocation to be a multiple of _Tp's alignment. That formula is taken from [1].
//
// [1]: https://en.wikipedia.org/wiki/Data_structure_alignment#Computing_padding
- size_t __bytes = __elements == 0 ? sizeof(__unbounded_array_control_block)
- : (__elements - 1) * sizeof(_Tp) + sizeof(__unbounded_array_control_block);
+ size_t __bytes = __elements == 0 ? sizeof(__unbounded_array_control_block)
+ : (__elements - 1) * sizeof(_Tp) + sizeof(__unbounded_array_control_block);
constexpr size_t __align = alignof(__unbounded_array_control_block);
return (__bytes + __align - 1) & ~(__align - 1);
}
@@ -1081,8 +1081,8 @@ template <class _Tp, class _Up>
#endif
template <class _Tp, class _Up>
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
-dynamic_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
+[[__nodiscard__]] inline
+ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
typedef typename shared_ptr<_Tp>::element_type _ET;
_ET* __p = dynamic_cast<_ET*>(__r.get());
return __p ? shared_ptr<_Tp>(__r, __p) : shared_ptr<_Tp>();
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 704af8af9a5e2..3238146851ede 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -888,6 +888,7 @@ module std [system] {
module is_always_lock_free { header "__atomic/is_always_lock_free.h" }
module kill_dependency { header "__atomic/kill_dependency.h" }
module memory_order { header "__atomic/memory_order.h" }
+ module to_failure_order { header "__atomic/to_failure_order.h" }
module to_gcc_order { header "__atomic/to_gcc_order.h" }
module support {
>From fdad7d0ab977a656d8bad32245f7589675492b26 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Sat, 30 May 2026 18:39:48 +0300
Subject: [PATCH 14/46] [libc++] Implemented lock-free DWCAS-based approach
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/include/CMakeLists.txt | 2 +
libcxx/include/__memory/atomic_shared_ptr.h | 481 +-----------
.../__memory/atomic_shared_ptr_lock_based.h | 491 ++++++++++++
.../__memory/atomic_shared_ptr_lock_free.h | 710 ++++++++++++++++++
libcxx/include/module.modulemap.in | 2 +
.../atomic_shared_ptr_dwcas.pass.cpp | 31 +
6 files changed, 1250 insertions(+), 467 deletions(-)
create mode 100644 libcxx/include/__memory/atomic_shared_ptr_lock_based.h
create mode 100644 libcxx/include/__memory/atomic_shared_ptr_lock_free.h
create mode 100644 libcxx/test/libcxx/utilities/memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 0c7278c30330e..cef420a25790c 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -599,6 +599,8 @@ set(files
__memory/array_cookie.h
__memory/assume_aligned.h
__memory/atomic_shared_ptr.h
+ __memory/atomic_shared_ptr_lock_based.h
+ __memory/atomic_shared_ptr_lock_free.h
__memory/auto_ptr.h
__memory/compressed_pair.h
__memory/concepts.h
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index 156885d9a688d..273b8da9140c8 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -10,483 +10,30 @@
#ifndef _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_H
#define _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_H
-#include <__atomic/atomic_sync_lite.h>
-#include <__atomic/check_memory_order.h>
-#include <__atomic/memory_order.h>
-#include <__atomic/support.h>
-#include <__atomic/to_failure_order.h>
#include <__config>
-#include <__cstddef/nullptr_t.h>
-#include <__memory/shared_count.h>
-#include <__utility/move.h>
-
-#include <cstdint>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif
-#if defined(__SANITIZE_THREAD__) || (defined(__has_feature) && __has_feature(thread_sanitizer))
-# if __has_include(<sanitizer/tsan_interface.h>)
-# include <sanitizer/tsan_interface.h>
-# define _LIBCPP_ATOMIC_SHARED_PTR_TSAN 1
+// The lock-free path requires a hardware 16-byte CAS to update both fields of
+// the (ptr, ctrl) pair as one linearizable step; without it the spinlock path
+// is the only correct option. is_always_lock_free stays false because the same
+// source built without cx16/LSE flags must still see a non-lock-free type.
+#if !defined(_LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR)
+# if _LIBCPP_HAS_THREADS && _LIBCPP_HAS_INT128 && \
+ ((defined(__x86_64__) && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)) || \
+ (defined(__aarch64__) && defined(__ARM_FEATURE_ATOMICS)))
+# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 1
+# else
+# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 0
# endif
#endif
-_LIBCPP_PUSH_MACROS
-#include <__undef_macros>
-
-// TSAN annotations model the lock-bit protocol on __ctrl_.
-#if defined(_LIBCPP_ATOMIC_SHARED_PTR_TSAN)
-# define _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(addr) \
- ::__tsan_mutex_pre_lock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
-# define _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(addr) \
- ::__tsan_mutex_post_lock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0, 0)
-# define _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(addr) \
- ::__tsan_mutex_pre_unlock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
-# define _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(addr) \
- ::__tsan_mutex_post_unlock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
+#if _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR
+# include <__memory/atomic_shared_ptr_lock_free.h>
#else
-# define _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(addr) ((void)(addr))
-# define _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(addr) ((void)(addr))
-# define _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(addr) ((void)(addr))
-# define _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(addr) ((void)(addr))
+# include <__memory/atomic_shared_ptr_lock_based.h>
#endif
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
-
-template <class _Tp>
-class shared_ptr;
-
-template <class _Tp>
-class weak_ptr;
-
-template <class _Tp>
-struct atomic;
-
-// Split state into pointer word and control word.
-// The control word stores control-block pointer plus lock/notify bits.
-struct __atomic_smart_ptr_storage {
- static constexpr uintptr_t __lock_bit_ = uintptr_t{1};
- static constexpr uintptr_t __notify_bit_ = uintptr_t{2};
- static constexpr uintptr_t __ptr_mask_ = ~(__lock_bit_ | __notify_bit_);
-
- _LIBCPP_HIDE_FROM_ABI static uintptr_t __encode(__shared_weak_count* __ctrl, uintptr_t __bits) noexcept {
- return (reinterpret_cast<uintptr_t>(__ctrl) & __ptr_mask_) | (__bits & ~__ptr_mask_);
- }
-
- _LIBCPP_HIDE_FROM_ABI static __shared_weak_count* __decode(uintptr_t __word) noexcept {
- return reinterpret_cast<__shared_weak_count*>(__word & __ptr_mask_);
- }
-
- _LIBCPP_HIDE_FROM_ABI static bool __has_lock(uintptr_t __word) noexcept { return (__word & __lock_bit_) != 0; }
- _LIBCPP_HIDE_FROM_ABI static bool __has_notify(uintptr_t __word) noexcept { return (__word & __notify_bit_) != 0; }
-};
-
-_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_one(const void* __address) noexcept {
-# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
- std::__atomic_notify_one_global_table(__address);
-# else
- std::__cxx_atomic_notify_one(reinterpret_cast<void const volatile*>(__address));
-# endif
-}
-
-_LIBCPP_HIDE_FROM_ABI inline void __atomic_smart_ptr_notify_all(const void* __address) noexcept {
-# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
- std::__atomic_notify_all_global_table(__address);
-# else
- std::__cxx_atomic_notify_all(reinterpret_cast<void const volatile*>(__address));
-# endif
-}
-
-template <class _Poll>
-_LIBCPP_HIDE_FROM_ABI inline void
-__atomic_smart_ptr_wait_on_address(const void* __address, const _Poll& __poll) noexcept {
- while (!__poll()) {
-# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
- auto __monitor_value = std::__atomic_monitor_global(__address);
- if (__poll())
- return;
- std::__atomic_wait_global_table(__address, __monitor_value);
-# else
- void const volatile* __volatile_address = reinterpret_cast<void const volatile*>(__address);
- auto __monitor_value = std::__libcpp_atomic_monitor(__volatile_address);
- if (__poll())
- return;
- std::__libcpp_atomic_wait(__volatile_address, __monitor_value);
-# endif
- }
-}
-
-template <class _Element>
-struct __atomic_smart_ptr_fields {
- mutable __cxx_atomic_impl<_Element*> __ptr_;
- mutable __cxx_atomic_impl<uintptr_t> __ctrl_;
-
- _LIBCPP_HIDE_FROM_ABI __atomic_smart_ptr_fields(_Element* __p, __shared_weak_count* __c) noexcept
- : __ptr_(__p), __ctrl_(__atomic_smart_ptr_storage::__encode(__c, 0)) {}
-
- _LIBCPP_HIDE_FROM_ABI const void* __ctrl_address() const noexcept {
- return static_cast<const void*>(__builtin_addressof(__ctrl_));
- }
-
- // Acquire lock bit on __ctrl_. Contended path sets notify bit and waits.
- _LIBCPP_HIDE_FROM_ABI void __lock() const noexcept {
- _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(__builtin_addressof(__ctrl_));
- uintptr_t __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
- for (;;) {
- if (!__atomic_smart_ptr_storage::__has_lock(__expected)) {
- uintptr_t __desired = __expected | __atomic_smart_ptr_storage::__lock_bit_;
- if (std::__cxx_atomic_compare_exchange_weak(
- __builtin_addressof(__ctrl_),
- __builtin_addressof(__expected),
- __desired,
- memory_order_acquire,
- memory_order_relaxed)) {
- _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(__builtin_addressof(__ctrl_));
- return;
- }
- continue;
- }
-
- uintptr_t __with_notify = __expected | __atomic_smart_ptr_storage::__notify_bit_;
- if (!__atomic_smart_ptr_storage::__has_notify(__expected)) {
- if (!std::__cxx_atomic_compare_exchange_weak(
- __builtin_addressof(__ctrl_),
- __builtin_addressof(__expected),
- __with_notify,
- memory_order_relaxed,
- memory_order_relaxed))
- continue;
- __expected = __with_notify;
- }
-
- std::__atomic_smart_ptr_wait_on_address(__ctrl_address(), [&] {
- __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
- return !__atomic_smart_ptr_storage::__has_lock(__expected);
- });
- }
- }
-
- // Publish new control pointer, clear bits, and notify waiters if needed.
- _LIBCPP_HIDE_FROM_ABI void __unlock(__shared_weak_count* __ctrl_to_publish) const noexcept {
- _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(__builtin_addressof(__ctrl_));
- uintptr_t __new_word = __atomic_smart_ptr_storage::__encode(__ctrl_to_publish, 0);
- uintptr_t __previous = std::__cxx_atomic_exchange(__builtin_addressof(__ctrl_), __new_word, memory_order_release);
- if (__atomic_smart_ptr_storage::__has_notify(__previous))
- std::__atomic_smart_ptr_notify_all(__ctrl_address());
- _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(__builtin_addressof(__ctrl_));
- }
-};
-
-// [util.smartptr.atomic.shared]: same stored pointer and same ownership, or both empty.
-template <class _Element>
-_LIBCPP_HIDE_FROM_ABI inline bool __atomic_smart_ptr_equivalent(
- _Element* __ptr,
- __shared_weak_count* __ctrl,
- _Element* __expected_ptr,
- __shared_weak_count* __expected_ctrl) noexcept {
- if (__ctrl == nullptr && __expected_ctrl == nullptr)
- return true;
- return __ptr == __expected_ptr && __ctrl == __expected_ctrl;
-}
-
-template <class _Tp>
-struct atomic<shared_ptr<_Tp>> {
- using value_type = shared_ptr<_Tp>;
-
- static constexpr bool is_always_lock_free = false;
-
- _LIBCPP_HIDE_FROM_ABI atomic() noexcept : __fields_(nullptr, nullptr) {}
- _LIBCPP_HIDE_FROM_ABI constexpr atomic(nullptr_t) noexcept : __fields_(nullptr, nullptr) {}
- _LIBCPP_HIDE_FROM_ABI atomic(shared_ptr<_Tp> __desired) noexcept : __fields_(__desired.__ptr_, __desired.__cntrl_) {
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
- }
-
- atomic(const atomic&) = delete;
- atomic& operator=(const atomic&) = delete;
-
- _LIBCPP_HIDE_FROM_ABI ~atomic() {
- if (auto* __c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed)))
- __c->__release_shared();
- }
-
- [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return false; }
-
- _LIBCPP_HIDE_FROM_ABI void operator=(shared_ptr<_Tp> __desired) noexcept { store(std::move(__desired)); }
- _LIBCPP_HIDE_FROM_ABI void operator=(nullptr_t) noexcept { store(nullptr); }
- _LIBCPP_HIDE_FROM_ABI operator shared_ptr<_Tp>() const noexcept { return load(); }
-
- _LIBCPP_HIDE_FROM_ABI void store(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept
- _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
- (void)__m;
- _Tp* __desired_ptr = __desired.__ptr_;
- __shared_weak_count* __desired_c = __desired.__cntrl_;
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
-
- __fields_.__lock();
- __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
- std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
-
- if (__old_c)
- __old_c->__release_shared();
- }
-
- [[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
- _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
- (void)__m;
- __fields_.__lock();
- _Tp* __ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
- __shared_weak_count* __c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
- if (__c)
- __c->__add_shared();
- __fields_.__unlock(__c);
- return shared_ptr<_Tp>::__create_with_control_block(__ptr, __c);
- }
-
- _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
- exchange(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
- (void)__m;
- _Tp* __desired_ptr = __desired.__ptr_;
- __shared_weak_count* __desired_c = __desired.__cntrl_;
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
-
- __fields_.__lock();
- _Tp* __old_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
- __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
- std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
-
- return shared_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_c);
- }
-
- _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
- shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
- _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
- (void)__success;
- (void)__failure;
- __fields_.__lock();
- _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
- __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
-
- if (__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
- _Tp* __desired_ptr = __desired.__ptr_;
- __shared_weak_count* __desired_c = __desired.__cntrl_;
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
-
- std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
- if (__cur_c)
- __cur_c->__release_shared();
- return true;
- }
-
- if (__cur_c)
- __cur_c->__add_shared();
- __fields_.__unlock(__cur_c);
- __expected = shared_ptr<_Tp>::__create_with_control_block(__cur_ptr, __cur_c);
- return false;
- }
-
- _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
- shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
- return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
- }
-
- _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
- shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
- _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
- return compare_exchange_strong(__expected, std::move(__desired), __success, __failure);
- }
-
- _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
- shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
- return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
- }
-
- // Wait until the stored value is not equivalent to __old.
- // __ctrl_ is the wait address; pointer changes are published with control updates.
- _LIBCPP_HIDE_FROM_ABI void wait(shared_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const noexcept
- _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
- _Tp* __old_ptr = __old.__ptr_;
- __shared_weak_count* __old_c = __old.__cntrl_;
-
- std::__atomic_smart_ptr_wait_on_address(__fields_.__ctrl_address(), [&] {
- uintptr_t __word = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), __m);
- __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(__word);
- if (__cur_c != __old_c)
- return true;
- _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), __m);
- return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
- });
- }
-
- _LIBCPP_HIDE_FROM_ABI void notify_one() noexcept { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
- _LIBCPP_HIDE_FROM_ABI void notify_all() noexcept { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
-
-private:
- __atomic_smart_ptr_fields<_Tp> __fields_;
-};
-
-template <class _Tp>
-struct atomic<weak_ptr<_Tp>> {
- using value_type = weak_ptr<_Tp>;
-
- static constexpr bool is_always_lock_free = false;
-
- _LIBCPP_HIDE_FROM_ABI atomic() noexcept : __fields_(nullptr, nullptr) {}
- _LIBCPP_HIDE_FROM_ABI atomic(weak_ptr<_Tp> __desired) noexcept : __fields_(__desired.__ptr_, __desired.__cntrl_) {
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
- }
-
- atomic(const atomic&) = delete;
- atomic& operator=(const atomic&) = delete;
-
- _LIBCPP_HIDE_FROM_ABI ~atomic() {
- if (auto* __c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed)))
- __c->__release_weak();
- }
-
- [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return false; }
-
- _LIBCPP_HIDE_FROM_ABI void operator=(weak_ptr<_Tp> __desired) noexcept { store(std::move(__desired)); }
- _LIBCPP_HIDE_FROM_ABI operator weak_ptr<_Tp>() const noexcept { return load(); }
-
- _LIBCPP_HIDE_FROM_ABI void store(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept
- _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
- (void)__m;
- _Tp* __desired_ptr = __desired.__ptr_;
- __shared_weak_count* __desired_c = __desired.__cntrl_;
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
-
- __fields_.__lock();
- __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
- std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
-
- if (__old_c)
- __old_c->__release_weak();
- }
-
- [[nodiscard]] _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
- _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
- (void)__m;
- __fields_.__lock();
- _Tp* __ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
- __shared_weak_count* __c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
- if (__c)
- __c->__add_weak();
- __fields_.__unlock(__c);
- return weak_ptr<_Tp>::__create_with_control_block(__ptr, __c);
- }
-
- _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp>
- exchange(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
- (void)__m;
- _Tp* __desired_ptr = __desired.__ptr_;
- __shared_weak_count* __desired_c = __desired.__cntrl_;
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
-
- __fields_.__lock();
- _Tp* __old_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
- __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
- std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
-
- return weak_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_c);
- }
-
- _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
- weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
- _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
- (void)__success;
- (void)__failure;
- __fields_.__lock();
- _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
- __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(
- std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
-
- if (__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
- _Tp* __desired_ptr = __desired.__ptr_;
- __shared_weak_count* __desired_c = __desired.__cntrl_;
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
-
- std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
- if (__cur_c)
- __cur_c->__release_weak();
- return true;
- }
-
- if (__cur_c)
- __cur_c->__add_weak();
- __fields_.__unlock(__cur_c);
- __expected = weak_ptr<_Tp>::__create_with_control_block(__cur_ptr, __cur_c);
- return false;
- }
-
- _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(
- weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
- return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
- }
-
- _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
- weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
- _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
- return compare_exchange_strong(__expected, std::move(__desired), __success, __failure);
- }
-
- _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(
- weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
- return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
- }
-
- _LIBCPP_HIDE_FROM_ABI void wait(weak_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const noexcept
- _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
- _Tp* __old_ptr = __old.__ptr_;
- __shared_weak_count* __old_c = __old.__cntrl_;
-
- std::__atomic_smart_ptr_wait_on_address(__fields_.__ctrl_address(), [&] {
- uintptr_t __word = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), __m);
- __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(__word);
- if (__cur_c != __old_c)
- return true;
- _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), __m);
- return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
- });
- }
-
- _LIBCPP_HIDE_FROM_ABI void notify_one() noexcept { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
- _LIBCPP_HIDE_FROM_ABI void notify_all() noexcept { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
-
-private:
- __atomic_smart_ptr_fields<_Tp> __fields_;
-};
-
-#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
-
-_LIBCPP_END_NAMESPACE_STD
-
-_LIBCPP_POP_MACROS
-
#endif // _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_H
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_based.h b/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
new file mode 100644
index 0000000000000..e4584612b5009
--- /dev/null
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
@@ -0,0 +1,491 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_LOCK_BASED_H
+#define _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_LOCK_BASED_H
+
+#include <__atomic/atomic_sync_lite.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/memory_order.h>
+#include <__atomic/support.h>
+#include <__atomic/to_failure_order.h>
+#include <__config>
+#include <__cstddef/nullptr_t.h>
+#include <__memory/shared_count.h>
+#include <__utility/move.h>
+
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+#if defined(__SANITIZE_THREAD__) || (defined(__has_feature) && __has_feature(thread_sanitizer))
+# if __has_include(<sanitizer/tsan_interface.h>)
+# include <sanitizer/tsan_interface.h>
+# define _LIBCPP_ATOMIC_SHARED_PTR_TSAN 1
+# endif
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+// TSAN annotations model the lock-bit protocol on __ctrl_.
+#if defined(_LIBCPP_ATOMIC_SHARED_PTR_TSAN)
+# define _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(addr) \
+ ::__tsan_mutex_pre_lock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
+# define _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(addr) \
+ ::__tsan_mutex_post_lock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0, 0)
+# define _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(addr) \
+ ::__tsan_mutex_pre_unlock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
+# define _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(addr) \
+ ::__tsan_mutex_post_unlock(reinterpret_cast<void*>(const_cast<__cxx_atomic_impl<uintptr_t>*>(addr)), 0)
+#else
+# define _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(addr) ((void)(addr))
+# define _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(addr) ((void)(addr))
+# define _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(addr) ((void)(addr))
+# define _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(addr) ((void)(addr))
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+
+template <class _Tp>
+class shared_ptr;
+
+template <class _Tp>
+class weak_ptr;
+
+template <class _Tp>
+struct atomic;
+
+// CB is at least 4-byte aligned, so bits 0..1 of its pointer are always zero
+// and can be stolen for the spinlock and notify bit without aliasing the address.
+struct __atomic_smart_ptr_storage {
+ static constexpr uintptr_t __lock_bit_ = uintptr_t{1};
+ static constexpr uintptr_t __notify_bit_ = uintptr_t{2};
+ static constexpr uintptr_t __ptr_mask_ = ~(__lock_bit_ | __notify_bit_);
+
+ static uintptr_t __encode(__shared_weak_count* __ctrl, uintptr_t __bits) noexcept {
+ return (reinterpret_cast<uintptr_t>(__ctrl) & __ptr_mask_) | (__bits & ~__ptr_mask_);
+ }
+
+ static __shared_weak_count* __decode(uintptr_t __word) noexcept {
+ return reinterpret_cast<__shared_weak_count*>(__word & __ptr_mask_);
+ }
+
+ static bool __has_lock(uintptr_t __word) noexcept { return (__word & __lock_bit_) != 0; }
+ static bool __has_notify(uintptr_t __word) noexcept { return (__word & __notify_bit_) != 0; }
+};
+
+inline void __atomic_smart_ptr_notify_one(const void* __address) noexcept {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ std::__atomic_notify_one_global_table(__address);
+# else
+ std::__cxx_atomic_notify_one(reinterpret_cast<void const volatile*>(__address));
+# endif
+}
+
+inline void __atomic_smart_ptr_notify_all(const void* __address) noexcept {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ std::__atomic_notify_all_global_table(__address);
+# else
+ std::__cxx_atomic_notify_all(reinterpret_cast<void const volatile*>(__address));
+# endif
+}
+
+template <class _Poll>
+inline void __atomic_smart_ptr_wait_on_address(const void* __address, const _Poll& __poll) noexcept {
+ while (!__poll()) {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ auto __monitor_value = std::__atomic_monitor_global(__address);
+ if (__poll())
+ return;
+ std::__atomic_wait_global_table(__address, __monitor_value);
+# else
+ void const volatile* __volatile_address = reinterpret_cast<void const volatile*>(__address);
+ auto __monitor_value = std::__libcpp_atomic_monitor(__volatile_address);
+ if (__poll())
+ return;
+ std::__libcpp_atomic_wait(__volatile_address, __monitor_value);
+# endif
+ }
+}
+
+template <class _Element>
+struct __atomic_smart_ptr_fields {
+ mutable __cxx_atomic_impl<_Element*> __ptr_;
+ mutable __cxx_atomic_impl<uintptr_t> __ctrl_;
+
+ __atomic_smart_ptr_fields(_Element* __p, __shared_weak_count* __c) noexcept
+ : __ptr_(__p), __ctrl_(__atomic_smart_ptr_storage::__encode(__c, 0)) {}
+
+ const void* __ctrl_address() const noexcept { return static_cast<const void*>(__builtin_addressof(__ctrl_)); }
+
+ // Notify bit is set before sleeping so __unlock wakes waiters; without it
+ // sleeping threads would remain blocked after the lock is released.
+ void __lock() const noexcept {
+ _LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(__builtin_addressof(__ctrl_));
+ uintptr_t __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
+ for (;;) {
+ if (!__atomic_smart_ptr_storage::__has_lock(__expected)) {
+ uintptr_t __desired = __expected | __atomic_smart_ptr_storage::__lock_bit_;
+ if (std::__cxx_atomic_compare_exchange_weak(
+ __builtin_addressof(__ctrl_),
+ __builtin_addressof(__expected),
+ __desired,
+ memory_order_acquire,
+ memory_order_relaxed)) {
+ _LIBCPP_ATOMIC_SP_TSAN_POST_LOCK(__builtin_addressof(__ctrl_));
+ return;
+ }
+ continue;
+ }
+
+ uintptr_t __with_notify = __expected | __atomic_smart_ptr_storage::__notify_bit_;
+ if (!__atomic_smart_ptr_storage::__has_notify(__expected)) {
+ if (!std::__cxx_atomic_compare_exchange_weak(
+ __builtin_addressof(__ctrl_),
+ __builtin_addressof(__expected),
+ __with_notify,
+ memory_order_relaxed,
+ memory_order_relaxed))
+ continue;
+ __expected = __with_notify;
+ }
+
+ std::__atomic_smart_ptr_wait_on_address(__ctrl_address(), [&] {
+ __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
+ return !__atomic_smart_ptr_storage::__has_lock(__expected);
+ });
+ }
+ }
+
+ // Notify bit means at least one waiter is sleeping on __ctrl_; wake them
+ // after publishing the new CB so they don't miss the change.
+ void __unlock(__shared_weak_count* __ctrl_to_publish) const noexcept {
+ _LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(__builtin_addressof(__ctrl_));
+ uintptr_t __new_word = __atomic_smart_ptr_storage::__encode(__ctrl_to_publish, 0);
+ uintptr_t __previous = std::__cxx_atomic_exchange(__builtin_addressof(__ctrl_), __new_word, memory_order_release);
+ if (__atomic_smart_ptr_storage::__has_notify(__previous))
+ std::__atomic_smart_ptr_notify_all(__ctrl_address());
+ _LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(__builtin_addressof(__ctrl_));
+ }
+};
+
+// [util.smartptr.atomic.shared]: same stored pointer and same ownership, or both empty.
+template <class _Element>
+inline bool __atomic_smart_ptr_equivalent(
+ _Element* __ptr,
+ __shared_weak_count* __ctrl,
+ _Element* __expected_ptr,
+ __shared_weak_count* __expected_ctrl) noexcept {
+ if (__ctrl == nullptr && __expected_ctrl == nullptr)
+ return true;
+ return __ptr == __expected_ptr && __ctrl == __expected_ctrl;
+}
+
+template <class _Tp>
+struct atomic<shared_ptr<_Tp>> {
+ using value_type = shared_ptr<_Tp>;
+
+ static constexpr bool is_always_lock_free = false;
+
+ atomic() noexcept : __fields_(nullptr, nullptr) {}
+ constexpr atomic(nullptr_t) noexcept : __fields_(nullptr, nullptr) {}
+ atomic(shared_ptr<_Tp> __desired) noexcept : __fields_(__desired.__ptr_, __desired.__cntrl_) {
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ }
+
+ atomic(const atomic&) = delete;
+ atomic& operator=(const atomic&) = delete;
+
+ ~atomic() {
+ if (auto* __c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed)))
+ __c->__release_shared();
+ }
+
+ [[nodiscard]] bool is_lock_free() const noexcept { return false; }
+
+ void operator=(shared_ptr<_Tp> __desired) noexcept { store(std::move(__desired)); }
+ void operator=(nullptr_t) noexcept { store(nullptr); }
+ operator shared_ptr<_Tp>() const noexcept { return load(); }
+
+ void store(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept
+ _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+ (void)__m;
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ __fields_.__lock();
+ __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+
+ if (__old_c)
+ __old_c->__release_shared();
+ }
+
+ [[nodiscard]] shared_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
+ _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+ (void)__m;
+ __fields_.__lock();
+ _Tp* __ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ if (__c)
+ __c->__add_shared();
+ __fields_.__unlock(__c);
+ return shared_ptr<_Tp>::__create_with_control_block(__ptr, __c);
+ }
+
+ shared_ptr<_Tp> exchange(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ (void)__m;
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ __fields_.__lock();
+ _Tp* __old_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+
+ return shared_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_c);
+ }
+
+ bool compare_exchange_strong(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ (void)__success;
+ (void)__failure;
+ __fields_.__lock();
+ _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+
+ if (__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+ if (__cur_c)
+ __cur_c->__release_shared();
+ return true;
+ }
+
+ if (__cur_c)
+ __cur_c->__add_shared();
+ __fields_.__unlock(__cur_c);
+ __expected = shared_ptr<_Tp>::__create_with_control_block(__cur_ptr, __cur_c);
+ return false;
+ }
+
+ bool compare_exchange_strong(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ bool compare_exchange_weak(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ return compare_exchange_strong(__expected, std::move(__desired), __success, __failure);
+ }
+
+ bool compare_exchange_weak(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ // __ctrl_ is the wait address: __ptr_ changes are always published together
+ // with __ctrl_ through __unlock, so watching __ctrl_ is sufficient.
+ void wait(shared_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const noexcept
+ _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
+ _Tp* __old_ptr = __old.__ptr_;
+ __shared_weak_count* __old_c = __old.__cntrl_;
+
+ std::__atomic_smart_ptr_wait_on_address(__fields_.__ctrl_address(), [&] {
+ uintptr_t __word = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), __m);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(__word);
+ if (__cur_c != __old_c)
+ return true;
+ _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), __m);
+ return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ });
+ }
+
+ void notify_one() noexcept { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
+ void notify_all() noexcept { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
+
+private:
+ __atomic_smart_ptr_fields<_Tp> __fields_;
+};
+
+template <class _Tp>
+struct atomic<weak_ptr<_Tp>> {
+ using value_type = weak_ptr<_Tp>;
+
+ static constexpr bool is_always_lock_free = false;
+
+ atomic() noexcept : __fields_(nullptr, nullptr) {}
+ atomic(weak_ptr<_Tp> __desired) noexcept : __fields_(__desired.__ptr_, __desired.__cntrl_) {
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ }
+
+ atomic(const atomic&) = delete;
+ atomic& operator=(const atomic&) = delete;
+
+ ~atomic() {
+ if (auto* __c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed)))
+ __c->__release_weak();
+ }
+
+ [[nodiscard]] bool is_lock_free() const noexcept { return false; }
+
+ void operator=(weak_ptr<_Tp> __desired) noexcept { store(std::move(__desired)); }
+ operator weak_ptr<_Tp>() const noexcept { return load(); }
+
+ void store(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept
+ _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+ (void)__m;
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ __fields_.__lock();
+ __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+
+ if (__old_c)
+ __old_c->__release_weak();
+ }
+
+ [[nodiscard]] weak_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
+ _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+ (void)__m;
+ __fields_.__lock();
+ _Tp* __ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ if (__c)
+ __c->__add_weak();
+ __fields_.__unlock(__c);
+ return weak_ptr<_Tp>::__create_with_control_block(__ptr, __c);
+ }
+
+ weak_ptr<_Tp> exchange(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ (void)__m;
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ __fields_.__lock();
+ _Tp* __old_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+
+ return weak_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_c);
+ }
+
+ bool compare_exchange_strong(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ (void)__success;
+ (void)__failure;
+ __fields_.__lock();
+ _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), memory_order_relaxed);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(
+ std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
+
+ if (__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
+ __fields_.__unlock(__desired_c);
+ if (__cur_c)
+ __cur_c->__release_weak();
+ return true;
+ }
+
+ if (__cur_c)
+ __cur_c->__add_weak();
+ __fields_.__unlock(__cur_c);
+ __expected = weak_ptr<_Tp>::__create_with_control_block(__cur_ptr, __cur_c);
+ return false;
+ }
+
+ bool compare_exchange_strong(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ bool compare_exchange_weak(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ return compare_exchange_strong(__expected, std::move(__desired), __success, __failure);
+ }
+
+ bool compare_exchange_weak(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ // __ctrl_ is the wait address: __ptr_ changes are always published together
+ // with __ctrl_ through __unlock, so watching __ctrl_ is sufficient.
+ void wait(weak_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const noexcept
+ _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
+ _Tp* __old_ptr = __old.__ptr_;
+ __shared_weak_count* __old_c = __old.__cntrl_;
+
+ std::__atomic_smart_ptr_wait_on_address(__fields_.__ctrl_address(), [&] {
+ uintptr_t __word = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), __m);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(__word);
+ if (__cur_c != __old_c)
+ return true;
+ _Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), __m);
+ return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ });
+ }
+
+ void notify_one() noexcept { std::__atomic_smart_ptr_notify_one(__fields_.__ctrl_address()); }
+ void notify_all() noexcept { std::__atomic_smart_ptr_notify_all(__fields_.__ctrl_address()); }
+
+private:
+ __atomic_smart_ptr_fields<_Tp> __fields_;
+};
+
+#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_LOCK_BASED_H
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
new file mode 100644
index 0000000000000..354fbf579040f
--- /dev/null
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
@@ -0,0 +1,710 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_LOCK_FREE_H
+#define _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_LOCK_FREE_H
+
+#include <__atomic/atomic_sync_lite.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/memory_order.h>
+#include <__atomic/support.h>
+#include <__atomic/to_failure_order.h>
+#include <__atomic/to_gcc_order.h>
+#include <__config>
+#include <__cstddef/nullptr_t.h>
+#include <__memory/shared_count.h>
+#include <__utility/move.h>
+
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+
+template <class _Tp>
+class shared_ptr;
+
+template <class _Tp>
+class weak_ptr;
+
+template <class _Tp>
+struct atomic;
+
+// __ctrl_ word layout (8 bytes):
+// bits 0..1 always zero (CBs are at least 4-byte aligned)
+// bits 2..47 control-block pointer
+// bits 48..63 16-bit local refcount (Williams split-refcount aspiration)
+// Bits 48..63 are available on both DWCAS-capable ISAs (x86-64 without LAMA,
+// AArch64 without LVA): user-space pointers stay within 48 bits.
+struct __atomic_smart_ptr_storage {
+ static constexpr uintptr_t __lock_bit_ = uintptr_t{1};
+ static constexpr uintptr_t __notify_bit_ = uintptr_t{2};
+ static constexpr uintptr_t __ptr_mask_ = ~(__lock_bit_ | __notify_bit_);
+ static constexpr unsigned __local_count_shift = 48;
+ static constexpr uintptr_t __local_count_unit = uintptr_t{1} << __local_count_shift;
+ static constexpr uintptr_t __local_count_mask = uintptr_t{0xFFFF} << __local_count_shift;
+ static constexpr uintptr_t __dwcas_ptr_mask = __ptr_mask_ & ~__local_count_mask;
+ static constexpr uint16_t __local_count_max = 0xFFFF;
+
+ static uintptr_t __encode_dwcas(__shared_weak_count* __ctrl, uint16_t __local) noexcept {
+ return (reinterpret_cast<uintptr_t>(__ctrl) & __dwcas_ptr_mask) |
+ (static_cast<uintptr_t>(__local) << __local_count_shift);
+ }
+
+ static __shared_weak_count* __decode_dwcas(uintptr_t __word) noexcept {
+ return reinterpret_cast<__shared_weak_count*>(__word & __dwcas_ptr_mask);
+ }
+
+ static uint16_t __decode_local(uintptr_t __word) noexcept {
+ return static_cast<uint16_t>((__word & __local_count_mask) >> __local_count_shift);
+ }
+};
+
+inline void __atomic_smart_ptr_notify_one(const void* __address) noexcept {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ std::__atomic_notify_one_global_table(__address);
+# else
+ std::__cxx_atomic_notify_one(reinterpret_cast<void const volatile*>(__address));
+# endif
+}
+
+inline void __atomic_smart_ptr_notify_all(const void* __address) noexcept {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ std::__atomic_notify_all_global_table(__address);
+# else
+ std::__cxx_atomic_notify_all(reinterpret_cast<void const volatile*>(__address));
+# endif
+}
+
+template <class _Poll>
+inline void __atomic_smart_ptr_wait_on_address(const void* __address, const _Poll& __poll) noexcept {
+ while (!__poll()) {
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ auto __monitor_value = std::__atomic_monitor_global(__address);
+ if (__poll())
+ return;
+ std::__atomic_wait_global_table(__address, __monitor_value);
+# else
+ void const volatile* __volatile_address = reinterpret_cast<void const volatile*>(__address);
+ auto __monitor_value = std::__libcpp_atomic_monitor(__volatile_address);
+ if (__poll())
+ return;
+ std::__libcpp_atomic_wait(__volatile_address, __monitor_value);
+# endif
+ }
+}
+
+// CMPXCHG16B/CASP require 16-byte alignment; the public ABI size is 16 bytes.
+template <class _Element>
+struct alignas(16) __atomic_smart_ptr_fields {
+ mutable __cxx_atomic_impl<_Element*> __ptr_;
+ mutable __cxx_atomic_impl<uintptr_t> __ctrl_;
+
+ __atomic_smart_ptr_fields(_Element* __p, __shared_weak_count* __c) noexcept
+ : __ptr_(__p), __ctrl_(__atomic_smart_ptr_storage::__encode_dwcas(__c, 0)) {}
+
+ // DWCAS updates __ptr_ and __ctrl_ as one 16-byte pair, so waiters observe
+ // the struct base; watching only __ctrl_ would miss stores where the pointer
+ // changes but the CB stays the same (e.g., aliasing constructor).
+ const void* __wait_address() const noexcept { return static_cast<const void*>(this); }
+
+ __extension__ using __sp_u128 = unsigned __int128;
+
+ // Storage layout: low 64 bits = __ptr_, high 64 bits = __ctrl_. Going
+ // through `this` instead of __builtin_addressof(__ptr_) keeps the 16-byte alignment explicit
+ // and avoids implementation-defined aliasing between the sub-atomics and
+ // the wide view (sub-atomics are never accessed when DWCAS is enabled).
+ // TODO(review): this cast bypasses strict aliasing - valid only because
+ // __ptr_ and __ctrl_ are never touched via __cxx_atomic_* on this path.
+ __sp_u128* __dwcas_address() const noexcept {
+ return const_cast<__sp_u128*>(reinterpret_cast<const __sp_u128*>(this));
+ }
+
+ __sp_u128 __dwcas_load(memory_order __o) const noexcept {
+ __sp_u128 __word;
+ __atomic_load(__dwcas_address(), __builtin_addressof(__word), std::__to_gcc_order(__o));
+ return __word;
+ }
+
+ void __dwcas_store(__sp_u128 __word, memory_order __o) const noexcept {
+ __atomic_store(__dwcas_address(), __builtin_addressof(__word), std::__to_gcc_order(__o));
+ }
+
+ __sp_u128 __dwcas_exchange(__sp_u128 __new, memory_order __o) const noexcept {
+ __sp_u128 __old;
+ __atomic_exchange(
+ __dwcas_address(), __builtin_addressof(__new), __builtin_addressof(__old), std::__to_gcc_order(__o));
+ return __old;
+ }
+
+ // Weak CAS suffices for both strong and weak public variants - the caller
+ // controls retry policy (see __cas_dwcas below).
+ bool __dwcas_compare_exchange_weak(
+ __sp_u128& __expected, __sp_u128 __desired, memory_order __s, memory_order __f) const noexcept {
+ return __atomic_compare_exchange(
+ __dwcas_address(),
+ __builtin_addressof(__expected),
+ __builtin_addressof(__desired),
+ /*weak=*/true,
+ std::__to_gcc_order(__s),
+ std::__to_gcc_failure_order(__f));
+ }
+
+ static _Element* __pair_ptr(__sp_u128 __w) noexcept {
+ return reinterpret_cast<_Element*>(static_cast<uintptr_t>(__w));
+ }
+ static uintptr_t __pair_ctrl(__sp_u128 __w) noexcept { return static_cast<uintptr_t>(__w >> 64); }
+ static __sp_u128 __pair_make(_Element* __p, uintptr_t __c) noexcept {
+ return static_cast<__sp_u128>(reinterpret_cast<uintptr_t>(__p)) | (static_cast<__sp_u128>(__c) << 64);
+ }
+};
+
+// [util.smartptr.atomic.shared]: same stored pointer and same ownership, or both empty.
+template <class _Element>
+inline bool __atomic_smart_ptr_equivalent(
+ _Element* __ptr,
+ __shared_weak_count* __ctrl,
+ _Element* __expected_ptr,
+ __shared_weak_count* __expected_ctrl) noexcept {
+ if (__ctrl == nullptr && __expected_ctrl == nullptr)
+ return true;
+ return __ptr == __expected_ptr && __ctrl == __expected_ctrl;
+}
+
+// Pre-pay the per-in-flight-load fetch_add(1) tick that each aspirational
+// loader still owes the retiring control block; without this, the racing
+// loader would double-count its own contribution on the new block.
+template <class _Counter>
+inline void __atomic_smart_ptr_drain_shared(_Counter* __cb, uint16_t __local) noexcept {
+ for (uint16_t __i = 0; __i < __local; ++__i)
+ __cb->__add_shared();
+}
+
+template <class _Counter>
+inline void __atomic_smart_ptr_drain_weak(_Counter* __cb, uint16_t __local) noexcept {
+ for (uint16_t __i = 0; __i < __local; ++__i)
+ __cb->__add_weak();
+}
+
+template <class _Tp>
+struct atomic<shared_ptr<_Tp>> {
+ using value_type = shared_ptr<_Tp>;
+
+ // is_always_lock_free stays false - the umbrella header can dispatch to
+ // the lock-based implementation on cx16/LSE-absent builds; within this TU
+ // is_lock_free() is always true.
+ static constexpr bool is_always_lock_free = false;
+
+ atomic() noexcept : __fields_(nullptr, nullptr) {}
+ constexpr atomic(nullptr_t) noexcept : __fields_(nullptr, nullptr) {}
+ atomic(shared_ptr<_Tp> __desired) noexcept : __fields_(__desired.__ptr_, __desired.__cntrl_) {
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ }
+
+ atomic(const atomic&) = delete;
+ atomic& operator=(const atomic&) = delete;
+
+ ~atomic() {
+ using __fields = __atomic_smart_ptr_fields<_Tp>;
+ auto __word = __fields_.__dwcas_load(memory_order_relaxed);
+ if (auto* __c = __atomic_smart_ptr_storage::__decode_dwcas(__fields::__pair_ctrl(__word))) {
+ // No in-flight loads can exist at destruction time (the standard
+ // forbids concurrent operations on a destroyed atomic), so local
+ // count must be zero by construction.
+ __c->__release_shared();
+ }
+ }
+
+ [[nodiscard]] bool is_lock_free() const noexcept { return true; }
+
+ void operator=(shared_ptr<_Tp> __desired) noexcept { store(std::move(__desired)); }
+ void operator=(nullptr_t) noexcept { store(nullptr); }
+ operator shared_ptr<_Tp>() const noexcept { return load(); }
+
+ void store(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept
+ _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+ __store_dwcas(std::move(__desired), __m);
+ }
+
+ [[nodiscard]] shared_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
+ _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+ return __load_dwcas(__m);
+ }
+
+ shared_ptr<_Tp> exchange(shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return __exchange_dwcas(std::move(__desired), __m);
+ }
+
+ bool compare_exchange_strong(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__strong=*/true);
+ }
+
+ bool compare_exchange_strong(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ bool compare_exchange_weak(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__strong=*/false);
+ }
+
+ bool compare_exchange_weak(
+ shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return compare_exchange_weak(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ void wait(shared_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const noexcept
+ _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
+ _Tp* __old_ptr = __old.__ptr_;
+ __shared_weak_count* __old_c = __old.__cntrl_;
+
+ std::__atomic_smart_ptr_wait_on_address(__fields_.__wait_address(), [&] {
+ using __fields = __atomic_smart_ptr_fields<_Tp>;
+ auto __word = __fields_.__dwcas_load(__m);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode_dwcas(__fields::__pair_ctrl(__word));
+ _Tp* __cur_ptr = __fields::__pair_ptr(__word);
+ return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ });
+ }
+
+ void notify_one() noexcept { std::__atomic_smart_ptr_notify_one(__fields_.__wait_address()); }
+ void notify_all() noexcept { std::__atomic_smart_ptr_notify_all(__fields_.__wait_address()); }
+
+private:
+ __atomic_smart_ptr_fields<_Tp> __fields_;
+
+ using __fields_type = __atomic_smart_ptr_fields<_Tp>;
+ using __u128 = typename __fields_type::__sp_u128;
+
+ shared_ptr<_Tp> __load_dwcas(memory_order __m) const noexcept {
+ auto __expected = __fields_.__dwcas_load(memory_order_acquire);
+
+ __shared_weak_count* __claimed_cb;
+ _Tp* __claimed_ptr;
+
+ while (true) {
+ uintptr_t __ctrl_word = __fields_type::__pair_ctrl(__expected);
+ __claimed_cb = __atomic_smart_ptr_storage::__decode_dwcas(__ctrl_word);
+ __claimed_ptr = __fields_type::__pair_ptr(__expected);
+
+ if (__claimed_cb == nullptr)
+ return shared_ptr<_Tp>{};
+
+ uint16_t __local = __atomic_smart_ptr_storage::__decode_local(__ctrl_word);
+ if (__local == __atomic_smart_ptr_storage::__local_count_max) {
+ // Local count is saturated; yielding here is bounded because progress
+ // on another thread will lower it back. Wider counters would only push
+ // the threshold further out - they cannot eliminate the retry.
+ __expected = __fields_.__dwcas_load(memory_order_acquire);
+ continue;
+ }
+
+ // TODO(question): ABA: the 16-bit local count is not a version tag across CB
+ // lifecycles. A freed CB reallocated at the same address with the same count
+ // lets this CAS succeed spuriously. Is this risk acceptable?
+ auto __desired =
+ __fields_type::__pair_make(__claimed_ptr, __ctrl_word + __atomic_smart_ptr_storage::__local_count_unit);
+ if (__fields_.__dwcas_compare_exchange_weak(__expected, __desired, memory_order_acq_rel, memory_order_acquire))
+ break;
+ }
+
+ __claimed_cb->__add_shared();
+
+ auto __cur = __fields_.__dwcas_load(memory_order_acquire);
+ while (true) {
+ uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
+ __shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
+
+ // TODO(review): the store that replaced this CB pre-paid our tick via
+ // drain_shared, so __release_shared below cannot be the last ref - verify
+ // no concurrent release can create a zero-count window between drain and here.
+ if (__cur_cb != __claimed_cb) {
+ // A racing store already drained our local tick via __drain_shared,
+ // so __add_shared above became a double-count. Cancel it.
+ __claimed_cb->__release_shared();
+ break;
+ }
+
+ auto __dec = __fields_type::__pair_make(
+ __fields_type::__pair_ptr(__cur), __cur_ctrl - __atomic_smart_ptr_storage::__local_count_unit);
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __dec, memory_order_acq_rel, memory_order_acquire))
+ break;
+ }
+
+ (void)__m;
+ return shared_ptr<_Tp>::__create_with_control_block(__claimed_ptr, __claimed_cb);
+ }
+
+ void __store_dwcas(shared_ptr<_Tp> __desired, memory_order __m) noexcept {
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ auto __new_pair =
+ __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
+ auto __old_pair = __fields_.__dwcas_exchange(__new_pair, memory_order_acq_rel);
+
+ __retire_old_pair(__old_pair);
+
+ (void)__m;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ }
+
+ shared_ptr<_Tp> __exchange_dwcas(shared_ptr<_Tp> __desired, memory_order __m) noexcept {
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ auto __new_pair =
+ __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
+ auto __old_pair = __fields_.__dwcas_exchange(__new_pair, memory_order_acq_rel);
+
+ uintptr_t __old_ctrl = __fields_type::__pair_ctrl(__old_pair);
+ _Tp* __old_ptr = __fields_type::__pair_ptr(__old_pair);
+ __shared_weak_count* __old_cb = __atomic_smart_ptr_storage::__decode_dwcas(__old_ctrl);
+ uint16_t __old_local = __atomic_smart_ptr_storage::__decode_local(__old_ctrl);
+
+ if (__old_cb) {
+ // Drain in-flight loaders' aspirational ticks. After this, __old_cb's
+ // global count includes one ref per in-flight load that paired up with
+ // this CB. We then transfer ownership of the atomic's own ref to the
+ // returned shared_ptr (no add/release).
+ std::__atomic_smart_ptr_drain_shared(__old_cb, __old_local);
+ }
+
+ (void)__m;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+
+ return shared_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_cb);
+ }
+
+ // __strong: when true, retry on benign DWCAS failures (e.g., local-count
+ // changes by concurrent loads); only return false on a real
+ // ownership-equivalence mismatch.
+ bool __cas_dwcas(shared_ptr<_Tp>& __expected,
+ shared_ptr<_Tp> __desired,
+ memory_order __success,
+ memory_order __failure,
+ bool __strong) noexcept {
+ _Tp* __exp_ptr = __expected.__ptr_;
+ __shared_weak_count* __exp_c = __expected.__cntrl_;
+
+ _Tp* __des_ptr = __desired.__ptr_;
+ __shared_weak_count* __des_c = __desired.__cntrl_;
+
+ auto __cur = __fields_.__dwcas_load(memory_order_acquire);
+
+ while (true) {
+ uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
+ _Tp* __cur_ptr = __fields_type::__pair_ptr(__cur);
+ __shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
+
+ if (!__atomic_smart_ptr_equivalent(__cur_ptr, __cur_cb, __exp_ptr, __exp_c)) {
+ // Ownership mismatch. Update __expected to the actual current value
+ // (with a fresh shared_ptr reference). __desired is NOT consumed;
+ // its destructor will release its own ref.
+ (void)__success;
+ __expected = __load_dwcas(__failure);
+ return false;
+ }
+
+ // Equivalence holds. Try to publish __desired with local_count=0.
+ auto __desired_pair =
+ __fields_type::__pair_make(__des_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__des_c, 0));
+
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __desired_pair, __success, __failure)) {
+ uint16_t __cur_local = __atomic_smart_ptr_storage::__decode_local(__cur_ctrl);
+ if (__cur_cb)
+ std::__atomic_smart_ptr_drain_shared(__cur_cb, __cur_local);
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return true;
+ }
+
+ if (!__strong) {
+ // weak: a single underlying CAS attempt - if it failed for any
+ // reason (real ownership change or concurrent local-count update),
+ // report failure with the current pair as the new __expected.
+ __expected = __load_dwcas(__failure);
+ return false;
+ }
+ // Strong CAS: loop and re-check equivalence.
+ }
+ }
+
+ // Retire path used by __store_dwcas - drains in-flight ticks and releases
+ // the atomic's own ref on the old control block.
+ void __retire_old_pair(__u128 __old_pair) noexcept {
+ uintptr_t __old_ctrl = __fields_type::__pair_ctrl(__old_pair);
+ __shared_weak_count* __old_cb = __atomic_smart_ptr_storage::__decode_dwcas(__old_ctrl);
+ uint16_t __old_local = __atomic_smart_ptr_storage::__decode_local(__old_ctrl);
+
+ if (__old_cb == nullptr)
+ return;
+
+ // Pre-pay aspirational ticks of in-flight loaders.
+ std::__atomic_smart_ptr_drain_shared(__old_cb, __old_local);
+ // Release the ref this atomic held on the old control block.
+ __old_cb->__release_shared();
+ }
+};
+
+template <class _Tp>
+struct atomic<weak_ptr<_Tp>> {
+ using value_type = weak_ptr<_Tp>;
+
+ static constexpr bool is_always_lock_free = false;
+
+ atomic() noexcept : __fields_(nullptr, nullptr) {}
+ atomic(weak_ptr<_Tp> __desired) noexcept : __fields_(__desired.__ptr_, __desired.__cntrl_) {
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ }
+
+ atomic(const atomic&) = delete;
+ atomic& operator=(const atomic&) = delete;
+
+ ~atomic() {
+ using __fields = __atomic_smart_ptr_fields<_Tp>;
+ auto __word = __fields_.__dwcas_load(memory_order_relaxed);
+ if (auto* __c = __atomic_smart_ptr_storage::__decode_dwcas(__fields::__pair_ctrl(__word)))
+ __c->__release_weak();
+ }
+
+ [[nodiscard]] bool is_lock_free() const noexcept { return true; }
+
+ void operator=(weak_ptr<_Tp> __desired) noexcept { store(std::move(__desired)); }
+ operator weak_ptr<_Tp>() const noexcept { return load(); }
+
+ void store(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept
+ _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
+ __store_dwcas(std::move(__desired), __m);
+ }
+
+ [[nodiscard]] weak_ptr<_Tp> load(memory_order __m = memory_order_seq_cst) const noexcept
+ _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
+ return __load_dwcas(__m);
+ }
+
+ weak_ptr<_Tp> exchange(weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return __exchange_dwcas(std::move(__desired), __m);
+ }
+
+ bool compare_exchange_strong(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__strong=*/true);
+ }
+
+ bool compare_exchange_strong(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return compare_exchange_strong(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ bool compare_exchange_weak(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__strong=*/false);
+ }
+
+ bool compare_exchange_weak(
+ weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __m = memory_order_seq_cst) noexcept {
+ return compare_exchange_weak(__expected, std::move(__desired), __m, std::__to_failure_order(__m));
+ }
+
+ void wait(weak_ptr<_Tp> __old, memory_order __m = memory_order_seq_cst) const noexcept
+ _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) {
+ _Tp* __old_ptr = __old.__ptr_;
+ __shared_weak_count* __old_c = __old.__cntrl_;
+
+ std::__atomic_smart_ptr_wait_on_address(__fields_.__wait_address(), [&] {
+ using __fields = __atomic_smart_ptr_fields<_Tp>;
+ auto __word = __fields_.__dwcas_load(__m);
+ __shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode_dwcas(__fields::__pair_ctrl(__word));
+ _Tp* __cur_ptr = __fields::__pair_ptr(__word);
+ return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ });
+ }
+
+ void notify_one() noexcept { std::__atomic_smart_ptr_notify_one(__fields_.__wait_address()); }
+ void notify_all() noexcept { std::__atomic_smart_ptr_notify_all(__fields_.__wait_address()); }
+
+private:
+ __atomic_smart_ptr_fields<_Tp> __fields_;
+
+ using __fields_type = __atomic_smart_ptr_fields<_Tp>;
+ using __u128 = typename __fields_type::__sp_u128;
+
+ weak_ptr<_Tp> __load_dwcas(memory_order __m) const noexcept {
+ auto __expected = __fields_.__dwcas_load(memory_order_acquire);
+
+ __shared_weak_count* __claimed_cb;
+ _Tp* __claimed_ptr;
+
+ while (true) {
+ uintptr_t __ctrl_word = __fields_type::__pair_ctrl(__expected);
+ __claimed_cb = __atomic_smart_ptr_storage::__decode_dwcas(__ctrl_word);
+ __claimed_ptr = __fields_type::__pair_ptr(__expected);
+
+ if (__claimed_cb == nullptr)
+ return weak_ptr<_Tp>{};
+
+ uint16_t __local = __atomic_smart_ptr_storage::__decode_local(__ctrl_word);
+ if (__local == __atomic_smart_ptr_storage::__local_count_max) {
+ __expected = __fields_.__dwcas_load(memory_order_acquire);
+ continue;
+ }
+
+ auto __desired =
+ __fields_type::__pair_make(__claimed_ptr, __ctrl_word + __atomic_smart_ptr_storage::__local_count_unit);
+ if (__fields_.__dwcas_compare_exchange_weak(__expected, __desired, memory_order_acq_rel, memory_order_acquire))
+ break;
+ }
+
+ __claimed_cb->__add_weak();
+
+ auto __cur = __fields_.__dwcas_load(memory_order_acquire);
+ while (true) {
+ uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
+ __shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
+
+ if (__cur_cb != __claimed_cb) {
+ __claimed_cb->__release_weak();
+ break;
+ }
+
+ auto __dec = __fields_type::__pair_make(
+ __fields_type::__pair_ptr(__cur), __cur_ctrl - __atomic_smart_ptr_storage::__local_count_unit);
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __dec, memory_order_acq_rel, memory_order_acquire))
+ break;
+ }
+
+ (void)__m;
+ return weak_ptr<_Tp>::__create_with_control_block(__claimed_ptr, __claimed_cb);
+ }
+
+ void __store_dwcas(weak_ptr<_Tp> __desired, memory_order __m) noexcept {
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ auto __new_pair =
+ __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
+ auto __old_pair = __fields_.__dwcas_exchange(__new_pair, memory_order_acq_rel);
+
+ __retire_old_pair_weak(__old_pair);
+
+ (void)__m;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ }
+
+ weak_ptr<_Tp> __exchange_dwcas(weak_ptr<_Tp> __desired, memory_order __m) noexcept {
+ _Tp* __desired_ptr = __desired.__ptr_;
+ __shared_weak_count* __desired_c = __desired.__cntrl_;
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+
+ auto __new_pair =
+ __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
+ auto __old_pair = __fields_.__dwcas_exchange(__new_pair, memory_order_acq_rel);
+
+ uintptr_t __old_ctrl = __fields_type::__pair_ctrl(__old_pair);
+ _Tp* __old_ptr = __fields_type::__pair_ptr(__old_pair);
+ __shared_weak_count* __old_cb = __atomic_smart_ptr_storage::__decode_dwcas(__old_ctrl);
+ uint16_t __old_local = __atomic_smart_ptr_storage::__decode_local(__old_ctrl);
+
+ if (__old_cb)
+ std::__atomic_smart_ptr_drain_weak(__old_cb, __old_local);
+
+ (void)__m;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+
+ return weak_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_cb);
+ }
+
+ bool __cas_dwcas(weak_ptr<_Tp>& __expected,
+ weak_ptr<_Tp> __desired,
+ memory_order __success,
+ memory_order __failure,
+ bool __strong) noexcept {
+ _Tp* __exp_ptr = __expected.__ptr_;
+ __shared_weak_count* __exp_c = __expected.__cntrl_;
+
+ _Tp* __des_ptr = __desired.__ptr_;
+ __shared_weak_count* __des_c = __desired.__cntrl_;
+
+ auto __cur = __fields_.__dwcas_load(memory_order_acquire);
+
+ while (true) {
+ uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
+ _Tp* __cur_ptr = __fields_type::__pair_ptr(__cur);
+ __shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
+
+ if (!__atomic_smart_ptr_equivalent(__cur_ptr, __cur_cb, __exp_ptr, __exp_c)) {
+ (void)__success;
+ __expected = __load_dwcas(__failure);
+ return false;
+ }
+
+ auto __desired_pair =
+ __fields_type::__pair_make(__des_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__des_c, 0));
+
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __desired_pair, __success, __failure)) {
+ uint16_t __cur_local = __atomic_smart_ptr_storage::__decode_local(__cur_ctrl);
+ if (__cur_cb)
+ std::__atomic_smart_ptr_drain_weak(__cur_cb, __cur_local);
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return true;
+ }
+
+ if (!__strong) {
+ __expected = __load_dwcas(__failure);
+ return false;
+ }
+ }
+ }
+
+ void __retire_old_pair_weak(__u128 __old_pair) noexcept {
+ uintptr_t __old_ctrl = __fields_type::__pair_ctrl(__old_pair);
+ __shared_weak_count* __old_cb = __atomic_smart_ptr_storage::__decode_dwcas(__old_ctrl);
+ uint16_t __old_local = __atomic_smart_ptr_storage::__decode_local(__old_ctrl);
+
+ if (__old_cb == nullptr)
+ return;
+
+ std::__atomic_smart_ptr_drain_weak(__old_cb, __old_local);
+ __old_cb->__release_weak();
+ }
+};
+
+#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_LOCK_FREE_H
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 3238146851ede..c882b124f24db 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -1692,6 +1692,8 @@ module std [system] {
}
module atomic_shared_ptr {
header "__memory/atomic_shared_ptr.h"
+ textual header "__memory/atomic_shared_ptr_lock_based.h"
+ textual header "__memory/atomic_shared_ptr_lock_free.h"
export std.memory.shared_ptr
}
module swap_allocator { header "__memory/swap_allocator.h" }
diff --git a/libcxx/test/libcxx/utilities/memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp b/libcxx/test/libcxx/utilities/memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp
new file mode 100644
index 0000000000000..6218a3f271f08
--- /dev/null
+++ b/libcxx/test/libcxx/utilities/memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp
@@ -0,0 +1,31 @@
+// REQUIRES: std-at-least-c++20
+// UNSUPPORTED: no-threads
+// ADDITIONAL_COMPILE_FLAGS: -march=x86-64-v2
+
+// Verify that with cx16/DWCAS available, is_lock_free() returns true.
+// is_always_lock_free remains false (matches standard expectations).
+
+#include <atomic>
+#include <cassert>
+#include <cstdio>
+#include <memory>
+
+int main(int, char**) {
+ using A = std::atomic<std::shared_ptr<int>>;
+
+ // is_always_lock_free must remain false to match existing libc++ tests.
+ static_assert(!A::is_always_lock_free, "is_always_lock_free should be false");
+
+ A a(std::make_shared<int>(42));
+ bool lf = a.is_lock_free();
+
+ // On x86-64 with cx16 (-march=x86-64-v2), DWCAS is available and
+ // is_lock_free() should return true.
+ std::printf("sizeof=%zu alignof=%zu is_lock_free=%d\n", sizeof(A), alignof(A), (int)lf);
+
+ assert(sizeof(A) == 16);
+ assert(alignof(A) == 16);
+ assert(lf == true);
+
+ return 0;
+}
>From 3a919b03d44cd5a841ea00a52042c30dbcf256d8 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Sat, 30 May 2026 19:21:05 +0300
Subject: [PATCH 15/46] [libc++] Fix refcount leak in
atomic<shared_ptr>::compare_exchange DWCAS path; gate
atomic_shared_ptr_dwcas.pass.cpp on target=x86_64 - the -march=x86-64-v2 flag
fails on AIX PowerPC and Android x86
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/include/__memory/atomic_shared_ptr_lock_free.h | 8 ++++++--
.../memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp | 1 +
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
index 354fbf579040f..5cd71de4ef373 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
@@ -434,8 +434,10 @@ struct atomic<shared_ptr<_Tp>> {
if (__fields_.__dwcas_compare_exchange_weak(__cur, __desired_pair, __success, __failure)) {
uint16_t __cur_local = __atomic_smart_ptr_storage::__decode_local(__cur_ctrl);
- if (__cur_cb)
+ if (__cur_cb) {
std::__atomic_smart_ptr_drain_shared(__cur_cb, __cur_local);
+ __cur_cb->__release_shared();
+ }
__desired.__ptr_ = nullptr;
__desired.__cntrl_ = nullptr;
std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
@@ -673,8 +675,10 @@ struct atomic<weak_ptr<_Tp>> {
if (__fields_.__dwcas_compare_exchange_weak(__cur, __desired_pair, __success, __failure)) {
uint16_t __cur_local = __atomic_smart_ptr_storage::__decode_local(__cur_ctrl);
- if (__cur_cb)
+ if (__cur_cb) {
std::__atomic_smart_ptr_drain_weak(__cur_cb, __cur_local);
+ __cur_cb->__release_weak();
+ }
__desired.__ptr_ = nullptr;
__desired.__cntrl_ = nullptr;
std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
diff --git a/libcxx/test/libcxx/utilities/memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp b/libcxx/test/libcxx/utilities/memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp
index 6218a3f271f08..c8df1e8921d43 100644
--- a/libcxx/test/libcxx/utilities/memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp
+++ b/libcxx/test/libcxx/utilities/memory/util.smartptr/atomic_shared_ptr_dwcas.pass.cpp
@@ -1,4 +1,5 @@
// REQUIRES: std-at-least-c++20
+// REQUIRES: target=x86_64-{{.*}}
// UNSUPPORTED: no-threads
// ADDITIONAL_COMPILE_FLAGS: -march=x86-64-v2
>From 3ef06a894b2926a55b24e763333362fd8d23bd86 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Sat, 30 May 2026 21:00:38 +0300
Subject: [PATCH 16/46] [libc++] Rename __strong parameter to __is_strong to
avoid Apple Clang ARC keyword clash
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
.../__memory/atomic_shared_ptr_lock_free.h | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
index 5cd71de4ef373..91dcdd75a8acf 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
@@ -252,7 +252,7 @@ struct atomic<shared_ptr<_Tp>> {
bool compare_exchange_strong(
shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
- return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__strong=*/true);
+ return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__is_strong=*/true);
}
bool compare_exchange_strong(
@@ -263,7 +263,7 @@ struct atomic<shared_ptr<_Tp>> {
bool compare_exchange_weak(
shared_ptr<_Tp>& __expected, shared_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
- return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__strong=*/false);
+ return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__is_strong=*/false);
}
bool compare_exchange_weak(
@@ -398,14 +398,14 @@ struct atomic<shared_ptr<_Tp>> {
return shared_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_cb);
}
- // __strong: when true, retry on benign DWCAS failures (e.g., local-count
+ // __is_strong: when true, retry on benign DWCAS failures (e.g., local-count
// changes by concurrent loads); only return false on a real
// ownership-equivalence mismatch.
bool __cas_dwcas(shared_ptr<_Tp>& __expected,
shared_ptr<_Tp> __desired,
memory_order __success,
memory_order __failure,
- bool __strong) noexcept {
+ bool __is_strong) noexcept {
_Tp* __exp_ptr = __expected.__ptr_;
__shared_weak_count* __exp_c = __expected.__cntrl_;
@@ -444,7 +444,7 @@ struct atomic<shared_ptr<_Tp>> {
return true;
}
- if (!__strong) {
+ if (!__is_strong) {
// weak: a single underlying CAS attempt - if it failed for any
// reason (real ownership change or concurrent local-count update),
// report failure with the current pair as the new __expected.
@@ -516,7 +516,7 @@ struct atomic<weak_ptr<_Tp>> {
bool compare_exchange_strong(
weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
- return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__strong=*/true);
+ return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__is_strong=*/true);
}
bool compare_exchange_strong(
@@ -527,7 +527,7 @@ struct atomic<weak_ptr<_Tp>> {
bool compare_exchange_weak(
weak_ptr<_Tp>& __expected, weak_ptr<_Tp> __desired, memory_order __success, memory_order __failure) noexcept
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
- return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__strong=*/false);
+ return __cas_dwcas(__expected, std::move(__desired), __success, __failure, /*__is_strong=*/false);
}
bool compare_exchange_weak(
@@ -650,7 +650,7 @@ struct atomic<weak_ptr<_Tp>> {
weak_ptr<_Tp> __desired,
memory_order __success,
memory_order __failure,
- bool __strong) noexcept {
+ bool __is_strong) noexcept {
_Tp* __exp_ptr = __expected.__ptr_;
__shared_weak_count* __exp_c = __expected.__cntrl_;
@@ -685,7 +685,7 @@ struct atomic<weak_ptr<_Tp>> {
return true;
}
- if (!__strong) {
+ if (!__is_strong) {
__expected = __load_dwcas(__failure);
return false;
}
>From 1ff9509816ea5256a088a7e54a9c540c351d2072 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Sat, 30 May 2026 23:43:35 +0300
Subject: [PATCH 17/46] [libc++] Enable lock-free atomic<shared_ptr> on Windows
x64 (clang-cl)
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/include/__memory/atomic_shared_ptr.h | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index 273b8da9140c8..bd7573da54c33 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -16,14 +16,17 @@
# pragma GCC system_header
#endif
-// The lock-free path requires a hardware 16-byte CAS to update both fields of
-// the (ptr, ctrl) pair as one linearizable step; without it the spinlock path
-// is the only correct option. is_always_lock_free stays false because the same
-// source built without cx16/LSE flags must still see a non-lock-free type.
+// __atomic_always_lock_free works in both GCC-compat and MSVC-compat (clang-cl):
+// x86-64 + cx16 (-march=x86-64-v2 or -mcx16) - CMPXCHG16B
+// AArch64 + LSE (-march=armv8.1-a or later) - CASP
+// Windows x64 with clang-cl + cx16 - also CMPXCHG16B
+// ISA whitelist: big-endian 64-bit (e.g. PowerPC) can pass __atomic_always_lock_free
+// but our unsigned __int128 layout (low-64=ptr, high-64=ctrl) requires little-endian.
+// _LIBCPP_HAS_INT128 is conservatively 0 in MSVC mode; __extension__ unsigned
+// __int128 compiles correctly in clang-cl and is used directly in the impl.
#if !defined(_LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR)
-# if _LIBCPP_HAS_THREADS && _LIBCPP_HAS_INT128 && \
- ((defined(__x86_64__) && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)) || \
- (defined(__aarch64__) && defined(__ARM_FEATURE_ATOMICS)))
+# if _LIBCPP_HAS_THREADS && sizeof(void*) == 8 && __atomic_always_lock_free(16, (void*)0) && \
+ (defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__) || defined(_M_ARM64))
# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 1
# else
# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 0
>From 355ec11cfb7363c12f71d75d15dfa1e467eb631b Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Sun, 31 May 2026 00:03:49 +0300
Subject: [PATCH 18/46] [libc++] Enable lock-free atomic<shared_ptr> on Windows
x64 (clang-cl) (Attempt #2)
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/include/__memory/atomic_shared_ptr.h | 13 +++----------
1 file changed, 3 insertions(+), 10 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index bd7573da54c33..d9e837f3dd046 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -16,17 +16,10 @@
# pragma GCC system_header
#endif
-// __atomic_always_lock_free works in both GCC-compat and MSVC-compat (clang-cl):
-// x86-64 + cx16 (-march=x86-64-v2 or -mcx16) - CMPXCHG16B
-// AArch64 + LSE (-march=armv8.1-a or later) - CASP
-// Windows x64 with clang-cl + cx16 - also CMPXCHG16B
-// ISA whitelist: big-endian 64-bit (e.g. PowerPC) can pass __atomic_always_lock_free
-// but our unsigned __int128 layout (low-64=ptr, high-64=ctrl) requires little-endian.
-// _LIBCPP_HAS_INT128 is conservatively 0 in MSVC mode; __extension__ unsigned
-// __int128 compiles correctly in clang-cl and is used directly in the impl.
#if !defined(_LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR)
-# if _LIBCPP_HAS_THREADS && sizeof(void*) == 8 && __atomic_always_lock_free(16, (void*)0) && \
- (defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__) || defined(_M_ARM64))
+# if _LIBCPP_HAS_THREADS && \
+ (((defined(__x86_64__) || defined(_M_X64)) && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)) || \
+ ((defined(__aarch64__) || defined(_M_ARM64)) && defined(__ARM_FEATURE_ATOMICS)))
# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 1
# else
# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 0
>From b14dc0b49609b2029c1d16d5780b15774dd50004 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Thu, 13 Aug 2026 22:12:52 +0300
Subject: [PATCH 19/46] fix: removed _LIBCPP_HAS_ATOMIC_HEADER after #199674
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/include/__memory/atomic_shared_ptr_lock_based.h | 4 ++--
libcxx/include/__memory/atomic_shared_ptr_lock_free.h | 4 ++--
libcxx/include/__memory/shared_ptr.h | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_based.h b/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
index e4584612b5009..29dfa4acfe615 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
@@ -55,7 +55,7 @@ _LIBCPP_PUSH_MACROS
_LIBCPP_BEGIN_NAMESPACE_STD
-#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
template <class _Tp>
class shared_ptr;
@@ -482,7 +482,7 @@ struct atomic<weak_ptr<_Tp>> {
__atomic_smart_ptr_fields<_Tp> __fields_;
};
-#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
index 91dcdd75a8acf..0860aaaf401ca 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
@@ -32,7 +32,7 @@ _LIBCPP_PUSH_MACROS
_LIBCPP_BEGIN_NAMESPACE_STD
-#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
template <class _Tp>
class shared_ptr;
@@ -705,7 +705,7 @@ struct atomic<weak_ptr<_Tp>> {
}
};
-#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 71f31994963d3..abd992281a6d3 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -1557,7 +1557,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
_LIBCPP_END_NAMESPACE_STD
-#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_ATOMIC_HEADER
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
# include <__memory/atomic_shared_ptr.h>
#endif
>From e1e4c680a3d2efe0470b1679934bee9c71a50c73 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Thu, 13 Aug 2026 22:45:46 +0300
Subject: [PATCH 20/46] fix: qualify __atomic_smart_ptr_equivalent calls;
always define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/include/__memory/atomic_shared_ptr.h | 14 ++++++--------
.../__memory/atomic_shared_ptr_lock_based.h | 8 ++++----
.../include/__memory/atomic_shared_ptr_lock_free.h | 8 ++++----
3 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index d9e837f3dd046..95e12fcff0a0c 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -16,14 +16,12 @@
# pragma GCC system_header
#endif
-#if !defined(_LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR)
-# if _LIBCPP_HAS_THREADS && \
- (((defined(__x86_64__) || defined(_M_X64)) && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)) || \
- ((defined(__aarch64__) || defined(_M_ARM64)) && defined(__ARM_FEATURE_ATOMICS)))
-# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 1
-# else
-# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 0
-# endif
+#if _LIBCPP_HAS_THREADS && \
+ (((defined(__x86_64__) || defined(_M_X64)) && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)) || \
+ ((defined(__aarch64__) || defined(_M_ARM64)) && defined(__ARM_FEATURE_ATOMICS)))
+# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 1
+#else
+# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 0
#endif
#if _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_based.h b/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
index 29dfa4acfe615..d1df87a9812b1 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
@@ -278,7 +278,7 @@ struct atomic<shared_ptr<_Tp>> {
__shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(
std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
- if (__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
+ if (std::__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
_Tp* __desired_ptr = __desired.__ptr_;
__shared_weak_count* __desired_c = __desired.__cntrl_;
__desired.__ptr_ = nullptr;
@@ -327,7 +327,7 @@ struct atomic<shared_ptr<_Tp>> {
if (__cur_c != __old_c)
return true;
_Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), __m);
- return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ return !std::__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
});
}
@@ -422,7 +422,7 @@ struct atomic<weak_ptr<_Tp>> {
__shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode(
std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
- if (__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
+ if (std::__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __expected.__ptr_, __expected.__cntrl_)) {
_Tp* __desired_ptr = __desired.__ptr_;
__shared_weak_count* __desired_c = __desired.__cntrl_;
__desired.__ptr_ = nullptr;
@@ -471,7 +471,7 @@ struct atomic<weak_ptr<_Tp>> {
if (__cur_c != __old_c)
return true;
_Tp* __cur_ptr = std::__cxx_atomic_load(__builtin_addressof(__fields_.__ptr_), __m);
- return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ return !std::__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
});
}
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
index 0860aaaf401ca..d56e7854e3eee 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
@@ -281,7 +281,7 @@ struct atomic<shared_ptr<_Tp>> {
auto __word = __fields_.__dwcas_load(__m);
__shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode_dwcas(__fields::__pair_ctrl(__word));
_Tp* __cur_ptr = __fields::__pair_ptr(__word);
- return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ return !std::__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
});
}
@@ -419,7 +419,7 @@ struct atomic<shared_ptr<_Tp>> {
_Tp* __cur_ptr = __fields_type::__pair_ptr(__cur);
__shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
- if (!__atomic_smart_ptr_equivalent(__cur_ptr, __cur_cb, __exp_ptr, __exp_c)) {
+ if (!std::__atomic_smart_ptr_equivalent(__cur_ptr, __cur_cb, __exp_ptr, __exp_c)) {
// Ownership mismatch. Update __expected to the actual current value
// (with a fresh shared_ptr reference). __desired is NOT consumed;
// its destructor will release its own ref.
@@ -545,7 +545,7 @@ struct atomic<weak_ptr<_Tp>> {
auto __word = __fields_.__dwcas_load(__m);
__shared_weak_count* __cur_c = __atomic_smart_ptr_storage::__decode_dwcas(__fields::__pair_ctrl(__word));
_Tp* __cur_ptr = __fields::__pair_ptr(__word);
- return !__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
+ return !std::__atomic_smart_ptr_equivalent(__cur_ptr, __cur_c, __old_ptr, __old_c);
});
}
@@ -664,7 +664,7 @@ struct atomic<weak_ptr<_Tp>> {
_Tp* __cur_ptr = __fields_type::__pair_ptr(__cur);
__shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
- if (!__atomic_smart_ptr_equivalent(__cur_ptr, __cur_cb, __exp_ptr, __exp_c)) {
+ if (!std::__atomic_smart_ptr_equivalent(__cur_ptr, __cur_cb, __exp_ptr, __exp_c)) {
(void)__success;
__expected = __load_dwcas(__failure);
return false;
>From 6d69e6b6533810d1786c4138ac0319421e53ddaf Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Fri, 14 Aug 2026 16:54:21 +0300
Subject: [PATCH 21/46] fix: export std::atomic from the memory module
partition (module_std)
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/modules/std/memory.inc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/modules/std/memory.inc b/libcxx/modules/std/memory.inc
index 0338ec0db325e..6e2fb59adf1c2 100644
--- a/libcxx/modules/std/memory.inc
+++ b/libcxx/modules/std/memory.inc
@@ -186,7 +186,7 @@ export namespace std {
using std::hash;
// [util.smartptr.atomic], atomic smart pointers
- // using std::atomic;
+ using std::atomic;
#if _LIBCPP_STD_VER >= 23
// [out.ptr.t], class template out_ptr_t
>From a9b76f407ec4c0b7787075f9cb323e7c19c36402 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:29:30 +0300
Subject: [PATCH 22/46] Update
libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../atomic_shared_ptr_memory_order.verify.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
index 7b92e1b23dfcd..c1181834c88cb 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
@@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
-// REQUIRES: diagnose-if-support
// REQUIRES: std-at-least-c++20
+// REQUIRES: diagnose-if-support
// UNSUPPORTED: no-threads
#include <atomic>
>From f46c307b72c418d0609b0923c8822d52bd4813b3 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:29:49 +0300
Subject: [PATCH 23/46] Update
libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../atomic_shared_ptr_refcount.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
index 7957fb7a7b787..87cd3f3914683 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
@@ -12,7 +12,7 @@
#include <atomic>
#include <cassert>
#include <memory>
-#include <utility> // Armv7, Armv8 require for std::move
+#include <utility>
namespace {
struct Tracker {
>From 7af680062fb008b9e0fb80123c146aee845c84d1 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:30:02 +0300
Subject: [PATCH 24/46] Update
libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../atomic_shared_ptr_nullptr.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
index 3889e86717948..47b84a74bdee1 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
@@ -13,7 +13,7 @@
#include <atomic>
#include <cassert>
-#include <cstddef> // Armv7, Armv8 (-fno-exceptions) require for std::nullptr_t
+#include <cstddef>
#include <memory>
#include <type_traits>
>From 5e6b8fc291c33233aefe9503ff1a1aeddbd700ee Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:30:22 +0300
Subject: [PATCH 25/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
index 16ccf9bac493b..136e1bd1e2ca8 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/is_lock_free.pass.cpp
@@ -16,7 +16,7 @@
// bool is_lock_free() const noexcept;
#include <atomic>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "atomic_smart_ptr_test_types.h"
>From 204143df8b191bf0e2f4da7697dbcf84c63b23bd Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:30:57 +0300
Subject: [PATCH 26/46] Update libcxx/docs/Status/Cxx20Papers.csv
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
libcxx/docs/Status/Cxx20Papers.csv | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index c3ab16fc3d424..47f6ae5923bc4 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -12,7 +12,7 @@
"`P0600R1 <https://wg21.link/P0600R1>`__","nodiscard in the Library","2017-11 (Albuquerque)","|Complete|","16","`#104010 <https://github.com/llvm/llvm-project/issues/104010>`__",""
"`P0616R0 <https://wg21.link/P0616R0>`__","de-pessimize legacy <numeric> algorithms with std::move","2017-11 (Albuquerque)","|Complete|","12","`#104011 <https://github.com/llvm/llvm-project/issues/104011>`__",""
"`P0653R2 <https://wg21.link/P0653R2>`__","Utility to convert a pointer to a raw pointer","2017-11 (Albuquerque)","|Complete|","6","`#104012 <https://github.com/llvm/llvm-project/issues/104012>`__",""
-"`P0718R2 <https://wg21.link/P0718R2>`__","Atomic shared_ptr","2017-11 (Albuquerque)","|Complete|","23","`#99980 <https://github.com/llvm/llvm-project/issues/99980>`__",""
+"`P0718R2 <https://wg21.link/P0718R2>`__","Atomic shared_ptr","2017-11 (Albuquerque)","|Complete|","24","`#99980 <https://github.com/llvm/llvm-project/issues/99980>`__",""
"`P0767R1 <https://wg21.link/P0767R1>`__","Deprecate POD","2017-11 (Albuquerque)","|Complete|","21","`#104013 <https://github.com/llvm/llvm-project/issues/104013>`__","It was previously erroneously marked as complete in LLVM 7."
"`P0768R1 <https://wg21.link/P0768R1>`__","Library Support for the Spaceship (Comparison) Operator","2017-11 (Albuquerque)","|Complete|","","`#104014 <https://github.com/llvm/llvm-project/issues/104014>`__",""
"`P0777R1 <https://wg21.link/P0777R1>`__","Treating Unnecessary ``decay``\ ","2017-11 (Albuquerque)","|Complete|","7","`#104016 <https://github.com/llvm/llvm-project/issues/104016>`__",""
>From c1b2c2e2f9c3ff636b9212e160d6704d3065a1a5 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:31:33 +0300
Subject: [PATCH 27/46] Update libcxx/docs/Status/Cxx23Issues.csv
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
libcxx/docs/Status/Cxx23Issues.csv | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index 3fc83a0e4d593..e7be13331fcf9 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -157,7 +157,7 @@
"`LWG3654 <https://wg21.link/LWG3654>`__","``basic_format_context::arg(size_t)`` should be ``noexcept`` ","2022-02 (Virtual)","|Complete|","15","`#104959 <https://github.com/llvm/llvm-project/issues/104959>`__",""
"`LWG3657 <https://wg21.link/LWG3657>`__","``std::hash<std::filesystem::path>`` is not enabled","2022-02 (Virtual)","|Complete|","17","`#104960 <https://github.com/llvm/llvm-project/issues/104960>`__",""
"`LWG3660 <https://wg21.link/LWG3660>`__","``iterator_traits<common_iterator>::pointer`` should conform to ยง[iterator.traits]","2022-02 (Virtual)","|Complete|","14","`#104961 <https://github.com/llvm/llvm-project/issues/104961>`__",""
-"`LWG3661 <https://wg21.link/LWG3661>`__","``constinit atomic<shared_ptr<T>> a(nullptr);`` should work","2022-02 (Virtual)","|Complete|","23","`#104962 <https://github.com/llvm/llvm-project/issues/104962>`__",""
+"`LWG3661 <https://wg21.link/LWG3661>`__","``constinit atomic<shared_ptr<T>> a(nullptr);`` should work","2022-02 (Virtual)","|Complete|","24","`#104962 <https://github.com/llvm/llvm-project/issues/104962>`__",""
"","","","","","",""
"`LWG3564 <https://wg21.link/LWG3564>`__","``transform_view::iterator<true>::value_type`` and ``iterator_category`` should use ``const F&``","2022-07 (Virtual)","|Complete|","20","`#104963 <https://github.com/llvm/llvm-project/issues/104963>`__",""
"`LWG3617 <https://wg21.link/LWG3617>`__","``function``/``packaged_task`` deduction guides and deducing ``this``","2022-07 (Virtual)","","","`#104965 <https://github.com/llvm/llvm-project/issues/104965>`__",""
>From 0d034c1ff49ef9bc83e4222d8290b21f354a5b11 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:31:46 +0300
Subject: [PATCH 28/46] Update libcxx/docs/Status/Cxx26Issues.csv
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
libcxx/docs/Status/Cxx26Issues.csv | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/Status/Cxx26Issues.csv b/libcxx/docs/Status/Cxx26Issues.csv
index fcf3cbd068206..e8b2f89582aa4 100644
--- a/libcxx/docs/Status/Cxx26Issues.csv
+++ b/libcxx/docs/Status/Cxx26Issues.csv
@@ -3,7 +3,7 @@
"`LWG3884 <https://wg21.link/LWG3884>`__","``flat_foo`` is missing allocator-extended copy/move constructors","2023-06 (Varna)","|Complete|","21","`#105269 <https://github.com/llvm/llvm-project/issues/105269>`__",""
"`LWG3885 <https://wg21.link/LWG3885>`__","``op`` should be in [zombie.names]","2023-06 (Varna)","|Nothing To Do|","","`#105270 <https://github.com/llvm/llvm-project/issues/105270>`__",""
"`LWG3887 <https://wg21.link/LWG3887>`__","Version macro for ``allocate_at_least``","2023-06 (Varna)","|Complete|","19","`#105271 <https://github.com/llvm/llvm-project/issues/105271>`__",""
-"`LWG3893 <https://wg21.link/LWG3893>`__","LWG 3661 broke ``atomic<shared_ptr<T>> a; a = nullptr;``","2023-06 (Varna)","|Complete|","23","`#105273 <https://github.com/llvm/llvm-project/issues/105273>`__",""
+"`LWG3893 <https://wg21.link/LWG3893>`__","LWG 3661 broke ``atomic<shared_ptr<T>> a; a = nullptr;``","2023-06 (Varna)","|Complete|","24","`#105273 <https://github.com/llvm/llvm-project/issues/105273>`__",""
"`LWG3894 <https://wg21.link/LWG3894>`__","``generator::promise_type::yield_value(ranges::elements_of<Rng, Alloc>)`` should not be ``noexcept``","2023-06 (Varna)","","","`#105274 <https://github.com/llvm/llvm-project/issues/105274>`__",""
"`LWG3903 <https://wg21.link/LWG3903>`__","span destructor is redundantly noexcept","2023-06 (Varna)","|Complete|","7","`#105275 <https://github.com/llvm/llvm-project/issues/105275>`__",""
"`LWG3904 <https://wg21.link/LWG3904>`__","``lazy_split_view::outer-iterator``'s const-converting constructor isn't setting ``trailing_empty_``","2023-06 (Varna)","","","`#105276 <https://github.com/llvm/llvm-project/issues/105276>`__",""
>From fa7788abde4a9c0c6d551d781f0444e4420b37f0 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:32:27 +0300
Subject: [PATCH 29/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../memory/util.smartptr/atomic/shared/convert.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
index 7c5ddcea6c8c7..d1ab9c74e8cb1 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/convert.pass.cpp
@@ -13,7 +13,7 @@
#include <atomic>
#include <cassert>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From 908004a6a330023db03674843967818b345e6df2 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:32:38 +0300
Subject: [PATCH 30/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../atomic/shared/compare_exchange_strong.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
index a94a9563839c8..c6bb1f7230430 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/compare_exchange_strong.pass.cpp
@@ -14,7 +14,7 @@
#include <atomic>
#include <cassert>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From 0b3645a87daec096cb0fae4d505742bfff58daf5 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:32:54 +0300
Subject: [PATCH 31/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../memory/util.smartptr/atomic/shared/exchange.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
index 39fa057141d92..eec9d88463406 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/exchange.pass.cpp
@@ -13,7 +13,7 @@
#include <atomic>
#include <cassert>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From eac63f29a27a8cf0a757a36508d85b8b09d3f558 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:33:13 +0300
Subject: [PATCH 32/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../util.smartptr/atomic/shared/is_always_lock_free.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
index ac685fbf8bfc1..90975ad3ea500 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/is_always_lock_free.pass.cpp
@@ -13,7 +13,7 @@
// bool is_lock_free() const noexcept;
#include <atomic>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From 20e8de1559650c08f41c2c92b28d4b6d7cdbea6e Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:33:50 +0300
Subject: [PATCH 33/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../utilities/memory/util.smartptr/atomic/shared/load.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
index 2942cfe6c1017..6278adc33534b 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/shared/load.pass.cpp
@@ -12,7 +12,7 @@
// shared_ptr<T> load(memory_order order = memory_order::seq_cst) const noexcept;
#include <atomic>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From 874c353c498b58da790a44266d56e73396755af4 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:34:37 +0300
Subject: [PATCH 34/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
index 8f282baef9bb2..bb4de8a987387 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/compare_exchange_strong.pass.cpp
@@ -14,7 +14,7 @@
#include <atomic>
#include <cassert>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From 99c31a770ffca8fe87c1e6a64e13dab3046f2437 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:35:01 +0300
Subject: [PATCH 35/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
index 97d46023e4e37..5e2a1382391b8 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/convert.pass.cpp
@@ -13,7 +13,7 @@
#include <atomic>
#include <cassert>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From bfa71d5fc77bb21471d45dc85b483f324f96c050 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:35:17 +0300
Subject: [PATCH 36/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../memory/util.smartptr/atomic/weak/exchange.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
index d553dbc34390a..1413df02765d5 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/exchange.pass.cpp
@@ -13,7 +13,7 @@
#include <atomic>
#include <cassert>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From acbf4c35534635e3a5197fd4d9845f1496335c38 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:35:33 +0300
Subject: [PATCH 37/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../util.smartptr/atomic/weak/is_always_lock_free.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
index 31b7fe877a92c..b0ca1d3581cfc 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_always_lock_free.pass.cpp
@@ -13,7 +13,7 @@
// bool is_lock_free() const noexcept;
#include <atomic>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From eeb4fc46bd293f12ff49e1510ac38a5d8d746dd3 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:35:48 +0300
Subject: [PATCH 38/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
index f65b24d55449d..8ef3b84ba58a0 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/weak/is_lock_free.pass.cpp
@@ -12,7 +12,7 @@
// bool is_lock_free() const noexcept;
#include <atomic>
-#include <concepts> // needed on Armv7/Armv8 with -fmodules
+#include <concepts>
#include <memory>
#include "../atomic_smart_ptr_test_types.h"
>From 8cec3600826075e63e1736098959d3c9b40787fa Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:36:23 +0300
Subject: [PATCH 39/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../util.smartptr/atomic/atomic_smart_ptr_test_types.h | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
index b8a235e8e1416..e803569b4f5a7 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
@@ -5,11 +5,10 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-//
+
// Shared fixtures for libc++ tests under util.smartptr/atomic/{shared,weak}/:
// heterogeneous value types (built-in, standard library, and user-defined).
-//
-//===----------------------------------------------------------------------===//
+
#ifndef TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
#define TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
>From d98ca0c93c0a0f228ef17cdf990089dd735cef88 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:36:39 +0300
Subject: [PATCH 40/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
index e803569b4f5a7..1160902d5559d 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
@@ -17,7 +17,7 @@
#include <memory>
#include <string>
-// --- User-defined and semi-random scalar-like types ---------------------------------
+// User-defined and semi-random scalar-like types.
struct TrackedPod {
std::uint32_t gen{};
>From 3c5f20d6db5ace19b604658b818002f354640ec8 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <34096407+ViNN280801 at users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:37:00 +0300
Subject: [PATCH 41/46] Update
libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
Co-authored-by: Hristo Hristov <zingam at outlook.com>
---
.../memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
index 1160902d5559d..6fc8fe285ad99 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
@@ -38,7 +38,7 @@ class Handle {
enum class Flag : std::uint16_t { Off = 0, Stale = 4099, On = 60000 };
-// --- Distinct shared states for compare/exchange style tests ------------------------
+// Distinct shared states for compare/exchange style tests.
template <class T>
struct SpValues;
>From 4258d961329f6a0bfccc332858b885a52c1142c5 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Wed, 26 Aug 2026 21:18:56 +0300
Subject: [PATCH 42/46] [libc++] Formatted with clang-version 22.1.8
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
.../memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
index 6fc8fe285ad99..264b697308ab8 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
+++ b/libcxx/test/std/utilities/memory/util.smartptr/atomic/atomic_smart_ptr_test_types.h
@@ -9,7 +9,6 @@
// Shared fixtures for libc++ tests under util.smartptr/atomic/{shared,weak}/:
// heterogeneous value types (built-in, standard library, and user-defined).
-
#ifndef TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
#define TEST_STD_UTILITIES_MEMORY_UTIL_SMARTPTR_ATOMIC_ATOMIC_SMART_PTR_TEST_TYPES_H
>From 866b4403596bbe5ed849adc17548f4a1e965a82a Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Wed, 26 Aug 2026 22:19:22 +0300
Subject: [PATCH 43/46] [libc++] Fix atomic<shared_ptr<T>> CAS livelock and
lost wakeup; rework benchmark harness.
__cas_dwcas waited for local_count == 0 before its DWCAS, which
livelocks under load()/compare_exchange_strong() contention. The CAS
now swaps unconditionally. This reopens a control-block A->B->A cycle
race; compensating with an extra release in the loader causes a
double-free (confirmed under ASan), so the fix instead accepts a
bounded tick-leak (confirmed under LSan, no corruption). Closing the
leak needs a version tag on local_count, an ABI-affecting change left
for later.
Also fix a lost-wakeup race in the lock-based fallback's notify-bit
re-arming, and add _LIBCPP_FORCE_LOCK_BASED_ATOMIC_SHARED_PTR to force
the spinlock path on DWCAS-capable targets for A/B benchmarking.
Implement and rework atomic_shared_ptr_bench.runner.py: render matplotlib/PNG
comparison charts, and add --swap-order/--interleave/--settle-seconds
to cancel thermal and frequency-drift bias from always benchmarking
the lock-free binary first.
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/include/__memory/atomic_shared_ptr.h | 8 +-
.../__memory/atomic_shared_ptr_lock_based.h | 53 +-
.../__memory/atomic_shared_ptr_lock_free.h | 273 ++--
.../benchmarks/atomic_shared_ptr.bench.cpp | 42 +-
.../test/benchmarks/atomic_shared_ptr_bench.h | 297 ++++
.../atomic_shared_ptr_bench.runner.py | 1267 +++++++++++++++++
.../atomic_shared_ptr_lock_based.bench.cpp | 23 +
.../atomic_shared_ptr_memory_order.verify.cpp | 2 +-
.../atomic_shared_ptr_nullptr.pass.cpp | 2 +-
.../atomic_shared_ptr_refcount.pass.cpp | 2 +-
libcxx/utils/requirements.txt | 1 +
11 files changed, 1838 insertions(+), 132 deletions(-)
create mode 100644 libcxx/test/benchmarks/atomic_shared_ptr_bench.h
create mode 100755 libcxx/test/benchmarks/atomic_shared_ptr_bench.runner.py
create mode 100644 libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp
diff --git a/libcxx/include/__memory/atomic_shared_ptr.h b/libcxx/include/__memory/atomic_shared_ptr.h
index 95e12fcff0a0c..57db16ca10e92 100644
--- a/libcxx/include/__memory/atomic_shared_ptr.h
+++ b/libcxx/include/__memory/atomic_shared_ptr.h
@@ -16,7 +16,13 @@
# pragma GCC system_header
#endif
-#if _LIBCPP_HAS_THREADS && \
+// _LIBCPP_FORCE_LOCK_BASED_ATOMIC_SHARED_PTR selects the stolen-bit spinlock
+// even on DWCAS targets. Local A/B-benchmark hook for
+// libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp; not a
+// supported public configuration. Must be consistent across the whole program.
+#if defined(_LIBCPP_FORCE_LOCK_BASED_ATOMIC_SHARED_PTR)
+# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 0
+#elif _LIBCPP_HAS_THREADS && \
(((defined(__x86_64__) || defined(_M_X64)) && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)) || \
((defined(__aarch64__) || defined(_M_ARM64)) && defined(__ARM_FEATURE_ATOMICS)))
# define _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR 1
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_based.h b/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
index d1df87a9812b1..b8d4b93a17a40 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_based.h
@@ -129,8 +129,13 @@ struct __atomic_smart_ptr_fields {
const void* __ctrl_address() const noexcept { return static_cast<const void*>(__builtin_addressof(__ctrl_)); }
- // Notify bit is set before sleeping so __unlock wakes waiters; without it
- // sleeping threads would remain blocked after the lock is released.
+ // Stolen-bit lock. Park via the global wait table (futex on Linux), not a
+ // pure CAS spin. The notify bit must be set on the *current* locked word
+ // before every park: __unlock publishes bits=0, so a waiter that loops
+ // inside wait_on_address after an unlock+reacquire can sleep on a word
+ // whose notify bit was already cleared. The next unlock then skips
+ // notify_all. On Linux that is a 2s FUTEX_WAIT timeout; on AIX the
+ // fallback wait polls platform_state forever (no native wake).
void __lock() const noexcept {
_LIBCPP_ATOMIC_SP_TSAN_PRE_LOCK(__builtin_addressof(__ctrl_));
uintptr_t __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
@@ -161,20 +166,36 @@ struct __atomic_smart_ptr_fields {
__expected = __with_notify;
}
- std::__atomic_smart_ptr_wait_on_address(__ctrl_address(), [&] {
- __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
- return !__atomic_smart_ptr_storage::__has_lock(__expected);
- });
+ // Snapshot the wait generation, then refuse to park unless this word
+ // still has both the lock and the notify bit. Otherwise go back and
+ // re-arm: parking after unlock cleared the bit is a missed wakeup.
+ const void* __address = __ctrl_address();
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ auto __monitor_value = std::__atomic_monitor_global(__address);
+ __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
+ if (!__atomic_smart_ptr_storage::__has_lock(__expected) || !__atomic_smart_ptr_storage::__has_notify(__expected))
+ continue;
+ std::__atomic_wait_global_table(__address, __monitor_value);
+# else
+ void const volatile* __volatile_address = reinterpret_cast<void const volatile*>(__address);
+ auto __monitor_value = std::__libcpp_atomic_monitor(__volatile_address);
+ __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
+ if (!__atomic_smart_ptr_storage::__has_lock(__expected) || !__atomic_smart_ptr_storage::__has_notify(__expected))
+ continue;
+ std::__libcpp_atomic_wait(__volatile_address, __monitor_value);
+# endif
+ __expected = std::__cxx_atomic_load(__builtin_addressof(__ctrl_), memory_order_relaxed);
}
}
- // Notify bit means at least one waiter is sleeping on __ctrl_; wake them
- // after publishing the new CB so they don't miss the change.
- void __unlock(__shared_weak_count* __ctrl_to_publish) const noexcept {
+ // Publish the new CB with both stolen bits clear. Wake if a locker armed
+ // the notify bit, or if the ownership word changed (atomic wait() watchers
+ // never set the lock notify bit).
+ void __unlock(__shared_weak_count* __ctrl_to_publish, bool __wake_value_waiters = false) const noexcept {
_LIBCPP_ATOMIC_SP_TSAN_PRE_UNLOCK(__builtin_addressof(__ctrl_));
uintptr_t __new_word = __atomic_smart_ptr_storage::__encode(__ctrl_to_publish, 0);
uintptr_t __previous = std::__cxx_atomic_exchange(__builtin_addressof(__ctrl_), __new_word, memory_order_release);
- if (__atomic_smart_ptr_storage::__has_notify(__previous))
+ if (__wake_value_waiters || __atomic_smart_ptr_storage::__has_notify(__previous))
std::__atomic_smart_ptr_notify_all(__ctrl_address());
_LIBCPP_ATOMIC_SP_TSAN_POST_UNLOCK(__builtin_addressof(__ctrl_));
}
@@ -232,7 +253,7 @@ struct atomic<shared_ptr<_Tp>> {
__shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
+ __fields_.__unlock(__desired_c, true);
if (__old_c)
__old_c->__release_shared();
@@ -263,7 +284,7 @@ struct atomic<shared_ptr<_Tp>> {
__shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
+ __fields_.__unlock(__desired_c, true);
return shared_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_c);
}
@@ -285,7 +306,7 @@ struct atomic<shared_ptr<_Tp>> {
__desired.__cntrl_ = nullptr;
std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
+ __fields_.__unlock(__desired_c, true);
if (__cur_c)
__cur_c->__release_shared();
return true;
@@ -376,7 +397,7 @@ struct atomic<weak_ptr<_Tp>> {
__shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
+ __fields_.__unlock(__desired_c, true);
if (__old_c)
__old_c->__release_weak();
@@ -407,7 +428,7 @@ struct atomic<weak_ptr<_Tp>> {
__shared_weak_count* __old_c = __atomic_smart_ptr_storage::__decode(
std::__cxx_atomic_load(__builtin_addressof(__fields_.__ctrl_), memory_order_relaxed));
std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
+ __fields_.__unlock(__desired_c, true);
return weak_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_c);
}
@@ -429,7 +450,7 @@ struct atomic<weak_ptr<_Tp>> {
__desired.__cntrl_ = nullptr;
std::__cxx_atomic_store(__builtin_addressof(__fields_.__ptr_), __desired_ptr, memory_order_relaxed);
- __fields_.__unlock(__desired_c);
+ __fields_.__unlock(__desired_c, true);
if (__cur_c)
__cur_c->__release_weak();
return true;
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
index d56e7854e3eee..773744afc1f82 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
@@ -46,9 +46,12 @@ struct atomic;
// __ctrl_ word layout (8 bytes):
// bits 0..1 always zero (CBs are at least 4-byte aligned)
// bits 2..47 control-block pointer
-// bits 48..63 16-bit local refcount (Williams split-refcount aspiration)
+// bits 48..63 16-bit local refcount (Williams split-refcount)
// Bits 48..63 are available on both DWCAS-capable ISAs (x86-64 without LAMA,
// AArch64 without LVA): user-space pointers stay within 48 bits.
+// The local count pins the CB in the atomic word so load() can __add_shared
+// without a use-after-free if a concurrent store retires the pair. Dropping
+// the pin (plain load-and-validate) is not safe: the snapshot is not a ref.
struct __atomic_smart_ptr_storage {
static constexpr uintptr_t __lock_bit_ = uintptr_t{1};
static constexpr uintptr_t __notify_bit_ = uintptr_t{2};
@@ -123,12 +126,9 @@ struct alignas(16) __atomic_smart_ptr_fields {
__extension__ using __sp_u128 = unsigned __int128;
- // Storage layout: low 64 bits = __ptr_, high 64 bits = __ctrl_. Going
- // through `this` instead of __builtin_addressof(__ptr_) keeps the 16-byte alignment explicit
- // and avoids implementation-defined aliasing between the sub-atomics and
- // the wide view (sub-atomics are never accessed when DWCAS is enabled).
- // TODO(review): this cast bypasses strict aliasing - valid only because
- // __ptr_ and __ctrl_ are never touched via __cxx_atomic_* on this path.
+ // Storage layout: low 64 bits = __ptr_, high 64 bits = __ctrl_. The wide
+ // view is the only access path while DWCAS is enabled; the sub-atomics are
+ // not read or written through __cxx_atomic_*.
__sp_u128* __dwcas_address() const noexcept {
return const_cast<__sp_u128*>(reinterpret_cast<const __sp_u128*>(this));
}
@@ -184,6 +184,13 @@ inline bool __atomic_smart_ptr_equivalent(
return __ptr == __expected_ptr && __ctrl == __expected_ctrl;
}
+// Split-refcount identity: the control-block pointer after DWCAS masking.
+// Aliasing constructors can change the stored T* while ownership stays the
+// same; store/exchange must not zero local_count in that case.
+inline bool __atomic_smart_ptr_same_control_block(__shared_weak_count* __a, __shared_weak_count* __b) noexcept {
+ return __atomic_smart_ptr_storage::__encode_dwcas(__a, 0) == __atomic_smart_ptr_storage::__encode_dwcas(__b, 0);
+}
+
// Pre-pay the per-in-flight-load fetch_add(1) tick that each aspirational
// loader still owes the retiring control block; without this, the racing
// loader would double-count its own contribution on the new block.
@@ -317,9 +324,9 @@ struct atomic<shared_ptr<_Tp>> {
continue;
}
- // TODO(question): ABA: the 16-bit local count is not a version tag across CB
- // lifecycles. A freed CB reallocated at the same address with the same count
- // lets this CAS succeed spuriously. Is this risk acceptable?
+ // Increment CAS is on the 16-byte word (pointer + CB + local_count), not
+ // a standalone version tag. Heap reuse of a retired CB at the same
+ // address while a loader is still in this loop is a client lifetime bug.
auto __desired =
__fields_type::__pair_make(__claimed_ptr, __ctrl_word + __atomic_smart_ptr_storage::__local_count_unit);
if (__fields_.__dwcas_compare_exchange_weak(__expected, __desired, memory_order_acq_rel, memory_order_acquire))
@@ -333,15 +340,21 @@ struct atomic<shared_ptr<_Tp>> {
uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
__shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
- // TODO(review): the store that replaced this CB pre-paid our tick via
- // drain_shared, so __release_shared below cannot be the last ref - verify
- // no concurrent release can create a zero-count window between drain and here.
if (__cur_cb != __claimed_cb) {
- // A racing store already drained our local tick via __drain_shared,
- // so __add_shared above became a double-count. Cancel it.
- __claimed_cb->__release_shared();
+ // Store already swapped this CB out and __drain_shared prepaid one
+ // heap ref per tick that was in the word. Extra-release here was
+ // meant to cancel a duplicate __add_shared, but it also fires when
+ // the tick was not in the drained word (A->B with a lost update, or
+ // A->B->A). That steals a live owner (readers-heavy: keep_a) and
+ // the returned shared_ptr plus keep_a both call __release_shared on
+ // a freed block. Keep the __add_shared; the drain ref may leak.
break;
}
+ // Same ownership and no ticks in the word. Extra-release here steals a
+ // live ref whenever we were not drained (readers-heavy double-free).
+ // Keep the add_shared; same-CB store/exchange must not zero local_count.
+ if (__atomic_smart_ptr_storage::__decode_local(__cur_ctrl) == 0)
+ break;
auto __dec = __fields_type::__pair_make(
__fields_type::__pair_ptr(__cur), __cur_ctrl - __atomic_smart_ptr_storage::__local_count_unit);
@@ -359,14 +372,34 @@ struct atomic<shared_ptr<_Tp>> {
__desired.__ptr_ = nullptr;
__desired.__cntrl_ = nullptr;
- auto __new_pair =
- __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
- auto __old_pair = __fields_.__dwcas_exchange(__new_pair, memory_order_acq_rel);
+ auto __cur = __fields_.__dwcas_load(memory_order_acquire);
+ while (true) {
+ uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
+ __shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
- __retire_old_pair(__old_pair);
+ if (std::__atomic_smart_ptr_same_control_block(__cur_cb, __desired_c)) {
+ // Same ownership, possibly a different T* (aliasing). Publishing
+ // local_count=0 retires in-flight load ticks while load() still
+ // treats this CB as live. Keep local_count; drop the extra ref
+ // taken from __desired.
+ auto __keep_local = __fields_type::__pair_make(__desired_ptr, __cur_ctrl);
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __keep_local, __m, memory_order_acquire)) {
+ if (__desired_c)
+ __desired_c->__release_shared();
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return;
+ }
+ continue;
+ }
- (void)__m;
- std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ auto __new_pair =
+ __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __new_pair, __m, memory_order_acquire)) {
+ __retire_old_pair(__cur);
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return;
+ }
+ }
}
shared_ptr<_Tp> __exchange_dwcas(shared_ptr<_Tp> __desired, memory_order __m) noexcept {
@@ -375,32 +408,42 @@ struct atomic<shared_ptr<_Tp>> {
__desired.__ptr_ = nullptr;
__desired.__cntrl_ = nullptr;
- auto __new_pair =
- __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
- auto __old_pair = __fields_.__dwcas_exchange(__new_pair, memory_order_acq_rel);
+ auto __cur = __fields_.__dwcas_load(memory_order_acquire);
+ while (true) {
+ uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
+ _Tp* __cur_ptr = __fields_type::__pair_ptr(__cur);
+ __shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
- uintptr_t __old_ctrl = __fields_type::__pair_ctrl(__old_pair);
- _Tp* __old_ptr = __fields_type::__pair_ptr(__old_pair);
- __shared_weak_count* __old_cb = __atomic_smart_ptr_storage::__decode_dwcas(__old_ctrl);
- uint16_t __old_local = __atomic_smart_ptr_storage::__decode_local(__old_ctrl);
+ if (std::__atomic_smart_ptr_same_control_block(__cur_cb, __desired_c)) {
+ auto __keep_local = __fields_type::__pair_make(__desired_ptr, __cur_ctrl);
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __keep_local, __m, memory_order_acquire)) {
+ if (__desired_c)
+ __desired_c->__release_shared();
+ if (__cur_cb)
+ __cur_cb->__add_shared();
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return shared_ptr<_Tp>::__create_with_control_block(__desired_ptr, __cur_cb);
+ }
+ continue;
+ }
- if (__old_cb) {
- // Drain in-flight loaders' aspirational ticks. After this, __old_cb's
- // global count includes one ref per in-flight load that paired up with
- // this CB. We then transfer ownership of the atomic's own ref to the
- // returned shared_ptr (no add/release).
- std::__atomic_smart_ptr_drain_shared(__old_cb, __old_local);
+ auto __new_pair =
+ __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __new_pair, __m, memory_order_acquire)) {
+ uint16_t __cur_local = __atomic_smart_ptr_storage::__decode_local(__cur_ctrl);
+ if (__cur_cb)
+ std::__atomic_smart_ptr_drain_shared(__cur_cb, __cur_local);
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return shared_ptr<_Tp>::__create_with_control_block(__cur_ptr, __cur_cb);
+ }
}
-
- (void)__m;
- std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
-
- return shared_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_cb);
}
- // __is_strong: when true, retry on benign DWCAS failures (e.g., local-count
- // changes by concurrent loads); only return false on a real
- // ownership-equivalence mismatch.
+ // __is_strong: retry on a failed DWCAS while equivalence still holds
+ // (concurrent local_count updates, spurious weak failure). Do not wait
+ // for local_count==0: that starves against load() on the same word
+ // (contended CAS bench: load then CAS). Swap the current word, including
+ // a non-zero local_count, and drain like store().
bool __cas_dwcas(shared_ptr<_Tp>& __expected,
shared_ptr<_Tp> __desired,
memory_order __success,
@@ -428,30 +471,36 @@ struct atomic<shared_ptr<_Tp>> {
return false;
}
- // Equivalence holds. Try to publish __desired with local_count=0.
- auto __desired_pair =
- __fields_type::__pair_make(__des_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__des_c, 0));
-
- if (__fields_.__dwcas_compare_exchange_weak(__cur, __desired_pair, __success, __failure)) {
+ if (std::__atomic_smart_ptr_same_control_block(__cur_cb, __des_c)) {
+ auto __keep_local = __fields_type::__pair_make(__des_ptr, __cur_ctrl);
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __keep_local, __success, __failure)) {
+ if (__des_c)
+ __des_c->__release_shared();
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return true;
+ }
+ } else {
+ auto __desired_pair =
+ __fields_type::__pair_make(__des_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__des_c, 0));
uint16_t __cur_local = __atomic_smart_ptr_storage::__decode_local(__cur_ctrl);
- if (__cur_cb) {
- std::__atomic_smart_ptr_drain_shared(__cur_cb, __cur_local);
- __cur_cb->__release_shared();
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __desired_pair, __success, __failure)) {
+ if (__cur_cb) {
+ std::__atomic_smart_ptr_drain_shared(__cur_cb, __cur_local);
+ __cur_cb->__release_shared();
+ }
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return true;
}
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
- std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
- return true;
}
if (!__is_strong) {
- // weak: a single underlying CAS attempt - if it failed for any
- // reason (real ownership change or concurrent local-count update),
- // report failure with the current pair as the new __expected.
__expected = __load_dwcas(__failure);
return false;
}
- // Strong CAS: loop and re-check equivalence.
}
}
@@ -592,9 +641,11 @@ struct atomic<weak_ptr<_Tp>> {
__shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
if (__cur_cb != __claimed_cb) {
- __claimed_cb->__release_weak();
+ // Same as atomic<shared_ptr>: do not extra-release on CB change.
break;
}
+ if (__atomic_smart_ptr_storage::__decode_local(__cur_ctrl) == 0)
+ break;
auto __dec = __fields_type::__pair_make(
__fields_type::__pair_ptr(__cur), __cur_ctrl - __atomic_smart_ptr_storage::__local_count_unit);
@@ -612,14 +663,30 @@ struct atomic<weak_ptr<_Tp>> {
__desired.__ptr_ = nullptr;
__desired.__cntrl_ = nullptr;
- auto __new_pair =
- __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
- auto __old_pair = __fields_.__dwcas_exchange(__new_pair, memory_order_acq_rel);
+ auto __cur = __fields_.__dwcas_load(memory_order_acquire);
+ while (true) {
+ uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
+ __shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
- __retire_old_pair_weak(__old_pair);
+ if (std::__atomic_smart_ptr_same_control_block(__cur_cb, __desired_c)) {
+ auto __keep_local = __fields_type::__pair_make(__desired_ptr, __cur_ctrl);
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __keep_local, __m, memory_order_acquire)) {
+ if (__desired_c)
+ __desired_c->__release_weak();
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return;
+ }
+ continue;
+ }
- (void)__m;
- std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ auto __new_pair =
+ __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __new_pair, __m, memory_order_acquire)) {
+ __retire_old_pair_weak(__cur);
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return;
+ }
+ }
}
weak_ptr<_Tp> __exchange_dwcas(weak_ptr<_Tp> __desired, memory_order __m) noexcept {
@@ -628,22 +695,35 @@ struct atomic<weak_ptr<_Tp>> {
__desired.__ptr_ = nullptr;
__desired.__cntrl_ = nullptr;
- auto __new_pair =
- __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
- auto __old_pair = __fields_.__dwcas_exchange(__new_pair, memory_order_acq_rel);
-
- uintptr_t __old_ctrl = __fields_type::__pair_ctrl(__old_pair);
- _Tp* __old_ptr = __fields_type::__pair_ptr(__old_pair);
- __shared_weak_count* __old_cb = __atomic_smart_ptr_storage::__decode_dwcas(__old_ctrl);
- uint16_t __old_local = __atomic_smart_ptr_storage::__decode_local(__old_ctrl);
-
- if (__old_cb)
- std::__atomic_smart_ptr_drain_weak(__old_cb, __old_local);
+ auto __cur = __fields_.__dwcas_load(memory_order_acquire);
+ while (true) {
+ uintptr_t __cur_ctrl = __fields_type::__pair_ctrl(__cur);
+ _Tp* __cur_ptr = __fields_type::__pair_ptr(__cur);
+ __shared_weak_count* __cur_cb = __atomic_smart_ptr_storage::__decode_dwcas(__cur_ctrl);
- (void)__m;
- std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ if (std::__atomic_smart_ptr_same_control_block(__cur_cb, __desired_c)) {
+ auto __keep_local = __fields_type::__pair_make(__desired_ptr, __cur_ctrl);
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __keep_local, __m, memory_order_acquire)) {
+ if (__desired_c)
+ __desired_c->__release_weak();
+ if (__cur_cb)
+ __cur_cb->__add_weak();
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return weak_ptr<_Tp>::__create_with_control_block(__desired_ptr, __cur_cb);
+ }
+ continue;
+ }
- return weak_ptr<_Tp>::__create_with_control_block(__old_ptr, __old_cb);
+ auto __new_pair =
+ __fields_type::__pair_make(__desired_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__desired_c, 0));
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __new_pair, __m, memory_order_acquire)) {
+ uint16_t __cur_local = __atomic_smart_ptr_storage::__decode_local(__cur_ctrl);
+ if (__cur_cb)
+ std::__atomic_smart_ptr_drain_weak(__cur_cb, __cur_local);
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return weak_ptr<_Tp>::__create_with_control_block(__cur_ptr, __cur_cb);
+ }
+ }
}
bool __cas_dwcas(weak_ptr<_Tp>& __expected,
@@ -670,19 +750,30 @@ struct atomic<weak_ptr<_Tp>> {
return false;
}
- auto __desired_pair =
- __fields_type::__pair_make(__des_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__des_c, 0));
-
- if (__fields_.__dwcas_compare_exchange_weak(__cur, __desired_pair, __success, __failure)) {
+ if (std::__atomic_smart_ptr_same_control_block(__cur_cb, __des_c)) {
+ auto __keep_local = __fields_type::__pair_make(__des_ptr, __cur_ctrl);
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __keep_local, __success, __failure)) {
+ if (__des_c)
+ __des_c->__release_weak();
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return true;
+ }
+ } else {
+ auto __desired_pair =
+ __fields_type::__pair_make(__des_ptr, __atomic_smart_ptr_storage::__encode_dwcas(__des_c, 0));
uint16_t __cur_local = __atomic_smart_ptr_storage::__decode_local(__cur_ctrl);
- if (__cur_cb) {
- std::__atomic_smart_ptr_drain_weak(__cur_cb, __cur_local);
- __cur_cb->__release_weak();
+ if (__fields_.__dwcas_compare_exchange_weak(__cur, __desired_pair, __success, __failure)) {
+ if (__cur_cb) {
+ std::__atomic_smart_ptr_drain_weak(__cur_cb, __cur_local);
+ __cur_cb->__release_weak();
+ }
+ __desired.__ptr_ = nullptr;
+ __desired.__cntrl_ = nullptr;
+ std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
+ return true;
}
- __desired.__ptr_ = nullptr;
- __desired.__cntrl_ = nullptr;
- std::__atomic_smart_ptr_notify_all(__fields_.__wait_address());
- return true;
}
if (!__is_strong) {
diff --git a/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp b/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
index 47239b5c9a613..763e3d95cbf76 100644
--- a/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
+++ b/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
@@ -7,28 +7,28 @@
//===----------------------------------------------------------------------===//
// REQUIRES: std-at-least-c++20
+// REQUIRES: stdlib=libc++
+// UNSUPPORTED: no-threads
+// Clang's x86-64 baseline does not define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16;
+// without this, the "lock-free" file compiles the stolen-bit path. Same flag
+// as atomic_shared_ptr_dwcas.pass.cpp. Keep in sync with the lock-based file
+// so the A/B differs only by _LIBCPP_FORCE_LOCK_BASED_ATOMIC_SHARED_PTR.
+// ADDITIONAL_COMPILE_FLAGS(target=x86_64-unknown-linux-gnu): -march=x86-64-v2
-#include <atomic>
-#include <memory>
-
-#include "benchmark/benchmark.h"
-
-static void BM_AtomicSharedPtrLoadUncontended(benchmark::State& st) {
- std::atomic<std::shared_ptr<int>> atom(std::make_shared<int>(42));
- while (st.KeepRunning()) {
- auto snap = atom.load();
- benchmark::DoNotOptimize(snap);
- }
-}
-BENCHMARK(BM_AtomicSharedPtrLoadUncontended);
+// libc++ default path: lock-free on x86_64/cx16 and AArch64/LSE, otherwise
+// lock-based. Pair with atomic_shared_ptr_lock_based.bench.cpp on the same
+// machine. Contended thread counts default to 2,4,8; override at process
+// start with ATOMIC_SP_BENCH_THREADS=2,4,6,8.
+//
+// Same-machine A/B (each file separately - names collide if consolidated
+// together). Driver next to these benches:
+// python3 libcxx/test/benchmarks/run_atomic_shared_ptr_bench.py --build <build>
+// python3 libcxx/test/benchmarks/run_atomic_shared_ptr_bench.py --build <build> \
+// --threads 2,4,6,8,10,12,14,16,18,20
+// Needs: pip install -r libcxx/utils/requirements.txt
+//
+// Report ratios to std::atomic<uint64_t>::* in the same JSON, not raw ns/op.
-static void BM_AtomicSharedPtrStoreUncontended(benchmark::State& st) {
- std::atomic<std::shared_ptr<int>> atom;
- auto keep = std::make_shared<int>(7);
- while (st.KeepRunning()) {
- atom.store(keep);
- }
-}
-BENCHMARK(BM_AtomicSharedPtrStoreUncontended);
+#include "atomic_shared_ptr_bench.h"
BENCHMARK_MAIN();
diff --git a/libcxx/test/benchmarks/atomic_shared_ptr_bench.h b/libcxx/test/benchmarks/atomic_shared_ptr_bench.h
new file mode 100644
index 0000000000000..e2abf708fb83b
--- /dev/null
+++ b/libcxx/test/benchmarks/atomic_shared_ptr_bench.h
@@ -0,0 +1,297 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_BENCHMARK_ATOMIC_SHARED_PTR_BENCH_H
+#define TEST_BENCHMARK_ATOMIC_SHARED_PTR_BENCH_H
+
+// libc++ lock-free vs lock-based microbenchmarks for
+// std::atomic<std::shared_ptr<T>>. Names match so compare-benchmarks can pair
+// the two binaries:
+//
+// lock-free (default on x86_64/cx16 and AArch64/LSE):
+// atomic_shared_ptr.bench.cpp
+// lock-based (forced, same machine):
+// atomic_shared_ptr_lock_based.bench.cpp
+//
+// Normalize on the same machine: divide each shared_ptr result by the matching
+// std::atomic<uint64_t> bench in the same run. Absolute ns/op are host-specific;
+// the ratio is the number to report.
+//
+// Driver: libcxx/test/benchmarks/run_atomic_shared_ptr_bench.py
+
+#include <atomic>
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "test_macros.h"
+
+#ifdef _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR
+# if _LIBCPP_HAS_LOCKFREE_ATOMIC_SHARED_PTR
+# define ATOMIC_SP_IMPL_LABEL "libcxx_lock_free"
+# else
+# define ATOMIC_SP_IMPL_LABEL "libcxx_lock_based"
+# endif
+#else
+# define ATOMIC_SP_IMPL_LABEL "libcxx_lock_based"
+#endif
+
+inline void annotate_atomic_sp(benchmark::State& state) { state.SetLabel(ATOMIC_SP_IMPL_LABEL); }
+
+// Contended thread counts. Override at process start (before BENCHMARK
+// registration) with ATOMIC_SP_BENCH_THREADS=2,4,6,8. Default: 2,4,8.
+// Google Benchmark has no --threads CLI that overrides ->Threads(N).
+inline std::vector<int> atomic_sp_thread_counts() {
+ std::vector<int> counts;
+ const char* env = std::getenv("ATOMIC_SP_BENCH_THREADS");
+ const char* p = (env != nullptr && env[0] != '\0') ? env : "2,4,8";
+ while (*p != '\0') {
+ while (*p == ' ' || *p == '\t' || *p == ',')
+ ++p;
+ if (*p == '\0')
+ break;
+ char* end = nullptr;
+ const long v = std::strtol(p, &end, 10);
+ if (end == p)
+ break;
+ if (v >= 1 && v <= 1024) {
+ const int t = static_cast<int>(v);
+ bool seen = false;
+ for (int existing : counts) {
+ if (existing == t) {
+ seen = true;
+ break;
+ }
+ }
+ if (!seen)
+ counts.push_back(t);
+ }
+ p = end;
+ }
+ if (counts.empty()) {
+ counts.push_back(2);
+ counts.push_back(4);
+ counts.push_back(8);
+ }
+ return counts;
+}
+
+inline void apply_contended_threads(benchmark::Benchmark* b) {
+ for (int t : atomic_sp_thread_counts())
+ b->Threads(t);
+ b->UseRealTime();
+}
+
+// Baseline: lock-free scalar CAS/load/store on the same binary. Use these as
+// the denominator when reporting normalized numbers.
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_u64_load_uncontended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ std::atomic<std::uint64_t> atom{42};
+ for (auto _ : state) {
+ auto snap = atom.load();
+ benchmark::DoNotOptimize(snap);
+ }
+}
+BENCHMARK(bm_atomic_u64_load_uncontended)->Name("std::atomic<uint64_t>::load() (uncontended)");
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_u64_store_uncontended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ std::atomic<std::uint64_t> atom{0};
+ for (auto _ : state) {
+ atom.store(7);
+ // Without this the store is not observed and -O3 DSE's the whole loop
+ // (the previous 0.000 ns / 1e12-iteration row).
+ benchmark::DoNotOptimize(atom);
+ }
+}
+BENCHMARK(bm_atomic_u64_store_uncontended)->Name("std::atomic<uint64_t>::store() (uncontended)");
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_u64_cas_uncontended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ std::atomic<std::uint64_t> atom{1};
+ for (auto _ : state) {
+ std::uint64_t expected = atom.load(std::memory_order_relaxed);
+ (void)atom.compare_exchange_strong(expected, expected ^ 1);
+ }
+}
+BENCHMARK(bm_atomic_u64_cas_uncontended)->Name("std::atomic<uint64_t>::compare_exchange_strong() (uncontended)");
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_u64_load_contended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ static std::atomic<std::uint64_t> atom{42};
+ if (state.thread_index() == 0)
+ atom.store(42);
+ for (auto _ : state) {
+ auto snap = atom.load();
+ benchmark::DoNotOptimize(snap);
+ }
+}
+BENCHMARK(bm_atomic_u64_load_contended)
+ ->Name("std::atomic<uint64_t>::load() (contended)")
+ ->Apply(apply_contended_threads);
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_u64_cas_contended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ static std::atomic<std::uint64_t> atom{1};
+ if (state.thread_index() == 0)
+ atom.store(1);
+ for (auto _ : state) {
+ std::uint64_t expected = atom.load(std::memory_order_relaxed);
+ (void)atom.compare_exchange_strong(expected, expected ^ 1);
+ }
+}
+BENCHMARK(bm_atomic_u64_cas_contended)
+ ->Name("std::atomic<uint64_t>::compare_exchange_strong() (contended)")
+ ->Apply(apply_contended_threads);
+
+// std::atomic<shared_ptr<T>>. Uncontended benches use a per-thread object.
+// Contended benches share one atomic (function-local static) so Threads(N)
+// is real contention, not N independent copies.
+//
+// Google Benchmark registers each ->Threads(K) as a separate invocation of
+// the same C++ function in one process. The static is initialized once;
+// without an explicit reset, CAS/store leftovers leak from threads:2 into
+// threads:4/8 (and across KeepRunning calibration). Thread 0 resets before
+// the timed loop; the loop begin is a barrier, so other threads wait.
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_shared_ptr_load_uncontended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ std::atomic<std::shared_ptr<int>> atom(std::make_shared<int>(42));
+ state.counters["lock_free"] = atom.is_lock_free() ? 1 : 0;
+ for (auto _ : state) {
+ auto snap = atom.load();
+ benchmark::DoNotOptimize(snap);
+ }
+}
+BENCHMARK(bm_atomic_shared_ptr_load_uncontended)->Name("std::atomic<shared_ptr<T>>::load() (uncontended)");
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_shared_ptr_store_uncontended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ std::atomic<std::shared_ptr<int>> atom;
+ auto keep = std::make_shared<int>(7);
+ state.counters["lock_free"] = atom.is_lock_free() ? 1 : 0;
+ for (auto _ : state) {
+ atom.store(keep);
+ }
+}
+BENCHMARK(bm_atomic_shared_ptr_store_uncontended)->Name("std::atomic<shared_ptr<T>>::store() (uncontended)");
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_shared_ptr_exchange_uncontended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ auto keep = std::make_shared<int>(7);
+ std::atomic<std::shared_ptr<int>> atom(keep);
+ state.counters["lock_free"] = atom.is_lock_free() ? 1 : 0;
+ for (auto _ : state) {
+ auto prev = atom.exchange(keep);
+ benchmark::DoNotOptimize(prev);
+ }
+}
+BENCHMARK(bm_atomic_shared_ptr_exchange_uncontended)->Name("std::atomic<shared_ptr<T>>::exchange() (uncontended)");
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_shared_ptr_cas_uncontended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ auto keep_a = std::make_shared<int>(1);
+ auto keep_b = std::make_shared<int>(2);
+ std::atomic<std::shared_ptr<int>> atom(keep_a);
+ state.counters["lock_free"] = atom.is_lock_free() ? 1 : 0;
+ for (auto _ : state) {
+ auto expected = atom.load(std::memory_order_relaxed);
+ auto desired = expected == keep_a ? keep_b : keep_a;
+ (void)atom.compare_exchange_strong(expected, desired);
+ }
+}
+BENCHMARK(bm_atomic_shared_ptr_cas_uncontended)
+ ->Name("std::atomic<shared_ptr<T>>::compare_exchange_strong() (uncontended)");
+
+struct AtomicSharedPtrSharedState {
+ std::shared_ptr<int> keep_a = std::make_shared<int>(1);
+ std::shared_ptr<int> keep_b = std::make_shared<int>(2);
+ std::atomic<std::shared_ptr<int>> atom{keep_a};
+
+ void reset() { atom.store(keep_a); }
+};
+
+inline void reset_shared_if_leader(benchmark::State& state, AtomicSharedPtrSharedState& shared) {
+ if (state.thread_index() == 0)
+ shared.reset();
+}
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_shared_ptr_load_contended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ static AtomicSharedPtrSharedState shared;
+ reset_shared_if_leader(state, shared);
+ // Set only on thread 0: Google Benchmark sums counters across threads.
+ if (state.thread_index() == 0)
+ state.counters["lock_free"] = shared.atom.is_lock_free() ? 1 : 0;
+ for (auto _ : state) {
+ auto snap = shared.atom.load();
+ benchmark::DoNotOptimize(snap);
+ }
+}
+BENCHMARK(bm_atomic_shared_ptr_load_contended)
+ ->Name("std::atomic<shared_ptr<T>>::load() (contended)")
+ ->Apply(apply_contended_threads);
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_shared_ptr_store_contended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ static AtomicSharedPtrSharedState shared;
+ reset_shared_if_leader(state, shared);
+ if (state.thread_index() == 0)
+ state.counters["lock_free"] = shared.atom.is_lock_free() ? 1 : 0;
+ for (auto _ : state) {
+ shared.atom.store(shared.keep_a);
+ }
+}
+BENCHMARK(bm_atomic_shared_ptr_store_contended)
+ ->Name("std::atomic<shared_ptr<T>>::store() (contended)")
+ ->Apply(apply_contended_threads);
+
+static TEST_ALIGN_BENCHMARK void bm_atomic_shared_ptr_cas_contended(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ static AtomicSharedPtrSharedState shared;
+ reset_shared_if_leader(state, shared);
+ if (state.thread_index() == 0)
+ state.counters["lock_free"] = shared.atom.is_lock_free() ? 1 : 0;
+ // load() then strong CAS is a real client pattern and the lock-free
+ // livelock trigger: load mutates local_count in the same DWCAS word
+ // that CAS tries to replace. Do not "fix" the hang by dropping load().
+ for (auto _ : state) {
+ auto expected = shared.atom.load(std::memory_order_relaxed);
+ auto desired = expected == shared.keep_a ? shared.keep_b : shared.keep_a;
+ (void)shared.atom.compare_exchange_strong(expected, desired);
+ }
+}
+BENCHMARK(bm_atomic_shared_ptr_cas_contended)
+ ->Name("std::atomic<shared_ptr<T>>::compare_exchange_strong() (contended)")
+ ->Apply(apply_contended_threads);
+
+// One writer, the rest load. Typical SMR-style read-mostly workload.
+static TEST_ALIGN_BENCHMARK void bm_atomic_shared_ptr_readers_heavy(benchmark::State& state) {
+ annotate_atomic_sp(state);
+ static AtomicSharedPtrSharedState shared;
+ reset_shared_if_leader(state, shared);
+ if (state.thread_index() == 0)
+ state.counters["lock_free"] = shared.atom.is_lock_free() ? 1 : 0;
+ const bool is_writer = state.thread_index() == 0;
+ for (auto _ : state) {
+ if (is_writer) {
+ shared.atom.store(shared.keep_b);
+ } else {
+ auto snap = shared.atom.load();
+ benchmark::DoNotOptimize(snap);
+ }
+ }
+}
+BENCHMARK(bm_atomic_shared_ptr_readers_heavy)
+ ->Name("std::atomic<shared_ptr<T>>::load() (readers-heavy, 1 writer)")
+ ->Apply(apply_contended_threads);
+
+#endif // TEST_BENCHMARK_ATOMIC_SHARED_PTR_BENCH_H
diff --git a/libcxx/test/benchmarks/atomic_shared_ptr_bench.runner.py b/libcxx/test/benchmarks/atomic_shared_ptr_bench.runner.py
new file mode 100755
index 0000000000000..20b103af7809a
--- /dev/null
+++ b/libcxx/test/benchmarks/atomic_shared_ptr_bench.runner.py
@@ -0,0 +1,1267 @@
+#!/usr/bin/env python3
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+"""Same-machine A/B of libc++ lock-free vs lock-based atomic<shared_ptr>.
+
+Runs atomic_shared_ptr.bench.cpp and atomic_shared_ptr_lock_based.bench.cpp
+separately (identical Google Benchmark names), writes two LNT files, a
+normalized ratio table (same-run uint64 CAS), PNG line charts of the same
+comparison, then optionally libcxx/utils/compare-benchmarks. Do not publish
+that tool's Geomean.
+
+Contended thread counts are not a Google Benchmark CLI flag. The bench
+header reads ATOMIC_SP_BENCH_THREADS at registration time. libc++'s
+%{exec} (libcxx/utils/run.py) starts the test with a clean environment,
+so this script compiles via lit (dry-run) and then runs t.tmp.exe itself.
+
+Requires a configured libc++ build (bootstrap or runtimes). Charts and
+compare-benchmarks both need: pip install -r libcxx/utils/requirements.txt
+(charts use the matplotlib dependency listed there). Pass --no-chart to
+skip charts entirely without needing matplotlib installed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import statistics
+import subprocess
+import sys
+import tempfile
+import time
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, NoReturn
+
+THREADS_ENV = "ATOMIC_SP_BENCH_THREADS"
+DEFAULT_THREADS = ",".join(str(i) for i in range(1, (os.cpu_count() or 1) + 1))
+DEFAULT_MIN_TIME = "1s"
+DEFAULT_REPETITIONS = 1
+MAX_THREADS = 1024
+THREADS_RE = re.compile(r"^[1-9][0-9]*(,[1-9][0-9]*)*$")
+MIN_TIME_RE = re.compile(r"^[0-9]+([.][0-9]+)?(s|ms)?$")
+AGGREGATE_SUFFIXES = ("_mean", "_median", "_stddev", "_cv")
+U64_CAS_CONTENDED = "std::atomic<uint64_t>::compare_exchange_strong() (contended)"
+SP_CONTENDED_OPS = ("store", "compare_exchange_strong", "load")
+SP_UNCONTENDED = (
+ "std::atomic<shared_ptr<T>>::load() (uncontended)",
+ "std::atomic<shared_ptr<T>>::store() (uncontended)",
+ "std::atomic<shared_ptr<T>>::exchange() (uncontended)",
+ "std::atomic<shared_ptr<T>>::compare_exchange_strong() (uncontended)",
+)
+
+LOCK_FREE_SRC = "atomic_shared_ptr.bench.cpp"
+LOCK_BASED_SRC = "atomic_shared_ptr_lock_based.bench.cpp"
+
+LOCK_FREE_COLOR = "#2a78d6"
+LOCK_BASED_COLOR = "#e34948"
+CHARTS_DIR_NAME = "charts"
+
+LIBCXX_TEST = Path("libcxx") / "test"
+BOOTSTRAP_LIBCXX_TEST = Path("runtimes") / "runtimes-bins" / LIBCXX_TEST
+BENCHMARKS_DIR = Path("benchmarks")
+LIT_OUTPUT_DIR = BENCHMARKS_DIR / "Output"
+LIBCXX_UTILS = Path("libcxx") / "utils"
+LIBCXX_LIT = LIBCXX_UTILS / "libcxx-lit"
+LLVM_LIT = Path("bin") / "llvm-lit"
+LIT_BENCH_EXE = "t.tmp.exe"
+BENCH_HEADER = BENCHMARKS_DIR / "atomic_shared_ptr_bench.h"
+ATOMIC_SP_HEADERS = (
+ Path("libcxx") / "include" / "__memory" / "atomic_shared_ptr.h",
+ Path("libcxx") / "include" / "__memory" / "atomic_shared_ptr_lock_free.h",
+ Path("libcxx") / "include" / "__memory" / "atomic_shared_ptr_lock_based.h",
+)
+
+
+def die(message: str, code: int = 1) -> NoReturn:
+ print(f"{Path(sys.argv[0]).name}: error: {message}", file=sys.stderr)
+ raise SystemExit(code)
+
+
+def log(message: str) -> None:
+ print(f"{Path(sys.argv[0]).name}: {message}", file=sys.stderr)
+
+
+def strip_aggregate_suffix(name: str) -> str:
+ for suffix in AGGREGATE_SUFFIXES:
+ if name.endswith(suffix):
+ return name[: -len(suffix)]
+ return name
+
+
+def selected_benchmarks(js: dict[str, Any]) -> list[dict[str, Any]]:
+ benches = js["benchmarks"]
+ have_median = any(
+ bm.get("aggregate_name") == "median" or bm["name"].endswith("_median")
+ for bm in benches
+ )
+ if have_median:
+ return [
+ bm
+ for bm in benches
+ if bm.get("aggregate_name") == "median" or bm["name"].endswith("_median")
+ ]
+ return [bm for bm in benches if bm.get("run_type", "iteration") != "aggregate"]
+
+
+def benchmarks_for_repetition(
+ js: dict[str, Any], repetition_index: int
+) -> list[dict[str, Any]]:
+ """Raw (non-aggregate) rows belonging to one --benchmark_repetitions run.
+
+ Only present when run_exe_to_lnt was invoked with repetitions > 1, since
+ that is the only case where Google Benchmark's JSON carries per-repetition
+ rows alongside the aggregate ones.
+ """
+ return [
+ bm
+ for bm in js["benchmarks"]
+ if bm.get("run_type", "iteration") != "aggregate"
+ and bm.get("repetition_index") == repetition_index
+ ]
+
+
+def thread_count(name: str) -> int | None:
+ marker = "/threads:"
+ if marker not in name:
+ return None
+ tail = strip_aggregate_suffix(name.rsplit(marker, 1)[1])
+ return int(tail)
+
+
+def times_by_threads(benches: list[dict[str, Any]], prefix: str) -> dict[int, float]:
+ out: dict[int, float] = {}
+ for bm in benches:
+ name = strip_aggregate_suffix(bm["name"])
+ if not name.startswith(prefix):
+ continue
+ threads = thread_count(name)
+ if threads is None:
+ continue
+ out[threads] = float(bm["real_time"])
+ return out
+
+
+def uncontended_time(benches: list[dict[str, Any]], name: str) -> float | None:
+ for bm in benches:
+ if strip_aggregate_suffix(bm["name"]) == name:
+ return float(bm["real_time"])
+ return None
+
+
+def format_ratio_row(values: list[float | None]) -> str:
+ cells = []
+ for value in values:
+ cells.append(f"{value:.1f}" if value is not None else "n/a")
+ return " | ".join(cells)
+
+
+def print_normalized_report(lf_json: Path, lb_json: Path, report_out: Path) -> None:
+ """Ratios to same-run same-thread uint64 CAS. Do not publish geomean."""
+ lf_js = json.loads(lf_json.read_text(encoding="utf-8"))
+ lb_js = json.loads(lb_json.read_text(encoding="utf-8"))
+ lf = selected_benchmarks(lf_js)
+ lb = selected_benchmarks(lb_js)
+ lf_base = times_by_threads(lf, U64_CAS_CONTENDED)
+ lb_base = times_by_threads(lb, U64_CAS_CONTENDED)
+ threads = sorted(set(lf_base) & set(lb_base))
+ if not threads:
+ die("normalized report: no overlapping uint64 CAS contended thread counts")
+
+ lines: list[str] = []
+ lf_load = lf_js.get("context", {}).get("load_avg")
+ lb_load = lb_js.get("context", {}).get("load_avg")
+ lf_scale = lf_js.get("context", {}).get("cpu_scaling_enabled")
+ lb_scale = lb_js.get("context", {}).get("cpu_scaling_enabled")
+ lines.append(
+ "Normalized to same-run std::atomic<uint64_t>::compare_exchange_strong (contended)."
+ )
+ lines.append(
+ "Do not compare raw ns across the two processes. Do not publish geomean."
+ )
+ lines.append(f"lock-free load_avg={lf_load} cpu_scaling={lf_scale}")
+ lines.append(f"lock-based load_avg={lb_load} cpu_scaling={lb_scale}")
+ if lf_scale or lb_scale:
+ lines.append(
+ "CPU scaling is enabled; pin with: sudo cpupower frequency-set -g performance"
+ )
+ header = "threads | " + " | ".join(str(t) for t in threads)
+ sep = "|".join(["---"] * (len(threads) + 1))
+
+ lines.append("")
+ lines.append("uint64 CAS contended (ns, denominator):")
+ lines.append(header)
+ lines.append(sep)
+ lines.append("lock-free | " + " | ".join(f"{lf_base[t]:.2f}" for t in threads))
+ lines.append("lock-based | " + " | ".join(f"{lb_base[t]:.2f}" for t in threads))
+ lines.append(
+ "lock-based/lock-free | "
+ + " | ".join(f"{lb_base[t] / lf_base[t]:.2f}" for t in threads)
+ )
+
+ for op in SP_CONTENDED_OPS:
+ prefix = f"std::atomic<shared_ptr<T>>::{op}() (contended)"
+ lf_row = times_by_threads(lf, prefix)
+ lb_row = times_by_threads(lb, prefix)
+ lf_x = [(lf_row[t] / lf_base[t]) if t in lf_row else None for t in threads]
+ lb_x = [(lb_row[t] / lb_base[t]) if t in lb_row else None for t in threads]
+ lines.append("")
+ lines.append(f"{op}() contended / uint64 CAS:")
+ lines.append(header)
+ lines.append(sep)
+ lines.append("lock-free | " + format_ratio_row(lf_x))
+ lines.append("lock-based | " + format_ratio_row(lb_x))
+
+ lines.append("")
+ lines.append(
+ "Uncontended shared_ptr (raw ns; DWCAS bookkeeping is expected to lose):"
+ )
+ for name in SP_UNCONTENDED:
+ lf_t = uncontended_time(lf, name)
+ lb_t = uncontended_time(lb, name)
+ short = name.removeprefix("std::atomic<shared_ptr<T>>::").removesuffix(
+ " (uncontended)"
+ )
+ lf_s = f"{lf_t:.2f}" if lf_t is not None else "n/a"
+ lb_s = f"{lb_t:.2f}" if lb_t is not None else "n/a"
+ lines.append(f" {short}: lock-free {lf_s} lock-based {lb_s}")
+
+ text = "\n".join(lines) + "\n"
+ report_out.write_text(text, encoding="utf-8")
+ sys.stdout.write(text)
+ log(f"wrote {report_out}")
+
+
+def write_lnt(benchmarks: list[dict[str, Any]], lnt_out: Path) -> None:
+ rows = []
+ for bm in benchmarks:
+ name = strip_aggregate_suffix(bm["name"])
+ escaped = name.replace(".", "_").replace(" ", "_")
+ rows.append(f"{escaped}.execution_time {bm['real_time']}")
+ lnt_out.write_text("\n".join(rows) + "\n", encoding="utf-8")
+
+
+ at dataclass(frozen=True)
+class ChartSeries:
+ """One line in a chart: a label, a color, and threads -> value samples."""
+
+ name: str
+ color: str
+ threads: list[int]
+ values: list[float]
+
+
+ at dataclass(frozen=True)
+class ChartSpec:
+ """Everything needed to render one lock-free vs lock-based comparison chart."""
+
+ title: str
+ y_title: str
+ filename: str
+ series: tuple[ChartSeries, ChartSeries]
+ note: str = ""
+
+
+class BenchmarkChartRenderer:
+ """Renders ChartSpec objects as matplotlib line charts saved to PNG.
+
+ Each chart plots one metric (baseline ns, or a ratio to baseline) against
+ thread count, with one line per implementation: major gridlines on the
+ labeled ticks, lighter dotted minor gridlines between them, and a
+ bordered legend.
+ """
+
+ MAJOR_GRID_COLOR = "#d9d9d9"
+ MINOR_GRID_COLOR = "#eeeeee"
+ DPI = 150
+ FIGSIZE = (9.0, 5.5)
+
+ def __init__(self, output_dir: Path) -> None:
+ self._output_dir = output_dir
+ self._output_dir.mkdir(parents=True, exist_ok=True)
+
+ def render(self, spec: ChartSpec) -> Path:
+ import matplotlib
+
+ matplotlib.use("Agg") # headless: no display on a build/CI box
+ import matplotlib.pyplot as plt
+ import matplotlib.ticker as mticker
+
+ figure, axes = plt.subplots(figsize=self.FIGSIZE, dpi=self.DPI)
+ all_threads: set[int] = set()
+ for series in spec.series:
+ all_threads.update(series.threads)
+ axes.plot(
+ series.threads,
+ series.values,
+ label=series.name,
+ color=series.color,
+ linewidth=2.5,
+ marker="o",
+ markersize=7,
+ markeredgecolor="white",
+ markeredgewidth=1,
+ )
+
+ axes.set_title(spec.title, fontsize=13, fontweight="bold")
+ axes.set_xlabel("threads")
+ axes.set_ylabel(spec.y_title)
+ axes.xaxis.set_major_locator(mticker.FixedLocator(sorted(all_threads)))
+ axes.xaxis.set_minor_locator(mticker.AutoMinorLocator(2))
+ axes.yaxis.set_minor_locator(mticker.AutoMinorLocator(2))
+ axes.set_axisbelow(True)
+ axes.grid(True, which="major", color=self.MAJOR_GRID_COLOR, linewidth=0.9)
+ axes.grid(
+ True,
+ which="minor",
+ color=self.MINOR_GRID_COLOR,
+ linewidth=0.6,
+ linestyle=":",
+ )
+ for side in ("top", "right"):
+ axes.spines[side].set_visible(False)
+ legend = axes.legend(
+ frameon=True, framealpha=0.9, edgecolor="#c9c9c9", loc="best"
+ )
+ legend.get_frame().set_linewidth(0.8)
+ if spec.note:
+ figure.text(0.01, 0.01, spec.note, fontsize=8.5, color="#666666", ha="left")
+ figure.tight_layout(rect=(0, 0.04, 1, 1))
+
+ out_path = self._output_dir / spec.filename
+ figure.savefig(out_path)
+ plt.close(figure)
+ return out_path
+
+ def render_all(self, specs: list[ChartSpec]) -> list[Path]:
+ return [self.render(spec) for spec in specs]
+
+
+def build_chart_specs(
+ lf_benches: list[dict[str, Any]],
+ lb_benches: list[dict[str, Any]],
+ title_suffix: str,
+ filename_suffix: str,
+) -> list[ChartSpec]:
+ """Same comparisons as print_normalized_report, as ChartSpecs instead of text."""
+ lf_base = times_by_threads(lf_benches, U64_CAS_CONTENDED)
+ lb_base = times_by_threads(lb_benches, U64_CAS_CONTENDED)
+ threads = sorted(set(lf_base) & set(lb_base))
+ if not threads:
+ return []
+
+ specs = [
+ ChartSpec(
+ title=f"uint64 CAS contended (ns, baseline){title_suffix}",
+ y_title="ns/op",
+ filename=f"baseline_cas{filename_suffix}.png",
+ series=(
+ ChartSeries(
+ "lock-free", LOCK_FREE_COLOR, threads, [lf_base[t] for t in threads]
+ ),
+ ChartSeries(
+ "lock-based",
+ LOCK_BASED_COLOR,
+ threads,
+ [lb_base[t] for t in threads],
+ ),
+ ),
+ note="Denominator for the ratio charts below; same code in both binaries, should track near 1x.",
+ )
+ ]
+
+ for op in SP_CONTENDED_OPS:
+ prefix = f"std::atomic<shared_ptr<T>>::{op}() (contended)"
+ lf_row = times_by_threads(lf_benches, prefix)
+ lb_row = times_by_threads(lb_benches, prefix)
+ op_threads = sorted(t for t in threads if t in lf_row and t in lb_row)
+ if not op_threads:
+ continue
+ specs.append(
+ ChartSpec(
+ title=f"{op}() contended / uint64 CAS{title_suffix}",
+ y_title="ratio to baseline CAS",
+ filename=f"{op}_ratio{filename_suffix}.png",
+ series=(
+ ChartSeries(
+ "lock-free",
+ LOCK_FREE_COLOR,
+ op_threads,
+ [lf_row[t] / lf_base[t] for t in op_threads],
+ ),
+ ChartSeries(
+ "lock-based",
+ LOCK_BASED_COLOR,
+ op_threads,
+ [lb_row[t] / lb_base[t] for t in op_threads],
+ ),
+ ),
+ note="Do not compare raw ns across the two processes; only this ratio.",
+ )
+ )
+ return specs
+
+
+def render_charts(
+ lf_json: Path,
+ lb_json: Path,
+ charts_dir: Path,
+ repetitions: int,
+ separate_charts: bool,
+) -> None:
+ """Write the aggregate comparison charts, and per-repetition ones if asked.
+
+ With --separate-charts and --repetitions N, this writes N per-repetition
+ chart sets plus one aggregate (median) set per metric - N+1 charts per
+ metric in total. Without --separate-charts, only the aggregate set.
+ """
+ try:
+ import matplotlib # noqa: F401
+ except ImportError:
+ log(
+ "charts skipped: matplotlib not installed "
+ "(pip install -r libcxx/utils/requirements.txt, or pass --no-chart)"
+ )
+ return
+
+ lf_js = json.loads(lf_json.read_text(encoding="utf-8"))
+ lb_js = json.loads(lb_json.read_text(encoding="utf-8"))
+ renderer = BenchmarkChartRenderer(charts_dir)
+
+ agg_suffix = " (median)" if repetitions > 1 else ""
+ agg_specs = build_chart_specs(
+ selected_benchmarks(lf_js), selected_benchmarks(lb_js), agg_suffix, ""
+ )
+ for path in renderer.render_all(agg_specs):
+ log(f"wrote chart {path}")
+
+ if not separate_charts:
+ return
+ if repetitions <= 1:
+ log(
+ "--separate-charts has nothing to separate at --repetitions 1 (only the aggregate chart was written)"
+ )
+ return
+ for rep in range(repetitions):
+ lf_rep = benchmarks_for_repetition(lf_js, rep)
+ lb_rep = benchmarks_for_repetition(lb_js, rep)
+ rep_specs = build_chart_specs(
+ lf_rep, lb_rep, f" (repetition {rep + 1}/{repetitions})", f"_rep{rep + 1}"
+ )
+ for path in renderer.render_all(rep_specs):
+ log(f"wrote chart {path}")
+
+
+def find_repo_root(start: Path) -> Path:
+ for directory in [start, *start.parents]:
+ if (directory / LIBCXX_LIT).is_file():
+ return directory
+ die(f"cannot find llvm-project root from {start}")
+
+
+def libcxx_test_roots(repo: Path, build: Path) -> list[Path]:
+ return [
+ build / BOOTSTRAP_LIBCXX_TEST,
+ build / LIBCXX_TEST,
+ repo / LIBCXX_TEST,
+ ]
+
+
+def normalize_threads(raw: str) -> str:
+ compact = re.sub(r"\s+", "", raw)
+ if not compact:
+ die("--threads is empty")
+ if not THREADS_RE.fullmatch(compact):
+ die(
+ f"invalid --threads '{raw}' "
+ "(expected e.g. 2,4,8 or 2,4,6,8,10,12,14,16,18,20)"
+ )
+ seen = []
+ max_n = 0
+ for part in compact.split(","):
+ value = int(part)
+ if value > MAX_THREADS:
+ die(f"thread count {value} exceeds {MAX_THREADS}")
+ if value not in seen:
+ seen.append(value)
+ max_n = max(max_n, value)
+ cpus = os.cpu_count() or 0
+ if cpus and max_n > cpus:
+ log(
+ f"warning: max thread count {max_n} > os.cpu_count()={cpus} (oversubscription)"
+ )
+ return ",".join(str(v) for v in seen)
+
+
+def is_bootstrap_build(build: Path) -> bool:
+ return (build / BOOTSTRAP_LIBCXX_TEST).is_dir()
+
+
+def lit_test_path(repo: Path, build: Path, name: str) -> Path:
+ root = (
+ build / BOOTSTRAP_LIBCXX_TEST
+ if is_bootstrap_build(build)
+ else repo / LIBCXX_TEST
+ )
+ return root / BENCHMARKS_DIR / name
+
+
+def find_output_dir(repo: Path, build: Path, name: str) -> Path | None:
+ for root in libcxx_test_roots(repo, build):
+ path = root / LIT_OUTPUT_DIR / f"{name}.dir"
+ if path.is_dir():
+ return path
+ return None
+
+
+def bench_inputs(repo: Path, src_name: str) -> list[Path]:
+ return [
+ repo / LIBCXX_TEST / BENCHMARKS_DIR / src_name,
+ repo / LIBCXX_TEST / BENCH_HEADER,
+ *[repo / rel for rel in ATOMIC_SP_HEADERS],
+ ]
+
+
+def exe_is_fresh(repo: Path, src_name: str, exe: Path) -> bool:
+ if not exe.is_file():
+ return False
+ exe_mtime = exe.stat().st_mtime
+ for src in bench_inputs(repo, src_name):
+ if src.is_file() and src.stat().st_mtime > exe_mtime:
+ return False
+ return True
+
+
+def find_bench_exe(output_dir: Path) -> Path | None:
+ named = output_dir / LIT_BENCH_EXE
+ if named.is_file() and os.access(named, os.X_OK):
+ return named
+ matches = sorted(output_dir.glob("*.exe"))
+ if matches:
+ return matches[0]
+ return None
+
+
+def elf_runpath_dirs(exe: Path) -> list[str]:
+ readelf = shutil.which("readelf") or shutil.which("llvm-readelf")
+ if readelf is None:
+ return []
+ try:
+ out = subprocess.check_output(
+ [readelf, "-d", str(exe)],
+ text=True,
+ stderr=subprocess.DEVNULL,
+ )
+ except (OSError, subprocess.CalledProcessError):
+ return []
+ dirs: list[str] = []
+ origin = str(exe.parent)
+ for line in out.splitlines():
+ if "RPATH" not in line and "RUNPATH" not in line:
+ continue
+ start = line.find("[")
+ end = line.rfind("]")
+ if start == -1 or end <= start:
+ continue
+ for part in line[start + 1 : end].split(":"): # noqa: E203
+ part = part.replace("$ORIGIN", origin).replace("${ORIGIN}", origin)
+ if part and part not in dirs:
+ dirs.append(part)
+ return dirs
+
+
+def infer_libcxx_lib_dirs(exe: Path) -> list[str]:
+ dirs: list[str] = []
+ for parent in exe.parents:
+ install_lib = parent / "test-suite-install" / "lib"
+ if not install_lib.is_dir():
+ continue
+ if (install_lib / "libc++.so.1").is_file() or (
+ install_lib / "libc++.so"
+ ).is_file():
+ dirs.append(str(install_lib))
+ for sub in sorted(install_lib.iterdir()):
+ if not sub.is_dir():
+ continue
+ if (sub / "libc++.so.1").is_file() or (sub / "libc++.so").is_file():
+ dirs.append(str(sub))
+ return dirs
+
+
+def env_for_exe(base: dict[str, str], exe: Path) -> dict[str, str]:
+ """Prepend the exe RUNPATH so LD_LIBRARY_PATH cannot pick system libc++."""
+ env = base.copy()
+ extra = elf_runpath_dirs(exe) + infer_libcxx_lib_dirs(exe)
+ if not extra:
+ return env
+ key = "DYLD_LIBRARY_PATH" if sys.platform == "darwin" else "LD_LIBRARY_PATH"
+ seen: list[str] = []
+ for directory in extra:
+ if directory not in seen:
+ seen.append(directory)
+ prefix = os.pathsep.join(seen)
+ old = env.get(key, "")
+ env[key] = prefix + (os.pathsep + old if old else "")
+ log(f"{key} prefix={prefix}")
+ return env
+
+
+def run_command(
+ argv: list[str],
+ env: dict[str, str] | None = None,
+ cwd: Path | None = None,
+) -> None:
+ log(" ".join(argv))
+ try:
+ subprocess.run(argv, check=True, env=env, cwd=cwd)
+ except FileNotFoundError as exc:
+ die(f"command not found: {argv[0]} ({exc})")
+ except subprocess.CalledProcessError as exc:
+ die(f"command failed with exit {exc.returncode}: {argv[0]}", exc.returncode)
+
+
+def python_tool(repo: Path, name: str) -> list[str]:
+ script = repo / LIBCXX_UTILS / name
+ if not script.is_file():
+ die(f"missing {script}")
+ return [sys.executable, str(script)]
+
+
+def build_lit_command(
+ repo: Path,
+ build: Path,
+ test_path: Path,
+ min_time: str,
+ extra: list[str],
+) -> list[str]:
+ llvm_lit = build / LLVM_LIT
+ if os.name != "nt":
+ lit = repo / LIBCXX_LIT
+ if not os.access(lit, os.X_OK):
+ die(f"missing executable {lit}")
+ cmd = [str(lit)]
+ if is_bootstrap_build(build):
+ cmd.append("-b")
+ cmd.append(str(build))
+ else:
+ if not llvm_lit.is_file():
+ die(f"llvm-lit not found: {llvm_lit}")
+ cmake = shutil.which("cmake") or os.environ.get("CMAKE", "cmake")
+ target = (
+ "runtimes-test-depends" if is_bootstrap_build(build) else "cxx-test-depends"
+ )
+ run_command([cmake, "--build", str(build), "--target", target])
+ cmd = [str(llvm_lit)]
+ cmd += [
+ "--show-all",
+ "--param",
+ "optimization=speed",
+ "--param",
+ "enable_benchmarks=dry-run",
+ "--param",
+ f"benchmark_min_time={min_time}",
+ str(test_path),
+ *extra,
+ ]
+ return cmd
+
+
+def validate_bench_output(exe: Path, text: str, threads: str, label: str) -> None:
+ """Catch the two ways a run can silently measure the wrong thing."""
+ missing = [part for part in threads.split(",") if f"threads:{part}" not in text]
+ if missing:
+ die(
+ f"{exe.name} did not register threads {','.join(missing)} "
+ f"(asked {THREADS_ENV}={threads}). "
+ "libcxx/utils/run.py uses a clean env; this script must run "
+ "t.tmp.exe itself, not rely on lit %{exec}."
+ )
+ if label == "lock-free" and "libcxx_lock_free" not in text:
+ die(
+ f"{exe.name} is not the lock-free path (no libcxx_lock_free label). "
+ "On x86-64 Clang needs -march=x86-64-v2 (see atomic_shared_ptr.bench.cpp). "
+ "Recompile with --rebuild after that flag is present."
+ )
+
+
+def ensure_bench_exe(
+ repo: Path,
+ build: Path,
+ src_name: str,
+ label: str,
+ min_time: str,
+ extra: list[str],
+ env: dict[str, str],
+ force_rebuild: bool,
+) -> Path:
+ """Return a fresh t.tmp.exe for src_name, compiling via lit if needed."""
+ output_dir = find_output_dir(repo, build, src_name)
+ exe = find_bench_exe(output_dir) if output_dir is not None else None
+ if exe is not None and not force_rebuild and exe_is_fresh(repo, src_name, exe):
+ log(f"reusing {exe} ({label}; skip lit / cmake)")
+ return exe
+ test_path = lit_test_path(repo, build, src_name)
+ log(f"compiling {label} via lit dry-run")
+ run_command(build_lit_command(repo, build, test_path, min_time, extra), env=env)
+ output_dir = find_output_dir(repo, build, src_name)
+ if output_dir is None:
+ die(f"no Output/{src_name}.dir after lit; check the run above")
+ exe = find_bench_exe(output_dir)
+ if exe is None:
+ die(f"no {LIT_BENCH_EXE} under {output_dir}")
+ return exe
+
+
+def run_exe_to_lnt(
+ exe: Path,
+ min_time: str,
+ repetitions: int,
+ filt: str | None,
+ json_out: Path,
+ lnt_out: Path,
+ env: dict[str, str],
+ threads: str,
+ label: str,
+) -> None:
+ argv = [
+ str(exe),
+ f"--benchmark_min_time={min_time}",
+ f"--benchmark_out={json_out}",
+ "--benchmark_out_format=json",
+ ]
+ if repetitions > 1:
+ argv.append(f"--benchmark_repetitions={repetitions}")
+ # Keep individual repetition rows in the JSON, not just the aggregates:
+ # --separate-charts needs them. write_lnt/selected_benchmarks() below
+ # still filter down to the median-only view for the .lnt file.
+ if filt:
+ argv.append(f"--benchmark_filter={filt}")
+ run_command(argv, env=env_for_exe(env, exe), cwd=exe.parent)
+ text = json_out.read_text(encoding="utf-8")
+ validate_bench_output(exe, text, threads, label)
+ js = json.loads(text)
+ scaling = js.get("context", {}).get("cpu_scaling_enabled")
+ if scaling:
+ log(f"{label}: CPU scaling is enabled (google-benchmark context)")
+ write_lnt(selected_benchmarks(js), lnt_out)
+
+
+def run_one_bench(
+ repo: Path,
+ build: Path,
+ src_name: str,
+ label: str,
+ lnt_out: Path,
+ min_time: str,
+ repetitions: int,
+ extra: list[str],
+ env: dict[str, str],
+ filt: str | None,
+ force_rebuild: bool,
+) -> None:
+ threads = env[THREADS_ENV]
+ exe = ensure_bench_exe(
+ repo, build, src_name, label, min_time, extra, env, force_rebuild
+ )
+ json_out = lnt_out.with_suffix(".json")
+ log(
+ f"running {label}: {THREADS_ENV}={threads} min_time={min_time} "
+ f"repetitions={repetitions}"
+ )
+ if filt:
+ log(f"filter {label}: --benchmark_filter={filt}")
+ run_exe_to_lnt(
+ exe, min_time, repetitions, filt, json_out, lnt_out, env, threads, label
+ )
+
+
+def run_bench_once(
+ exe: Path,
+ min_time: str,
+ filt: str | None,
+ env: dict[str, str],
+ threads: str,
+ label: str,
+ json_out: Path,
+) -> dict[str, Any]:
+ """Run exe for exactly one repetition (no --benchmark_repetitions) and
+ return the parsed JSON. Used by run_interleaved, which needs to
+ interleave individual repetitions across the two binaries rather than
+ letting Google Benchmark run all repetitions of one binary back to back.
+ """
+ argv = [
+ str(exe),
+ f"--benchmark_min_time={min_time}",
+ f"--benchmark_out={json_out}",
+ "--benchmark_out_format=json",
+ ]
+ if filt:
+ argv.append(f"--benchmark_filter={filt}")
+ run_command(argv, env=env_for_exe(env, exe), cwd=exe.parent)
+ text = json_out.read_text(encoding="utf-8")
+ validate_bench_output(exe, text, threads, label)
+ return json.loads(text)
+
+
+def _safe_stdev(values: list[float]) -> float:
+ return statistics.stdev(values) if len(values) > 1 else 0.0
+
+
+def _safe_cv(values: list[float]) -> float:
+ mean = statistics.mean(values)
+ return (_safe_stdev(values) / mean) if mean else 0.0
+
+
+def merge_repetition_runs(runs: list[dict[str, Any]]) -> dict[str, Any]:
+ """Combine N single-repetition Google Benchmark JSON runs into the same
+ shape --benchmark_repetitions=N would have produced: every individual
+ iteration row (tagged with its repetition_index, like run_exe_to_lnt's
+ output already is) plus computed mean/median/stddev/cv aggregate rows.
+ This is what lets selected_benchmarks(), times_by_threads(),
+ print_normalized_report(), and the chart builders treat an interleaved
+ run exactly like a normal --benchmark_repetitions run.
+ """
+ all_rows: list[dict[str, Any]] = []
+ by_key: dict[str, list[dict[str, Any]]] = {}
+ for rep_index, run_js in enumerate(runs):
+ for bm in run_js["benchmarks"]:
+ row = dict(bm)
+ row["run_type"] = "iteration"
+ row["repetition_index"] = rep_index
+ row["repetitions"] = len(runs)
+ all_rows.append(row)
+ by_key.setdefault(row["name"], []).append(row)
+
+ for name, rows in by_key.items():
+ real_times = [float(r["real_time"]) for r in rows]
+ cpu_times = [float(r["cpu_time"]) for r in rows]
+ label = rows[0].get("label", "")
+ time_unit = rows[0].get("time_unit", "ns")
+ iterations = rows[0].get("iterations", len(rows))
+ aggregates = (
+ ("mean", statistics.mean(real_times), statistics.mean(cpu_times)),
+ ("median", statistics.median(real_times), statistics.median(cpu_times)),
+ ("stddev", _safe_stdev(real_times), _safe_stdev(cpu_times)),
+ ("cv", _safe_cv(real_times), _safe_cv(cpu_times)),
+ )
+ for agg_name, real_value, cpu_value in aggregates:
+ all_rows.append(
+ {
+ "name": f"{name}_{agg_name}",
+ "run_type": "aggregate",
+ "aggregate_name": agg_name,
+ "repetitions": len(runs),
+ "iterations": iterations,
+ "real_time": real_value,
+ "cpu_time": cpu_value,
+ "time_unit": time_unit,
+ "label": label,
+ }
+ )
+
+ context = dict(runs[0].get("context", {})) if runs else {}
+ return {"context": context, "benchmarks": all_rows}
+
+
+def run_interleaved(
+ repo: Path,
+ build: Path,
+ min_time: str,
+ repetitions: int,
+ extra: list[str],
+ env: dict[str, str],
+ filt: str | None,
+ force_rebuild: bool,
+ swap_order: bool,
+ settle_seconds: float,
+ lf_lnt: Path,
+ lb_lnt: Path,
+) -> None:
+ """Alternate lock-free/lock-based one repetition at a time (rep 1 of
+ each, then rep 2 of each, ...) so residual thermal or clock-frequency
+ drift lands on both sides evenly instead of systematically favoring
+ whichever binary runs first (see --swap-order, --interleave help).
+ """
+ order = [
+ (LOCK_FREE_SRC, "lock-free", lf_lnt),
+ (LOCK_BASED_SRC, "lock-based", lb_lnt),
+ ]
+ if swap_order:
+ order.reverse()
+
+ exes = {
+ label: ensure_bench_exe(
+ repo, build, src, label, min_time, extra, env, force_rebuild
+ )
+ for src, label, _ in order
+ }
+ threads = env[THREADS_ENV]
+ runs: dict[str, list[dict[str, Any]]] = {label: [] for _, label, _ in order}
+
+ with tempfile.TemporaryDirectory(prefix="atomic_sp_interleave_") as scratch:
+ first = True
+ for rep in range(repetitions):
+ for _src, label, _lnt in order:
+ if not first and settle_seconds > 0:
+ log(f"settling {settle_seconds}s before {label}")
+ time.sleep(settle_seconds)
+ first = False
+ log(f"interleaved {rep + 1}/{repetitions}: running {label}")
+ json_out = Path(scratch) / f"{label}_{rep}.json"
+ runs[label].append(
+ run_bench_once(
+ exes[label], min_time, filt, env, threads, label, json_out
+ )
+ )
+
+ for _src, label, lnt_out in order:
+ merged = merge_repetition_runs(runs[label])
+ scaling = merged.get("context", {}).get("cpu_scaling_enabled")
+ if scaling:
+ log(f"{label}: CPU scaling is enabled (google-benchmark context)")
+ json_out = lnt_out.with_suffix(".json")
+ json_out.write_text(json.dumps(merged), encoding="utf-8")
+ write_lnt(selected_benchmarks(merged), lnt_out)
+ log(f"wrote {json_out} ({label}, interleaved x{repetitions})")
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Same-machine A/B of libc++ lock-free vs lock-based "
+ "std::atomic<std::shared_ptr<T>> benches.",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog="""
+examples:
+ %(prog)s --build <build>
+ %(prog)s --build <build> --threads 2,4,6,8,10,12,14,16,18,20 --min-time 1s
+ %(prog)s --build <build> --threads 2,4,6,8,10,12,14,16,18,20 --min-time 1s --repetitions 5
+ %(prog)s --build <build> --threads 8 --filter 'compare_exchange_strong.*contended'
+ %(prog)s --build <build> --repetitions 10 --separate-charts
+ %(prog)s --build <build> --no-chart
+ %(prog)s --build <build> --charts-out-dir /tmp/atomic_sp_charts
+ %(prog)s --build <build> --repetitions 10 --interleave
+ %(prog)s --build <build> --repetitions 10 --out-dir /tmp/ab_normal
+ %(prog)s --build <build> --repetitions 10 --out-dir /tmp/ab_swapped --swap-order
+
+notes:
+ Run order is a systematic bias, not noise. By default lock-free runs first
+ and lock-based second, so the second binary is always measured on an
+ already-warmed chip and any residual thermal/frequency drift always lands
+ on the same side. --swap-order reverses the order so you can measure that
+ bias (run both directions into separate --out-dir paths and compare the
+ normalized tables); --interleave alternates the two binaries one repetition
+ at a time so the drift is spread evenly across both and the bias cancels.
+ Prefer --interleave for numbers that will be published; use --swap-order
+ when you want the bias quantified rather than removed.
+ Charts (normalized-ratio line charts, lock-free vs lock-based) are written
+ as PNG to <charts-out-dir or out-dir>/charts/*.png by default; pass
+ --no-chart to skip them, --separate-charts to additionally get one chart
+ set per repetition, or --charts-out-dir to redirect the charts/ folder
+ somewhere other than --out-dir. Needs matplotlib
+ (libcxx/utils/requirements.txt); missing matplotlib only skips charts, it
+ does not fail the run.
+ Run lock-free and lock-based one file at a time. Consolidating both
+ Output dirs together mixes identical bench names.
+ Contended rows at lit's default 0.2s often look like plateaus
+ (1e6 / 5e6 / 2.5e6). This wrapper defaults to 1s.
+ --threads 8 (alone) is the cheap isolation check: no threads:2/4 in
+ the same process, so leaked static state cannot explain the number.
+ After the first compile, omit --rebuild so only t.tmp.exe is re-run
+ (libcxx-lit otherwise rebuilds runtimes-test-depends every time).
+ Direct t.tmp.exe runs prepend the binary RUNPATH to LD_LIBRARY_PATH
+ so a user path like /usr/lib cannot hide the just-built libc++.
+ Publish from --repetitions 5 (median in LNT / normalized.txt). Single
+ samples plus CPU scaling are not enough for an RFC table.
+ compare-benchmarks prints a Geomean that mixes baseline rows with
+ shared_ptr rows; do not publish it. Use the normalized table.
+""",
+ )
+ parser.add_argument(
+ "--build",
+ type=Path,
+ default=None,
+ help="Build directory (default: $ATOMIC_SP_BENCH_BUILD or <repo>/build)",
+ )
+ parser.add_argument(
+ "--threads",
+ default=None,
+ help=f"Comma-separated positive integers (default: {DEFAULT_THREADS})",
+ )
+ parser.add_argument(
+ "--min-time",
+ default=DEFAULT_MIN_TIME,
+ help=f"lit --param benchmark_min_time (default: {DEFAULT_MIN_TIME})",
+ )
+ parser.add_argument(
+ "--out-dir",
+ type=Path,
+ default=Path(tempfile.gettempdir()) / "atomic_shared_ptr_bench",
+ help="Where to write lock_free.lnt / lock_based.lnt",
+ )
+ parser.add_argument(
+ "--repetitions",
+ type=int,
+ default=DEFAULT_REPETITIONS,
+ help=(
+ "Google Benchmark --benchmark_repetitions. Values >1 keep the median "
+ "in LNT / normalized.txt / the aggregate charts, and also keep every "
+ "individual repetition in the JSON for --separate-charts. "
+ f"Default: {DEFAULT_REPETITIONS}. Use 5 before publishing."
+ ),
+ )
+ parser.add_argument(
+ "--filter",
+ default=None,
+ help="Pass --benchmark_filter to t.tmp.exe (after compile)",
+ )
+ parser.add_argument(
+ "--rebuild",
+ action="store_true",
+ help="Force lit dry-run compile even if t.tmp.exe is newer than sources",
+ )
+ parser.add_argument("--lock-free-only", action="store_true")
+ parser.add_argument("--lock-based-only", action="store_true")
+ parser.add_argument("--no-compare", action="store_true")
+ parser.add_argument(
+ "--no-chart",
+ action="store_true",
+ help="Skip rendering charts (normalized.txt / LNT are unaffected)",
+ )
+ parser.add_argument(
+ "--separate-charts",
+ action="store_true",
+ help=(
+ "Also render one chart set per repetition, in addition to the "
+ "aggregate (median) one - e.g. --repetitions 10 writes 10 "
+ "per-repetition chart sets plus 1 aggregate set per metric. "
+ "Needs --repetitions > 1; no-op otherwise."
+ ),
+ )
+ parser.add_argument(
+ "--charts-out-dir",
+ type=Path,
+ default=None,
+ help="Parent directory for the charts/ folder (default: --out-dir)",
+ )
+ parser.add_argument(
+ "--swap-order",
+ action="store_true",
+ help=(
+ "Run lock-based first and lock-free second, reversing the default "
+ "order. By default this script always runs lock-free first, so the "
+ "second binary is always measured on an already-warmed chip; any "
+ "residual thermal or clock drift that the CPU governor does not "
+ "remove therefore always lands on the same side of the comparison, "
+ "which is a systematic bias, not random noise. Use this to measure "
+ "that bias: run once normally, once with --swap-order (writing to "
+ "different --out-dir paths), and compare the two normalized tables. "
+ "Matching ratios mean run order does not affect the result and the "
+ "numbers are safe to publish; diverging ratios give you the "
+ "magnitude of the order effect to quote as an error bar. Cheaper "
+ "than --interleave (two normal runs), but only quantifies the bias "
+ "instead of cancelling it."
+ ),
+ )
+ parser.add_argument(
+ "--interleave",
+ action="store_true",
+ help=(
+ "Alternate the two binaries one repetition at a time (lock-free "
+ "rep 1, lock-based rep 1, lock-free rep 2, lock-based rep 2, ...) "
+ "instead of running all repetitions of one binary and then all "
+ "repetitions of the other. This spreads thermal and frequency "
+ "drift evenly across both sides rather than concentrating it in "
+ "whichever binary runs second, so it cancels the order bias that "
+ "--swap-order only measures. Needs --repetitions > 1 to do "
+ "anything. Each repetition is a separate process launch, so the "
+ "individual runs are recombined into the usual per-repetition plus "
+ "mean/median/stddev/cv layout; the resulting .json / .lnt / "
+ "normalized.txt / charts are identical in shape to a normal run, "
+ "including --separate-charts. Slower to start (2*N process "
+ "launches instead of 2) but the preferred mode for numbers that "
+ "will be published. Combine with --swap-order to flip which "
+ "binary leads each interleaved pair."
+ ),
+ )
+ parser.add_argument(
+ "--settle-seconds",
+ type=float,
+ default=0.0,
+ help=(
+ "Sleep this many seconds every time execution switches from one "
+ "binary to the other, to let the chip cool back toward a baseline "
+ "thermal/frequency state before the next measurement. In "
+ "sequential mode (the default, or with --swap-order) that is a "
+ "single pause before the second binary; with --interleave it is "
+ "before every switch, so total added wall-clock time is roughly "
+ "2*(--repetitions)*--settle-seconds. Cheap and does not change the "
+ "script's structure, but only reduces the thermal effect instead "
+ "of measuring (--swap-order) or cancelling (--interleave) it - use "
+ "it together with those, not instead of them. Default: 0 (no "
+ "pause)."
+ ),
+ )
+ parser.add_argument(
+ "lit_args",
+ nargs=argparse.REMAINDER,
+ help="Extra llvm-lit args after --",
+ )
+ return parser.parse_args(argv)
+
+
+def main(argv: list[str]) -> int:
+ start_time = time.monotonic()
+ args = parse_args(argv)
+ repo = find_repo_root(Path(__file__).resolve().parent)
+
+ build = args.build
+ if build is None:
+ env_build = os.environ.get("ATOMIC_SP_BENCH_BUILD")
+ build = Path(env_build) if env_build else repo / "build"
+ build = build.resolve()
+ if not build.is_dir():
+ die(f"build directory not found: {build}")
+ if not (build / LLVM_LIT).exists():
+ die(
+ f"llvm-lit not found under {build / LLVM_LIT.parent}; configure the build first"
+ )
+
+ extra = list(args.lit_args)
+ if extra and extra[0] == "--":
+ extra = extra[1:]
+
+ threads = args.threads
+ if threads is None:
+ threads = os.environ.get(THREADS_ENV, DEFAULT_THREADS)
+ threads = normalize_threads(threads)
+
+ if not MIN_TIME_RE.fullmatch(args.min_time):
+ die(f"invalid --min-time '{args.min_time}' (expected e.g. 1s, 0.5s, 500ms)")
+ if args.repetitions < 1:
+ die(f"invalid --repetitions {args.repetitions} (expected >= 1)")
+ if args.settle_seconds < 0:
+ die(f"invalid --settle-seconds {args.settle_seconds} (expected >= 0)")
+
+ out_dir = args.out_dir
+ out_dir.mkdir(parents=True, exist_ok=True)
+ out_dir = out_dir.resolve()
+
+ charts_parent = args.charts_out_dir if args.charts_out_dir is not None else out_dir
+ charts_parent.mkdir(parents=True, exist_ok=True)
+ charts_parent = charts_parent.resolve()
+
+ env = os.environ.copy()
+ env[THREADS_ENV] = threads
+
+ do_lock_free = not args.lock_based_only
+ do_lock_based = not args.lock_free_only
+ if args.lock_free_only and args.lock_based_only:
+ die("use only one of --lock-free-only / --lock-based-only")
+
+ interleave = args.interleave and do_lock_free and do_lock_based
+ if args.interleave and not interleave:
+ log("--interleave needs both implementations; ignoring it with --lock-*-only")
+ if interleave and args.repetitions <= 1:
+ log("--interleave with --repetitions 1 is just a plain sequential run")
+
+ log(f"repo={repo}")
+ log(f"build={build} bootstrap={'yes' if is_bootstrap_build(build) else 'no'}")
+ log(
+ f"threads={threads} min_time={args.min_time} "
+ f"repetitions={args.repetitions} out={out_dir} "
+ f"charts={'(disabled)' if args.no_chart else charts_parent / CHARTS_DIR_NAME}"
+ )
+ log(
+ f"order={'lock-based first' if args.swap_order else 'lock-free first'} "
+ f"mode={'interleaved' if interleave else 'sequential'}"
+ )
+
+ lf_lnt = out_dir / "lock_free.lnt"
+ lb_lnt = out_dir / "lock_based.lnt"
+
+ if interleave:
+ run_interleaved(
+ repo,
+ build,
+ args.min_time,
+ args.repetitions,
+ extra,
+ env,
+ args.filter,
+ args.rebuild,
+ args.swap_order,
+ args.settle_seconds,
+ lf_lnt,
+ lb_lnt,
+ )
+ else:
+ planned = [
+ (do_lock_free, LOCK_FREE_SRC, "lock-free", lf_lnt),
+ (do_lock_based, LOCK_BASED_SRC, "lock-based", lb_lnt),
+ ]
+ if args.swap_order:
+ planned.reverse()
+ first = True
+ for enabled, src, label, lnt_out in planned:
+ if not enabled:
+ continue
+ if not first and args.settle_seconds > 0:
+ log(f"settling {args.settle_seconds}s before {label}")
+ time.sleep(args.settle_seconds)
+ first = False
+ run_one_bench(
+ repo,
+ build,
+ src,
+ label,
+ lnt_out,
+ args.min_time,
+ args.repetitions,
+ extra,
+ env,
+ args.filter,
+ args.rebuild,
+ )
+
+ lf_json = lf_lnt.with_suffix(".json")
+ lb_json = lb_lnt.with_suffix(".json")
+ if do_lock_free and do_lock_based and lf_json.is_file() and lb_json.is_file():
+ print_normalized_report(lf_json, lb_json, out_dir / "normalized.txt")
+ if not args.no_chart:
+ render_charts(
+ lf_json,
+ lb_json,
+ charts_parent / CHARTS_DIR_NAME,
+ args.repetitions,
+ args.separate_charts,
+ )
+
+ if not args.no_compare and do_lock_free and do_lock_based:
+ compare = python_tool(repo, "compare-benchmarks")
+ log(f"compare-benchmarks {lf_lnt} {lb_lnt}")
+ log(
+ "compare-benchmarks Geomean mixes baseline and shared_ptr rows; "
+ "do not publish it. Use normalized.txt."
+ )
+ run_command([*compare, str(lf_lnt), str(lb_lnt)])
+
+ elapsed = time.monotonic() - start_time
+ log(f"done in {elapsed:.1f}s ({elapsed / 60:.1f} min)")
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main(sys.argv[1:]))
+ except KeyboardInterrupt:
+ log("Interrupted")
+ sys.exit(1)
diff --git a/libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp b/libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp
new file mode 100644
index 0000000000000..005da9427fa0c
--- /dev/null
+++ b/libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp
@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+// REQUIRES: stdlib=libc++
+// UNSUPPORTED: no-threads
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_FORCE_LOCK_BASED_ATOMIC_SHARED_PTR
+// ADDITIONAL_COMPILE_FLAGS(target=x86_64-unknown-linux-gnu): -march=x86-64-v2
+
+// Same benches as atomic_shared_ptr.bench.cpp, but forces libc++'s stolen-bit
+// spinlock path even on DWCAS targets. Same-machine A/B only; names match so
+// compare-benchmarks can pair the two files. Driver:
+// python3 libcxx/test/benchmarks/run_atomic_shared_ptr_bench.py --build <build>
+
+#include "atomic_shared_ptr_bench.h"
+
+BENCHMARK_MAIN();
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
index c1181834c88cb..7b92e1b23dfcd 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_memory_order.verify.cpp
@@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
-// REQUIRES: std-at-least-c++20
// REQUIRES: diagnose-if-support
+// REQUIRES: std-at-least-c++20
// UNSUPPORTED: no-threads
#include <atomic>
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
index 47b84a74bdee1..3889e86717948 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_nullptr.pass.cpp
@@ -13,7 +13,7 @@
#include <atomic>
#include <cassert>
-#include <cstddef>
+#include <cstddef> // Armv7, Armv8 (-fno-exceptions) require for std::nullptr_t
#include <memory>
#include <type_traits>
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
index 87cd3f3914683..7957fb7a7b787 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_shared_ptr_refcount.pass.cpp
@@ -12,7 +12,7 @@
#include <atomic>
#include <cassert>
#include <memory>
-#include <utility>
+#include <utility> // Armv7, Armv8 require for std::move
namespace {
struct Tracker {
diff --git a/libcxx/utils/requirements.txt b/libcxx/utils/requirements.txt
index 5ebc9537ba591..a302e53c074aa 100644
--- a/libcxx/utils/requirements.txt
+++ b/libcxx/utils/requirements.txt
@@ -1,5 +1,6 @@
-r ci/lnt/requirements.txt
click
+matplotlib
numpy
pandas
plotly
>From 5d024c6d8b39fe492188715663ae07b84e27aea8 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Wed, 26 Aug 2026 22:57:58 +0300
Subject: [PATCH 44/46] [libc++] Fix GCC -Wcomment error and stale driver
script name in atomic_shared_ptr benchmarks
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp | 5 ++---
libcxx/test/benchmarks/atomic_shared_ptr_bench.h | 2 +-
.../test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp | 2 +-
3 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp b/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
index 763e3d95cbf76..99c8d0fd88d1a 100644
--- a/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
+++ b/libcxx/test/benchmarks/atomic_shared_ptr.bench.cpp
@@ -22,9 +22,8 @@
//
// Same-machine A/B (each file separately - names collide if consolidated
// together). Driver next to these benches:
-// python3 libcxx/test/benchmarks/run_atomic_shared_ptr_bench.py --build <build>
-// python3 libcxx/test/benchmarks/run_atomic_shared_ptr_bench.py --build <build> \
-// --threads 2,4,6,8,10,12,14,16,18,20
+// python3 libcxx/test/benchmarks/atomic_shared_ptr_bench.runner.py --build <build>
+// Add --threads 2,4,6,8,10,12,14,16,18,20 to override the default 2,4,8.
// Needs: pip install -r libcxx/utils/requirements.txt
//
// Report ratios to std::atomic<uint64_t>::* in the same JSON, not raw ns/op.
diff --git a/libcxx/test/benchmarks/atomic_shared_ptr_bench.h b/libcxx/test/benchmarks/atomic_shared_ptr_bench.h
index e2abf708fb83b..33022b657ac82 100644
--- a/libcxx/test/benchmarks/atomic_shared_ptr_bench.h
+++ b/libcxx/test/benchmarks/atomic_shared_ptr_bench.h
@@ -22,7 +22,7 @@
// std::atomic<uint64_t> bench in the same run. Absolute ns/op are host-specific;
// the ratio is the number to report.
//
-// Driver: libcxx/test/benchmarks/run_atomic_shared_ptr_bench.py
+// Driver: libcxx/test/benchmarks/atomic_shared_ptr_bench.runner.py
#include <atomic>
#include <cstdint>
diff --git a/libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp b/libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp
index 005da9427fa0c..3c36964431f85 100644
--- a/libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp
+++ b/libcxx/test/benchmarks/atomic_shared_ptr_lock_based.bench.cpp
@@ -16,7 +16,7 @@
// Same benches as atomic_shared_ptr.bench.cpp, but forces libc++'s stolen-bit
// spinlock path even on DWCAS targets. Same-machine A/B only; names match so
// compare-benchmarks can pair the two files. Driver:
-// python3 libcxx/test/benchmarks/run_atomic_shared_ptr_bench.py --build <build>
+// python3 libcxx/test/benchmarks/atomic_shared_ptr_bench.runner.py --build <build>
#include "atomic_shared_ptr_bench.h"
>From 115dc5939283742a5cc0c5cbb741b2e698a47905 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Mon, 31 Aug 2026 17:31:22 +0300
Subject: [PATCH 45/46] [libc++] egenerate feature-test-macro tests for
__cpp_lib_atomic_shared_ptr.
atomic.version.compile.pass.cpp and version.version.compile.pass.cpp
were stale against version's now-unconditional __cpp_lib_atomic_shared_ptr;
regenerated via generate_feature_test_macro_components.py.
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
.../atomic.version.compile.pass.cpp | 16 +++++-----------
.../version.version.compile.pass.cpp | 16 +++++-----------
2 files changed, 10 insertions(+), 22 deletions(-)
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/atomic.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/atomic.version.compile.pass.cpp
index 289938121d98a..05594c80ec100 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/atomic.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/atomic.version.compile.pass.cpp
@@ -419,17 +419,11 @@
# error "__cpp_lib_atomic_ref should have the value 202603L in c++29"
# endif
-# if !defined(_LIBCPP_VERSION)
-# ifndef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should be defined in c++29"
-# endif
-# if __cpp_lib_atomic_shared_ptr != 201711L
-# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++29"
-# endif
-# else
-# ifdef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should not be defined because it is unimplemented in libc++!"
-# endif
+# ifndef __cpp_lib_atomic_shared_ptr
+# error "__cpp_lib_atomic_shared_ptr should be defined in c++29"
+# endif
+# if __cpp_lib_atomic_shared_ptr != 201711L
+# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++29"
# endif
# ifndef __cpp_lib_atomic_value_initialization
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index 69742779e3d5e..ece0f741c5afa 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -8747,17 +8747,11 @@
# error "__cpp_lib_atomic_ref should have the value 202603L in c++29"
# endif
-# if !defined(_LIBCPP_VERSION)
-# ifndef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should be defined in c++29"
-# endif
-# if __cpp_lib_atomic_shared_ptr != 201711L
-# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++29"
-# endif
-# else
-# ifdef __cpp_lib_atomic_shared_ptr
-# error "__cpp_lib_atomic_shared_ptr should not be defined because it is unimplemented in libc++!"
-# endif
+# ifndef __cpp_lib_atomic_shared_ptr
+# error "__cpp_lib_atomic_shared_ptr should be defined in c++29"
+# endif
+# if __cpp_lib_atomic_shared_ptr != 201711L
+# error "__cpp_lib_atomic_shared_ptr should have the value 201711L in c++29"
# endif
# ifndef __cpp_lib_atomic_value_initialization
>From 14f8af83161a90d399c537d3e43d26cac35252e8 Mon Sep 17 00:00:00 2001
From: Vladislav Semykin <vladislav.semykin at gmail.com>
Date: Mon, 31 Aug 2026 17:31:40 +0300
Subject: [PATCH 46/46] [libc++] Fix strict-aliasing UB in
atomic<shared_ptr<T>> DWCAS storage.
Store the pointer/control pair directly as unsigned __int128 instead
of reinterpret_cast'ing two separate sub-objects to that type.
Flagged in review of #194215 by jyknight.
Signed-off-by: Vladislav Semykin <vladislav.semykin at gmail.com>
---
.../__memory/atomic_shared_ptr_lock_free.h | 30 ++++++++++---------
1 file changed, 16 insertions(+), 14 deletions(-)
diff --git a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
index 773744afc1f82..b78e8f71306a4 100644
--- a/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
+++ b/libcxx/include/__memory/atomic_shared_ptr_lock_free.h
@@ -113,25 +113,27 @@ inline void __atomic_smart_ptr_wait_on_address(const void* __address, const _Pol
// CMPXCHG16B/CASP require 16-byte alignment; the public ABI size is 16 bytes.
template <class _Element>
struct alignas(16) __atomic_smart_ptr_fields {
- mutable __cxx_atomic_impl<_Element*> __ptr_;
- mutable __cxx_atomic_impl<uintptr_t> __ctrl_;
+ __extension__ using __sp_u128 = unsigned __int128;
+
+ // Sole storage: low 64 bits = pointer, high 64 bits = control word. This
+ // used to be two separate __cxx_atomic_impl<_Element*>/__cxx_atomic_impl<uintptr_t>
+ // sub-objects, accessed as one 16-byte pair by reinterpret_cast'ing the
+ // struct's address to __sp_u128* - a strict-aliasing violation, since the
+ // storage was never created as a __sp_u128. Storing it as a __sp_u128 to
+ // begin with removes the cast entirely: __dwcas_address() now just takes
+ // this object's own address.
+ mutable __sp_u128 __word_;
__atomic_smart_ptr_fields(_Element* __p, __shared_weak_count* __c) noexcept
- : __ptr_(__p), __ctrl_(__atomic_smart_ptr_storage::__encode_dwcas(__c, 0)) {}
+ : __word_(__pair_make(__p, __atomic_smart_ptr_storage::__encode_dwcas(__c, 0))) {}
- // DWCAS updates __ptr_ and __ctrl_ as one 16-byte pair, so waiters observe
- // the struct base; watching only __ctrl_ would miss stores where the pointer
- // changes but the CB stays the same (e.g., aliasing constructor).
+ // DWCAS updates the pointer and control word as one 16-byte pair, so
+ // waiters observe the struct base; watching only the control half would
+ // miss stores where the pointer changes but the CB stays the same (e.g.,
+ // aliasing constructor).
const void* __wait_address() const noexcept { return static_cast<const void*>(this); }
- __extension__ using __sp_u128 = unsigned __int128;
-
- // Storage layout: low 64 bits = __ptr_, high 64 bits = __ctrl_. The wide
- // view is the only access path while DWCAS is enabled; the sub-atomics are
- // not read or written through __cxx_atomic_*.
- __sp_u128* __dwcas_address() const noexcept {
- return const_cast<__sp_u128*>(reinterpret_cast<const __sp_u128*>(this));
- }
+ __sp_u128* __dwcas_address() const noexcept { return __builtin_addressof(__word_); }
__sp_u128 __dwcas_load(memory_order __o) const noexcept {
__sp_u128 __word;
More information about the libcxx-commits
mailing list