[libcxx-commits] [libcxx] [libc++] P0718R2: Implementation of std::atomic<shared_ptr<T>> and std::atomic<weak_ptr<T>> (PR #194215)

[Vladislav Semykin via libcxx-commits]

================
@@ -0,0 +1,805 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_LOCK_FREE_H
+#define _LIBCPP___MEMORY_ATOMIC_SHARED_PTR_LOCK_FREE_H
+
+#include <__atomic/atomic_sync_lite.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/memory_order.h>
+#include <__atomic/support.h>
+#include <__atomic/to_failure_order.h>
+#include <__atomic/to_gcc_order.h>
+#include <__config>
+#include <__cstddef/nullptr_t.h>
+#include <__memory/shared_count.h>
+#include <__utility/move.h>
+
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
+
+template <class _Tp>
+class shared_ptr;
+
+template <class _Tp>
+class weak_ptr;
+
+template <class _Tp>
+struct atomic;
+
+// __ctrl_ word layout (8 bytes):
+//   bits 0..1    always zero (CBs are at least 4-byte aligned)
+//   bits 2..47   control-block pointer
+//   bits 48..63  16-bit local refcount (Williams split-refcount)
+// Bits 48..63 are available on both DWCAS-capable ISAs (x86-64 without LAMA,
+// AArch64 without LVA): user-space pointers stay within 48 bits.
+// The local count pins the CB in the atomic word so load() can __add_shared
+// without a use-after-free if a concurrent store retires the pair. Dropping
+// the pin (plain load-and-validate) is not safe: the snapshot is not a ref.
+struct __atomic_smart_ptr_storage {
+  static constexpr uintptr_t __lock_bit_        = uintptr_t{1};
+  static constexpr uintptr_t __notify_bit_      = uintptr_t{2};
+  static constexpr uintptr_t __ptr_mask_        = ~(__lock_bit_ | __notify_bit_);
+  static constexpr unsigned __local_count_shift = 48;
+  static constexpr uintptr_t __local_count_unit = uintptr_t{1} << __local_count_shift;
+  static constexpr uintptr_t __local_count_mask = uintptr_t{0xFFFF} << __local_count_shift;
+  static constexpr uintptr_t __dwcas_ptr_mask   = __ptr_mask_ & ~__local_count_mask;
+  static constexpr uint16_t __local_count_max   = 0xFFFF;
+
+  static uintptr_t __encode_dwcas(__shared_weak_count* __ctrl, uint16_t __local) noexcept {
+    return (reinterpret_cast<uintptr_t>(__ctrl) & __dwcas_ptr_mask) |
+           (static_cast<uintptr_t>(__local) << __local_count_shift);
+  }
+
+  static __shared_weak_count* __decode_dwcas(uintptr_t __word) noexcept {
+    return reinterpret_cast<__shared_weak_count*>(__word & __dwcas_ptr_mask);
+  }
+
+  static uint16_t __decode_local(uintptr_t __word) noexcept {
+    return static_cast<uint16_t>((__word & __local_count_mask) >> __local_count_shift);
+  }
+};
+
+inline void __atomic_smart_ptr_notify_one(const void* __address) noexcept {
+#  if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+  std::__atomic_notify_one_global_table(__address);
+#  else
+  std::__cxx_atomic_notify_one(reinterpret_cast<void const volatile*>(__address));
+#  endif
+}
+
+inline void __atomic_smart_ptr_notify_all(const void* __address) noexcept {
+#  if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+  std::__atomic_notify_all_global_table(__address);
+#  else
+  std::__cxx_atomic_notify_all(reinterpret_cast<void const volatile*>(__address));
+#  endif
+}
+
+template <class _Poll>
+inline void __atomic_smart_ptr_wait_on_address(const void* __address, const _Poll& __poll) noexcept {
+  while (!__poll()) {
+#  if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+    auto __monitor_value = std::__atomic_monitor_global(__address);
+    if (__poll())
+      return;
+    std::__atomic_wait_global_table(__address, __monitor_value);
+#  else
+    void const volatile* __volatile_address = reinterpret_cast<void const volatile*>(__address);
+    auto __monitor_value                    = std::__libcpp_atomic_monitor(__volatile_address);
+    if (__poll())
+      return;
+    std::__libcpp_atomic_wait(__volatile_address, __monitor_value);
+#  endif
+  }
+}
+
+// CMPXCHG16B/CASP require 16-byte alignment; the public ABI size is 16 bytes.
+template <class _Element>
+struct alignas(16) __atomic_smart_ptr_fields {
+  mutable __cxx_atomic_impl<_Element*> __ptr_;
+  mutable __cxx_atomic_impl<uintptr_t> __ctrl_;
+
+  __atomic_smart_ptr_fields(_Element* __p, __shared_weak_count* __c) noexcept
+      : __ptr_(__p), __ctrl_(__atomic_smart_ptr_storage::__encode_dwcas(__c, 0)) {}
+
+  // DWCAS updates __ptr_ and __ctrl_ as one 16-byte pair, so waiters observe
+  // the struct base; watching only __ctrl_ would miss stores where the pointer
+  // changes but the CB stays the same (e.g., aliasing constructor).
+  const void* __wait_address() const noexcept { return static_cast<const void*>(this); }
+
+  __extension__ using __sp_u128 = unsigned __int128;
+
+  // Storage layout: low 64 bits = __ptr_, high 64 bits = __ctrl_. The wide
+  // view is the only access path while DWCAS is enabled; the sub-atomics are
+  // not read or written through __cxx_atomic_*.
+  __sp_u128* __dwcas_address() const noexcept {
+    return const_cast<__sp_u128*>(reinterpret_cast<const __sp_u128*>(this));
----------------
ViNN280801 wrote:

@jyknight, see 9535cac2c301ec0fe14e555a13e36f423f8b6902

https://github.com/llvm/llvm-project/pull/194215