[libcxx-commits] [libcxx] [libc++] Implement C++20 atomic_ref (PR #76647)

Wed Feb 7 06:10:49 PST 2024

https://github.com/dalg24 updated https://github.com/llvm/llvm-project/pull/76647

>From f1734703770c3da4e93422e4c8d4a695a7c06eee Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sat, 30 Dec 2023 21:40:37 -0500
Subject: [PATCH 01/45] [libc++][atomic_ref] Refactor atomic_base_impl class
 and friends

so it can be reused to implement atomic_ref.
Salvaged/adapted from https://reviews.llvm.org/D72240

Co-Authored-By: Benjamin Trapani
---
 libcxx/include/__atomic/cxx_atomic_impl.h | 409 ++++------------------
 libcxx/include/__config                   |   4 +-
 2 files changed, 66 insertions(+), 347 deletions(-)

diff --git a/libcxx/include/__atomic/cxx_atomic_impl.h b/libcxx/include/__atomic/cxx_atomic_impl.h
index 1a0b808a0cb1c4..18c85aec4eef27 100644
--- a/libcxx/include/__atomic/cxx_atomic_impl.h
+++ b/libcxx/include/__atomic/cxx_atomic_impl.h
@@ -15,8 +15,12 @@
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/is_assignable.h>
+#include <__type_traits/is_pointer.h>
+#include <__type_traits/is_reference.h>
 #include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/is_volatile.h>
 #include <__type_traits/remove_const.h>
+#include <__type_traits/remove_reference.h>
 #include <cstddef>
 #include <cstring>
 
@@ -58,9 +62,27 @@ struct __cxx_atomic_base_impl {
   }
 #  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {}
+  using __contained_t = _Tp;
   _Tp __a_value;
 };
 
+template <typename _Tp, template <typename> class _TemplateTp>
+struct __is_instantiation_of : false_type {};
+
+template <typename _Tp, template <typename> class _TemplateTp>
+struct __is_instantiation_of<_TemplateTp<_Tp>, _TemplateTp> : true_type {};
+
+template <typename _Tp,
+          typename = typename enable_if<__is_instantiation_of<typename remove_volatile<typename _Tp::__base>::type,
+                                                              __cxx_atomic_base_impl>::value,
+                                        bool>::type>
+struct __cxx_atomic_base_impl_traits {
+  static constexpr bool __is_value_volatile = is_volatile<_Tp>::value;
+  static constexpr bool __is_value_ref      = is_reference<typename _Tp::__contained_t>::value;
+  using __underlying_t = typename remove_volatile<typename remove_reference<typename _Tp::__contained_t>::type>::type;
+  static constexpr bool __is_value_pointer = is_pointer<__underlying_t>::value;
+};
+
 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
   // Avoid switch statement to make this a constexpr.
   return __order == memory_order_relaxed
@@ -87,13 +109,15 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory
                                 : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME))));
 }
 
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
+template <typename _Tp, typename enable_if<__cxx_atomic_base_impl_traits<_Tp>::__is_value_volatile, bool>::type = 0>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_init(_Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __val) {
   __cxx_atomic_assign_volatile(__a->__a_value, __val);
 }
 
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
+template <typename _Tp, typename enable_if<!__cxx_atomic_base_impl_traits<_Tp>::__is_value_volatile, bool>::type = 0>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_init(_Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __val) {
   __a->__a_value = __val;
 }
 
@@ -107,53 +131,28 @@ _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
+__cxx_atomic_store(_Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __val, memory_order __order) {
   __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
-  __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
-  _Tp __ret;
+_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t
+__cxx_atomic_load(const _Tp* __a, memory_order __order) {
+  typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __ret;
   __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
   return __ret;
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load_inplace(
+    const _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t* __dst, memory_order __order) {
   __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
-  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
-  _Tp __ret;
-  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
-  return __ret;
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
-  _Tp __ret;
-  __atomic_exchange(
-      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
-  return __ret;
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
-  _Tp __ret;
+_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_exchange(
+    _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __value, memory_order __order) {
+  typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __ret;
   __atomic_exchange(
       std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
   return __ret;
@@ -161,9 +160,9 @@ _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
-    volatile __cxx_atomic_base_impl<_Tp>* __a,
-    _Tp* __expected,
-    _Tp __value,
+    _Tp* __a,
+    typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t* __expected,
+    typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __value,
     memory_order __success,
     memory_order __failure) {
   return __atomic_compare_exchange(
@@ -175,23 +174,11 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
       __to_gcc_failure_order(__failure));
 }
 
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
-    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      false,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
-}
-
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
-    volatile __cxx_atomic_base_impl<_Tp>* __a,
-    _Tp* __expected,
-    _Tp __value,
+    _Tp* __a,
+    typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t* __expected,
+    typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __value,
     memory_order __success,
     memory_order __failure) {
   return __atomic_compare_exchange(
@@ -203,18 +190,6 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
       __to_gcc_failure_order(__failure));
 }
 
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
-    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      true,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
-}
-
 template <typename _Tp>
 struct __skip_amt {
   enum { value = 1 };
@@ -233,302 +208,44 @@ template <typename _Tp, int n>
 struct __skip_amt<_Tp[n]> {};
 
 template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
-}
-
-template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
-}
-
-template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t
+__cxx_atomic_fetch_add(_Tp* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_add(
+      std::addressof(__a->__a_value),
+      __delta * __skip_amt<typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t>::value,
+      __to_gcc_order(__order));
 }
 
 template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t
+__cxx_atomic_fetch_sub(_Tp* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_sub(
+      std::addressof(__a->__a_value),
+      __delta * __skip_amt<typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t>::value,
+      __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_fetch_and(
+    _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __pattern, memory_order __order) {
   return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_fetch_or(
+    _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __pattern, memory_order __order) {
   return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_fetch_xor(
+    _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __pattern, memory_order __order) {
   return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 }
 
 #  define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
 
-#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP)
-
-template <typename _Tp>
-struct __cxx_atomic_base_impl {
-  _LIBCPP_HIDE_FROM_ABI
-#  ifndef _LIBCPP_CXX03_LANG
-  __cxx_atomic_base_impl() _NOEXCEPT = default;
-#  else
-  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
-  }
-#  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
-  _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
-};
-
-#  define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
-
-_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
-  __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
-}
-
-_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
-  __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
-  __c11_atomic_init(std::addressof(__a->__a_value), __val);
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
-  __c11_atomic_init(std::addressof(__a->__a_value), __val);
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
-  __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
-  __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
-  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
-  return __c11_atomic_load(
-      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
-  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
-  return __c11_atomic_load(
-      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
-  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
-  *__dst           = __c11_atomic_load(
-      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
-  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
-  *__dst           = __c11_atomic_load(
-      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_exchange(
-      std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_exchange(
-      std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
-}
-
-_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
-  // Avoid switch statement to make this a constexpr.
-  return __order == memory_order_release
-           ? memory_order_relaxed
-           : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
-    __cxx_atomic_base_impl<_Tp> volatile* __a,
-    _Tp* __expected,
-    _Tp __value,
-    memory_order __success,
-    memory_order __failure) _NOEXCEPT {
-  return __c11_atomic_compare_exchange_strong(
-      std::addressof(__a->__a_value),
-      __expected,
-      __value,
-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
-    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
-    _NOEXCEPT {
-  return __c11_atomic_compare_exchange_strong(
-      std::addressof(__a->__a_value),
-      __expected,
-      __value,
-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
-    __cxx_atomic_base_impl<_Tp> volatile* __a,
-    _Tp* __expected,
-    _Tp __value,
-    memory_order __success,
-    memory_order __failure) _NOEXCEPT {
-  return __c11_atomic_compare_exchange_weak(
-      std::addressof(__a->__a_value),
-      __expected,
-      __value,
-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
-    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
-    _NOEXCEPT {
-  return __c11_atomic_compare_exchange_weak(
-      std::addressof(__a->__a_value),
-      __expected,
-      __value,
-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_add(
-      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_add(
-      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp*
-__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_add(
-      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp*
-__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_add(
-      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_sub(
-      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_sub(
-      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp*
-__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_sub(
-      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp*
-__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_sub(
-      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_and(
-      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_and(
-      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_or(
-      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_or(
-      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_xor(
-      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
-}
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
-  return __c11_atomic_fetch_xor(
-      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
-}
-
-#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
-
+#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP
 #ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS
 
 template <typename _Tp>
@@ -813,7 +530,7 @@ template <typename _Tp,
 template <typename _Tp, typename _Base = __cxx_atomic_base_impl<_Tp> >
 #endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS
 struct __cxx_atomic_impl : public _Base {
-  static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
+  using __base = _Base;
 
   _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {}
diff --git a/libcxx/include/__config b/libcxx/include/__config
index d356960e9e62b4..e7d6551c867deb 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -1183,7 +1183,9 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c
 
 #  if __has_feature(cxx_atomic) || __has_extension(c_atomic) || __has_keyword(_Atomic)
 #    define _LIBCPP_HAS_C_ATOMIC_IMP
-#  elif defined(_LIBCPP_COMPILER_GCC)
+#  endif
+
+#  if defined(_LIBCPP_COMPILER_GCC) || (__has_builtin(__atomic_load) && __has_builtin(__atomic_store) && __has_builtin(__atomic_exchange) && __has_builtin(__atomic_compare_exchange))
 #    define _LIBCPP_HAS_GCC_ATOMIC_IMP
 #  endif
 

>From 3d8d9db866d483759ac2f1d7da8cf448459fb1cc Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sat, 30 Dec 2023 21:47:43 -0500
Subject: [PATCH 02/45] [libc++][atomic_ref] Enable atomic load/exchange for
 non-default constructible types

---
 libcxx/include/__atomic/cxx_atomic_impl.h | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/libcxx/include/__atomic/cxx_atomic_impl.h b/libcxx/include/__atomic/cxx_atomic_impl.h
index 18c85aec4eef27..56cd703c258944 100644
--- a/libcxx/include/__atomic/cxx_atomic_impl.h
+++ b/libcxx/include/__atomic/cxx_atomic_impl.h
@@ -138,9 +138,11 @@ __cxx_atomic_store(_Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__unde
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t
 __cxx_atomic_load(const _Tp* __a, memory_order __order) {
-  typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __ret;
-  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
-  return __ret;
+  using _Ret = typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t;
+  alignas(alignof(_Ret)) unsigned char __mem[sizeof(_Ret)];
+  __atomic_load(
+      std::addressof(__a->__a_value), std::addressof(*reinterpret_cast<_Ret*>(__mem)), __to_gcc_order(__order));
+  return *reinterpret_cast<_Ret*>(__mem);
 }
 
 template <typename _Tp>
@@ -152,10 +154,14 @@ _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load_inplace(
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_exchange(
     _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __value, memory_order __order) {
-  typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __ret;
+  using _Ret = typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t;
+  alignas(alignof(_Ret)) unsigned char __mem[sizeof(_Ret)];
   __atomic_exchange(
-      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
-  return __ret;
+      std::addressof(__a->__a_value),
+      std::addressof(__value),
+      std::addressof(*reinterpret_cast<_Ret*>(__mem)),
+      __to_gcc_order(__order));
+  return *reinterpret_cast<_Ret*>(__mem);
 }
 
 template <typename _Tp>

>From f05a599cbb48e680f22d97e8d19b23399a3ba288 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sat, 30 Dec 2023 21:50:09 -0500
Subject: [PATCH 03/45] [libc++][atomic_ref] Add
 _LIBCPP_CHECK_WAIT_MEMORY_ORDER macro

---
 libcxx/include/__atomic/check_memory_order.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcxx/include/__atomic/check_memory_order.h b/libcxx/include/__atomic/check_memory_order.h
index 3012aec0521b38..536f764a619026 100644
--- a/libcxx/include/__atomic/check_memory_order.h
+++ b/libcxx/include/__atomic/check_memory_order.h
@@ -27,4 +27,8 @@
   _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || __f == memory_order_acq_rel,                                 \
                            "memory order argument to atomic operation is invalid")
 
+#define _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m)                                                                           \
+  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel,                                 \
+                           "memory order argument to atomic operation is invalid")
+
 #endif // _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H

>From 6e01477cdeb41ba2cb62f922fd95dff91401016b Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sat, 30 Dec 2023 21:51:40 -0500
Subject: [PATCH 04/45] [libc++][atomic_ref] Implement C++20 atomic_ref

---
 libcxx/include/CMakeLists.txt        |   1 +
 libcxx/include/__atomic/atomic_ref.h | 238 +++++++++++++++++++++++++++
 libcxx/include/atomic                |   1 +
 3 files changed, 240 insertions(+)
 create mode 100644 libcxx/include/__atomic/atomic_ref.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 6ded426640f0d0..1da9cfbd3adf74 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -236,6 +236,7 @@ set(files
   __atomic/atomic_flag.h
   __atomic/atomic_init.h
   __atomic/atomic_lock_free.h
+  __atomic/atomic_ref.h
   __atomic/atomic_sync.h
   __atomic/check_memory_order.h
   __atomic/contention_t.h
diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
new file mode 100644
index 00000000000000..b4299804eccc31
--- /dev/null
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -0,0 +1,238 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H
+#define _LIBCPP___ATOMIC_ATOMIC_REF_H
+
+#include <__assert>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/cxx_atomic_impl.h>
+#include <__atomic/is_always_lock_free.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_function.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_same.h>
+#include <cinttypes>
+#include <concepts>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, bool = is_integral_v<_Tp> && !is_same_v<_Tp, bool>, bool = is_floating_point_v<_Tp>>
+struct __atomic_ref_base {
+  mutable __cxx_atomic_impl<_Tp&> __a_;
+
+  using value_type = _Tp;
+
+  static constexpr size_t required_alignment = alignof(_Tp);
+
+  static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<_Tp>::__value;
+
+  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __cxx_atomic_is_lock_free(sizeof(_Tp)); }
+
+  _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
+      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
+        "memory order argument to atomic store operation is invalid");
+    __cxx_atomic_store(&__a_, __desired, __order);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {
+    store(__desired);
+    return __desired;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
+      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
+            __order == memory_order::seq_cst,
+        "memory order argument to atomic load operation is invalid");
+    return __cxx_atomic_load(&__a_, __order);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+    return __cxx_atomic_exchange(&__a_, __desired, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
+      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        __failure == memory_order::relaxed || __failure == memory_order::consume ||
+            __failure == memory_order::acquire || __failure == memory_order::seq_cst,
+        "failure memory order argument to weak atomic compare-and-exchange operation is invalid");
+    return __cxx_atomic_compare_exchange_weak(&__a_, &__expected, __desired, __success, __failure);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
+      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        __failure == memory_order::relaxed || __failure == memory_order::consume ||
+            __failure == memory_order::acquire || __failure == memory_order::seq_cst,
+        "failure memory order argument to strong atomic compare-and-exchange operation is invalid");
+    return __cxx_atomic_compare_exchange_strong(&__a_, &__expected, __desired, __success, __failure);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+    return __cxx_atomic_compare_exchange_weak(&__a_, &__expected, __desired, __order, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI bool
+  compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+    return __cxx_atomic_compare_exchange_strong(&__a_, &__expected, __desired, __order, __order);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
+      _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) {
+    _LIBCPP_ASSERT_UNCATEGORIZED(
+        __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
+            __order == memory_order::seq_cst,
+        "memory order argument to atomic wait operation is invalid");
+    __cxx_atomic_wait(addressof(__a_), __old, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { __cxx_atomic_notify_one(addressof(__a_)); }
+  _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { __cxx_atomic_notify_all(addressof(__a_)); }
+
+  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __a_(__obj) {}
+};
+
+template <class _Tp>
+struct __atomic_ref_base<_Tp, /*_IsIntegral=*/true, /*_IsFloatingPoint=*/false>
+    : public __atomic_ref_base<_Tp, false, false> {
+  using __base = __atomic_ref_base<_Tp, false, false>;
+
+  using difference_type = __base::value_type;
+
+  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __base(__obj) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_add(&this->__a_, __arg, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_sub(&this->__a_, __arg, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_and(&this->__a_, __arg, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_or(&this->__a_, __arg, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_xor(&this->__a_, __arg, __order);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(_Tp(1)); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) const noexcept { return fetch_sub(_Tp(1)); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator++() const noexcept { return fetch_add(_Tp(1)) + _Tp(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator--() const noexcept { return fetch_sub(_Tp(1)) - _Tp(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __arg) const noexcept { return fetch_and(__arg) & __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __arg) const noexcept { return fetch_or(__arg) | __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __arg) const noexcept { return fetch_xor(__arg) ^ __arg; }
+};
+
+template <class _Tp>
+struct __atomic_ref_base<_Tp, /*_IsIntegral=*/false, /*_IsFloatingPoint=*/true>
+    : public __atomic_ref_base<_Tp, false, false> {
+  using __base = __atomic_ref_base<_Tp, false, false>;
+
+  using difference_type = __base::value_type;
+
+  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __base(__obj) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_add(&this->__a_, __arg, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_sub(&this->__a_, __arg, __order);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
+};
+
+template <class _Tp>
+struct atomic_ref : public __atomic_ref_base<_Tp> {
+  static_assert(is_trivially_copyable<_Tp>::value, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
+
+  using __base = __atomic_ref_base<_Tp>;
+
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+    _LIBCPP_ASSERT_UNCATEGORIZED((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
+                                 "atomic_ref ctor: referenced object must be aligned to required_alignment");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+
+  atomic_ref& operator=(const atomic_ref&) = delete;
+};
+
+template <class _Tp>
+struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
+  using __base = __atomic_ref_base<_Tp*>;
+
+  using difference_type = ptrdiff_t;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_add(&this->__a_, __arg, __order);
+  }
+  _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
+    return __cxx_atomic_fetch_sub(&this->__a_, __arg, __order);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) const noexcept { return fetch_sub(1); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator++() const noexcept { return fetch_add(1) + 1; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator--() const noexcept { return fetch_sub(1) - 1; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; }
+
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {}
+
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); }
+
+  atomic_ref& operator=(const atomic_ref&) = delete;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H
diff --git a/libcxx/include/atomic b/libcxx/include/atomic
index 2e8f5b521a55eb..b71033b8a43f3b 100644
--- a/libcxx/include/atomic
+++ b/libcxx/include/atomic
@@ -594,6 +594,7 @@ template <class T>
 #include <__atomic/atomic_flag.h>
 #include <__atomic/atomic_init.h>
 #include <__atomic/atomic_lock_free.h>
+#include <__atomic/atomic_ref.h>
 #include <__atomic/atomic_sync.h>
 #include <__atomic/check_memory_order.h>
 #include <__atomic/contention_t.h>

>From 1575d41fed20826be90b6b2ed0f0dd4ef3ca6e7f Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sat, 30 Dec 2023 21:54:12 -0500
Subject: [PATCH 05/45] [libc++][atomic_ref] Add tests for atomic_ref

---
 .../assert.compare_exchange_strong.pass.cpp   |  63 +++++++++
 .../assert.compare_exchange_weak.pass.cpp     |  63 +++++++++
 .../atomics/atomics.ref/assert.ctor.pass.cpp  |  38 +++++
 .../atomics/atomics.ref/assert.load.pass.cpp  |  60 ++++++++
 .../atomics/atomics.ref/assert.store.pass.cpp |  68 +++++++++
 .../atomics/atomics.ref/assert.wait.pass.cpp  |  60 ++++++++
 .../std/atomics/atomics.ref/assign.pass.cpp   |  50 +++++++
 .../atomics.ref/bitwise_and_assign.pass.cpp   |  47 +++++++
 .../atomics.ref/bitwise_or_assign.pass.cpp    |  47 +++++++
 .../atomics.ref/bitwise_xor_assign.pass.cpp   |  47 +++++++
 .../compare_exchange_strong.pass.cpp          |  83 +++++++++++
 .../compare_exchange_weak.pass.cpp            |  84 +++++++++++
 .../std/atomics/atomics.ref/convert.pass.cpp  |  47 +++++++
 .../atomics.ref/ctor.explicit.verify.cpp      |  34 +++++
 .../std/atomics/atomics.ref/ctor.pass.cpp     |  46 ++++++
 .../atomics/atomics.ref/deduction.pass.cpp    |  39 ++++++
 .../std/atomics/atomics.ref/exchange.pass.cpp |  48 +++++++
 .../atomics/atomics.ref/fetch_add.pass.cpp    |  75 ++++++++++
 .../atomics/atomics.ref/fetch_and.pass.cpp    |  56 ++++++++
 .../std/atomics/atomics.ref/fetch_or.pass.cpp |  54 +++++++
 .../atomics/atomics.ref/fetch_sub.pass.cpp    |  75 ++++++++++
 .../atomics/atomics.ref/fetch_xor.pass.cpp    |  54 +++++++
 .../atomics.ref/increment_decrement.pass.cpp  |  77 ++++++++++
 .../atomics.ref/is_always_lock_free.pass.cpp  |  47 +++++++
 .../std/atomics/atomics.ref/load.pass.cpp     |  48 +++++++
 .../atomics/atomics.ref/member_types.pass.cpp | 132 ++++++++++++++++++
 .../atomics/atomics.ref/notify_all.pass.cpp   |  86 ++++++++++++
 .../atomics/atomics.ref/notify_one.pass.cpp   |  54 +++++++
 .../operator_minus_equals.pass.cpp            |  64 +++++++++
 .../atomics.ref/operator_plus_equals.pass.cpp |  64 +++++++++
 .../atomics.ref/required_alignment.pass.cpp   |  34 +++++
 .../std/atomics/atomics.ref/store.pass.cpp    |  50 +++++++
 .../std/atomics/atomics.ref/type.verify.cpp   |  26 ++++
 .../std/atomics/atomics.ref/wait.pass.cpp     |  65 +++++++++
 34 files changed, 1985 insertions(+)
 create mode 100644 libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
 create mode 100644 libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
 create mode 100644 libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
 create mode 100644 libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
 create mode 100644 libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
 create mode 100644 libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/ctor.explicit.verify.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/load.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/member_types.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/store.pass.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/type.verify.cpp
 create mode 100644 libcxx/test/std/atomics/atomics.ref/wait.pass.cpp

diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
new file mode 100644
index 00000000000000..3a991c9351cd85
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -0,0 +1,63 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: availability-verbose_abort-missing
+// ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
+
+// <atomic>
+
+// bool compare_exchange_strong(T& expected, T desired, memory_order success, memory_order failure) const noexcept;
+//
+// Preconditions: failure is memory_order::relaxed, memory_order::consume, memory_order::acquire, or memory_order::seq_cst.
+
+#include <atomic>
+
+#include "check_assertion.h"
+
+template <typename T>
+void test_compare_exchange_strong_invalid_memory_order() {
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+    T t(T(2));
+    a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_relaxed);
+  }
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        T t(T(2));
+        a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_release);
+      }()),
+      "memory order argument to strong atomic compare-and-exchange operation is invalid");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        T t(T(2));
+        a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_acq_rel);
+      }()),
+      "memory order argument to strong atomic compare-and-exchange operation is invalid");
+}
+
+int main(int, char**) {
+  test_compare_exchange_strong_invalid_memory_order<int>();
+  test_compare_exchange_strong_invalid_memory_order<float>();
+  test_compare_exchange_strong_invalid_memory_order<int*>();
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_compare_exchange_strong_invalid_memory_order<X>();
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
new file mode 100644
index 00000000000000..c9506f556129ee
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -0,0 +1,63 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: availability-verbose_abort-missing
+// ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
+
+// <atomic>
+
+// bool compare_exchange_weak(T& expected, T desired, memory_order success, memory_order failure) const noexcept;
+//
+// Preconditions: failure is memory_order::relaxed, memory_order::consume, memory_order::acquire, or memory_order::seq_cst.
+
+#include <atomic>
+
+#include "check_assertion.h"
+
+template <typename T>
+void test_compare_exchange_weak_invalid_memory_order() {
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+    T t(T(2));
+    a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_relaxed);
+  }
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        T t(T(2));
+        a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_release);
+      }()),
+      "memory order argument to weak atomic compare-and-exchange operation is invalid");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        T t(T(2));
+        a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_acq_rel);
+      }()),
+      "memory order argument to weak atomic compare-and-exchange operation is invalid");
+}
+
+int main(int, char**) {
+  test_compare_exchange_weak_invalid_memory_order<int>();
+  test_compare_exchange_weak_invalid_memory_order<float>();
+  test_compare_exchange_weak_invalid_memory_order<int*>();
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_compare_exchange_weak_invalid_memory_order<X>();
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
new file mode 100644
index 00000000000000..3705167181519c
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
@@ -0,0 +1,38 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: availability-verbose_abort-missing
+
+// <atomic>
+
+// atomic_ref(T& obj);
+//
+// Preconditions: The referenced object is aligned to required_alignment.
+
+#include <atomic>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  {
+    char c[8];
+    float* f = new (c) float(3.14f);
+    [[maybe_unused]] std::atomic_ref<float> r(*f);
+  }
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        char c[8];
+        float* f = new (c + 1) float(3.14f);
+        [[maybe_unused]] std::atomic_ref<float> r(*f);
+      }()),
+      "atomic_ref ctor: referenced object must be aligned to required_alignment");
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
new file mode 100644
index 00000000000000..4181b1c12c7db4
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
@@ -0,0 +1,60 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: availability-verbose_abort-missing
+// ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
+
+// <atomic>
+
+// T load(memory_order order = memory_order::seq_cst) const noexcept;
+//
+// Preconditions: order is memory_order::relaxed, memory_order::consume, memory_order::acquire, or memory_order::seq_cst.
+
+#include <atomic>
+
+#include "check_assertion.h"
+
+template <typename T>
+void test_load_invalid_memory_order() {
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+    (void)a.load(std::memory_order_relaxed);
+  }
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        (void)a.load(std::memory_order_release);
+      }()),
+      "memory order argument to atomic load operation is invalid");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        (void)a.load(std::memory_order_acq_rel);
+      }()),
+      "memory order argument to atomic load operation is invalid");
+}
+
+int main(int, char**) {
+  test_load_invalid_memory_order<int>();
+  test_load_invalid_memory_order<float>();
+  test_load_invalid_memory_order<int*>();
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_load_invalid_memory_order<X>();
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
new file mode 100644
index 00000000000000..f543bcc35295f7
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
@@ -0,0 +1,68 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: availability-verbose_abort-missing
+// ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
+
+// <atomic>
+
+// void store(T desired, memory_order order = memory_order::seq_cst) const noexcept;
+//
+// Preconditions: order is memory_order::relaxed, memory_order::release, or memory_order::seq_cst.
+
+#include <atomic>
+
+#include "check_assertion.h"
+
+template <typename T>
+void test_store_invalid_memory_order() {
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+    a.store(T(2), std::memory_order_relaxed);
+  }
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        a.store(T(2), std::memory_order_consume);
+      }()),
+      "memory order argument to atomic store operation is invalid");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        a.store(T(2), std::memory_order_acquire);
+      }()),
+      "memory order argument to atomic store operation is invalid");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        a.store(T(2), std::memory_order_acq_rel);
+      }()),
+      "memory order argument to atomic store operation is invalid");
+}
+
+int main(int, char**) {
+  test_store_invalid_memory_order<int>();
+  test_store_invalid_memory_order<float>();
+  test_store_invalid_memory_order<int*>();
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_store_invalid_memory_order<X>();
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
new file mode 100644
index 00000000000000..2b1c9208527471
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
@@ -0,0 +1,60 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: availability-verbose_abort-missing
+// ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
+
+// <atomic>
+
+// void wait(T old, memory_order order = memory_order::seq_cst) const noexcept;
+//
+// Preconditions: order is memory_order::relaxed, memory_order::consume, memory_order::acquire, or memory_order::seq_cst.
+
+#include <atomic>
+
+#include "check_assertion.h"
+
+template <typename T>
+void test_wait_invalid_memory_order() {
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+    a.wait(T(2), std::memory_order_relaxed);
+  }
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        a.wait(T(2), std::memory_order_release);
+      }()),
+      "memory order argument to atomic wait operation is invalid");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([] {
+        T x(T(1));
+        std::atomic_ref<T> a(x);
+        a.wait(T(2), std::memory_order_acq_rel);
+      }()),
+      "memory order argument to atomic wait operation is invalid");
+}
+
+int main(int, char**) {
+  test_wait_invalid_memory_order<int>();
+  test_wait_invalid_memory_order<float>();
+  test_wait_invalid_memory_order<int*>();
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_wait_invalid_memory_order<X>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
new file mode 100644
index 00000000000000..95d29df70fe160
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
@@ -0,0 +1,50 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// T operator=(T) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+void test_assign() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  a = T(2);
+  assert(x == T(2));
+
+  ASSERT_NOEXCEPT(a = T(0));
+  static_assert(std::is_nothrow_assignable_v<std::atomic_ref<T>, T>);
+
+  static_assert(!std::is_copy_assignable_v<std::atomic_ref<T>>);
+}
+
+void test() {
+  test_assign<int>();
+
+  test_assign<float>();
+
+  test_assign<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_assign<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
new file mode 100644
index 00000000000000..68c64b9ab34589
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
@@ -0,0 +1,47 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type operator&=(integral-type) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_bitwise_and_assign = requires { std::declval<T const>() &= std::declval<T>(); };
+
+static_assert(!has_bitwise_and_assign<std::atomic_ref<float>>);
+static_assert(!has_bitwise_and_assign<std::atomic_ref<int*>>);
+static_assert(!has_bitwise_and_assign<std::atomic_ref<const int*>>);
+static_assert(!has_bitwise_and_assign<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_bitwise_and_assign<std::atomic_ref<X>>);
+
+template <typename T>
+void test_integral() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert((a &= T(2)) == T(0));
+  assert(x == T(0));
+  ASSERT_NOEXCEPT(a &= T(0));
+}
+
+void test() { test_integral<int>(); }
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
new file mode 100644
index 00000000000000..20ec80697c70ed
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
@@ -0,0 +1,47 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type operator|=(integral-type) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_bitwise_or_assign = requires { std::declval<T const>() |= std::declval<T>(); };
+
+static_assert(!has_bitwise_or_assign<std::atomic_ref<float>>);
+static_assert(!has_bitwise_or_assign<std::atomic_ref<int*>>);
+static_assert(!has_bitwise_or_assign<std::atomic_ref<const int*>>);
+static_assert(!has_bitwise_or_assign<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_bitwise_or_assign<std::atomic_ref<X>>);
+
+template <typename T>
+void test_integral() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert((a |= T(2)) == T(3));
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a &= T(0));
+}
+
+void test() { test_integral<int>(); }
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
new file mode 100644
index 00000000000000..2d41bf01f95629
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
@@ -0,0 +1,47 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type operator|=(integral-type) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_bitwise_xor_assign = requires { std::declval<T const>() ^= std::declval<T>(); };
+
+static_assert(!has_bitwise_xor_assign<std::atomic_ref<float>>);
+static_assert(!has_bitwise_xor_assign<std::atomic_ref<int*>>);
+static_assert(!has_bitwise_xor_assign<std::atomic_ref<const int*>>);
+static_assert(!has_bitwise_xor_assign<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_bitwise_xor_assign<std::atomic_ref<X>>);
+
+template <typename T>
+void test_integral() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert((a ^= T(2)) == T(3));
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a ^= T(0));
+}
+
+void test() { test_integral<int>(); }
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp b/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
new file mode 100644
index 00000000000000..b9dc84c04f7668
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
@@ -0,0 +1,83 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// bool compare_exchange_strong(T&, T, memory_order, memory_order) const noexcept;
+// bool compare_exchange_strong(T&, T, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+void test_compare_exchange_strong() {
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+
+    T t(T(1));
+    assert(a.compare_exchange_strong(t, T(2)) == true);
+    assert(a == T(2));
+    assert(t == T(1));
+    assert(a.compare_exchange_strong(t, T(3)) == false);
+    assert(a == T(2));
+    assert(t == T(2));
+
+    ASSERT_NOEXCEPT(a.compare_exchange_strong(t, T(2)));
+  }
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+
+    T t(T(1));
+    assert(a.compare_exchange_strong(t, T(2), std::memory_order_seq_cst) == true);
+    assert(a == T(2));
+    assert(t == T(1));
+    assert(a.compare_exchange_strong(t, T(3), std::memory_order_seq_cst) == false);
+    assert(a == T(2));
+    assert(t == T(2));
+
+    ASSERT_NOEXCEPT(a.compare_exchange_strong(t, T(2), std::memory_order_seq_cst));
+  }
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+
+    T t(T(1));
+    assert(a.compare_exchange_strong(t, T(2), std::memory_order_release, std::memory_order_relaxed) == true);
+    assert(a == T(2));
+    assert(t == T(1));
+    assert(a.compare_exchange_strong(t, T(3), std::memory_order_release, std::memory_order_relaxed) == false);
+    assert(a == T(2));
+    assert(t == T(2));
+
+    ASSERT_NOEXCEPT(a.compare_exchange_strong(t, T(2), std::memory_order_release, std::memory_order_relaxed));
+  }
+}
+
+void test() {
+  test_compare_exchange_strong<int>();
+
+  test_compare_exchange_strong<float>();
+
+  test_compare_exchange_strong<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_compare_exchange_strong<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp b/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
new file mode 100644
index 00000000000000..3d16be5e3a1499
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
@@ -0,0 +1,84 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// bool compare_exchange_weak(T&, T, memory_order, memory_order) const noexcept;
+// bool compare_exchange_weak(T&, T, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+void test_compare_exchange_weak() {
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+
+    T t(T(1));
+    assert(a.compare_exchange_weak(t, T(2)) == true);
+    assert(a == T(2));
+    assert(t == T(1));
+    assert(a.compare_exchange_weak(t, T(3)) == false);
+    assert(a == T(2));
+    assert(t == T(2));
+
+    ASSERT_NOEXCEPT(a.compare_exchange_weak(t, T(2)));
+  }
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+
+    T t(T(1));
+    assert(a.compare_exchange_weak(t, T(2), std::memory_order_seq_cst) == true);
+    assert(a == T(2));
+    assert(t == T(1));
+    assert(a.compare_exchange_weak(t, T(3), std::memory_order_seq_cst) == false);
+    assert(a == T(2));
+    assert(t == T(2));
+
+    ASSERT_NOEXCEPT(a.compare_exchange_weak(t, T(2), std::memory_order_seq_cst));
+  }
+  {
+    T x(T(1));
+    std::atomic_ref<T> a(x);
+
+    T t(T(1));
+    assert(a.compare_exchange_weak(t, T(2), std::memory_order_release, std::memory_order_relaxed) == true);
+    assert(a == T(2));
+    assert(t == T(1));
+    assert(a.compare_exchange_weak(t, T(3), std::memory_order_release, std::memory_order_relaxed) == false);
+    assert(a == T(2));
+    assert(t == T(2));
+
+    ASSERT_NOEXCEPT(a.compare_exchange_weak(t, T(2), std::memory_order_release, std::memory_order_relaxed));
+  }
+}
+
+void test() {
+  test_compare_exchange_weak<int>();
+
+  test_compare_exchange_weak<float>();
+
+  test_compare_exchange_weak<int*>();
+
+  struct X {
+    int i;
+    //X() = default;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_compare_exchange_weak<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp b/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
new file mode 100644
index 00000000000000..cdfd3442eac806
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
@@ -0,0 +1,47 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// operator T() const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+void test_convert() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(a == T(1));
+
+  ASSERT_NOEXCEPT(T(a));
+  static_assert(std::is_nothrow_convertible_v<std::atomic_ref<T>, T>);
+}
+
+void test() {
+  test_convert<int>();
+
+  test_convert<float>();
+
+  test_convert<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_convert<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/ctor.explicit.verify.cpp b/libcxx/test/std/atomics/atomics.ref/ctor.explicit.verify.cpp
new file mode 100644
index 00000000000000..3f1c133c643d65
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/ctor.explicit.verify.cpp
@@ -0,0 +1,34 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// explicit atomic_ref(T&);
+
+#include <atomic>
+
+template <class T>
+void test(std::atomic_ref<T>) {}
+
+void explicit_ctor() {
+  int i = 0;
+  // expected-error-re@*:* {{{{.*}}no matching function for call to 'test'}}
+  test<int>(i);
+
+  float f = 0.f;
+  // expected-error-re@*:* {{{{.*}}no matching function for call to 'test'}}
+  test<float>(f);
+
+  int* p = &i;
+  // expected-error-re@*:* {{{{.*}}no matching function for call to 'test'}}
+  test<int*>(p);
+
+  struct X {
+  } x;
+  // expected-error-re@*:* {{{{.*}}no matching function for call to 'test'}}
+  test<X>(x);
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp b/libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp
new file mode 100644
index 00000000000000..42890f4a537fc7
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp
@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <atomic>
+
+// explicit atomic_ref(T&);
+
+#include <atomic>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+auto makeAtomicRef(T& obj) {
+  // check that the constructor is explicit
+  static_assert(!std::is_convertible_v<T, std::atomic_ref<T>>);
+  static_assert(std::is_constructible_v<std::atomic_ref<T>, T&>);
+  return std::atomic_ref<T>(obj);
+}
+
+void test() {
+  int i = 0;
+  (void)makeAtomicRef(i);
+
+  float f = 0.f;
+  (void)makeAtomicRef(f);
+
+  int* p = &i;
+  (void)makeAtomicRef(p);
+
+  struct X {
+  } x;
+  (void)makeAtomicRef(x);
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp b/libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp
new file mode 100644
index 00000000000000..62cfcc08aa0420
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// ADDITIONAL_COMPILE_FLAGS: -Wno-ctad-maybe-unsupported
+
+// <atomic>
+
+#include <atomic>
+#include <type_traits>
+
+void test() {
+  int i = 0;
+  std::atomic_ref a0(i);
+  static_assert(std::is_same_v<decltype(a0), std::atomic_ref<int>>);
+
+  float f = 0.f;
+  std::atomic_ref a1(f);
+  static_assert(std::is_same_v<decltype(a1), std::atomic_ref<float>>);
+
+  int* p = &i;
+  std::atomic_ref a2(p);
+  static_assert(std::is_same_v<decltype(a2), std::atomic_ref<int*>>);
+
+  struct X {
+  } x;
+  std::atomic_ref a3(x);
+  static_assert(std::is_same_v<decltype(a3), std::atomic_ref<X>>);
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp b/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
new file mode 100644
index 00000000000000..75055bd8679bc3
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
@@ -0,0 +1,48 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// T exchange(T, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+void test_exchange() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(a.exchange(T(2)) == T(1));
+  ASSERT_NOEXCEPT(a.exchange(T(2)));
+
+  assert(a.exchange(T(3), std::memory_order_seq_cst) == T(2));
+  ASSERT_NOEXCEPT(a.exchange(T(3), std::memory_order_seq_cst));
+}
+
+void test() {
+  test_exchange<int>();
+
+  test_exchange<float>();
+
+  test_exchange<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_exchange<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
new file mode 100644
index 00000000000000..439d5a305dd6b7
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
@@ -0,0 +1,75 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type fetch_add(integral-type, memory_order = memory_order::seq_cst) const noexcept;
+// floating-point-type fetch_add(floating-point-type, memory_order = memory_order::seq_cst) const noexcept;
+// T* fetch_add(difference_type, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_fetch_add = requires {
+  std::declval<T const>().fetch_add(std::declval<T>());
+  std::declval<T const>().fetch_add(std::declval<T>(), std::declval<std::memory_order>());
+};
+
+static_assert(!has_fetch_add<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_fetch_add<std::atomic_ref<X>>);
+
+template <typename T>
+void test_arithmetic() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(a.fetch_add(T(2)) == T(1));
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a.fetch_add(T(0)));
+
+  assert(a.fetch_add(T(4), std::memory_order_relaxed) == T(3));
+  assert(x == T(7));
+  ASSERT_NOEXCEPT(a.fetch_add(T(0), std::memory_order_relaxed));
+}
+
+template <typename T>
+void test_pointer() {
+  using X = std::remove_pointer_t<T>;
+  X t[9]  = {};
+  T p{&t[1]};
+  std::atomic_ref<T> a(p);
+
+  assert(a.fetch_add(2) == &t[1]);
+  assert(a == &t[3]);
+  ASSERT_NOEXCEPT(a.fetch_add(0));
+
+  assert(a.fetch_add(4, std::memory_order_relaxed) == &t[3]);
+  assert(a == &t[7]);
+  ASSERT_NOEXCEPT(a.fetch_add(0, std::memory_order_relaxed));
+}
+
+void test() {
+  test_arithmetic<int>();
+  test_arithmetic<float>();
+
+  test_pointer<int*>();
+  test_pointer<const int*>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
new file mode 100644
index 00000000000000..d837bc8f423c98
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
@@ -0,0 +1,56 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type fetch_and(integral-type, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_fetch_and = requires {
+  std::declval<T const>().fetch_and(std::declval<T>());
+  std::declval<T const>().fetch_and(std::declval<T>(), std::declval<std::memory_order>());
+};
+
+static_assert(!has_fetch_and<std::atomic_ref<float>>);
+static_assert(!has_fetch_and<std::atomic_ref<int*>>);
+static_assert(!has_fetch_and<std::atomic_ref<const int*>>);
+static_assert(!has_fetch_and<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_fetch_and<std::atomic_ref<X>>);
+
+template <typename T>
+void test_integral() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(a.fetch_and(T(2)) == T(1));
+  assert(x == T(0));
+  ASSERT_NOEXCEPT(a.fetch_and(T(0)));
+
+  x = T(1);
+
+  assert(a.fetch_and(T(2), std::memory_order_relaxed) == T(1));
+  assert(x == T(0));
+  ASSERT_NOEXCEPT(a.fetch_and(T(0), std::memory_order_relaxed));
+}
+
+void test() { test_integral<int>(); }
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
new file mode 100644
index 00000000000000..88a836810b002e
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
@@ -0,0 +1,54 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type fetch_or(integral-type, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_fetch_or = requires {
+  std::declval<T const>().fetch_or(std::declval<T>());
+  std::declval<T const>().fetch_or(std::declval<T>(), std::declval<std::memory_order>());
+};
+
+static_assert(!has_fetch_or<std::atomic_ref<float>>);
+static_assert(!has_fetch_or<std::atomic_ref<int*>>);
+static_assert(!has_fetch_or<std::atomic_ref<const int*>>);
+static_assert(!has_fetch_or<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_fetch_or<std::atomic_ref<X>>);
+
+template <typename T>
+void test_integral() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(a.fetch_or(T(2)) == T(1));
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a.fetch_or(T(0)));
+
+  assert(a.fetch_or(T(2), std::memory_order_relaxed) == T(3));
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a.fetch_or(T(0), std::memory_order_relaxed));
+}
+
+void test() { test_integral<int>(); }
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
new file mode 100644
index 00000000000000..e9fc7ea7cb5ff1
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
@@ -0,0 +1,75 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type fetch_sub(integral-type, memory_order = memory_order::seq_cst) const noexcept;
+// floating-point-type fetch_sub(floating-point-type, memory_order = memory_order::seq_cst) const noexcept;
+// T* fetch_sub(difference_type, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_fetch_sub = requires {
+  std::declval<T const>().fetch_sub(std::declval<T>());
+  std::declval<T const>().fetch_sub(std::declval<T>(), std::declval<std::memory_order>());
+};
+
+static_assert(!has_fetch_sub<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_fetch_sub<std::atomic_ref<X>>);
+
+template <typename T>
+void test_arithmetic() {
+  T x(T(7));
+  std::atomic_ref<T> a(x);
+
+  assert(a.fetch_sub(T(4)) == T(7));
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a.fetch_sub(T(0)));
+
+  assert(a.fetch_sub(T(2), std::memory_order_relaxed) == T(3));
+  assert(x == T(1));
+  ASSERT_NOEXCEPT(a.fetch_sub(T(0), std::memory_order_relaxed));
+}
+
+template <typename T>
+void test_pointer() {
+  using X = std::remove_pointer_t<T>;
+  X t[9]  = {};
+  T p{&t[7]};
+  std::atomic_ref<T> a(p);
+
+  assert(a.fetch_sub(4) == &t[7]);
+  assert(a == &t[3]);
+  ASSERT_NOEXCEPT(a.fetch_sub(0));
+
+  assert(a.fetch_sub(2, std::memory_order_relaxed) == &t[3]);
+  assert(a == &t[1]);
+  ASSERT_NOEXCEPT(a.fetch_sub(0, std::memory_order_relaxed));
+}
+
+void test() {
+  test_arithmetic<int>();
+  test_arithmetic<float>();
+
+  test_pointer<int*>();
+  test_pointer<const int*>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
new file mode 100644
index 00000000000000..2e2f913e9e242a
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
@@ -0,0 +1,54 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type fetch_xor(integral-type, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_fetch_xor = requires {
+  std::declval<T const>().fetch_xor(std::declval<T>());
+  std::declval<T const>().fetch_xor(std::declval<T>(), std::declval<std::memory_order>());
+};
+
+static_assert(!has_fetch_xor<std::atomic_ref<float>>);
+static_assert(!has_fetch_xor<std::atomic_ref<int*>>);
+static_assert(!has_fetch_xor<std::atomic_ref<const int*>>);
+static_assert(!has_fetch_xor<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_fetch_xor<std::atomic_ref<X>>);
+
+template <typename T>
+void test_integral() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(a.fetch_xor(T(2)) == T(1));
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a.fetch_xor(T(0)));
+
+  assert(a.fetch_xor(T(2), std::memory_order_relaxed) == T(3));
+  assert(x == T(1));
+  ASSERT_NOEXCEPT(a.fetch_xor(T(0), std::memory_order_relaxed));
+}
+
+void test() { test_integral<int>(); }
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp b/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
new file mode 100644
index 00000000000000..dd1bcaa6f554f0
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
@@ -0,0 +1,77 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type operator++(int) const noexcept;
+// integral-type operator--(int) const noexcept;
+// integral-type operator++() const noexcept;
+// integral-type operator--() const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_pre_increment_operator = requires { ++std::declval<T const>(); };
+
+template <typename T>
+concept has_post_increment_operator = requires { std::declval<T const>()++; };
+
+template <typename T>
+concept has_pre_decrement_operator = requires { --std::declval<T const>(); };
+
+template <typename T>
+concept has_post_decrement_operator = requires { std::declval<T const>()--; };
+
+template <typename T>
+constexpr bool does_not_have_increment_nor_decrement_operators() {
+  return !has_pre_increment_operator<T> && !has_pre_decrement_operator<T> && !has_post_increment_operator<T> &&
+         !has_post_decrement_operator<T>;
+}
+
+static_assert(does_not_have_increment_nor_decrement_operators<float>());
+static_assert(does_not_have_increment_nor_decrement_operators<int*>());
+static_assert(does_not_have_increment_nor_decrement_operators<const int*>());
+static_assert(does_not_have_increment_nor_decrement_operators<bool>());
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(does_not_have_increment_nor_decrement_operators<X>());
+
+template <typename T>
+void test_integral() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(++a == T(2));
+  assert(x == T(2));
+  ASSERT_NOEXCEPT(++a);
+
+  assert(--a == T(1));
+  assert(x == T(1));
+  ASSERT_NOEXCEPT(--a);
+
+  assert(a++ == T(1));
+  assert(x == T(2));
+  ASSERT_NOEXCEPT(++a);
+
+  assert(a-- == T(2));
+  assert(x == T(1));
+  ASSERT_NOEXCEPT(--a);
+}
+
+void test() { test_integral<int>(); }
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
new file mode 100644
index 00000000000000..7f0548d2512481
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <atomic>
+
+// static constexpr bool is_always_lock_free;
+// bool is_lock_free() const noexcept;
+
+#include <atomic>
+#include <cassert>
+
+#include "test_macros.h"
+
+template <typename T>
+void checkAlwaysLockFree(std::atomic_ref<T> a) {
+  if (std::atomic_ref<T>::is_always_lock_free) {
+    assert(a.is_lock_free());
+  }
+  ASSERT_NOEXCEPT(a.is_lock_free());
+}
+
+void test() {
+  int i = 0;
+  checkAlwaysLockFree(std::atomic_ref<int>(i));
+
+  float f = 0.f;
+  checkAlwaysLockFree(std::atomic_ref<float>(f));
+
+  int* p = &i;
+  checkAlwaysLockFree(std::atomic_ref<int*>(p));
+
+  struct X {
+  } x;
+  checkAlwaysLockFree(std::atomic_ref<X>(x));
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/load.pass.cpp b/libcxx/test/std/atomics/atomics.ref/load.pass.cpp
new file mode 100644
index 00000000000000..fe3aaaf1edcf8a
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/load.pass.cpp
@@ -0,0 +1,48 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// T load(memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+void test_load() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(a.load() == T(1));
+  ASSERT_NOEXCEPT(a.load());
+
+  assert(a.load(std::memory_order_seq_cst) == T(1));
+  ASSERT_NOEXCEPT(a.load(std::memory_order_seq_cst));
+}
+
+void test() {
+  test_load<int>();
+
+  test_load<float>();
+
+  test_load<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_load<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/member_types.pass.cpp b/libcxx/test/std/atomics/atomics.ref/member_types.pass.cpp
new file mode 100644
index 00000000000000..d4e2f0126d6216
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/member_types.pass.cpp
@@ -0,0 +1,132 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <atomic>
+
+// template <class T>
+// struct atomic_ref
+// {
+//    using value_type = T;
+//    using difference_type = value_type;      // only for atomic_ref<Integral> and
+//                                             // atomic_ref<Floating> specializations
+//    using difference_type = std::ptrdiff_t;  // only for atomic_ref<T*> specializations
+//
+//    explicit atomic_ref(T&);
+//    atomic_ref(const atomic_ref&) noexcept;
+//    atomic_ref& operator=(const atomic_ref&) = delete;
+// };
+
+#include <atomic>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <class T>
+concept has_difference_type = requires { typename T::difference_type; };
+
+template <class T>
+void check_member_types() {
+  if constexpr ((std::is_integral_v<T> && !std::is_same_v<T, bool>) || std::is_floating_point_v<T>) {
+    ASSERT_SAME_TYPE(typename std::atomic_ref<T>::value_type, T);
+    ASSERT_SAME_TYPE(typename std::atomic_ref<T>::difference_type, T);
+  } else if constexpr (std::is_pointer_v<T>) {
+    ASSERT_SAME_TYPE(typename std::atomic_ref<T>::value_type, T);
+    ASSERT_SAME_TYPE(typename std::atomic_ref<T>::difference_type, std::ptrdiff_t);
+  } else {
+    ASSERT_SAME_TYPE(typename std::atomic_ref<T>::value_type, T);
+    static_assert(!has_difference_type<std::atomic_ref<T>>);
+  }
+}
+
+template <class T>
+void test() {
+  // value_type and difference_type (except for primary template)
+  check_member_types<T>();
+
+  static_assert(std::is_nothrow_copy_constructible_v<std::atomic_ref<T>>);
+
+  static_assert(!std::is_copy_assignable_v<std::atomic_ref<T>>);
+
+  // explicit constructor
+  static_assert(!std::is_convertible_v<T, std::atomic_ref<T>>);
+  static_assert(std::is_constructible_v<std::atomic_ref<T>, T&>);
+}
+
+void testall() {
+  // Primary template
+  struct Empty {};
+  test<Empty>();
+  struct Trivial {
+    int a;
+    float b;
+  };
+  test<Trivial>();
+  test<bool>();
+
+  // Partial specialization for pointer types
+  test<void*>();
+
+  // Specialization for integral types
+  // + character types
+  test<char>();
+  test<char8_t>();
+  test<char16_t>();
+  test<char32_t>();
+  test<wchar_t>();
+  // + standard signed integer types
+  test<signed char>();
+  test<short>();
+  test<int>();
+  test<long>();
+  test<long long>();
+  // + standard unsigned integer types
+  test<unsigned char>();
+  test<unsigned short>();
+  test<unsigned int>();
+  test<unsigned long>();
+  test<unsigned long long>();
+  // + any other types needed by the typedefs in the header <cstdint>
+  test<int8_t>();
+  test<int16_t>();
+  test<int32_t>();
+  test<int64_t>();
+  test<int_fast8_t>();
+  test<int_fast16_t>();
+  test<int_fast32_t>();
+  test<int_fast64_t>();
+  test<int_least8_t>();
+  test<int_least16_t>();
+  test<int_least32_t>();
+  test<int_least64_t>();
+  test<intmax_t>();
+  test<intptr_t>();
+  test<uint8_t>();
+  test<uint16_t>();
+  test<uint32_t>();
+  test<uint64_t>();
+  test<uint_fast8_t>();
+  test<uint_fast16_t>();
+  test<uint_fast32_t>();
+  test<uint_fast64_t>();
+  test<uint_least8_t>();
+  test<uint_least16_t>();
+  test<uint_least32_t>();
+  test<uint_least64_t>();
+  test<uintmax_t>();
+  test<uintptr_t>();
+
+  // Specialization for floating-point types
+  // + floating-point types
+  test<float>();
+  test<double>();
+  test<long double>();
+  // + TODO extended floating-point types
+}
+
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp b/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
new file mode 100644
index 00000000000000..3ea9de240a192a
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
@@ -0,0 +1,86 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// void notify_all() const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <thread>
+#include <type_traits>
+#include <vector>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+template <typename T>
+void test_notify_all() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  bool done                      = false;
+  std::atomic<int> started_num   = 0;
+  std::atomic<int> wait_done_num = 0;
+
+  constexpr auto number_of_threads = 8;
+  std::vector<std::thread> threads;
+  threads.reserve(number_of_threads);
+
+  for (auto j = 0; j < number_of_threads; ++j) {
+    threads.push_back(support::make_test_thread([&a, &started_num, &done, &wait_done_num] {
+      started_num.fetch_add(1, std::memory_order::relaxed);
+
+      a.wait(T(1));
+      wait_done_num.fetch_add(1, std::memory_order::relaxed);
+
+      // likely to fail if wait did not block
+      assert(done);
+    }));
+  }
+
+  while (started_num.load(std::memory_order::relaxed) != number_of_threads) {
+    std::this_thread::yield();
+  }
+
+  std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+  done = true;
+  a.store(T(3));
+  a.notify_all();
+
+  // notify_all should unblock all the threads so that the loop below won't stuck
+  while (wait_done_num.load(std::memory_order::relaxed) != number_of_threads) {
+    std::this_thread::yield();
+  }
+
+  for (auto& thread : threads) {
+    thread.join();
+  }
+
+  ASSERT_NOEXCEPT(a.notify_all());
+}
+
+void test() {
+  test_notify_all<int>();
+
+  test_notify_all<float>();
+
+  test_notify_all<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_notify_all<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp b/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
new file mode 100644
index 00000000000000..a3ae251ab770c8
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
@@ -0,0 +1,54 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// void notify_one() const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <thread>
+#include <type_traits>
+#include <vector>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+template <typename T>
+void test_notify_one() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  std::thread t = support::make_test_thread([&]() {
+    a.store(T(3));
+    a.notify_one();
+  });
+  a.wait(T(1));
+  assert(a.load() == T(3));
+  t.join();
+  ASSERT_NOEXCEPT(a.notify_one());
+}
+
+void test() {
+  test_notify_one<int>();
+
+  test_notify_one<float>();
+
+  test_notify_one<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_notify_one<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
new file mode 100644
index 00000000000000..2570d07da0f582
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
@@ -0,0 +1,64 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type operator-=(integral-type) const noexcept;
+// floating-point-type operator-=(floating-point-type) const noexcept;
+// T* operator-=(difference_type) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_operator_minus_equals = requires { std::declval<T const>() -= std::declval<T>(); };
+
+static_assert(!has_operator_minus_equals<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_operator_minus_equals<std::atomic_ref<X>>);
+
+template <typename T>
+void test_arithmetic() {
+  T x(T(3));
+  std::atomic_ref<T> a(x);
+
+  assert((a -= T(2)) == T(1));
+  assert(x == T(1));
+  ASSERT_NOEXCEPT(a -= T(0));
+}
+
+template <typename T>
+void test_pointer() {
+  using X = std::remove_pointer_t<T>;
+  X t[9]  = {};
+  T p{&t[3]};
+  std::atomic_ref<T> a(p);
+
+  assert((a -= 2) == &t[1]);
+  assert(a == &t[1]);
+  ASSERT_NOEXCEPT(a -= 0);
+}
+
+void test() {
+  test_arithmetic<int>();
+  test_arithmetic<float>();
+
+  test_pointer<int*>();
+  test_pointer<const int*>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
new file mode 100644
index 00000000000000..7c097e568349c7
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
@@ -0,0 +1,64 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// integral-type operator+=(integral-type) const noexcept;
+// floating-point-type operator+=(floating-point-type) const noexcept;
+// T* operator+=(difference_type) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+concept has_operator_plus_equals = requires { std::declval<T const>() += std::declval<T>(); };
+
+static_assert(!has_operator_plus_equals<std::atomic_ref<bool>>);
+struct X {
+  int i;
+  X(int ii) noexcept : i(ii) {}
+  bool operator==(X o) const { return i == o.i; }
+};
+static_assert(!has_operator_plus_equals<std::atomic_ref<X>>);
+
+template <typename T>
+void test_arithmetic() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert((a += T(2)) == T(3));
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a += T(0));
+}
+
+template <typename T>
+void test_pointer() {
+  using X = std::remove_pointer_t<T>;
+  X t[9]  = {};
+  T p{&t[1]};
+  std::atomic_ref<T> a(p);
+
+  assert((a += 2) == &t[3]);
+  assert(a == &t[3]);
+  ASSERT_NOEXCEPT(a += 0);
+}
+
+void test() {
+  test_arithmetic<int>();
+  test_arithmetic<float>();
+
+  test_pointer<int*>();
+  test_pointer<const int*>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp b/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
new file mode 100644
index 00000000000000..341c6d18707cdd
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
@@ -0,0 +1,34 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// static constexpr size_t required_alignment;
+
+#include <atomic>
+
+template <typename T>
+void checkRequiredAlignment() {
+  static_assert(std::atomic_ref<T>::required_alignment >= alignof(T));
+}
+
+void test() {
+  checkRequiredAlignment<int>();
+  checkRequiredAlignment<float>();
+  checkRequiredAlignment<int*>();
+  struct Empty {};
+  checkRequiredAlignment<Empty>();
+  struct Trivial {
+    int a;
+  };
+  checkRequiredAlignment<Trivial>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/store.pass.cpp b/libcxx/test/std/atomics/atomics.ref/store.pass.cpp
new file mode 100644
index 00000000000000..102311bb43b236
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/store.pass.cpp
@@ -0,0 +1,50 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// void store(T, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <typename T>
+void test_store() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  a.store(T(2));
+  assert(x == T(2));
+  ASSERT_NOEXCEPT(a.store(T(1)));
+
+  a.store(T(3), std::memory_order_seq_cst);
+  assert(x == T(3));
+  ASSERT_NOEXCEPT(a.store(T(0), std::memory_order_seq_cst));
+}
+
+void test() {
+  test_store<int>();
+
+  test_store<float>();
+
+  test_store<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_store<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/type.verify.cpp b/libcxx/test/std/atomics/atomics.ref/type.verify.cpp
new file mode 100644
index 00000000000000..9a8b036ffd1f8c
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/type.verify.cpp
@@ -0,0 +1,26 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <atomic_ref>
+
+// template<class T>
+// class atomic_ref;
+
+// The program is ill-formed if is_trivially_copyable_v<T> is false.
+
+#include <atomic>
+
+void trivially_copyable() {
+  struct X {
+    X() = default;
+    X(X const&) {} // -> not trivially copyable
+  } x;
+  // expected-error-re@*:* {{static assertion failed {{.*}}atomic_ref<T> requires that 'T' be a trivially copyable type}}
+  std::atomic_ref<X> r(x);
+}
diff --git a/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp b/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
new file mode 100644
index 00000000000000..51d3a79dd59a5e
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
@@ -0,0 +1,65 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// void wait(T, memory_order = memory_order::seq_cst) const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <type_traits>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+template <typename T>
+void test_wait() {
+  T x(T(1));
+  std::atomic_ref<T> a(x);
+
+  assert(a.load() == T(1));
+  a.wait(T(0));
+  std::thread t1 = support::make_test_thread([&]() {
+    a.store(T(3));
+    a.notify_one();
+  });
+  a.wait(T(1));
+  assert(a.load() == T(3));
+  t1.join();
+  ASSERT_NOEXCEPT(a.wait(T(0)));
+
+  assert(a.load() == T(3));
+  a.wait(T(0), std::memory_order_seq_cst);
+  std::thread t2 = support::make_test_thread([&]() {
+    a.store(T(5));
+    a.notify_one();
+  });
+  a.wait(T(3), std::memory_order_seq_cst);
+  assert(a.load() == T(5));
+  t2.join();
+  ASSERT_NOEXCEPT(a.wait(T(0), std::memory_order_seq_cst));
+}
+
+void test() {
+  test_wait<int>();
+
+  test_wait<float>();
+
+  test_wait<int*>();
+
+  struct X {
+    int i;
+    X(int ii) noexcept : i(ii) {}
+    bool operator==(X o) const { return i == o.i; }
+  };
+  test_wait<X>();
+}
+
+int main(int, char**) {
+  test();
+  return 0;
+}

>From 2767cf0ff06858db923a748afacd1469fe89b5d0 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sun, 31 Dec 2023 08:41:09 -0500
Subject: [PATCH 06/45] [libc++][atomic_ref] Fix shadow warning

---
 libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp        | 4 ++--
 libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp        | 4 ++--
 .../std/atomics/atomics.ref/operator_minus_equals.pass.cpp    | 4 ++--
 .../std/atomics/atomics.ref/operator_plus_equals.pass.cpp     | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
index 439d5a305dd6b7..84ad4f54a6b5cd 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
@@ -47,8 +47,8 @@ void test_arithmetic() {
 
 template <typename T>
 void test_pointer() {
-  using X = std::remove_pointer_t<T>;
-  X t[9]  = {};
+  using U = std::remove_pointer_t<T>;
+  U t[9]  = {};
   T p{&t[1]};
   std::atomic_ref<T> a(p);
 
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
index e9fc7ea7cb5ff1..031f6f78a37c0c 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
@@ -47,8 +47,8 @@ void test_arithmetic() {
 
 template <typename T>
 void test_pointer() {
-  using X = std::remove_pointer_t<T>;
-  X t[9]  = {};
+  using U = std::remove_pointer_t<T>;
+  U t[9]  = {};
   T p{&t[7]};
   std::atomic_ref<T> a(p);
 
diff --git a/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
index 2570d07da0f582..e7e25efb7acd27 100644
--- a/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
@@ -40,8 +40,8 @@ void test_arithmetic() {
 
 template <typename T>
 void test_pointer() {
-  using X = std::remove_pointer_t<T>;
-  X t[9]  = {};
+  using U = std::remove_pointer_t<T>;
+  U t[9]  = {};
   T p{&t[3]};
   std::atomic_ref<T> a(p);
 
diff --git a/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
index 7c097e568349c7..1aeadffafaef84 100644
--- a/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
@@ -40,8 +40,8 @@ void test_arithmetic() {
 
 template <typename T>
 void test_pointer() {
-  using X = std::remove_pointer_t<T>;
-  X t[9]  = {};
+  using U = std::remove_pointer_t<T>;
+  U t[9]  = {};
   T p{&t[1]};
   std::atomic_ref<T> a(p);
 

>From d79c4275c7b896e3ed4dc94c972047a520fb9b25 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sun, 31 Dec 2023 21:53:56 -0500
Subject: [PATCH 07/45] [libc++][atomic_ref] Revert all changes to
 __atomic/cxx_atomic_impl.h

---
 libcxx/include/__atomic/cxx_atomic_impl.h | 423 ++++++++++++++++++----
 libcxx/include/__config                   |   4 +-
 2 files changed, 351 insertions(+), 76 deletions(-)

diff --git a/libcxx/include/__atomic/cxx_atomic_impl.h b/libcxx/include/__atomic/cxx_atomic_impl.h
index 56cd703c258944..1a0b808a0cb1c4 100644
--- a/libcxx/include/__atomic/cxx_atomic_impl.h
+++ b/libcxx/include/__atomic/cxx_atomic_impl.h
@@ -15,12 +15,8 @@
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/is_assignable.h>
-#include <__type_traits/is_pointer.h>
-#include <__type_traits/is_reference.h>
 #include <__type_traits/is_trivially_copyable.h>
-#include <__type_traits/is_volatile.h>
 #include <__type_traits/remove_const.h>
-#include <__type_traits/remove_reference.h>
 #include <cstddef>
 #include <cstring>
 
@@ -62,27 +58,9 @@ struct __cxx_atomic_base_impl {
   }
 #  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {}
-  using __contained_t = _Tp;
   _Tp __a_value;
 };
 
-template <typename _Tp, template <typename> class _TemplateTp>
-struct __is_instantiation_of : false_type {};
-
-template <typename _Tp, template <typename> class _TemplateTp>
-struct __is_instantiation_of<_TemplateTp<_Tp>, _TemplateTp> : true_type {};
-
-template <typename _Tp,
-          typename = typename enable_if<__is_instantiation_of<typename remove_volatile<typename _Tp::__base>::type,
-                                                              __cxx_atomic_base_impl>::value,
-                                        bool>::type>
-struct __cxx_atomic_base_impl_traits {
-  static constexpr bool __is_value_volatile = is_volatile<_Tp>::value;
-  static constexpr bool __is_value_ref      = is_reference<typename _Tp::__contained_t>::value;
-  using __underlying_t = typename remove_volatile<typename remove_reference<typename _Tp::__contained_t>::type>::type;
-  static constexpr bool __is_value_pointer = is_pointer<__underlying_t>::value;
-};
-
 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
   // Avoid switch statement to make this a constexpr.
   return __order == memory_order_relaxed
@@ -109,15 +87,13 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory
                                 : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME))));
 }
 
-template <typename _Tp, typename enable_if<__cxx_atomic_base_impl_traits<_Tp>::__is_value_volatile, bool>::type = 0>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_init(_Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __val) {
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
   __cxx_atomic_assign_volatile(__a->__a_value, __val);
 }
 
-template <typename _Tp, typename enable_if<!__cxx_atomic_base_impl_traits<_Tp>::__is_value_volatile, bool>::type = 0>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_init(_Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __val) {
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
   __a->__a_value = __val;
 }
 
@@ -131,44 +107,63 @@ _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_store(_Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __val, memory_order __order) {
+__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
   __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t
-__cxx_atomic_load(const _Tp* __a, memory_order __order) {
-  using _Ret = typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t;
-  alignas(alignof(_Ret)) unsigned char __mem[sizeof(_Ret)];
-  __atomic_load(
-      std::addressof(__a->__a_value), std::addressof(*reinterpret_cast<_Ret*>(__mem)), __to_gcc_order(__order));
-  return *reinterpret_cast<_Ret*>(__mem);
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
+  __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load_inplace(
-    const _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t* __dst, memory_order __order) {
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
+  _Tp __ret;
+  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
   __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_exchange(
-    _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __value, memory_order __order) {
-  using _Ret = typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t;
-  alignas(alignof(_Ret)) unsigned char __mem[sizeof(_Ret)];
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
+  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
+  _Tp __ret;
+  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
+  _Tp __ret;
   __atomic_exchange(
-      std::addressof(__a->__a_value),
-      std::addressof(__value),
-      std::addressof(*reinterpret_cast<_Ret*>(__mem)),
-      __to_gcc_order(__order));
-  return *reinterpret_cast<_Ret*>(__mem);
+      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
+  _Tp __ret;
+  __atomic_exchange(
+      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
 }
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
-    _Tp* __a,
-    typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t* __expected,
-    typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __value,
+    volatile __cxx_atomic_base_impl<_Tp>* __a,
+    _Tp* __expected,
+    _Tp __value,
     memory_order __success,
     memory_order __failure) {
   return __atomic_compare_exchange(
@@ -180,11 +175,23 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
       __to_gcc_failure_order(__failure));
 }
 
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      false,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
-    _Tp* __a,
-    typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t* __expected,
-    typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __value,
+    volatile __cxx_atomic_base_impl<_Tp>* __a,
+    _Tp* __expected,
+    _Tp __value,
     memory_order __success,
     memory_order __failure) {
   return __atomic_compare_exchange(
@@ -196,6 +203,18 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
       __to_gcc_failure_order(__failure));
 }
 
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      true,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
 template <typename _Tp>
 struct __skip_amt {
   enum { value = 1 };
@@ -214,44 +233,302 @@ template <typename _Tp, int n>
 struct __skip_amt<_Tp[n]> {};
 
 template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t
-__cxx_atomic_fetch_add(_Tp* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_add(
-      std::addressof(__a->__a_value),
-      __delta * __skip_amt<typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t>::value,
-      __to_gcc_order(__order));
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
 }
 
 template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t
-__cxx_atomic_fetch_sub(_Tp* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_sub(
-      std::addressof(__a->__a_value),
-      __delta * __skip_amt<typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t>::value,
-      __to_gcc_order(__order));
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_fetch_and(
-    _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __pattern, memory_order __order) {
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
   return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_fetch_or(
-    _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __pattern, memory_order __order) {
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
   return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 }
 
 template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __cxx_atomic_fetch_xor(
-    _Tp* __a, typename __cxx_atomic_base_impl_traits<_Tp>::__underlying_t __pattern, memory_order __order) {
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
   return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 }
 
 #  define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
 
-#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP
+#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP)
+
+template <typename _Tp>
+struct __cxx_atomic_base_impl {
+  _LIBCPP_HIDE_FROM_ABI
+#  ifndef _LIBCPP_CXX03_LANG
+  __cxx_atomic_base_impl() _NOEXCEPT = default;
+#  else
+  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
+  }
+#  endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
+  _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
+};
+
+#  define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
+
+_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
+  __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
+  __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
+  __c11_atomic_init(std::addressof(__a->__a_value), __val);
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
+  __c11_atomic_init(std::addressof(__a->__a_value), __val);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
+  __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
+  __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
+  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+  return __c11_atomic_load(
+      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
+  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+  return __c11_atomic_load(
+      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
+  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+  *__dst           = __c11_atomic_load(
+      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
+  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+  *__dst           = __c11_atomic_load(
+      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_exchange(
+      std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_exchange(
+      std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
+  // Avoid switch statement to make this a constexpr.
+  return __order == memory_order_release
+           ? memory_order_relaxed
+           : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    __cxx_atomic_base_impl<_Tp> volatile* __a,
+    _Tp* __expected,
+    _Tp __value,
+    memory_order __success,
+    memory_order __failure) _NOEXCEPT {
+  return __c11_atomic_compare_exchange_strong(
+      std::addressof(__a->__a_value),
+      __expected,
+      __value,
+      static_cast<__memory_order_underlying_t>(__success),
+      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
+    _NOEXCEPT {
+  return __c11_atomic_compare_exchange_strong(
+      std::addressof(__a->__a_value),
+      __expected,
+      __value,
+      static_cast<__memory_order_underlying_t>(__success),
+      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    __cxx_atomic_base_impl<_Tp> volatile* __a,
+    _Tp* __expected,
+    _Tp __value,
+    memory_order __success,
+    memory_order __failure) _NOEXCEPT {
+  return __c11_atomic_compare_exchange_weak(
+      std::addressof(__a->__a_value),
+      __expected,
+      __value,
+      static_cast<__memory_order_underlying_t>(__success),
+      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
+    _NOEXCEPT {
+  return __c11_atomic_compare_exchange_weak(
+      std::addressof(__a->__a_value),
+      __expected,
+      __value,
+      static_cast<__memory_order_underlying_t>(__success),
+      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_add(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_add(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp*
+__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_add(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp*
+__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_add(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_sub(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_sub(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp*
+__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_sub(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp*
+__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_sub(
+      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_and(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_and(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_or(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_or(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_xor(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
+  return __c11_atomic_fetch_xor(
+      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
+}
+
+#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
+
 #ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS
 
 template <typename _Tp>
@@ -536,7 +813,7 @@ template <typename _Tp,
 template <typename _Tp, typename _Base = __cxx_atomic_base_impl<_Tp> >
 #endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS
 struct __cxx_atomic_impl : public _Base {
-  using __base = _Base;
+  static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
 
   _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {}
diff --git a/libcxx/include/__config b/libcxx/include/__config
index e7d6551c867deb..d356960e9e62b4 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -1183,9 +1183,7 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c
 
 #  if __has_feature(cxx_atomic) || __has_extension(c_atomic) || __has_keyword(_Atomic)
 #    define _LIBCPP_HAS_C_ATOMIC_IMP
-#  endif
-
-#  if defined(_LIBCPP_COMPILER_GCC) || (__has_builtin(__atomic_load) && __has_builtin(__atomic_store) && __has_builtin(__atomic_exchange) && __has_builtin(__atomic_compare_exchange))
+#  elif defined(_LIBCPP_COMPILER_GCC)
 #    define _LIBCPP_HAS_GCC_ATOMIC_IMP
 #  endif
 

>From 722217bc0963f2de9b0bf405551929a1b95b8753 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 1 Jan 2024 11:20:44 -0500
Subject: [PATCH 08/45] [libc++][atomic_ref] move __to_gcc_[failure_]order to
 its own header file

---
 libcxx/include/CMakeLists.txt             |  1 +
 libcxx/include/__atomic/atomic_ref.h      |  1 +
 libcxx/include/__atomic/cxx_atomic_impl.h | 27 +-----------
 libcxx/include/__atomic/to_gcc_order.h    | 54 +++++++++++++++++++++++
 libcxx/include/module.modulemap.in        |  1 +
 5 files changed, 58 insertions(+), 26 deletions(-)
 create mode 100644 libcxx/include/__atomic/to_gcc_order.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 1da9cfbd3adf74..8e86e37efd20ce 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -245,6 +245,7 @@ set(files
   __atomic/is_always_lock_free.h
   __atomic/kill_dependency.h
   __atomic/memory_order.h
+  __atomic/to_gcc_order.h
   __availability
   __bit/bit_cast.h
   __bit/bit_ceil.h
diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index b4299804eccc31..ab232b221291f3 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -21,6 +21,7 @@
 #include <__atomic/check_memory_order.h>
 #include <__atomic/cxx_atomic_impl.h>
 #include <__atomic/is_always_lock_free.h>
+#include <__atomic/to_gcc_order.h>
 #include <__config>
 #include <__memory/addressof.h>
 #include <__type_traits/is_floating_point.h>
diff --git a/libcxx/include/__atomic/cxx_atomic_impl.h b/libcxx/include/__atomic/cxx_atomic_impl.h
index 1a0b808a0cb1c4..be5c1497873cca 100644
--- a/libcxx/include/__atomic/cxx_atomic_impl.h
+++ b/libcxx/include/__atomic/cxx_atomic_impl.h
@@ -11,6 +11,7 @@
 
 #include <__atomic/is_always_lock_free.h>
 #include <__atomic/memory_order.h>
+#include <__atomic/to_gcc_order.h>
 #include <__config>
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
@@ -61,32 +62,6 @@ struct __cxx_atomic_base_impl {
   _Tp __a_value;
 };
 
-_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
-  // Avoid switch statement to make this a constexpr.
-  return __order == memory_order_relaxed
-           ? __ATOMIC_RELAXED
-           : (__order == memory_order_acquire
-                  ? __ATOMIC_ACQUIRE
-                  : (__order == memory_order_release
-                         ? __ATOMIC_RELEASE
-                         : (__order == memory_order_seq_cst
-                                ? __ATOMIC_SEQ_CST
-                                : (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL : __ATOMIC_CONSUME))));
-}
-
-_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) {
-  // Avoid switch statement to make this a constexpr.
-  return __order == memory_order_relaxed
-           ? __ATOMIC_RELAXED
-           : (__order == memory_order_acquire
-                  ? __ATOMIC_ACQUIRE
-                  : (__order == memory_order_release
-                         ? __ATOMIC_RELAXED
-                         : (__order == memory_order_seq_cst
-                                ? __ATOMIC_SEQ_CST
-                                : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME))));
-}
-
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
   __cxx_atomic_assign_volatile(__a->__a_value, __val);
diff --git a/libcxx/include/__atomic/to_gcc_order.h b/libcxx/include/__atomic/to_gcc_order.h
new file mode 100644
index 00000000000000..d04c111addd314
--- /dev/null
+++ b/libcxx/include/__atomic/to_gcc_order.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_TO_GCC_ORDER_H
+#define _LIBCPP___ATOMIC_TO_GCC_ORDER_H
+
+#include <__atomic/memory_order.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if defined(__ATOMIC_RELAXED) && defined(__ATOMIC_CONSUME) && defined(__ATOMIC_ACQUIRE) &&                             \
+    defined(__ATOMIC_RELEASE) && defined(__ATOMIC_ACQ_REL) && defined(__ATOMIC_SEQ_CST)
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
+  // Avoid switch statement to make this a constexpr.
+  return __order == memory_order_relaxed
+           ? __ATOMIC_RELAXED
+           : (__order == memory_order_acquire
+                  ? __ATOMIC_ACQUIRE
+                  : (__order == memory_order_release
+                         ? __ATOMIC_RELEASE
+                         : (__order == memory_order_seq_cst
+                                ? __ATOMIC_SEQ_CST
+                                : (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL : __ATOMIC_CONSUME))));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) {
+  // Avoid switch statement to make this a constexpr.
+  return __order == memory_order_relaxed
+           ? __ATOMIC_RELAXED
+           : (__order == memory_order_acquire
+                  ? __ATOMIC_ACQUIRE
+                  : (__order == memory_order_release
+                         ? __ATOMIC_RELAXED
+                         : (__order == memory_order_seq_cst
+                                ? __ATOMIC_SEQ_CST
+                                : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME))));
+}
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_TO_GCC_ORDER_H
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 9828c48e0587bb..f320f2cc2a2860 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -1094,6 +1094,7 @@ module std_private_atomic_fence               [system] { header "__atomic/fence.
 module std_private_atomic_is_always_lock_free [system] { header "__atomic/is_always_lock_free.h" }
 module std_private_atomic_kill_dependency     [system] { header "__atomic/kill_dependency.h" }
 module std_private_atomic_memory_order        [system] { header "__atomic/memory_order.h" }
+module std_private_atomic_to_gcc_order        [system] { header "__atomic/to_gcc_order.h" }
 
 module std_private_bit_bit_cast       [system] { header "__bit/bit_cast.h" }
 module std_private_bit_bit_ceil       [system] { header "__bit/bit_ceil.h" }

>From 1a0463faf88150d0b09786c6ac783d81edfb1513 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sun, 31 Dec 2023 21:51:25 -0500
Subject: [PATCH 09/45] [libc++][atomic_ref] Reimplement atomic_ref in terms of
 the GCC __atomic builtins

---
 libcxx/include/__atomic/atomic_ref.h | 84 +++++++++++++++++++---------
 1 file changed, 59 insertions(+), 25 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index ab232b221291f3..2b54e170245160 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -19,8 +19,6 @@
 
 #include <__assert>
 #include <__atomic/check_memory_order.h>
-#include <__atomic/cxx_atomic_impl.h>
-#include <__atomic/is_always_lock_free.h>
 #include <__atomic/to_gcc_order.h>
 #include <__config>
 #include <__memory/addressof.h>
@@ -46,22 +44,22 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp, bool = is_integral_v<_Tp> && !is_same_v<_Tp, bool>, bool = is_floating_point_v<_Tp>>
 struct __atomic_ref_base {
-  mutable __cxx_atomic_impl<_Tp&> __a_;
+  _Tp* __ptr_;
 
   using value_type = _Tp;
 
   static constexpr size_t required_alignment = alignof(_Tp);
 
-  static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<_Tp>::__value;
+  static constexpr bool is_always_lock_free = __atomic_always_lock_free(sizeof(_Tp), 0);
 
-  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __cxx_atomic_is_lock_free(sizeof(_Tp)); }
+  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), 0); }
 
   _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
       _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
     _LIBCPP_ASSERT_UNCATEGORIZED(
         __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
         "memory order argument to atomic store operation is invalid");
-    __cxx_atomic_store(&__a_, __desired, __order);
+    __atomic_store(__ptr_, std::addressof(__desired), __to_gcc_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {
@@ -75,13 +73,19 @@ struct __atomic_ref_base {
         __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
             __order == memory_order::seq_cst,
         "memory order argument to atomic load operation is invalid");
-    return __cxx_atomic_load(&__a_, __order);
+    alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
+    auto* __ret = reinterpret_cast<_Tp*>(__mem);
+    __atomic_load(__ptr_, __ret, __to_gcc_order(__order));
+    return *__ret;
   }
 
   _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); }
 
   _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
-    return __cxx_atomic_exchange(&__a_, __desired, __order);
+    alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
+    auto* __ret = reinterpret_cast<_Tp*>(__mem);
+    __atomic_exchange(__ptr_, std::addressof(__desired), __ret, __to_gcc_order(__order));
+    return *__ret;
   }
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
@@ -90,7 +94,13 @@ struct __atomic_ref_base {
         __failure == memory_order::relaxed || __failure == memory_order::consume ||
             __failure == memory_order::acquire || __failure == memory_order::seq_cst,
         "failure memory order argument to weak atomic compare-and-exchange operation is invalid");
-    return __cxx_atomic_compare_exchange_weak(&__a_, &__expected, __desired, __success, __failure);
+    return __atomic_compare_exchange(
+        __ptr_,
+        std::addressof(__expected),
+        std::addressof(__desired),
+        true,
+        __to_gcc_order(__success),
+        __to_gcc_order(__failure));
   }
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
@@ -99,16 +109,34 @@ struct __atomic_ref_base {
         __failure == memory_order::relaxed || __failure == memory_order::consume ||
             __failure == memory_order::acquire || __failure == memory_order::seq_cst,
         "failure memory order argument to strong atomic compare-and-exchange operation is invalid");
-    return __cxx_atomic_compare_exchange_strong(&__a_, &__expected, __desired, __success, __failure);
+    return __atomic_compare_exchange(
+        __ptr_,
+        std::addressof(__expected),
+        std::addressof(__desired),
+        false,
+        __to_gcc_order(__success),
+        __to_gcc_order(__failure));
   }
 
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
-    return __cxx_atomic_compare_exchange_weak(&__a_, &__expected, __desired, __order, __order);
+    return __atomic_compare_exchange(
+        __ptr_,
+        std::addressof(__expected),
+        std::addressof(__desired),
+        true,
+        __to_gcc_order(__order),
+        __to_gcc_failure_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
-    return __cxx_atomic_compare_exchange_strong(&__a_, &__expected, __desired, __order, __order);
+    return __atomic_compare_exchange(
+        __ptr_,
+        std::addressof(__expected),
+        std::addressof(__desired),
+        false,
+        __to_gcc_order(__order),
+        __to_gcc_failure_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
@@ -117,12 +145,18 @@ struct __atomic_ref_base {
         __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
             __order == memory_order::seq_cst,
         "memory order argument to atomic wait operation is invalid");
-    __cxx_atomic_wait(addressof(__a_), __old, __order);
+    // FIXME
+    (void)__old;
+    (void)__order;
+  }
+  _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept {
+    // FIXME
+  }
+  _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept {
+    // FIXME
   }
-  _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { __cxx_atomic_notify_one(addressof(__a_)); }
-  _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { __cxx_atomic_notify_all(addressof(__a_)); }
 
-  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __a_(__obj) {}
+  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(&__obj) {}
 };
 
 template <class _Tp>
@@ -137,19 +171,19 @@ struct __atomic_ref_base<_Tp, /*_IsIntegral=*/true, /*_IsFloatingPoint=*/false>
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
 
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_add(&this->__a_, __arg, __order);
+    return __atomic_fetch_add(this->__ptr_, __arg, __to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_sub(&this->__a_, __arg, __order);
+    return __atomic_fetch_sub(this->__ptr_, __arg, __to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_and(&this->__a_, __arg, __order);
+    return __atomic_fetch_and(this->__ptr_, __arg, __to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_or(&this->__a_, __arg, __order);
+    return __atomic_fetch_or(this->__ptr_, __arg, __to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_xor(&this->__a_, __arg, __order);
+    return __atomic_fetch_xor(this->__ptr_, __arg, __to_gcc_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(_Tp(1)); }
@@ -175,10 +209,10 @@ struct __atomic_ref_base<_Tp, /*_IsIntegral=*/false, /*_IsFloatingPoint=*/true>
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
 
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_add(&this->__a_, __arg, __order);
+    return __atomic_fetch_add(this->__ptr_, __arg, __to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_sub(&this->__a_, __arg, __order);
+    return __atomic_fetch_sub(this->__ptr_, __arg, __to_gcc_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
@@ -210,10 +244,10 @@ struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
   using difference_type = ptrdiff_t;
 
   _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_add(&this->__a_, __arg, __order);
+    return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), __to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __cxx_atomic_fetch_sub(&this->__a_, __arg, __order);
+    return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), __to_gcc_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(1); }

>From 5e5a5cdeee2c886c97b8619ebe2a8be28c6b281d Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 1 Jan 2024 12:08:43 -0500
Subject: [PATCH 10/45] [libc++][atomic_ref] add header to module map

---
 libcxx/include/module.modulemap.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index f320f2cc2a2860..c361a542518a4b 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -1086,6 +1086,7 @@ module std_private_atomic_atomic_flag         [system] {
 }
 module std_private_atomic_atomic_init         [system] { header "__atomic/atomic_init.h" }
 module std_private_atomic_atomic_lock_free    [system] { header "__atomic/atomic_lock_free.h" }
+module std_private_atomic_atomic_ref          [system] { header "__atomic/atomic_ref.h" }
 module std_private_atomic_atomic_sync         [system] { header "__atomic/atomic_sync.h" }
 module std_private_atomic_check_memory_order  [system] { header "__atomic/check_memory_order.h" }
 module std_private_atomic_contention_t        [system] { header "__atomic/contention_t.h" }

>From d5525ada82081d38ec4a31905ade3f09db00c27c Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 1 Jan 2024 13:59:09 -0500
Subject: [PATCH 11/45] [libc++][atomic_ref] fixup atomic_{add,sub} for
 floating points

---
 libcxx/include/__atomic/atomic_ref.h | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 2b54e170245160..f552bf4b257355 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -209,10 +209,20 @@ struct __atomic_ref_base<_Tp, /*_IsIntegral=*/false, /*_IsFloatingPoint=*/true>
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
 
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_add(this->__ptr_, __arg, __to_gcc_order(__order));
+    _Tp __old = this->load(memory_order_relaxed);
+    _Tp __new = __old + __arg;
+    while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
+      __new = __old + __arg;
+    }
+    return __old;
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_sub(this->__ptr_, __arg, __to_gcc_order(__order));
+    _Tp __old = this->load(memory_order_relaxed);
+    _Tp __new = __old - __arg;
+    while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
+      __new = __old - __arg;
+    }
+    return __old;
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }

>From a974df059d5c4e683d3bf15ae42c97c59aebc850 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 1 Jan 2024 17:01:02 -0500
Subject: [PATCH 12/45] [libc++][atomic_ref] cleanup header includes

---
 libcxx/include/__atomic/atomic_ref.h | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index f552bf4b257355..3a904b3a7cbe38 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -23,13 +23,10 @@
 #include <__config>
 #include <__memory/addressof.h>
 #include <__type_traits/is_floating_point.h>
-#include <__type_traits/is_function.h>
-#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_integral.h>
 #include <__type_traits/is_same.h>
-#include <cinttypes>
-#include <concepts>
+#include <__type_traits/is_trivially_copyable.h>
 #include <cstddef>
-#include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -231,7 +228,7 @@ struct __atomic_ref_base<_Tp, /*_IsIntegral=*/false, /*_IsFloatingPoint=*/true>
 
 template <class _Tp>
 struct atomic_ref : public __atomic_ref_base<_Tp> {
-  static_assert(is_trivially_copyable<_Tp>::value, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
+  static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
 
   using __base = __atomic_ref_base<_Tp>;
 

>From 41727813fb7bf67f70bb2d995b3a24a7d698bf55 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 1 Jan 2024 17:01:12 -0500
Subject: [PATCH 13/45] [libc++][atomic_ref] annotate wait, notify_one, and
 notify_all as expected failures

---
 libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp | 1 +
 libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp | 1 +
 libcxx/test/std/atomics/atomics.ref/wait.pass.cpp       | 1 +
 3 files changed, 3 insertions(+)

diff --git a/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp b/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
index 3ea9de240a192a..66579c3f6d2ada 100644
--- a/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
@@ -6,6 +6,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
+// XFAIL: *
 
 // void notify_all() const noexcept;
 
diff --git a/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp b/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
index a3ae251ab770c8..212596974f4828 100644
--- a/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
@@ -6,6 +6,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
+// XFAIL: *
 
 // void notify_one() const noexcept;
 
diff --git a/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp b/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
index 51d3a79dd59a5e..f32842989d0315 100644
--- a/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
@@ -6,6 +6,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
+// XFAIL: *
 
 // void wait(T, memory_order = memory_order::seq_cst) const noexcept;
 

>From 02220bad91d27f1f1e6026b24805b256138800ec Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Fri, 5 Jan 2024 12:09:25 -0500
Subject: [PATCH 14/45] [libc++][atomic_ref] fix generic-modules build

---
 libcxx/include/module.modulemap.in | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index c361a542518a4b..b2d41268442893 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -1095,7 +1095,10 @@ module std_private_atomic_fence               [system] { header "__atomic/fence.
 module std_private_atomic_is_always_lock_free [system] { header "__atomic/is_always_lock_free.h" }
 module std_private_atomic_kill_dependency     [system] { header "__atomic/kill_dependency.h" }
 module std_private_atomic_memory_order        [system] { header "__atomic/memory_order.h" }
-module std_private_atomic_to_gcc_order        [system] { header "__atomic/to_gcc_order.h" }
+module std_private_atomic_to_gcc_order        [system] {
+  header "__atomic/to_gcc_order.h"
+  export std_private_atomic_memory_order
+}
 
 module std_private_bit_bit_cast       [system] { header "__bit/bit_cast.h" }
 module std_private_bit_bit_ceil       [system] { header "__bit/bit_ceil.h" }

>From 8cc91fb05928aae3b0413954a6027ba119a788c4 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Fri, 5 Jan 2024 14:13:17 -0500
Subject: [PATCH 15/45] [libc++][atomic_ref] uncomment `using std::atomic_ref;`
 in atomic module

---
 libcxx/modules/std/atomic.inc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/modules/std/atomic.inc b/libcxx/modules/std/atomic.inc
index 88b31ccdb20840..efc45e3c0f4dd7 100644
--- a/libcxx/modules/std/atomic.inc
+++ b/libcxx/modules/std/atomic.inc
@@ -22,7 +22,7 @@ export namespace std {
 
   // [atomics.ref.generic], class template atomic_ref
   // [atomics.ref.pointer], partial specialization for pointers
-  // using std::atomic_ref;
+  using std::atomic_ref;
 
   // [atomics.types.generic], class template atomic
   using std::atomic;

>From 28e4f55266b032ba47def16e3f022fd65f922d42 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Fri, 5 Jan 2024 17:48:30 -0500
Subject: [PATCH 16/45] [libc++][atomic_ref] add missing header include for
 uintptr_t caught by hardening mode CI builds

---
 libcxx/include/__atomic/atomic_ref.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 3a904b3a7cbe38..32bdb615593df9 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -27,6 +27,7 @@
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_trivially_copyable.h>
 #include <cstddef>
+#include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header

>From 0391ec0d79fbf21329fa1a4eeeccc5a3158c5533 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sat, 6 Jan 2024 21:44:20 -0500
Subject: [PATCH 17/45] [libc++][atomic_ref] Attempt to use __cxx_atomic_wait

---
 libcxx/include/__atomic/atomic_ref.h              | 13 ++++---------
 libcxx/include/__atomic/atomic_sync.h             | 15 +++++++++++++++
 .../std/atomics/atomics.ref/notify_all.pass.cpp   |  1 -
 .../std/atomics/atomics.ref/notify_one.pass.cpp   |  1 -
 libcxx/test/std/atomics/atomics.ref/wait.pass.cpp |  1 -
 5 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 32bdb615593df9..5944384490b4b3 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -18,6 +18,7 @@
 #define _LIBCPP___ATOMIC_ATOMIC_REF_H
 
 #include <__assert>
+#include <__atomic/atomic_sync.h>
 #include <__atomic/check_memory_order.h>
 #include <__atomic/to_gcc_order.h>
 #include <__config>
@@ -143,16 +144,10 @@ struct __atomic_ref_base {
         __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
             __order == memory_order::seq_cst,
         "memory order argument to atomic wait operation is invalid");
-    // FIXME
-    (void)__old;
-    (void)__order;
-  }
-  _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept {
-    // FIXME
-  }
-  _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept {
-    // FIXME
+    __cxx_atomic_wait(__ptr_, __old, __order);
   }
+  _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { __cxx_atomic_notify_one(__ptr_); }
+  _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { __cxx_atomic_notify_all(__ptr_); }
 
   _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(&__obj) {}
 };
diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h
index e1994ddde86c19..21a69f4fd62bb2 100644
--- a/libcxx/include/__atomic/atomic_sync.h
+++ b/libcxx/include/__atomic/atomic_sync.h
@@ -27,6 +27,21 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+template <class _Tp>
+struct __is_cxx_atomic_impl : false_type {};
+
+template <class _Tp>
+struct __is_cxx_atomic_impl<__cxx_atomic_impl<_Tp> > : true_type {};
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI __enable_if_t<!__is_cxx_atomic_impl<__remove_cv_t<_Tp> >::value, _Tp>
+__cxx_atomic_load(_Tp* __ptr, memory_order __order) noexcept {
+  alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
+  auto* __ret = reinterpret_cast<_Tp*>(__mem);
+  __atomic_load(__ptr, __ret, __to_gcc_order(__order));
+  return *__ret;
+}
+
 #ifndef _LIBCPP_HAS_NO_THREADS
 
 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*);
diff --git a/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp b/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
index 66579c3f6d2ada..3ea9de240a192a 100644
--- a/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
@@ -6,7 +6,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// XFAIL: *
 
 // void notify_all() const noexcept;
 
diff --git a/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp b/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
index 212596974f4828..a3ae251ab770c8 100644
--- a/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
@@ -6,7 +6,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// XFAIL: *
 
 // void notify_one() const noexcept;
 
diff --git a/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp b/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
index f32842989d0315..51d3a79dd59a5e 100644
--- a/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
@@ -6,7 +6,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// XFAIL: *
 
 // void wait(T, memory_order = memory_order::seq_cst) const noexcept;
 

>From 82fcdadb5b725cd4818429233065cb338f89f0e1 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sun, 7 Jan 2024 20:55:40 -0500
Subject: [PATCH 18/45] [libc++][atomic_ref] Fiddle with hardening-mode
 annotations in preconditions tests

---
 .../atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp | 2 +-
 .../atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp   | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp     | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp     | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp    | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp     | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
index 3a991c9351cd85..8d78a58524a6fd 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
+// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
index c9506f556129ee..1c19116e90009f 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
+// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
index 3705167181519c..28152875660669 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
+// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
 // XFAIL: availability-verbose_abort-missing
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
index 4181b1c12c7db4..9a017e7ba036ce 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
+// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
index f543bcc35295f7..6de13404c2d2ae 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
+// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
index 2b1c9208527471..e0c33cd207d682 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
+// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 

>From 4fec87cf1fd94fa34ee5b234c62a14ca95df0b91 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Sun, 7 Jan 2024 20:57:52 -0500
Subject: [PATCH 19/45] [libc++][atomic_ref] Fix cxx03 CI build

---
 libcxx/include/__atomic/atomic_sync.h | 7 +++++--
 libcxx/include/module.modulemap.in    | 5 ++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h
index 21a69f4fd62bb2..4a2d0fc21810ee 100644
--- a/libcxx/include/__atomic/atomic_sync.h
+++ b/libcxx/include/__atomic/atomic_sync.h
@@ -12,6 +12,7 @@
 #include <__atomic/contention_t.h>
 #include <__atomic/cxx_atomic_impl.h>
 #include <__atomic/memory_order.h>
+#include <__atomic/to_gcc_order.h>
 #include <__availability>
 #include <__chrono/duration.h>
 #include <__config>
@@ -19,6 +20,8 @@
 #include <__thread/poll_with_backoff.h>
 #include <__thread/support.h>
 #include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/remove_cv.h>
 #include <cstring>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -35,8 +38,8 @@ struct __is_cxx_atomic_impl<__cxx_atomic_impl<_Tp> > : true_type {};
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI __enable_if_t<!__is_cxx_atomic_impl<__remove_cv_t<_Tp> >::value, _Tp>
-__cxx_atomic_load(_Tp* __ptr, memory_order __order) noexcept {
-  alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
+__cxx_atomic_load(_Tp* __ptr, memory_order __order) _NOEXCEPT {
+  _ALIGNAS_TYPE(_Tp) unsigned char __mem[sizeof(_Tp)];
   auto* __ret = reinterpret_cast<_Tp*>(__mem);
   __atomic_load(__ptr, __ret, __to_gcc_order(__order));
   return *__ret;
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index b2d41268442893..71c9cee8b027a2 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -1087,7 +1087,10 @@ module std_private_atomic_atomic_flag         [system] {
 module std_private_atomic_atomic_init         [system] { header "__atomic/atomic_init.h" }
 module std_private_atomic_atomic_lock_free    [system] { header "__atomic/atomic_lock_free.h" }
 module std_private_atomic_atomic_ref          [system] { header "__atomic/atomic_ref.h" }
-module std_private_atomic_atomic_sync         [system] { header "__atomic/atomic_sync.h" }
+module std_private_atomic_atomic_sync         [system] {
+  header "__atomic/atomic_sync.h"
+  export std_private_atomic_to_gcc_order
+}
 module std_private_atomic_check_memory_order  [system] { header "__atomic/check_memory_order.h" }
 module std_private_atomic_contention_t        [system] { header "__atomic/contention_t.h" }
 module std_private_atomic_cxx_atomic_impl     [system] { header "__atomic/cxx_atomic_impl.h" }

>From f763a3a853ff0f97b89c4ae425b73b73e78e5775 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 15 Jan 2024 11:50:21 -0500
Subject: [PATCH 20/45] Revert "[libc++][atomic_ref] Fiddle with hardening-mode
 annotations in preconditions tests"

This reverts commit a132470db3beae515ad47afef2f27e9d65e7303e.
---
 .../atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp | 2 +-
 .../atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp   | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp     | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp     | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp    | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp     | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
index 8d78a58524a6fd..3a991c9351cd85 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
+// UNSUPPORTED: libcpp-hardening-mode=none
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
index 1c19116e90009f..c9506f556129ee 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
+// UNSUPPORTED: libcpp-hardening-mode=none
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
index 28152875660669..3705167181519c 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
+// UNSUPPORTED: libcpp-hardening-mode=none
 // XFAIL: availability-verbose_abort-missing
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
index 9a017e7ba036ce..4181b1c12c7db4 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
+// UNSUPPORTED: libcpp-hardening-mode=none
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
index 6de13404c2d2ae..f543bcc35295f7 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
+// UNSUPPORTED: libcpp-hardening-mode=none
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
index e0c33cd207d682..2b1c9208527471 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
@@ -6,7 +6,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode={{extensive|debug}}
+// UNSUPPORTED: libcpp-hardening-mode=none
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 

>From 259afbf1493667b7f2e46eef07b08803250d0aa9 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 15 Jan 2024 13:46:12 -0500
Subject: [PATCH 21/45] [libc++][atomic_ref] Robust against ADL lookup

---
 libcxx/include/__atomic/atomic_ref.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 5944384490b4b3..3ba12cc3240e4e 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -144,10 +144,10 @@ struct __atomic_ref_base {
         __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
             __order == memory_order::seq_cst,
         "memory order argument to atomic wait operation is invalid");
-    __cxx_atomic_wait(__ptr_, __old, __order);
+    std::__cxx_atomic_wait(__ptr_, __old, __order);
   }
-  _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { __cxx_atomic_notify_one(__ptr_); }
-  _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { __cxx_atomic_notify_all(__ptr_); }
+  _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__cxx_atomic_notify_one(__ptr_); }
+  _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__cxx_atomic_notify_all(__ptr_); }
 
   _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(&__obj) {}
 };

>From f0cbfec31bdf47031a337eb912895ecbe6016be8 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 16:37:16 -0500
Subject: [PATCH 22/45] [libc++][atomic_ref] Qualify all __to_gcc_order calls

---
 libcxx/include/__atomic/atomic_ref.h | 36 ++++++++++++++--------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 3ba12cc3240e4e..d0f532b11059d5 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -58,7 +58,7 @@ struct __atomic_ref_base {
     _LIBCPP_ASSERT_UNCATEGORIZED(
         __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
         "memory order argument to atomic store operation is invalid");
-    __atomic_store(__ptr_, std::addressof(__desired), __to_gcc_order(__order));
+    __atomic_store(__ptr_, std::addressof(__desired), std::__to_gcc_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {
@@ -74,7 +74,7 @@ struct __atomic_ref_base {
         "memory order argument to atomic load operation is invalid");
     alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
     auto* __ret = reinterpret_cast<_Tp*>(__mem);
-    __atomic_load(__ptr_, __ret, __to_gcc_order(__order));
+    __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order));
     return *__ret;
   }
 
@@ -83,7 +83,7 @@ struct __atomic_ref_base {
   _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
     alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
     auto* __ret = reinterpret_cast<_Tp*>(__mem);
-    __atomic_exchange(__ptr_, std::addressof(__desired), __ret, __to_gcc_order(__order));
+    __atomic_exchange(__ptr_, std::addressof(__desired), __ret, std::__to_gcc_order(__order));
     return *__ret;
   }
   _LIBCPP_HIDE_FROM_ABI bool
@@ -98,8 +98,8 @@ struct __atomic_ref_base {
         std::addressof(__expected),
         std::addressof(__desired),
         true,
-        __to_gcc_order(__success),
-        __to_gcc_order(__failure));
+        std::__to_gcc_order(__success),
+        std::__to_gcc_order(__failure));
   }
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
@@ -113,8 +113,8 @@ struct __atomic_ref_base {
         std::addressof(__expected),
         std::addressof(__desired),
         false,
-        __to_gcc_order(__success),
-        __to_gcc_order(__failure));
+        std::__to_gcc_order(__success),
+        std::__to_gcc_order(__failure));
   }
 
   _LIBCPP_HIDE_FROM_ABI bool
@@ -124,8 +124,8 @@ struct __atomic_ref_base {
         std::addressof(__expected),
         std::addressof(__desired),
         true,
-        __to_gcc_order(__order),
-        __to_gcc_failure_order(__order));
+        std::__to_gcc_order(__order),
+        std::__to_gcc_failure_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
@@ -134,8 +134,8 @@ struct __atomic_ref_base {
         std::addressof(__expected),
         std::addressof(__desired),
         false,
-        __to_gcc_order(__order),
-        __to_gcc_failure_order(__order));
+        std::__to_gcc_order(__order),
+        std::__to_gcc_failure_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
@@ -164,19 +164,19 @@ struct __atomic_ref_base<_Tp, /*_IsIntegral=*/true, /*_IsFloatingPoint=*/false>
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
 
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_add(this->__ptr_, __arg, __to_gcc_order(__order));
+    return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_sub(this->__ptr_, __arg, __to_gcc_order(__order));
+    return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_and(this->__ptr_, __arg, __to_gcc_order(__order));
+    return __atomic_fetch_and(this->__ptr_, __arg, std::__to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_or(this->__ptr_, __arg, __to_gcc_order(__order));
+    return __atomic_fetch_or(this->__ptr_, __arg, std::__to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_xor(this->__ptr_, __arg, __to_gcc_order(__order));
+    return __atomic_fetch_xor(this->__ptr_, __arg, std::__to_gcc_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(_Tp(1)); }
@@ -247,10 +247,10 @@ struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
   using difference_type = ptrdiff_t;
 
   _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), __to_gcc_order(__order));
+    return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
   }
   _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
-    return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), __to_gcc_order(__order));
+    return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(1); }

>From 46ab57281d04c465f06b03dcae816b1ae29d0f3f Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 16:56:56 -0500
Subject: [PATCH 23/45] [libc++][atomic_ref] Simplify inheritance from
 __atomic_ref_base

---
 libcxx/include/__atomic/atomic_ref.h | 85 ++++++++++++++++------------
 1 file changed, 49 insertions(+), 36 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index d0f532b11059d5..be58359d6e957a 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -21,11 +21,10 @@
 #include <__atomic/atomic_sync.h>
 #include <__atomic/check_memory_order.h>
 #include <__atomic/to_gcc_order.h>
+#include <__concepts/arithmetic.h>
+#include <__concepts/same_as.h>
 #include <__config>
 #include <__memory/addressof.h>
-#include <__type_traits/is_floating_point.h>
-#include <__type_traits/is_integral.h>
-#include <__type_traits/is_same.h>
 #include <__type_traits/is_trivially_copyable.h>
 #include <cstddef>
 #include <cstdint>
@@ -41,7 +40,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 20
 
-template <class _Tp, bool = is_integral_v<_Tp> && !is_same_v<_Tp, bool>, bool = is_floating_point_v<_Tp>>
+template <class _Tp>
 struct __atomic_ref_base {
   _Tp* __ptr_;
 
@@ -153,16 +152,41 @@ struct __atomic_ref_base {
 };
 
 template <class _Tp>
-struct __atomic_ref_base<_Tp, /*_IsIntegral=*/true, /*_IsFloatingPoint=*/false>
-    : public __atomic_ref_base<_Tp, false, false> {
-  using __base = __atomic_ref_base<_Tp, false, false>;
+struct atomic_ref : public __atomic_ref_base<_Tp> {
+  static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
 
-  using difference_type = __base::value_type;
+  using __base = __atomic_ref_base<_Tp>;
+
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+    _LIBCPP_ASSERT_UNCATEGORIZED((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
+                                 "atomic_ref ctor: referenced object must be aligned to required_alignment");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
+
+  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+
+  atomic_ref& operator=(const atomic_ref&) = delete;
+};
 
-  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __base(__obj) {}
+template <class _Tp>
+  requires(std::integral<_Tp> && !std::same_as<bool, _Tp>)
+struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
+  using __base = __atomic_ref_base<_Tp>;
+
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+    _LIBCPP_ASSERT_UNCATEGORIZED((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
+                                 "atomic_ref ctor: referenced object must be aligned to required_alignment");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
 
+  atomic_ref& operator=(const atomic_ref&) = delete;
+
+  using difference_type = __base::value_type;
+
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
     return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
   }
@@ -191,16 +215,23 @@ struct __atomic_ref_base<_Tp, /*_IsIntegral=*/true, /*_IsFloatingPoint=*/false>
 };
 
 template <class _Tp>
-struct __atomic_ref_base<_Tp, /*_IsIntegral=*/false, /*_IsFloatingPoint=*/true>
-    : public __atomic_ref_base<_Tp, false, false> {
-  using __base = __atomic_ref_base<_Tp, false, false>;
+  requires std::floating_point<_Tp>
+struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
+  using __base = __atomic_ref_base<_Tp>;
 
-  using difference_type = __base::value_type;
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+    _LIBCPP_ASSERT_UNCATEGORIZED((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
+                                 "atomic_ref ctor: referenced object must be aligned to required_alignment");
+  }
 
-  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __base(__obj) {}
+  _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
 
+  atomic_ref& operator=(const atomic_ref&) = delete;
+
+  using difference_type = __base::value_type;
+
   _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
     _Tp __old = this->load(memory_order_relaxed);
     _Tp __new = __old + __arg;
@@ -223,26 +254,14 @@ struct __atomic_ref_base<_Tp, /*_IsIntegral=*/false, /*_IsFloatingPoint=*/true>
 };
 
 template <class _Tp>
-struct atomic_ref : public __atomic_ref_base<_Tp> {
-  static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
-
-  using __base = __atomic_ref_base<_Tp>;
-
-  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
-    _LIBCPP_ASSERT_UNCATEGORIZED((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
-                                 "atomic_ref ctor: referenced object must be aligned to required_alignment");
-  }
+struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
+  using __base = __atomic_ref_base<_Tp*>;
 
-  _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
+  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {}
 
-  _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+  _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); }
 
   atomic_ref& operator=(const atomic_ref&) = delete;
-};
-
-template <class _Tp>
-struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
-  using __base = __atomic_ref_base<_Tp*>;
 
   using difference_type = ptrdiff_t;
 
@@ -259,12 +278,6 @@ struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
   _LIBCPP_HIDE_FROM_ABI _Tp* operator--() const noexcept { return fetch_sub(1) - 1; }
   _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; }
   _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; }
-
-  _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {}
-
-  _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); }
-
-  atomic_ref& operator=(const atomic_ref&) = delete;
 };
 
 #endif // _LIBCPP_STD_VER >= 20

>From f9edc9f08d91c7db347be1781802435f9e5b740b Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 17:05:22 -0500
Subject: [PATCH 24/45] [libc++][atomic_ref] Prefer snake_case

---
 libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp    | 10 +++++-----
 .../atomics/atomics.ref/is_always_lock_free.pass.cpp | 10 +++++-----
 .../atomics/atomics.ref/required_alignment.pass.cpp  | 12 ++++++------
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp b/libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp
index 42890f4a537fc7..d20f2cd5e42352 100644
--- a/libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/ctor.pass.cpp
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 
 template <typename T>
-auto makeAtomicRef(T& obj) {
+auto make_atomic_ref(T& obj) {
   // check that the constructor is explicit
   static_assert(!std::is_convertible_v<T, std::atomic_ref<T>>);
   static_assert(std::is_constructible_v<std::atomic_ref<T>, T&>);
@@ -27,17 +27,17 @@ auto makeAtomicRef(T& obj) {
 
 void test() {
   int i = 0;
-  (void)makeAtomicRef(i);
+  (void)make_atomic_ref(i);
 
   float f = 0.f;
-  (void)makeAtomicRef(f);
+  (void)make_atomic_ref(f);
 
   int* p = &i;
-  (void)makeAtomicRef(p);
+  (void)make_atomic_ref(p);
 
   struct X {
   } x;
-  (void)makeAtomicRef(x);
+  (void)make_atomic_ref(x);
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
index 7f0548d2512481..ad958b5d9cf354 100644
--- a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
@@ -19,7 +19,7 @@
 #include "test_macros.h"
 
 template <typename T>
-void checkAlwaysLockFree(std::atomic_ref<T> a) {
+void check_always_lock_free(std::atomic_ref<T> a) {
   if (std::atomic_ref<T>::is_always_lock_free) {
     assert(a.is_lock_free());
   }
@@ -28,17 +28,17 @@ void checkAlwaysLockFree(std::atomic_ref<T> a) {
 
 void test() {
   int i = 0;
-  checkAlwaysLockFree(std::atomic_ref<int>(i));
+  check_always_lock_free(std::atomic_ref<int>(i));
 
   float f = 0.f;
-  checkAlwaysLockFree(std::atomic_ref<float>(f));
+  check_always_lock_free(std::atomic_ref<float>(f));
 
   int* p = &i;
-  checkAlwaysLockFree(std::atomic_ref<int*>(p));
+  check_always_lock_free(std::atomic_ref<int*>(p));
 
   struct X {
   } x;
-  checkAlwaysLockFree(std::atomic_ref<X>(x));
+  check_always_lock_free(std::atomic_ref<X>(x));
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp b/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
index 341c6d18707cdd..f51a030b532836 100644
--- a/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
@@ -12,20 +12,20 @@
 #include <atomic>
 
 template <typename T>
-void checkRequiredAlignment() {
+void check_required_alignment() {
   static_assert(std::atomic_ref<T>::required_alignment >= alignof(T));
 }
 
 void test() {
-  checkRequiredAlignment<int>();
-  checkRequiredAlignment<float>();
-  checkRequiredAlignment<int*>();
+  check_required_alignment<int>();
+  check_required_alignment<float>();
+  check_required_alignment<int*>();
   struct Empty {};
-  checkRequiredAlignment<Empty>();
+  check_required_alignment<Empty>();
   struct Trivial {
     int a;
   };
-  checkRequiredAlignment<Trivial>();
+  check_required_alignment<Trivial>();
 }
 
 int main(int, char**) {

>From 2438001fa5bca3e9e5910078a3e67e4812a65b4c Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 17:16:20 -0500
Subject: [PATCH 25/45] [libc++][atomic_ref] Use proper _LIBCPP_ASSERT_* macro

---
 libcxx/include/__atomic/atomic_ref.h | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index be58359d6e957a..08910340e654f8 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -54,7 +54,7 @@ struct __atomic_ref_base {
 
   _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
       _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
-    _LIBCPP_ASSERT_UNCATEGORIZED(
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
         "memory order argument to atomic store operation is invalid");
     __atomic_store(__ptr_, std::addressof(__desired), std::__to_gcc_order(__order));
@@ -67,7 +67,7 @@ struct __atomic_ref_base {
 
   _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
       _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {
-    _LIBCPP_ASSERT_UNCATEGORIZED(
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
             __order == memory_order::seq_cst,
         "memory order argument to atomic load operation is invalid");
@@ -88,7 +88,7 @@ struct __atomic_ref_base {
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
       _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
-    _LIBCPP_ASSERT_UNCATEGORIZED(
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __failure == memory_order::relaxed || __failure == memory_order::consume ||
             __failure == memory_order::acquire || __failure == memory_order::seq_cst,
         "failure memory order argument to weak atomic compare-and-exchange operation is invalid");
@@ -103,7 +103,7 @@ struct __atomic_ref_base {
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
       _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
-    _LIBCPP_ASSERT_UNCATEGORIZED(
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __failure == memory_order::relaxed || __failure == memory_order::consume ||
             __failure == memory_order::acquire || __failure == memory_order::seq_cst,
         "failure memory order argument to strong atomic compare-and-exchange operation is invalid");
@@ -139,7 +139,7 @@ struct __atomic_ref_base {
 
   _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
       _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) {
-    _LIBCPP_ASSERT_UNCATEGORIZED(
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
             __order == memory_order::seq_cst,
         "memory order argument to atomic wait operation is invalid");
@@ -158,8 +158,8 @@ struct atomic_ref : public __atomic_ref_base<_Tp> {
   using __base = __atomic_ref_base<_Tp>;
 
   _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
-    _LIBCPP_ASSERT_UNCATEGORIZED((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
-                                 "atomic_ref ctor: referenced object must be aligned to required_alignment");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
+                                          "atomic_ref ctor: referenced object must be aligned to required_alignment");
   }
 
   _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
@@ -175,8 +175,8 @@ struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
   using __base = __atomic_ref_base<_Tp>;
 
   _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
-    _LIBCPP_ASSERT_UNCATEGORIZED((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
-                                 "atomic_ref ctor: referenced object must be aligned to required_alignment");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
+                                          "atomic_ref ctor: referenced object must be aligned to required_alignment");
   }
 
   _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
@@ -220,8 +220,8 @@ struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
   using __base = __atomic_ref_base<_Tp>;
 
   _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
-    _LIBCPP_ASSERT_UNCATEGORIZED((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
-                                 "atomic_ref ctor: referenced object must be aligned to required_alignment");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
+                                          "atomic_ref ctor: referenced object must be aligned to required_alignment");
   }
 
   _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;

>From 255559f94d02c4dd1610368bfc80052044e62d84 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 17:34:00 -0500
Subject: [PATCH 26/45] [libc++][atomic_ref] Prefix assertion messages with
 "atomic_ref:"

---
 libcxx/include/__atomic/atomic_ref.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 08910340e654f8..5b788f3a40f003 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -56,7 +56,7 @@ struct __atomic_ref_base {
       _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
     _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
-        "memory order argument to atomic store operation is invalid");
+        "atomic_ref: memory order argument to atomic store operation is invalid");
     __atomic_store(__ptr_, std::addressof(__desired), std::__to_gcc_order(__order));
   }
 
@@ -70,7 +70,7 @@ struct __atomic_ref_base {
     _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
             __order == memory_order::seq_cst,
-        "memory order argument to atomic load operation is invalid");
+        "atomic_ref: memory order argument to atomic load operation is invalid");
     alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
     auto* __ret = reinterpret_cast<_Tp*>(__mem);
     __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order));
@@ -91,7 +91,7 @@ struct __atomic_ref_base {
     _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __failure == memory_order::relaxed || __failure == memory_order::consume ||
             __failure == memory_order::acquire || __failure == memory_order::seq_cst,
-        "failure memory order argument to weak atomic compare-and-exchange operation is invalid");
+        "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
     return __atomic_compare_exchange(
         __ptr_,
         std::addressof(__expected),
@@ -106,7 +106,7 @@ struct __atomic_ref_base {
     _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __failure == memory_order::relaxed || __failure == memory_order::consume ||
             __failure == memory_order::acquire || __failure == memory_order::seq_cst,
-        "failure memory order argument to strong atomic compare-and-exchange operation is invalid");
+        "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
     return __atomic_compare_exchange(
         __ptr_,
         std::addressof(__expected),
@@ -142,7 +142,7 @@ struct __atomic_ref_base {
     _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
             __order == memory_order::seq_cst,
-        "memory order argument to atomic wait operation is invalid");
+        "atomic_ref: memory order argument to atomic wait operation is invalid");
     std::__cxx_atomic_wait(__ptr_, __old, __order);
   }
   _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__cxx_atomic_notify_one(__ptr_); }

>From 8ebfe2df8ac8ae1e864b04752579d48fce8527cd Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 17:39:38 -0500
Subject: [PATCH 27/45] [libc++][atomic_ref] Fix XFAIL annotations

---
 .../atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp | 2 +-
 .../atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp   | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp     | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp     | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp    | 2 +-
 libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp     | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
index 3a991c9351cd85..a3433913080342 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -7,7 +7,7 @@
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: availability-verbose_abort-missing
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
index c9506f556129ee..9d8bbebb70166b 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -7,7 +7,7 @@
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: availability-verbose_abort-missing
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
index 3705167181519c..a19f6516ff3e4c 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
@@ -7,7 +7,7 @@
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: availability-verbose_abort-missing
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <atomic>
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
index 4181b1c12c7db4..f94ab808b43179 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
@@ -7,7 +7,7 @@
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: availability-verbose_abort-missing
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
index f543bcc35295f7..1d41c4341fb996 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
@@ -7,7 +7,7 @@
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: availability-verbose_abort-missing
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
index 2b1c9208527471..5a68cabadb17b6 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
@@ -7,7 +7,7 @@
 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: availability-verbose_abort-missing
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>

>From a745599d47b298d295c7003cb3807063b7dccfec Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 17:42:50 -0500
Subject: [PATCH 28/45] [libc++][atomic_ref] Fixup conversion test

---
 libcxx/test/std/atomics/atomics.ref/convert.pass.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp b/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
index cdfd3442eac806..d4cd38efffbb94 100644
--- a/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
@@ -20,7 +20,8 @@ void test_convert() {
   T x(T(1));
   std::atomic_ref<T> a(x);
 
-  assert(a == T(1));
+  T converted = a;
+  assert(converted == T(1));
 
   ASSERT_NOEXCEPT(T(a));
   static_assert(std::is_nothrow_convertible_v<std::atomic_ref<T>, T>);

>From 07ecf37fad8b286e5ce859a67c70f1b91fa7b3bd Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 17:44:28 -0500
Subject: [PATCH 29/45] [libc++][atomic_ref] Drop test that the ctor is
 explicit since there is already a check

---
 .../atomics.ref/ctor.explicit.verify.cpp      | 34 -------------------
 1 file changed, 34 deletions(-)
 delete mode 100644 libcxx/test/std/atomics/atomics.ref/ctor.explicit.verify.cpp

diff --git a/libcxx/test/std/atomics/atomics.ref/ctor.explicit.verify.cpp b/libcxx/test/std/atomics/atomics.ref/ctor.explicit.verify.cpp
deleted file mode 100644
index 3f1c133c643d65..00000000000000
--- a/libcxx/test/std/atomics/atomics.ref/ctor.explicit.verify.cpp
+++ /dev/null
@@ -1,34 +0,0 @@
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// explicit atomic_ref(T&);
-
-#include <atomic>
-
-template <class T>
-void test(std::atomic_ref<T>) {}
-
-void explicit_ctor() {
-  int i = 0;
-  // expected-error-re@*:* {{{{.*}}no matching function for call to 'test'}}
-  test<int>(i);
-
-  float f = 0.f;
-  // expected-error-re@*:* {{{{.*}}no matching function for call to 'test'}}
-  test<float>(f);
-
-  int* p = &i;
-  // expected-error-re@*:* {{{{.*}}no matching function for call to 'test'}}
-  test<int*>(p);
-
-  struct X {
-  } x;
-  // expected-error-re@*:* {{{{.*}}no matching function for call to 'test'}}
-  test<X>(x);
-}

>From ba4ec71734bbb73a887f36d1f1ba34ea7755e598 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 17:48:46 -0500
Subject: [PATCH 30/45] [libc++][atomic_ref] Fixup do not use
 -Wno-ctad-maybe-unsupported in CTAD tests

---
 libcxx/include/__atomic/atomic_ref.h                   | 2 ++
 libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 5b788f3a40f003..4ba8bdcccf1cc9 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -280,6 +280,8 @@ struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
   _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; }
 };
 
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(atomic_ref);
+
 #endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp b/libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp
index 62cfcc08aa0420..27aaf2ecdb3c91 100644
--- a/libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/deduction.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// ADDITIONAL_COMPILE_FLAGS: -Wno-ctad-maybe-unsupported
 
 // <atomic>
 

>From b476c80ecaab26d80c5f73c0819da98cd0a9f808 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 17:55:13 -0500
Subject: [PATCH 31/45] [libc++][atomic_ref] Rename {type ->
 requires-trivially-copyable}.verify.cpp

---
 .../{type.verify.cpp => requires-trivially-copyable.verify.cpp}   | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename libcxx/test/std/atomics/atomics.ref/{type.verify.cpp => requires-trivially-copyable.verify.cpp} (100%)

diff --git a/libcxx/test/std/atomics/atomics.ref/type.verify.cpp b/libcxx/test/std/atomics/atomics.ref/requires-trivially-copyable.verify.cpp
similarity index 100%
rename from libcxx/test/std/atomics/atomics.ref/type.verify.cpp
rename to libcxx/test/std/atomics/atomics.ref/requires-trivially-copyable.verify.cpp

>From 21bf0662d008b5e699e9642b4c46bb754bc1d054 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 18:09:46 -0500
Subject: [PATCH 32/45] [libc++][atomic_ref] Const the atomic_refs in the tests

---
 .../atomics.ref/assert.compare_exchange_strong.pass.cpp   | 6 +++---
 .../atomics.ref/assert.compare_exchange_weak.pass.cpp     | 6 +++---
 .../test/libcxx/atomics/atomics.ref/assert.load.pass.cpp  | 6 +++---
 .../test/libcxx/atomics/atomics.ref/assert.store.pass.cpp | 8 ++++----
 .../test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp  | 6 +++---
 libcxx/test/std/atomics/atomics.ref/assign.pass.cpp       | 2 +-
 .../std/atomics/atomics.ref/bitwise_and_assign.pass.cpp   | 2 +-
 .../std/atomics/atomics.ref/bitwise_or_assign.pass.cpp    | 2 +-
 .../std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp   | 2 +-
 .../atomics/atomics.ref/compare_exchange_strong.pass.cpp  | 6 +++---
 .../atomics/atomics.ref/compare_exchange_weak.pass.cpp    | 6 +++---
 libcxx/test/std/atomics/atomics.ref/convert.pass.cpp      | 2 +-
 libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp     | 2 +-
 libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp    | 4 ++--
 libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp    | 2 +-
 libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp     | 2 +-
 libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp    | 4 ++--
 libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp    | 2 +-
 .../std/atomics/atomics.ref/increment_decrement.pass.cpp  | 2 +-
 .../std/atomics/atomics.ref/is_always_lock_free.pass.cpp  | 2 +-
 libcxx/test/std/atomics/atomics.ref/load.pass.cpp         | 2 +-
 libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp   | 2 +-
 libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp   | 2 +-
 .../atomics/atomics.ref/operator_minus_equals.pass.cpp    | 4 ++--
 .../std/atomics/atomics.ref/operator_plus_equals.pass.cpp | 4 ++--
 libcxx/test/std/atomics/atomics.ref/store.pass.cpp        | 2 +-
 libcxx/test/std/atomics/atomics.ref/wait.pass.cpp         | 2 +-
 27 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
index a3433913080342..aeab9f5ff3fb7f 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -24,7 +24,7 @@ template <typename T>
 void test_compare_exchange_strong_invalid_memory_order() {
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
     T t(T(2));
     a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_relaxed);
   }
@@ -32,7 +32,7 @@ void test_compare_exchange_strong_invalid_memory_order() {
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         T t(T(2));
         a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_release);
       }()),
@@ -41,7 +41,7 @@ void test_compare_exchange_strong_invalid_memory_order() {
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         T t(T(2));
         a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_acq_rel);
       }()),
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
index 9d8bbebb70166b..407146722f17c2 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -24,7 +24,7 @@ template <typename T>
 void test_compare_exchange_weak_invalid_memory_order() {
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
     T t(T(2));
     a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_relaxed);
   }
@@ -32,7 +32,7 @@ void test_compare_exchange_weak_invalid_memory_order() {
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         T t(T(2));
         a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_release);
       }()),
@@ -41,7 +41,7 @@ void test_compare_exchange_weak_invalid_memory_order() {
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         T t(T(2));
         a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_acq_rel);
       }()),
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
index f94ab808b43179..34e7a2b56f0f02 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
@@ -24,14 +24,14 @@ template <typename T>
 void test_load_invalid_memory_order() {
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
     (void)a.load(std::memory_order_relaxed);
   }
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         (void)a.load(std::memory_order_release);
       }()),
       "memory order argument to atomic load operation is invalid");
@@ -39,7 +39,7 @@ void test_load_invalid_memory_order() {
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         (void)a.load(std::memory_order_acq_rel);
       }()),
       "memory order argument to atomic load operation is invalid");
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
index 1d41c4341fb996..12e21ea38b94ab 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
@@ -24,14 +24,14 @@ template <typename T>
 void test_store_invalid_memory_order() {
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
     a.store(T(2), std::memory_order_relaxed);
   }
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         a.store(T(2), std::memory_order_consume);
       }()),
       "memory order argument to atomic store operation is invalid");
@@ -39,7 +39,7 @@ void test_store_invalid_memory_order() {
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         a.store(T(2), std::memory_order_acquire);
       }()),
       "memory order argument to atomic store operation is invalid");
@@ -47,7 +47,7 @@ void test_store_invalid_memory_order() {
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         a.store(T(2), std::memory_order_acq_rel);
       }()),
       "memory order argument to atomic store operation is invalid");
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
index 5a68cabadb17b6..7604132ff694c3 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
@@ -24,14 +24,14 @@ template <typename T>
 void test_wait_invalid_memory_order() {
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
     a.wait(T(2), std::memory_order_relaxed);
   }
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         a.wait(T(2), std::memory_order_release);
       }()),
       "memory order argument to atomic wait operation is invalid");
@@ -39,7 +39,7 @@ void test_wait_invalid_memory_order() {
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
         T x(T(1));
-        std::atomic_ref<T> a(x);
+        std::atomic_ref<T> const a(x);
         a.wait(T(2), std::memory_order_acq_rel);
       }()),
       "memory order argument to atomic wait operation is invalid");
diff --git a/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
index 95d29df70fe160..d971dd74e02793 100644
--- a/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/assign.pass.cpp
@@ -18,7 +18,7 @@
 template <typename T>
 void test_assign() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   a = T(2);
   assert(x == T(2));
diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
index 68c64b9ab34589..d32648e1edee6f 100644
--- a/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
@@ -32,7 +32,7 @@ static_assert(!has_bitwise_and_assign<std::atomic_ref<X>>);
 template <typename T>
 void test_integral() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert((a &= T(2)) == T(0));
   assert(x == T(0));
diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
index 20ec80697c70ed..ffbf9fba7a097b 100644
--- a/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
@@ -32,7 +32,7 @@ static_assert(!has_bitwise_or_assign<std::atomic_ref<X>>);
 template <typename T>
 void test_integral() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert((a |= T(2)) == T(3));
   assert(x == T(3));
diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
index 2d41bf01f95629..a31faa7e58d908 100644
--- a/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
@@ -32,7 +32,7 @@ static_assert(!has_bitwise_xor_assign<std::atomic_ref<X>>);
 template <typename T>
 void test_integral() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert((a ^= T(2)) == T(3));
   assert(x == T(3));
diff --git a/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp b/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
index b9dc84c04f7668..7a9628b42e41eb 100644
--- a/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
@@ -20,7 +20,7 @@ template <typename T>
 void test_compare_exchange_strong() {
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
 
     T t(T(1));
     assert(a.compare_exchange_strong(t, T(2)) == true);
@@ -34,7 +34,7 @@ void test_compare_exchange_strong() {
   }
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
 
     T t(T(1));
     assert(a.compare_exchange_strong(t, T(2), std::memory_order_seq_cst) == true);
@@ -48,7 +48,7 @@ void test_compare_exchange_strong() {
   }
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
 
     T t(T(1));
     assert(a.compare_exchange_strong(t, T(2), std::memory_order_release, std::memory_order_relaxed) == true);
diff --git a/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp b/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
index 3d16be5e3a1499..6eac7967b46db5 100644
--- a/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
@@ -20,7 +20,7 @@ template <typename T>
 void test_compare_exchange_weak() {
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
 
     T t(T(1));
     assert(a.compare_exchange_weak(t, T(2)) == true);
@@ -34,7 +34,7 @@ void test_compare_exchange_weak() {
   }
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
 
     T t(T(1));
     assert(a.compare_exchange_weak(t, T(2), std::memory_order_seq_cst) == true);
@@ -48,7 +48,7 @@ void test_compare_exchange_weak() {
   }
   {
     T x(T(1));
-    std::atomic_ref<T> a(x);
+    std::atomic_ref<T> const a(x);
 
     T t(T(1));
     assert(a.compare_exchange_weak(t, T(2), std::memory_order_release, std::memory_order_relaxed) == true);
diff --git a/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp b/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
index d4cd38efffbb94..b965607950b0f6 100644
--- a/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
@@ -18,7 +18,7 @@
 template <typename T>
 void test_convert() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   T converted = a;
   assert(converted == T(1));
diff --git a/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp b/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
index 75055bd8679bc3..a4b9edd4620f10 100644
--- a/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
@@ -18,7 +18,7 @@
 template <typename T>
 void test_exchange() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(a.exchange(T(2)) == T(1));
   ASSERT_NOEXCEPT(a.exchange(T(2)));
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
index 84ad4f54a6b5cd..e93538273b6969 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
@@ -34,7 +34,7 @@ static_assert(!has_fetch_add<std::atomic_ref<X>>);
 template <typename T>
 void test_arithmetic() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(a.fetch_add(T(2)) == T(1));
   assert(x == T(3));
@@ -50,7 +50,7 @@ void test_pointer() {
   using U = std::remove_pointer_t<T>;
   U t[9]  = {};
   T p{&t[1]};
-  std::atomic_ref<T> a(p);
+  std::atomic_ref<T> const a(p);
 
   assert(a.fetch_add(2) == &t[1]);
   assert(a == &t[3]);
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
index d837bc8f423c98..f83d9a65ad9909 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
@@ -35,7 +35,7 @@ static_assert(!has_fetch_and<std::atomic_ref<X>>);
 template <typename T>
 void test_integral() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(a.fetch_and(T(2)) == T(1));
   assert(x == T(0));
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
index 88a836810b002e..ecb4ba3859f29a 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
@@ -35,7 +35,7 @@ static_assert(!has_fetch_or<std::atomic_ref<X>>);
 template <typename T>
 void test_integral() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(a.fetch_or(T(2)) == T(1));
   assert(x == T(3));
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
index 031f6f78a37c0c..21041355f26d23 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
@@ -34,7 +34,7 @@ static_assert(!has_fetch_sub<std::atomic_ref<X>>);
 template <typename T>
 void test_arithmetic() {
   T x(T(7));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(a.fetch_sub(T(4)) == T(7));
   assert(x == T(3));
@@ -50,7 +50,7 @@ void test_pointer() {
   using U = std::remove_pointer_t<T>;
   U t[9]  = {};
   T p{&t[7]};
-  std::atomic_ref<T> a(p);
+  std::atomic_ref<T> const a(p);
 
   assert(a.fetch_sub(4) == &t[7]);
   assert(a == &t[3]);
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
index 2e2f913e9e242a..49856567f8e8ac 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
@@ -35,7 +35,7 @@ static_assert(!has_fetch_xor<std::atomic_ref<X>>);
 template <typename T>
 void test_integral() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(a.fetch_xor(T(2)) == T(1));
   assert(x == T(3));
diff --git a/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp b/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
index dd1bcaa6f554f0..586b5a17f82ad4 100644
--- a/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
@@ -50,7 +50,7 @@ static_assert(does_not_have_increment_nor_decrement_operators<X>());
 template <typename T>
 void test_integral() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(++a == T(2));
   assert(x == T(2));
diff --git a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
index ad958b5d9cf354..25dec05f7fff0a 100644
--- a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
@@ -19,7 +19,7 @@
 #include "test_macros.h"
 
 template <typename T>
-void check_always_lock_free(std::atomic_ref<T> a) {
+void check_always_lock_free(std::atomic_ref<T> const a) {
   if (std::atomic_ref<T>::is_always_lock_free) {
     assert(a.is_lock_free());
   }
diff --git a/libcxx/test/std/atomics/atomics.ref/load.pass.cpp b/libcxx/test/std/atomics/atomics.ref/load.pass.cpp
index fe3aaaf1edcf8a..25171fc247c1c6 100644
--- a/libcxx/test/std/atomics/atomics.ref/load.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/load.pass.cpp
@@ -18,7 +18,7 @@
 template <typename T>
 void test_load() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(a.load() == T(1));
   ASSERT_NOEXCEPT(a.load());
diff --git a/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp b/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
index 3ea9de240a192a..7f7575583b6fe8 100644
--- a/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/notify_all.pass.cpp
@@ -21,7 +21,7 @@
 template <typename T>
 void test_notify_all() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   bool done                      = false;
   std::atomic<int> started_num   = 0;
diff --git a/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp b/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
index a3ae251ab770c8..7139aae3385436 100644
--- a/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/notify_one.pass.cpp
@@ -21,7 +21,7 @@
 template <typename T>
 void test_notify_one() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   std::thread t = support::make_test_thread([&]() {
     a.store(T(3));
diff --git a/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
index e7e25efb7acd27..583818aeb18b8c 100644
--- a/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
@@ -31,7 +31,7 @@ static_assert(!has_operator_minus_equals<std::atomic_ref<X>>);
 template <typename T>
 void test_arithmetic() {
   T x(T(3));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert((a -= T(2)) == T(1));
   assert(x == T(1));
@@ -43,7 +43,7 @@ void test_pointer() {
   using U = std::remove_pointer_t<T>;
   U t[9]  = {};
   T p{&t[3]};
-  std::atomic_ref<T> a(p);
+  std::atomic_ref<T> const a(p);
 
   assert((a -= 2) == &t[1]);
   assert(a == &t[1]);
diff --git a/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
index 1aeadffafaef84..196d30654f0793 100644
--- a/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
@@ -31,7 +31,7 @@ static_assert(!has_operator_plus_equals<std::atomic_ref<X>>);
 template <typename T>
 void test_arithmetic() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert((a += T(2)) == T(3));
   assert(x == T(3));
@@ -43,7 +43,7 @@ void test_pointer() {
   using U = std::remove_pointer_t<T>;
   U t[9]  = {};
   T p{&t[1]};
-  std::atomic_ref<T> a(p);
+  std::atomic_ref<T> const a(p);
 
   assert((a += 2) == &t[3]);
   assert(a == &t[3]);
diff --git a/libcxx/test/std/atomics/atomics.ref/store.pass.cpp b/libcxx/test/std/atomics/atomics.ref/store.pass.cpp
index 102311bb43b236..0ecafe83f7a245 100644
--- a/libcxx/test/std/atomics/atomics.ref/store.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/store.pass.cpp
@@ -18,7 +18,7 @@
 template <typename T>
 void test_store() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   a.store(T(2));
   assert(x == T(2));
diff --git a/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp b/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
index 51d3a79dd59a5e..77d6b47146ac31 100644
--- a/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/wait.pass.cpp
@@ -19,7 +19,7 @@
 template <typename T>
 void test_wait() {
   T x(T(1));
-  std::atomic_ref<T> a(x);
+  std::atomic_ref<T> const a(x);
 
   assert(a.load() == T(1));
   a.wait(T(0));

>From 6df2fa87e4d3b1430beaf42b86086292a62b78a2 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 18:20:59 -0500
Subject: [PATCH 33/45] [libc++][atomic_ref] Let required_alignement test check
 both at runtime and compile time

---
 .../std/atomics/atomics.ref/required_alignment.pass.cpp  | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp b/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
index f51a030b532836..e595f27262aaf4 100644
--- a/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/required_alignment.pass.cpp
@@ -10,13 +10,14 @@
 // static constexpr size_t required_alignment;
 
 #include <atomic>
+#include <cassert>
 
 template <typename T>
-void check_required_alignment() {
-  static_assert(std::atomic_ref<T>::required_alignment >= alignof(T));
+constexpr void check_required_alignment() {
+  assert(std::atomic_ref<T>::required_alignment >= alignof(T));
 }
 
-void test() {
+constexpr bool test() {
   check_required_alignment<int>();
   check_required_alignment<float>();
   check_required_alignment<int*>();
@@ -26,9 +27,11 @@ void test() {
     int a;
   };
   check_required_alignment<Trivial>();
+  return true;
 }
 
 int main(int, char**) {
   test();
+  static_assert(test());
   return 0;
 }

>From 05dad2c0d1deabd79d0816a6fc4a9a5c23114833 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 18:26:58 -0500
Subject: [PATCH 34/45] [libc++][atomic_ref] Add a release note and update the
 status page

---
 libcxx/docs/ReleaseNotes/19.rst    | 1 +
 libcxx/docs/Status/Cxx20Papers.csv | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 908d46b711a5a6..99e04d8cbf960d 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -40,6 +40,7 @@ Implemented Papers
 
 - P2637R3 - Member ``visit``
 - P2652R2 - Disallow User Specialization of ``allocator_traits``
+- P0019R8 - ``atomic_ref``
 
 
 Improvements and New Features
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index d73088687975c2..e7de3e0eea7823 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -26,7 +26,7 @@
 "`P0905R1 <https://wg21.link/P0905R1>`__","CWG","Symmetry for spaceship","Jacksonville","","","|spaceship|"
 "`P0966R1 <https://wg21.link/P0966R1>`__","LWG","``string::reserve``\  Should Not Shrink","Jacksonville","|Complete| [#note-P0966]_","12.0"
 "","","","","","",""
-"`P0019R8 <https://wg21.link/P0019R8>`__","LWG","Atomic Ref","Rapperswil","",""
+"`P0019R8 <https://wg21.link/P0019R8>`__","LWG","Atomic Ref","Rapperswil","[In Progress]",""
 "`P0458R2 <https://wg21.link/P0458R2>`__","LWG","Checking for Existence of an Element in Associative Containers","Rapperswil","|Complete|","13.0"
 "`P0475R1 <https://wg21.link/P0475R1>`__","LWG","LWG 2511: guaranteed copy elision for piecewise construction","Rapperswil","|Complete|",""
 "`P0476R2 <https://wg21.link/P0476R2>`__","LWG","Bit-casting object representations","Rapperswil","|Complete|","14.0"

>From 991ccb67f5293ca20397ac41a6269e6867eae132 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 18:51:02 -0500
Subject: [PATCH 35/45] [libc++][atomic_ref] Update generated files

---
 libcxx/include/libcxx.imp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp
index 69de4705f37886..76f785bd06c7d5 100644
--- a/libcxx/include/libcxx.imp
+++ b/libcxx/include/libcxx.imp
@@ -235,6 +235,7 @@
   { include: [ "<__atomic/atomic_flag.h>", "private", "<atomic>", "public" ] },
   { include: [ "<__atomic/atomic_init.h>", "private", "<atomic>", "public" ] },
   { include: [ "<__atomic/atomic_lock_free.h>", "private", "<atomic>", "public" ] },
+  { include: [ "<__atomic/atomic_ref.h>", "private", "<atomic>", "public" ] },
   { include: [ "<__atomic/atomic_sync.h>", "private", "<atomic>", "public" ] },
   { include: [ "<__atomic/check_memory_order.h>", "private", "<atomic>", "public" ] },
   { include: [ "<__atomic/contention_t.h>", "private", "<atomic>", "public" ] },
@@ -243,6 +244,7 @@
   { include: [ "<__atomic/is_always_lock_free.h>", "private", "<atomic>", "public" ] },
   { include: [ "<__atomic/kill_dependency.h>", "private", "<atomic>", "public" ] },
   { include: [ "<__atomic/memory_order.h>", "private", "<atomic>", "public" ] },
+  { include: [ "<__atomic/to_gcc_order.h>", "private", "<atomic>", "public" ] },
   { include: [ "<__bit/bit_cast.h>", "private", "<bit>", "public" ] },
   { include: [ "<__bit/bit_ceil.h>", "private", "<bit>", "public" ] },
   { include: [ "<__bit/bit_floor.h>", "private", "<bit>", "public" ] },

>From bd870b0fd6ff061b1d3b254c264cd96858b57225 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 19:11:29 -0500
Subject: [PATCH 36/45] [libc++][atomic_ref] Fix clang-tidy warnings
 modernize-use-nullptr

---
 libcxx/include/__atomic/atomic_ref.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 4ba8bdcccf1cc9..d8aaac92b303de 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -48,9 +48,9 @@ struct __atomic_ref_base {
 
   static constexpr size_t required_alignment = alignof(_Tp);
 
-  static constexpr bool is_always_lock_free = __atomic_always_lock_free(sizeof(_Tp), 0);
+  static constexpr bool is_always_lock_free = __atomic_always_lock_free(sizeof(_Tp), nullptr);
 
-  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), 0); }
+  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); }
 
   _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
       _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {

>From c10501cd405f168f6c5223894d377f572467660f Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Mon, 5 Feb 2024 22:39:26 -0500
Subject: [PATCH 37/45] [libc++][atomic_ref] Use std::same_as<T> trick

---
 .../atomics.ref/bitwise_and_assign.pass.cpp   |  4 +-
 .../atomics.ref/bitwise_or_assign.pass.cpp    |  4 +-
 .../atomics.ref/bitwise_xor_assign.pass.cpp   |  4 +-
 .../compare_exchange_strong.pass.cpp          | 20 ++++++---
 .../compare_exchange_weak.pass.cpp            | 19 +++++---
 .../std/atomics/atomics.ref/exchange.pass.cpp | 17 +++++---
 .../atomics/atomics.ref/fetch_add.pass.cpp    | 37 ++++++++++------
 .../atomics/atomics.ref/fetch_and.pass.cpp    | 19 +++++---
 .../std/atomics/atomics.ref/fetch_or.pass.cpp | 21 ++++++---
 .../atomics/atomics.ref/fetch_sub.pass.cpp    | 37 ++++++++++------
 .../atomics/atomics.ref/fetch_xor.pass.cpp    | 21 ++++++---
 .../atomics.ref/increment_decrement.pass.cpp  | 43 ++++++++++++-------
 .../atomics.ref/is_always_lock_free.pass.cpp  |  4 +-
 .../std/atomics/atomics.ref/load.pass.cpp     | 17 +++++---
 .../operator_minus_equals.pass.cpp            |  7 ++-
 .../atomics.ref/operator_plus_equals.pass.cpp |  7 ++-
 16 files changed, 192 insertions(+), 89 deletions(-)

diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
index d32648e1edee6f..396448cb4bc3f1 100644
--- a/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_and_assign.pass.cpp
@@ -11,6 +11,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -34,7 +35,8 @@ void test_integral() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert((a &= T(2)) == T(0));
+  std::same_as<T> auto y = (a &= T(2));
+  assert(y == T(0));
   assert(x == T(0));
   ASSERT_NOEXCEPT(a &= T(0));
 }
diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
index ffbf9fba7a097b..425a7538c3d543 100644
--- a/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_or_assign.pass.cpp
@@ -11,6 +11,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -34,7 +35,8 @@ void test_integral() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert((a |= T(2)) == T(3));
+  std::same_as<T> auto y = (a |= T(2));
+  assert(y == T(3));
   assert(x == T(3));
   ASSERT_NOEXCEPT(a &= T(0));
 }
diff --git a/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp b/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
index a31faa7e58d908..6eaea88fdce672 100644
--- a/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/bitwise_xor_assign.pass.cpp
@@ -11,6 +11,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -34,7 +35,8 @@ void test_integral() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert((a ^= T(2)) == T(3));
+  std::same_as<T> auto y = (a ^= T(2));
+  assert(y == T(3));
   assert(x == T(3));
   ASSERT_NOEXCEPT(a ^= T(0));
 }
diff --git a/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp b/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
index 7a9628b42e41eb..8ea62925eda43a 100644
--- a/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/compare_exchange_strong.pass.cpp
@@ -12,6 +12,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -23,10 +24,12 @@ void test_compare_exchange_strong() {
     std::atomic_ref<T> const a(x);
 
     T t(T(1));
-    assert(a.compare_exchange_strong(t, T(2)) == true);
+    std::same_as<bool> auto y = a.compare_exchange_strong(t, T(2));
+    assert(y == true);
     assert(a == T(2));
     assert(t == T(1));
-    assert(a.compare_exchange_strong(t, T(3)) == false);
+    y = a.compare_exchange_strong(t, T(3));
+    assert(y == false);
     assert(a == T(2));
     assert(t == T(2));
 
@@ -37,10 +40,12 @@ void test_compare_exchange_strong() {
     std::atomic_ref<T> const a(x);
 
     T t(T(1));
-    assert(a.compare_exchange_strong(t, T(2), std::memory_order_seq_cst) == true);
+    std::same_as<bool> auto y = a.compare_exchange_strong(t, T(2), std::memory_order_seq_cst);
+    assert(y == true);
     assert(a == T(2));
     assert(t == T(1));
-    assert(a.compare_exchange_strong(t, T(3), std::memory_order_seq_cst) == false);
+    y = a.compare_exchange_strong(t, T(3), std::memory_order_seq_cst);
+    assert(y == false);
     assert(a == T(2));
     assert(t == T(2));
 
@@ -51,10 +56,13 @@ void test_compare_exchange_strong() {
     std::atomic_ref<T> const a(x);
 
     T t(T(1));
-    assert(a.compare_exchange_strong(t, T(2), std::memory_order_release, std::memory_order_relaxed) == true);
+    std::same_as<bool> auto y =
+        a.compare_exchange_strong(t, T(2), std::memory_order_release, std::memory_order_relaxed);
+    assert(y == true);
     assert(a == T(2));
     assert(t == T(1));
-    assert(a.compare_exchange_strong(t, T(3), std::memory_order_release, std::memory_order_relaxed) == false);
+    y = a.compare_exchange_strong(t, T(3), std::memory_order_release, std::memory_order_relaxed);
+    assert(y == false);
     assert(a == T(2));
     assert(t == T(2));
 
diff --git a/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp b/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
index 6eac7967b46db5..cd5455315b9faa 100644
--- a/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/compare_exchange_weak.pass.cpp
@@ -11,6 +11,7 @@
 // bool compare_exchange_weak(T&, T, memory_order = memory_order::seq_cst) const noexcept;
 
 #include <atomic>
+#include <concepts>
 #include <cassert>
 #include <type_traits>
 
@@ -23,10 +24,12 @@ void test_compare_exchange_weak() {
     std::atomic_ref<T> const a(x);
 
     T t(T(1));
-    assert(a.compare_exchange_weak(t, T(2)) == true);
+    std::same_as<bool> auto y = a.compare_exchange_weak(t, T(2));
+    assert(y == true);
     assert(a == T(2));
     assert(t == T(1));
-    assert(a.compare_exchange_weak(t, T(3)) == false);
+    y = a.compare_exchange_weak(t, T(3));
+    assert(y == false);
     assert(a == T(2));
     assert(t == T(2));
 
@@ -37,10 +40,12 @@ void test_compare_exchange_weak() {
     std::atomic_ref<T> const a(x);
 
     T t(T(1));
-    assert(a.compare_exchange_weak(t, T(2), std::memory_order_seq_cst) == true);
+    std::same_as<bool> auto y = a.compare_exchange_weak(t, T(2), std::memory_order_seq_cst);
+    assert(y == true);
     assert(a == T(2));
     assert(t == T(1));
-    assert(a.compare_exchange_weak(t, T(3), std::memory_order_seq_cst) == false);
+    y = a.compare_exchange_weak(t, T(3), std::memory_order_seq_cst);
+    assert(y == false);
     assert(a == T(2));
     assert(t == T(2));
 
@@ -51,10 +56,12 @@ void test_compare_exchange_weak() {
     std::atomic_ref<T> const a(x);
 
     T t(T(1));
-    assert(a.compare_exchange_weak(t, T(2), std::memory_order_release, std::memory_order_relaxed) == true);
+    std::same_as<bool> auto y = a.compare_exchange_weak(t, T(2), std::memory_order_release, std::memory_order_relaxed);
+    assert(y == true);
     assert(a == T(2));
     assert(t == T(1));
-    assert(a.compare_exchange_weak(t, T(3), std::memory_order_release, std::memory_order_relaxed) == false);
+    y = a.compare_exchange_weak(t, T(3), std::memory_order_release, std::memory_order_relaxed);
+    assert(y == false);
     assert(a == T(2));
     assert(t == T(2));
 
diff --git a/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp b/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
index a4b9edd4620f10..f1e41ef8051a14 100644
--- a/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/exchange.pass.cpp
@@ -11,6 +11,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -20,11 +21,17 @@ void test_exchange() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert(a.exchange(T(2)) == T(1));
-  ASSERT_NOEXCEPT(a.exchange(T(2)));
-
-  assert(a.exchange(T(3), std::memory_order_seq_cst) == T(2));
-  ASSERT_NOEXCEPT(a.exchange(T(3), std::memory_order_seq_cst));
+  {
+    std::same_as<T> auto y = a.exchange(T(2));
+    assert(y == T(1));
+    ASSERT_NOEXCEPT(a.exchange(T(2)));
+  }
+
+  {
+    std::same_as<T> auto y = a.exchange(T(3), std::memory_order_seq_cst);
+    assert(y == T(2));
+    ASSERT_NOEXCEPT(a.exchange(T(3), std::memory_order_seq_cst));
+  }
 }
 
 void test() {
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
index e93538273b6969..11e0f30d3b822f 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_add.pass.cpp
@@ -13,6 +13,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -36,13 +37,19 @@ void test_arithmetic() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert(a.fetch_add(T(2)) == T(1));
-  assert(x == T(3));
-  ASSERT_NOEXCEPT(a.fetch_add(T(0)));
+  {
+    std::same_as<T> auto y = a.fetch_add(T(2));
+    assert(y == T(1));
+    assert(x == T(3));
+    ASSERT_NOEXCEPT(a.fetch_add(T(0)));
+  }
 
-  assert(a.fetch_add(T(4), std::memory_order_relaxed) == T(3));
-  assert(x == T(7));
-  ASSERT_NOEXCEPT(a.fetch_add(T(0), std::memory_order_relaxed));
+  {
+    std::same_as<T> auto y = a.fetch_add(T(4), std::memory_order_relaxed);
+    assert(y == T(3));
+    assert(x == T(7));
+    ASSERT_NOEXCEPT(a.fetch_add(T(0), std::memory_order_relaxed));
+  }
 }
 
 template <typename T>
@@ -52,13 +59,19 @@ void test_pointer() {
   T p{&t[1]};
   std::atomic_ref<T> const a(p);
 
-  assert(a.fetch_add(2) == &t[1]);
-  assert(a == &t[3]);
-  ASSERT_NOEXCEPT(a.fetch_add(0));
+  {
+    std::same_as<T> auto y = a.fetch_add(2);
+    assert(y == &t[1]);
+    assert(a == &t[3]);
+    ASSERT_NOEXCEPT(a.fetch_add(0));
+  }
 
-  assert(a.fetch_add(4, std::memory_order_relaxed) == &t[3]);
-  assert(a == &t[7]);
-  ASSERT_NOEXCEPT(a.fetch_add(0, std::memory_order_relaxed));
+  {
+    std::same_as<T> auto y = a.fetch_add(4, std::memory_order_relaxed);
+    assert(y == &t[3]);
+    assert(a == &t[7]);
+    ASSERT_NOEXCEPT(a.fetch_add(0, std::memory_order_relaxed));
+  }
 }
 
 void test() {
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
index f83d9a65ad9909..b0eef527a58eeb 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_and.pass.cpp
@@ -11,6 +11,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -37,15 +38,21 @@ void test_integral() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert(a.fetch_and(T(2)) == T(1));
-  assert(x == T(0));
-  ASSERT_NOEXCEPT(a.fetch_and(T(0)));
+  {
+    std::same_as<T> auto y = a.fetch_and(T(2));
+    assert(y == T(1));
+    assert(x == T(0));
+    ASSERT_NOEXCEPT(a.fetch_and(T(0)));
+  }
 
   x = T(1);
 
-  assert(a.fetch_and(T(2), std::memory_order_relaxed) == T(1));
-  assert(x == T(0));
-  ASSERT_NOEXCEPT(a.fetch_and(T(0), std::memory_order_relaxed));
+  {
+    std::same_as<T> auto y = a.fetch_and(T(2), std::memory_order_relaxed);
+    assert(y == T(1));
+    assert(x == T(0));
+    ASSERT_NOEXCEPT(a.fetch_and(T(0), std::memory_order_relaxed));
+  }
 }
 
 void test() { test_integral<int>(); }
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
index ecb4ba3859f29a..53112b32e662f5 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_or.pass.cpp
@@ -10,6 +10,7 @@
 // integral-type fetch_or(integral-type, memory_order = memory_order::seq_cst) const noexcept;
 
 #include <atomic>
+#include <concepts>
 #include <cassert>
 #include <type_traits>
 
@@ -37,13 +38,19 @@ void test_integral() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert(a.fetch_or(T(2)) == T(1));
-  assert(x == T(3));
-  ASSERT_NOEXCEPT(a.fetch_or(T(0)));
-
-  assert(a.fetch_or(T(2), std::memory_order_relaxed) == T(3));
-  assert(x == T(3));
-  ASSERT_NOEXCEPT(a.fetch_or(T(0), std::memory_order_relaxed));
+  {
+    std::same_as<T> auto y = a.fetch_or(T(2));
+    assert(y == T(1));
+    assert(x == T(3));
+    ASSERT_NOEXCEPT(a.fetch_or(T(0)));
+  }
+
+  {
+    std::same_as<T> auto y = a.fetch_or(T(2), std::memory_order_relaxed);
+    assert(y == T(3));
+    assert(x == T(3));
+    ASSERT_NOEXCEPT(a.fetch_or(T(0), std::memory_order_relaxed));
+  }
 }
 
 void test() { test_integral<int>(); }
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
index 21041355f26d23..96625020563d32 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_sub.pass.cpp
@@ -13,6 +13,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -36,13 +37,19 @@ void test_arithmetic() {
   T x(T(7));
   std::atomic_ref<T> const a(x);
 
-  assert(a.fetch_sub(T(4)) == T(7));
-  assert(x == T(3));
-  ASSERT_NOEXCEPT(a.fetch_sub(T(0)));
+  {
+    std::same_as<T> auto y = a.fetch_sub(T(4));
+    assert(y == T(7));
+    assert(x == T(3));
+    ASSERT_NOEXCEPT(a.fetch_sub(T(0)));
+  }
 
-  assert(a.fetch_sub(T(2), std::memory_order_relaxed) == T(3));
-  assert(x == T(1));
-  ASSERT_NOEXCEPT(a.fetch_sub(T(0), std::memory_order_relaxed));
+  {
+    std::same_as<T> auto y = a.fetch_sub(T(2), std::memory_order_relaxed);
+    assert(y == T(3));
+    assert(x == T(1));
+    ASSERT_NOEXCEPT(a.fetch_sub(T(0), std::memory_order_relaxed));
+  }
 }
 
 template <typename T>
@@ -52,13 +59,19 @@ void test_pointer() {
   T p{&t[7]};
   std::atomic_ref<T> const a(p);
 
-  assert(a.fetch_sub(4) == &t[7]);
-  assert(a == &t[3]);
-  ASSERT_NOEXCEPT(a.fetch_sub(0));
+  {
+    std::same_as<T> auto y = a.fetch_sub(4);
+    assert(y == &t[7]);
+    assert(a == &t[3]);
+    ASSERT_NOEXCEPT(a.fetch_sub(0));
+  }
 
-  assert(a.fetch_sub(2, std::memory_order_relaxed) == &t[3]);
-  assert(a == &t[1]);
-  ASSERT_NOEXCEPT(a.fetch_sub(0, std::memory_order_relaxed));
+  {
+    std::same_as<T> auto y = a.fetch_sub(2, std::memory_order_relaxed);
+    assert(y == &t[3]);
+    assert(a == &t[1]);
+    ASSERT_NOEXCEPT(a.fetch_sub(0, std::memory_order_relaxed));
+  }
 }
 
 void test() {
diff --git a/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp b/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
index 49856567f8e8ac..cfeb544905603a 100644
--- a/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/fetch_xor.pass.cpp
@@ -10,6 +10,7 @@
 // integral-type fetch_xor(integral-type, memory_order = memory_order::seq_cst) const noexcept;
 
 #include <atomic>
+#include <concepts>
 #include <cassert>
 #include <type_traits>
 
@@ -37,13 +38,19 @@ void test_integral() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert(a.fetch_xor(T(2)) == T(1));
-  assert(x == T(3));
-  ASSERT_NOEXCEPT(a.fetch_xor(T(0)));
-
-  assert(a.fetch_xor(T(2), std::memory_order_relaxed) == T(3));
-  assert(x == T(1));
-  ASSERT_NOEXCEPT(a.fetch_xor(T(0), std::memory_order_relaxed));
+  {
+    std::same_as<T> auto y = a.fetch_xor(T(2));
+    assert(y == T(1));
+    assert(x == T(3));
+    ASSERT_NOEXCEPT(a.fetch_xor(T(0)));
+  }
+
+  {
+    std::same_as<T> auto y = a.fetch_xor(T(2), std::memory_order_relaxed);
+    assert(y == T(3));
+    assert(x == T(1));
+    ASSERT_NOEXCEPT(a.fetch_xor(T(0), std::memory_order_relaxed));
+  }
 }
 
 void test() { test_integral<int>(); }
diff --git a/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp b/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
index 586b5a17f82ad4..1bad4a73a83ab2 100644
--- a/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/increment_decrement.pass.cpp
@@ -14,6 +14,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -52,21 +53,33 @@ void test_integral() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert(++a == T(2));
-  assert(x == T(2));
-  ASSERT_NOEXCEPT(++a);
-
-  assert(--a == T(1));
-  assert(x == T(1));
-  ASSERT_NOEXCEPT(--a);
-
-  assert(a++ == T(1));
-  assert(x == T(2));
-  ASSERT_NOEXCEPT(++a);
-
-  assert(a-- == T(2));
-  assert(x == T(1));
-  ASSERT_NOEXCEPT(--a);
+  {
+    std::same_as<T> auto y = ++a;
+    assert(y == T(2));
+    assert(x == T(2));
+    ASSERT_NOEXCEPT(++a);
+  }
+
+  {
+    std::same_as<T> auto y = --a;
+    assert(y == T(1));
+    assert(x == T(1));
+    ASSERT_NOEXCEPT(--a);
+  }
+
+  {
+    std::same_as<T> auto y = a++;
+    assert(y == T(1));
+    assert(x == T(2));
+    ASSERT_NOEXCEPT(a++);
+  }
+
+  {
+    std::same_as<T> auto y = a--;
+    assert(y == T(2));
+    assert(x == T(1));
+    ASSERT_NOEXCEPT(a--);
+  }
 }
 
 void test() { test_integral<int>(); }
diff --git a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
index 25dec05f7fff0a..89da904e8fd469 100644
--- a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
@@ -15,13 +15,15 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 
 #include "test_macros.h"
 
 template <typename T>
 void check_always_lock_free(std::atomic_ref<T> const a) {
   if (std::atomic_ref<T>::is_always_lock_free) {
-    assert(a.is_lock_free());
+    std::same_as<bool> auto is_lock_free = a.is_lock_free();
+    assert(is_lock_free);
   }
   ASSERT_NOEXCEPT(a.is_lock_free());
 }
diff --git a/libcxx/test/std/atomics/atomics.ref/load.pass.cpp b/libcxx/test/std/atomics/atomics.ref/load.pass.cpp
index 25171fc247c1c6..e83b9cf88f41d5 100644
--- a/libcxx/test/std/atomics/atomics.ref/load.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/load.pass.cpp
@@ -10,6 +10,7 @@
 // T load(memory_order = memory_order::seq_cst) const noexcept;
 
 #include <atomic>
+#include <concepts>
 #include <cassert>
 #include <type_traits>
 
@@ -20,11 +21,17 @@ void test_load() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert(a.load() == T(1));
-  ASSERT_NOEXCEPT(a.load());
-
-  assert(a.load(std::memory_order_seq_cst) == T(1));
-  ASSERT_NOEXCEPT(a.load(std::memory_order_seq_cst));
+  {
+    std::same_as<T> auto y = a.load();
+    assert(y == T(1));
+    ASSERT_NOEXCEPT(a.load());
+  }
+
+  {
+    std::same_as<T> auto y = a.load(std::memory_order_seq_cst);
+    assert(y == T(1));
+    ASSERT_NOEXCEPT(a.load(std::memory_order_seq_cst));
+  }
 }
 
 void test() {
diff --git a/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
index 583818aeb18b8c..4265c0b6a5fb4f 100644
--- a/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/operator_minus_equals.pass.cpp
@@ -13,6 +13,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <concepts>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -33,7 +34,8 @@ void test_arithmetic() {
   T x(T(3));
   std::atomic_ref<T> const a(x);
 
-  assert((a -= T(2)) == T(1));
+  std::same_as<T> auto y = (a -= T(2));
+  assert(y == T(1));
   assert(x == T(1));
   ASSERT_NOEXCEPT(a -= T(0));
 }
@@ -45,7 +47,8 @@ void test_pointer() {
   T p{&t[3]};
   std::atomic_ref<T> const a(p);
 
-  assert((a -= 2) == &t[1]);
+  std::same_as<T> auto y = (a -= 2);
+  assert(y == &t[1]);
   assert(a == &t[1]);
   ASSERT_NOEXCEPT(a -= 0);
 }
diff --git a/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
index 196d30654f0793..a82ebae84481f0 100644
--- a/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/operator_plus_equals.pass.cpp
@@ -12,6 +12,7 @@
 // T* operator+=(difference_type) const noexcept;
 
 #include <atomic>
+#include <concepts>
 #include <cassert>
 #include <type_traits>
 
@@ -33,7 +34,8 @@ void test_arithmetic() {
   T x(T(1));
   std::atomic_ref<T> const a(x);
 
-  assert((a += T(2)) == T(3));
+  std::same_as<T> auto y = (a += T(2));
+  assert(y == T(3));
   assert(x == T(3));
   ASSERT_NOEXCEPT(a += T(0));
 }
@@ -45,7 +47,8 @@ void test_pointer() {
   T p{&t[1]};
   std::atomic_ref<T> const a(p);
 
-  assert((a += 2) == &t[3]);
+  std::same_as<T> auto y = (a += 2);
+  assert(y == &t[3]);
   assert(a == &t[3]);
   ASSERT_NOEXCEPT(a += 0);
 }

>From 6d310d82b54061c29acb9b9cbff5e0db670bd01a Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Tue, 6 Feb 2024 08:24:35 -0500
Subject: [PATCH 38/45] [libc++][atomic_ref] Forgot to update assertion msg in
 tests

---
 .../atomics.ref/assert.compare_exchange_strong.pass.cpp     | 4 ++--
 .../atomics.ref/assert.compare_exchange_weak.pass.cpp       | 4 ++--
 libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp | 4 ++--
 .../test/libcxx/atomics/atomics.ref/assert.store.pass.cpp   | 6 +++---
 libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp | 4 ++--
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
index aeab9f5ff3fb7f..5ccf2333c814f3 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -36,7 +36,7 @@ void test_compare_exchange_strong_invalid_memory_order() {
         T t(T(2));
         a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_release);
       }()),
-      "memory order argument to strong atomic compare-and-exchange operation is invalid");
+      "atomic_ref: memory order argument to strong atomic compare-and-exchange operation is invalid");
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
@@ -45,7 +45,7 @@ void test_compare_exchange_strong_invalid_memory_order() {
         T t(T(2));
         a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_acq_rel);
       }()),
-      "memory order argument to strong atomic compare-and-exchange operation is invalid");
+      "atomic_ref: memory order argument to strong atomic compare-and-exchange operation is invalid");
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
index 407146722f17c2..017d15f8c673a4 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -36,7 +36,7 @@ void test_compare_exchange_weak_invalid_memory_order() {
         T t(T(2));
         a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_release);
       }()),
-      "memory order argument to weak atomic compare-and-exchange operation is invalid");
+      "atomic_ref: memory order argument to weak atomic compare-and-exchange operation is invalid");
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
@@ -45,7 +45,7 @@ void test_compare_exchange_weak_invalid_memory_order() {
         T t(T(2));
         a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_acq_rel);
       }()),
-      "memory order argument to weak atomic compare-and-exchange operation is invalid");
+      "atomic_ref: memory order argument to weak atomic compare-and-exchange operation is invalid");
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
index 34e7a2b56f0f02..6bd5b603caf797 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
@@ -34,7 +34,7 @@ void test_load_invalid_memory_order() {
         std::atomic_ref<T> const a(x);
         (void)a.load(std::memory_order_release);
       }()),
-      "memory order argument to atomic load operation is invalid");
+      "atomic_ref: memory order argument to atomic load operation is invalid");
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
@@ -42,7 +42,7 @@ void test_load_invalid_memory_order() {
         std::atomic_ref<T> const a(x);
         (void)a.load(std::memory_order_acq_rel);
       }()),
-      "memory order argument to atomic load operation is invalid");
+      "atomic_ref: memory order argument to atomic load operation is invalid");
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
index 12e21ea38b94ab..d3290ad3afac9f 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
@@ -34,7 +34,7 @@ void test_store_invalid_memory_order() {
         std::atomic_ref<T> const a(x);
         a.store(T(2), std::memory_order_consume);
       }()),
-      "memory order argument to atomic store operation is invalid");
+      "atomic_ref: memory order argument to atomic store operation is invalid");
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
@@ -42,7 +42,7 @@ void test_store_invalid_memory_order() {
         std::atomic_ref<T> const a(x);
         a.store(T(2), std::memory_order_acquire);
       }()),
-      "memory order argument to atomic store operation is invalid");
+      "atomic_ref: memory order argument to atomic store operation is invalid");
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
@@ -50,7 +50,7 @@ void test_store_invalid_memory_order() {
         std::atomic_ref<T> const a(x);
         a.store(T(2), std::memory_order_acq_rel);
       }()),
-      "memory order argument to atomic store operation is invalid");
+      "atomic_ref: memory order argument to atomic store operation is invalid");
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
index 7604132ff694c3..4656e3e7e333b5 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
@@ -34,7 +34,7 @@ void test_wait_invalid_memory_order() {
         std::atomic_ref<T> const a(x);
         a.wait(T(2), std::memory_order_release);
       }()),
-      "memory order argument to atomic wait operation is invalid");
+      "atomic_ref: memory order argument to atomic wait operation is invalid");
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
@@ -42,7 +42,7 @@ void test_wait_invalid_memory_order() {
         std::atomic_ref<T> const a(x);
         a.wait(T(2), std::memory_order_acq_rel);
       }()),
-      "memory order argument to atomic wait operation is invalid");
+      "atomic_ref: memory order argument to atomic wait operation is invalid");
 }
 
 int main(int, char**) {

>From 7d8a1c85385ecab368e0e52ca1fab11d5720c3c6 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Tue, 6 Feb 2024 09:38:27 -0500
Subject: [PATCH 39/45] [libc++][atomic_ref] Fix test
 compare_exchange_{strong,weak} assertion msg

---
 .../atomics.ref/assert.compare_exchange_strong.pass.cpp       | 4 ++--
 .../atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
index 5ccf2333c814f3..bb9796d2d5c365 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -36,7 +36,7 @@ void test_compare_exchange_strong_invalid_memory_order() {
         T t(T(2));
         a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_release);
       }()),
-      "atomic_ref: memory order argument to strong atomic compare-and-exchange operation is invalid");
+      "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
@@ -45,7 +45,7 @@ void test_compare_exchange_strong_invalid_memory_order() {
         T t(T(2));
         a.compare_exchange_strong(t, T(3), std::memory_order_relaxed, std::memory_order_acq_rel);
       }()),
-      "atomic_ref: memory order argument to strong atomic compare-and-exchange operation is invalid");
+      "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
index 017d15f8c673a4..b5b6f27bfad6ce 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -36,7 +36,7 @@ void test_compare_exchange_weak_invalid_memory_order() {
         T t(T(2));
         a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_release);
       }()),
-      "atomic_ref: memory order argument to weak atomic compare-and-exchange operation is invalid");
+      "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
 
   TEST_LIBCPP_ASSERT_FAILURE(
       ([] {
@@ -45,7 +45,7 @@ void test_compare_exchange_weak_invalid_memory_order() {
         T t(T(2));
         a.compare_exchange_weak(t, T(3), std::memory_order_relaxed, std::memory_order_acq_rel);
       }()),
-      "atomic_ref: memory order argument to weak atomic compare-and-exchange operation is invalid");
+      "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
 }
 
 int main(int, char**) {

>From 89424db3793f88edac857c79737c823010091858 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Tue, 6 Feb 2024 15:48:04 -0500
Subject: [PATCH 40/45] [libc++][atomic_ref] Forgot to use addressof in
 __atomic_ref_base constructor

---
 libcxx/include/__atomic/atomic_ref.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index d8aaac92b303de..880118648061b1 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -44,6 +44,8 @@ template <class _Tp>
 struct __atomic_ref_base {
   _Tp* __ptr_;
 
+  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {}
+
   using value_type = _Tp;
 
   static constexpr size_t required_alignment = alignof(_Tp);
@@ -147,8 +149,6 @@ struct __atomic_ref_base {
   }
   _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__cxx_atomic_notify_one(__ptr_); }
   _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__cxx_atomic_notify_all(__ptr_); }
-
-  _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(&__obj) {}
 };
 
 template <class _Tp>

>From 956f4b3ae3d176ce88594993b68e3f89bc4e71ce Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Tue, 6 Feb 2024 22:46:41 -0500
Subject: [PATCH 41/45] [libc++][atomic_ref] Prepare the code for clearing
 padding bits

---
 libcxx/include/__atomic/atomic_ref.h | 56 +++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 6 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 880118648061b1..126fa60e1816bd 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -25,6 +25,7 @@
 #include <__concepts/same_as.h>
 #include <__config>
 #include <__memory/addressof.h>
+#include <__type_traits/has_unique_object_representation.h>
 #include <__type_traits/is_trivially_copyable.h>
 #include <cstddef>
 #include <cstdint>
@@ -44,6 +45,24 @@ template <class _Tp>
 struct __atomic_ref_base {
   _Tp* __ptr_;
 
+  // whether the object pointed to has padding AND we will actually ignore them
+  static constexpr bool __might_have_padding =
+#  if __has_builtin(__builtin_clear_padding)
+      !has_unique_object_representations_v<_Tp> && !same_as<_Tp, float> && !same_as<_Tp, double>;
+#  else
+      false;
+#  endif
+
+  _LIBCPP_HIDE_FROM_ABI static _Tp* __clear_padding(_Tp& __val) noexcept {
+    _Tp* __ptr = std::addressof(__val);
+#  if __has_builtin(__builtin_clear_padding)
+    if constexpr (__might_have_padding) {
+      __builtin_clear_padding(__ptr);
+    }
+#  endif
+    return __ptr;
+  }
+
   _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {}
 
   using value_type = _Tp;
@@ -59,7 +78,7 @@ struct __atomic_ref_base {
     _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
         __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
         "atomic_ref: memory order argument to atomic store operation is invalid");
-    __atomic_store(__ptr_, std::addressof(__desired), std::__to_gcc_order(__order));
+    __atomic_store(__ptr_, __clear_padding(__desired), std::__to_gcc_order(__order));
   }
 
   _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {
@@ -84,9 +103,34 @@ struct __atomic_ref_base {
   _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
     alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
     auto* __ret = reinterpret_cast<_Tp*>(__mem);
-    __atomic_exchange(__ptr_, std::addressof(__desired), __ret, std::__to_gcc_order(__order));
+    __atomic_exchange(__ptr_, __clear_padding(__desired), __ret, std::__to_gcc_order(__order));
     return *__ret;
   }
+
+  _LIBCPP_HIDE_FROM_ABI static bool __compare_exchange(
+      _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept {
+    if constexpr (!__might_have_padding) {
+      return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure);
+    } else { // _Tp has padding bits and __builtin_clear_padding is available
+      __clear_padding(*__desired);
+      _Tp __copy = *__expected;
+      __clear_padding(__copy);
+      while (true) {
+        _Tp __prev = __copy;
+        if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) {
+          return true;
+        }
+        _Tp __curr = __copy;
+        if (std::memcmp(__clear_padding(__prev), __clear_padding(__curr), sizeof(_Tp)) != 0) {
+          // Value representation without padding bits do not compare equal ->
+          // write the current content of *ptr into *expected
+          std::memcpy(__expected, std::addressof(__copy), sizeof(_Tp));
+          return false;
+        }
+      }
+    }
+  }
+
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
       _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
@@ -94,7 +138,7 @@ struct __atomic_ref_base {
         __failure == memory_order::relaxed || __failure == memory_order::consume ||
             __failure == memory_order::acquire || __failure == memory_order::seq_cst,
         "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
-    return __atomic_compare_exchange(
+    return __compare_exchange(
         __ptr_,
         std::addressof(__expected),
         std::addressof(__desired),
@@ -109,7 +153,7 @@ struct __atomic_ref_base {
         __failure == memory_order::relaxed || __failure == memory_order::consume ||
             __failure == memory_order::acquire || __failure == memory_order::seq_cst,
         "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
-    return __atomic_compare_exchange(
+    return __compare_exchange(
         __ptr_,
         std::addressof(__expected),
         std::addressof(__desired),
@@ -120,7 +164,7 @@ struct __atomic_ref_base {
 
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
-    return __atomic_compare_exchange(
+    return __compare_exchange(
         __ptr_,
         std::addressof(__expected),
         std::addressof(__desired),
@@ -130,7 +174,7 @@ struct __atomic_ref_base {
   }
   _LIBCPP_HIDE_FROM_ABI bool
   compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
-    return __atomic_compare_exchange(
+    return __compare_exchange(
         __ptr_,
         std::addressof(__expected),
         std::addressof(__desired),

>From 6d5c9de025876f2f8c2a11877a682be5e5eb0405 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Wed, 7 Feb 2024 09:04:54 -0500
Subject: [PATCH 42/45] [libc++][atomic_ref] Include missing <cstring> header
 for memcmp and memcpy

---
 libcxx/include/__atomic/atomic_ref.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 126fa60e1816bd..425acdfdb7cd38 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -29,6 +29,7 @@
 #include <__type_traits/is_trivially_copyable.h>
 #include <cstddef>
 #include <cstdint>
+#include <cstring>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header

>From 8ad10b6b0af000972e0656795e18735d84cb4fb7 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Wed, 7 Feb 2024 08:25:00 -0500
Subject: [PATCH 43/45] [libc++][atomic_ref] Avoid sneaky casting `T x(T(1))``

---
 .../std/atomics/atomics.ref/convert.pass.cpp  | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp b/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
index b965607950b0f6..1ac1d1ea3b28da 100644
--- a/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/convert.pass.cpp
@@ -16,30 +16,33 @@
 #include "test_macros.h"
 
 template <typename T>
-void test_convert() {
-  T x(T(1));
-  std::atomic_ref<T> const a(x);
+void test_convert(T const& x) {
+  T copy = x;
+  std::atomic_ref<T> const a(copy);
 
   T converted = a;
-  assert(converted == T(1));
+  assert(converted == x);
 
   ASSERT_NOEXCEPT(T(a));
   static_assert(std::is_nothrow_convertible_v<std::atomic_ref<T>, T>);
 }
 
 void test() {
-  test_convert<int>();
+  int i = 1;
+  test_convert<int>(i);
 
-  test_convert<float>();
+  float f = 1;
+  test_convert<float>(f);
 
-  test_convert<int*>();
+  int* p = &i;
+  test_convert<int*>(p);
 
   struct X {
     int i;
     X(int ii) noexcept : i(ii) {}
     bool operator==(X o) const { return i == o.i; }
-  };
-  test_convert<X>();
+  } x{1};
+  test_convert<X>(x);
 }
 
 int main(int, char**) {

>From ee7cb8080bded33f7d4aeae0de54379a43dafd13 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Wed, 7 Feb 2024 08:56:31 -0500
Subject: [PATCH 44/45] [libc++][atomic_ref] Check more types for
 is_always_lock_free

---
 .../atomics.ref/is_always_lock_free.pass.cpp  | 27 ++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
index 89da904e8fd469..ef035a558b39f8 100644
--- a/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.ref/is_always_lock_free.pass.cpp
@@ -28,6 +28,13 @@ void check_always_lock_free(std::atomic_ref<T> const a) {
   ASSERT_NOEXCEPT(a.is_lock_free());
 }
 
+#define CHECK_ALWAYS_LOCK_FREE(T)                                                                                      \
+  do {                                                                                                                 \
+    typedef T type;                                                                                                    \
+    type obj{};                                                                                                        \
+    check_always_lock_free(std::atomic_ref<type>(obj));                                                                \
+  } while (0)
+
 void test() {
   int i = 0;
   check_always_lock_free(std::atomic_ref<int>(i));
@@ -38,9 +45,23 @@ void test() {
   int* p = &i;
   check_always_lock_free(std::atomic_ref<int*>(p));
 
-  struct X {
-  } x;
-  check_always_lock_free(std::atomic_ref<X>(x));
+  CHECK_ALWAYS_LOCK_FREE(struct Empty{});
+  CHECK_ALWAYS_LOCK_FREE(struct OneInt { int i; });
+  CHECK_ALWAYS_LOCK_FREE(struct IntArr2 { int i[2]; });
+  CHECK_ALWAYS_LOCK_FREE(struct FloatArr3 { float i[3]; });
+  CHECK_ALWAYS_LOCK_FREE(struct LLIArr2 { long long int i[2]; });
+  CHECK_ALWAYS_LOCK_FREE(struct LLIArr4 { long long int i[4]; });
+  CHECK_ALWAYS_LOCK_FREE(struct LLIArr8 { long long int i[8]; });
+  CHECK_ALWAYS_LOCK_FREE(struct LLIArr16 { long long int i[16]; });
+  CHECK_ALWAYS_LOCK_FREE(struct Padding {
+    char c; /* padding */
+    long long int i;
+  });
+  CHECK_ALWAYS_LOCK_FREE(union IntFloat {
+    int i;
+    float f;
+  });
+  CHECK_ALWAYS_LOCK_FREE(enum class CharEnumClass : char{foo});
 }
 
 int main(int, char**) {

>From 6ef3362b1ba7b1a0bb49e674a7e060708c121940 Mon Sep 17 00:00:00 2001
From: Damien L-G <dalg24 at gmail.com>
Date: Wed, 7 Feb 2024 09:00:04 -0500
Subject: [PATCH 45/45] [libc++][atomic_ref] Use fake minimally aligned pointer
 in is_lock_free()

---
 libcxx/include/__atomic/atomic_ref.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index 425acdfdb7cd38..bf5a45d4eb8d70 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -70,9 +70,12 @@ struct __atomic_ref_base {
 
   static constexpr size_t required_alignment = alignof(_Tp);
 
-  static constexpr bool is_always_lock_free = __atomic_always_lock_free(sizeof(_Tp), nullptr);
+  static constexpr bool is_always_lock_free =
+      __atomic_always_lock_free(sizeof(_Tp), reinterpret_cast<void*>(-required_alignment));
 
-  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); }
+  _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept {
+    return __atomic_is_lock_free(sizeof(_Tp), reinterpret_cast<void*>(-required_alignment));
+  }
 
   _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
       _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {

