[libcxx-commits] [libcxx] [libc++] Implement C++20 atomic_ref (PR #76647)
Damien L-G via libcxx-commits
libcxx-commits at lists.llvm.org
Mon Mar 11 12:39:26 PDT 2024
================
@@ -0,0 +1,343 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// Kokkos v. 4.0
+// Copyright (2022) National Technology & Engineering
+// Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H
+#define _LIBCPP___ATOMIC_ATOMIC_REF_H
+
+#include <__assert>
+#include <__atomic/atomic_sync.h>
+#include <__atomic/check_memory_order.h>
+#include <__atomic/to_gcc_order.h>
+#include <__concepts/arithmetic.h>
+#include <__concepts/same_as.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__type_traits/has_unique_object_representation.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+struct __atomic_ref_base {
+ _Tp* __ptr_;
+
+ _LIBCPP_HIDE_FROM_ABI static _Tp* __clear_padding(_Tp& __val) noexcept {
+ _Tp* __ptr = std::addressof(__val);
+# if __has_builtin(__builtin_clear_padding)
+ __builtin_clear_padding(__ptr);
+# endif
+ return __ptr;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {}
+
+ using value_type = _Tp;
+
+ static constexpr size_t required_alignment = alignof(_Tp);
+
+ static constexpr bool is_always_lock_free =
+ __atomic_always_lock_free(sizeof(_Tp), reinterpret_cast<void*>(-required_alignment));
+
+ _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); }
+
+ _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
+ _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
+ _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+ __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
+ "atomic_ref: memory order argument to atomic store operation is invalid");
+ __atomic_store(__ptr_, __clear_padding(__desired), std::__to_gcc_order(__order));
+ }
+
+ _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {
+ store(__desired);
+ return __desired;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
+ _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {
+ _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+ __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
+ __order == memory_order::seq_cst,
+ "atomic_ref: memory order argument to atomic load operation is invalid");
+ alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
+ auto* __ret = reinterpret_cast<_Tp*>(__mem);
+ __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order));
+ return *__ret;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); }
+
+ _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+ alignas(_Tp) unsigned char __mem[sizeof(_Tp)];
+ auto* __ret = reinterpret_cast<_Tp*>(__mem);
+ __atomic_exchange(__ptr_, __clear_padding(__desired), __ret, std::__to_gcc_order(__order));
+ return *__ret;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI static bool __compare_exchange(
+ _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept {
+ if constexpr (
+# if __has_builtin(__builtin_clear_padding)
+ has_unique_object_representations_v<_Tp> || same_as<_Tp, float> || same_as<_Tp, double>
+# else
+ true // NOLINT(readability-simplify-boolean-expr)
+# endif
+ ) {
+ return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure);
+ } else { // _Tp has padding bits and __builtin_clear_padding is available
+ __clear_padding(*__desired);
+ _Tp __copy = *__expected;
+ __clear_padding(__copy);
+ while (true) {
+ _Tp __prev = __copy;
+ if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) {
+ return true;
+ }
+ _Tp __curr = __copy;
+ if (std::memcmp(__clear_padding(__prev), __clear_padding(__curr), sizeof(_Tp)) != 0) {
+ // Value representation without padding bits do not compare equal ->
+ // write the current content of *ptr into *expected
+ std::memcpy(__expected, std::addressof(__copy), sizeof(_Tp));
+ return false;
+ }
+ }
+ }
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool
+ compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+ __failure == memory_order::relaxed || __failure == memory_order::consume ||
+ __failure == memory_order::acquire || __failure == memory_order::seq_cst,
+ "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
+ return __compare_exchange(
+ __ptr_,
+ std::addressof(__expected),
+ std::addressof(__desired),
+ true,
+ std::__to_gcc_order(__success),
+ std::__to_gcc_order(__failure));
+ }
+ _LIBCPP_HIDE_FROM_ABI bool
+ compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
+ _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
+ _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+ __failure == memory_order::relaxed || __failure == memory_order::consume ||
+ __failure == memory_order::acquire || __failure == memory_order::seq_cst,
+ "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
+ return __compare_exchange(
+ __ptr_,
+ std::addressof(__expected),
+ std::addressof(__desired),
+ false,
+ std::__to_gcc_order(__success),
+ std::__to_gcc_order(__failure));
+ }
+
+ _LIBCPP_HIDE_FROM_ABI bool
+ compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+ return __compare_exchange(
+ __ptr_,
+ std::addressof(__expected),
+ std::addressof(__desired),
+ true,
+ std::__to_gcc_order(__order),
+ std::__to_gcc_failure_order(__order));
+ }
+ _LIBCPP_HIDE_FROM_ABI bool
+ compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
+ return __compare_exchange(
+ __ptr_,
+ std::addressof(__expected),
+ std::addressof(__desired),
+ false,
+ std::__to_gcc_order(__order),
+ std::__to_gcc_failure_order(__order));
+ }
+
+ _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
+ _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) {
+ _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+ __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
+ __order == memory_order::seq_cst,
+ "atomic_ref: memory order argument to atomic wait operation is invalid");
+ std::__atomic_wait(*this, __old, __order);
+ }
+ _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__atomic_notify_one(*this); }
+ _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__atomic_notify_all(*this); }
+};
+
+template <class _Tp>
+struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {
+ static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) {
+ return __a.load(__order);
+ }
+ static _LIBCPP_HIDE_FROM_ABI const _Tp* __atomic_contention_address(const __atomic_ref_base<_Tp>& __a) {
+ return __a.__ptr_;
+ }
+};
+
+template <class _Tp>
+struct atomic_ref : public __atomic_ref_base<_Tp> {
+ static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
+
+ using __base = __atomic_ref_base<_Tp>;
+
+ _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+ _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
+ "atomic_ref ctor: referenced object must be aligned to required_alignment");
+ }
+
+ _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
+
+ _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
+
+ atomic_ref& operator=(const atomic_ref&) = delete;
+};
+
+template <class _Tp>
+ requires(std::integral<_Tp> && !std::same_as<bool, _Tp>)
+struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
+ using __base = __atomic_ref_base<_Tp>;
+
+ using difference_type = __base::value_type;
+
+ _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
+ _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN((uintptr_t)addressof(__obj) % __base::required_alignment == 0,
----------------
dalg24 wrote:
Done
https://github.com/llvm/llvm-project/pull/76647
More information about the libcxx-commits
mailing list