[libcxx-commits] [libcxx] [libc++] Implement P0528R3 `std::atomic` CAS for types with paddings (PR #76180)
via libcxx-commits
libcxx-commits at lists.llvm.org
Sun Jun 7 13:41:04 PDT 2026
https://github.com/huixie90 updated https://github.com/llvm/llvm-project/pull/76180
>From 483a78c09bc191bcaa08956cc3670045cd854944 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie0621 at gmail.com>
Date: Thu, 21 Dec 2023 20:31:29 +0000
Subject: [PATCH] [libc++] make std::atomic works with types with paddings
---
libcxx/include/CMakeLists.txt | 1 +
libcxx/include/__atomic/support/c11.h | 83 ++++--
libcxx/include/__atomic/support/common.h | 75 ++++++
.../atomics.types.generic/padding.pass.cpp | 241 ++++++++++++++++++
4 files changed, 375 insertions(+), 25 deletions(-)
create mode 100644 libcxx/include/__atomic/support/common.h
create mode 100644 libcxx/test/std/atomics/atomics.types.generic/padding.pass.cpp
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 10dfb4b4d58cb..5e1a7afea06a2 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -226,6 +226,7 @@ set(files
__atomic/kill_dependency.h
__atomic/memory_order.h
__atomic/support.h
+ __atomic/support/common.h
__atomic/support/c11.h
__atomic/support/gcc.h
__atomic/to_gcc_order.h
diff --git a/libcxx/include/__atomic/support/c11.h b/libcxx/include/__atomic/support/c11.h
index 1ad299882a12a..8d8254f4936b4 100644
--- a/libcxx/include/__atomic/support/c11.h
+++ b/libcxx/include/__atomic/support/c11.h
@@ -9,6 +9,7 @@
#ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H
#define _LIBCPP___ATOMIC_SUPPORT_C11_H
+#include <__atomic/support/common.h>
#include <__atomic/memory_order.h>
#include <__config>
#include <__cstddef/ptrdiff_t.h>
@@ -30,11 +31,24 @@ struct __cxx_atomic_base_impl {
_LIBCPP_HIDE_FROM_ABI
#ifndef _LIBCPP_CXX03_LANG
__cxx_atomic_base_impl() _NOEXCEPT = default;
+
+# if _LIBCPP_STD_VER >= 20 && __has_builtin(__builtin_clear_padding)
+ __cxx_atomic_base_impl() noexcept
+ requires __needs_clear_padding<_Tp>::value
+ : __a_value() {
+ if (!__builtin_is_constant_evaluated()) {
+ __builtin_clear_padding(__a_value);
+ }
+ }
+# endif // _LIBCPP_STD_VER >= 20 && __has_builtin(__builtin_clear_padding)
+
#else
__cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
}
#endif // _LIBCPP_CXX03_LANG
- _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
+ _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {
+ std::__clear_padding_if_needed(__a_value);
+ }
_Atomic(_Tp) __a_value;
};
@@ -50,21 +64,25 @@ _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
+ std::__clear_padding_if_needed(__val);
__c11_atomic_init(std::addressof(__a->__a_value), __val);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
+ std::__clear_padding_if_needed(__val);
__c11_atomic_init(std::addressof(__a->__a_value), __val);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
+ std::__clear_padding_if_needed(__val);
__c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
+ std::__clear_padding_if_needed(__val);
__c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
}
@@ -100,12 +118,14 @@ __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, me
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
+ std::__clear_padding_if_needed(__value);
return __c11_atomic_exchange(
std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
+ std::__clear_padding_if_needed(__value);
return __c11_atomic_exchange(
std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
}
@@ -124,23 +144,29 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
_Tp __value,
memory_order __success,
memory_order __failure) _NOEXCEPT {
- return __c11_atomic_compare_exchange_strong(
- std::addressof(__a->__a_value),
- __expected,
- __value,
- static_cast<__memory_order_underlying_t>(__success),
- static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+ return __atomic_cas_with_clear_padding(
+ __expected, __value, [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) {
+ return __c11_atomic_compare_exchange_strong(
+ std::addressof(__a->__a_value),
+ __expected_or_copy,
+ __value_maybe_padding_cleared,
+ static_cast<__memory_order_underlying_t>(__success),
+ static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+ });
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
__cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
_NOEXCEPT {
- return __c11_atomic_compare_exchange_strong(
- std::addressof(__a->__a_value),
- __expected,
- __value,
- static_cast<__memory_order_underlying_t>(__success),
- static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+ return __atomic_cas_with_clear_padding(
+ __expected, __value, [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) {
+ return __c11_atomic_compare_exchange_strong(
+ std::addressof(__a->__a_value),
+ __expected_or_copy,
+ __value_maybe_padding_cleared,
+ static_cast<__memory_order_underlying_t>(__success),
+ static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+ });
}
template <class _Tp>
@@ -150,23 +176,30 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
_Tp __value,
memory_order __success,
memory_order __failure) _NOEXCEPT {
- return __c11_atomic_compare_exchange_weak(
- std::addressof(__a->__a_value),
- __expected,
- __value,
- static_cast<__memory_order_underlying_t>(__success),
- static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+ return __atomic_cas_with_clear_padding(
+ __expected, __value, [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) {
+ return __c11_atomic_compare_exchange_weak(
+ std::addressof(__a->__a_value),
+ __expected_or_copy,
+ __value_maybe_padding_cleared,
+ static_cast<__memory_order_underlying_t>(__success),
+ static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+ });
}
+
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
__cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
_NOEXCEPT {
- return __c11_atomic_compare_exchange_weak(
- std::addressof(__a->__a_value),
- __expected,
- __value,
- static_cast<__memory_order_underlying_t>(__success),
- static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+ return __atomic_cas_with_clear_padding(
+ __expected, __value, [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) {
+ return __c11_atomic_compare_exchange_weak(
+ std::addressof(__a->__a_value),
+ __expected_or_copy,
+ __value_maybe_padding_cleared,
+ static_cast<__memory_order_underlying_t>(__success),
+ static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+ });
}
template <class _Tp>
diff --git a/libcxx/include/__atomic/support/common.h b/libcxx/include/__atomic/support/common.h
new file mode 100644
index 0000000000000..fe02e3736f88c
--- /dev/null
+++ b/libcxx/include/__atomic/support/common.h
@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_SUPPORT_COMMON_H
+#define _LIBCPP___ATOMIC_SUPPORT_COMMON_H
+
+#include <__config>
+#include <__memory/addressof.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/has_unique_object_representation.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/negation.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/forward.h>
+#include <cstring>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 20 && __has_builtin(__builtin_clear_padding)
+
+template <class _Tp>
+struct __needs_clear_padding
+ : _And<_Not<has_unique_object_representations<_Tp>>, _Not<is_same<_Tp, float>>, _Not<is_same<_Tp, double>>> {};
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr void __clear_padding_if_needed(_Tp&& __obj) noexcept {
+ if constexpr (__needs_clear_padding<remove_cvref_t<_Tp>>::value) {
+ if (!__builtin_is_constant_evaluated()) {
+ __builtin_clear_padding(std::addressof(__obj));
+ }
+ }
+}
+
+template <class _Tp, class _Up, class _CasFunc>
+_LIBCPP_HIDE_FROM_ABI bool __atomic_cas_with_clear_padding(_Tp* __expected, _Up&& __value, _CasFunc&& __cas_func) {
+ if constexpr (!__needs_clear_padding<remove_cv_t<_Tp>>::value) {
+ return __cas_func(__expected, std::forward<_Up>(__value));
+ } else {
+ std::__clear_padding_if_needed(__value);
+ remove_cv_t<_Tp> __expected_copy = *__expected;
+ std::__clear_padding_if_needed(__expected_copy);
+ if (__cas_func(std::addressof(__expected_copy), std::forward<_Up>(__value))) {
+ return true;
+ } else {
+ std::memcpy(__expected, std::addressof(__expected_copy), sizeof(remove_cv_t<_Tp>));
+ return false;
+ }
+ }
+}
+
+#else // _LIBCPP_STD_VER >= 20 && __has_builtin(__builtin_clear_padding)
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR void __clear_padding_if_needed(_Tp&&) noexcept {}
+
+template <class _Tp, class _Up, class _CasFunc>
+_LIBCPP_HIDE_FROM_ABI bool __atomic_cas_with_clear_padding(_Tp* __expected, _Up&& __value, _CasFunc&& __cas_func) {
+ return __cas_func(__expected, std::forward<_Up>(__value));
+}
+
+#endif // _LIBCPP_STD_VER >= 20 && __has_builtin(__builtin_clear_padding)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_SUPPORT_COMMON_H
diff --git a/libcxx/test/std/atomics/atomics.types.generic/padding.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/padding.pass.cpp
new file mode 100644
index 0000000000000..e8ea1ecbddd81
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.types.generic/padding.pass.cpp
@@ -0,0 +1,241 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// XFAIL: !has-64-bit-atomics
+
+// atomic_init is deprecated
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
+#include <atomic>
+#include <cassert>
+#include <cstring>
+#include <type_traits>
+
+struct Foo {
+ int i;
+ char c;
+};
+
+static_assert(!std::has_unique_object_representations_v<Foo>);
+static_assert(sizeof(Foo) > sizeof(int) + sizeof(char));
+
+Foo make_foo(int i, char c, unsigned char pad_byte) {
+ Foo f;
+ std::memset(&f, pad_byte, sizeof(Foo));
+ f.i = i;
+ f.c = c;
+ return f;
+}
+
+void assert_foo_padding(const Foo& f, unsigned char pad_byte) {
+ alignas(Foo) unsigned char buf[sizeof(Foo)];
+ std::memset(buf, pad_byte, sizeof(Foo));
+ Foo& reference = *reinterpret_cast<Foo*>(buf);
+ reference.i = f.i;
+ reference.c = f.c;
+ assert(std::memcmp(&f, &reference, sizeof(Foo)) == 0);
+}
+
+#if __has_builtin(__builtin_clear_padding)
+
+void test_default_constructor() {
+ std::atomic<Foo> a;
+ Foo loaded = a.load();
+ assert(loaded.i == 0);
+ assert(loaded.c == '\0');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_value_constructor() {
+ Foo init = make_foo(10, 'a', 0xBB);
+ assert_foo_padding(init, 0xBB);
+ std::atomic<Foo> a(init);
+ Foo loaded = a.load();
+ assert(loaded.i == 10);
+ assert(loaded.c == 'a');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_store() {
+ std::atomic<Foo> a;
+ Foo value = make_foo(5, 'x', 0xAB);
+ assert_foo_padding(value, 0xAB);
+ a.store(value);
+ Foo loaded = a.load();
+ assert(loaded.i == 5);
+ assert(loaded.c == 'x');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_exchange() {
+ Foo initial = make_foo(1, 'a', 0x00);
+ assert_foo_padding(initial, 0x00);
+ std::atomic<Foo> a(initial);
+ Foo new_val = make_foo(2, 'b', 0xCD);
+ assert_foo_padding(new_val, 0xCD);
+ Foo old = a.exchange(new_val);
+ assert(old.i == 1);
+ assert(old.c == 'a');
+ assert_foo_padding(old, 0);
+ Foo loaded = a.load();
+ assert(loaded.i == 2);
+ assert(loaded.c == 'b');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_atomic_init() {
+ std::atomic<Foo> a;
+ Foo init = make_foo(7, 'z', 0xEF);
+ assert_foo_padding(init, 0xEF);
+ std::atomic_init(&a, init);
+ Foo loaded = a.load();
+ assert(loaded.i == 7);
+ assert(loaded.c == 'z');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_compare_exchange_strong_success_padding_only() {
+ // CAS should succeed when only padding differs in expected; expected is unchanged.
+ std::atomic<Foo> a;
+
+ Foo init = make_foo(10, 'a', 0xBB);
+ assert_foo_padding(init, 0xBB);
+ a.store(init);
+
+ Foo expected = make_foo(10, 'a', 0xAA);
+ assert_foo_padding(expected, 0xAA);
+
+ alignas(Foo) char original_expected[sizeof(Foo)];
+ std::memcpy(original_expected, &expected, sizeof(Foo));
+
+ Foo new_value = make_foo(42, 'b', 0xCC);
+ assert_foo_padding(new_value, 0xCC);
+
+ bool r = a.compare_exchange_strong(expected, new_value);
+
+ assert(r);
+ assert(std::memcmp(&expected, original_expected, sizeof(Foo)) == 0);
+ Foo loaded = a.load();
+ assert(loaded.i == 42);
+ assert(loaded.c == 'b');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_compare_exchange_strong_failure() {
+ std::atomic<Foo> a;
+ Foo stored = make_foo(10, 'a', 0xBB);
+ assert_foo_padding(stored, 0xBB);
+ a.store(stored);
+
+ Foo expected = make_foo(99, 'a', 0xAA);
+ assert_foo_padding(expected, 0xAA);
+ Foo new_value = make_foo(42, 'b', 0xCC);
+ assert_foo_padding(new_value, 0xCC);
+
+ bool r = a.compare_exchange_strong(expected, new_value);
+
+ assert(!r);
+ assert(expected.i == 10);
+ assert(expected.c == 'a');
+ assert_foo_padding(expected, 0);
+ Foo loaded = a.load();
+ assert(loaded.i == 10);
+ assert(loaded.c == 'a');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_compare_exchange_weak_success_padding_only() {
+ std::atomic<Foo> a;
+ Foo stored = make_foo(10, 'a', 0xBB);
+ assert_foo_padding(stored, 0xBB);
+ a.store(stored);
+
+ Foo new_value = make_foo(42, 'b', 0xCC);
+ assert_foo_padding(new_value, 0xCC);
+
+ Foo original_expected = make_foo(10, 'a', 0xAA);
+ assert_foo_padding(original_expected, 0xAA);
+
+ bool r = false;
+ while (!r) {
+ Foo expected = make_foo(10, 'a', 0xAA);
+ assert_foo_padding(expected, 0xAA);
+ r = a.compare_exchange_weak(expected, new_value);
+ if (r) {
+ assert(std::memcmp(&expected, &original_expected, sizeof(Foo)) == 0);
+ } else {
+ // Spurious failure: expected is updated to the current atomic value.
+ assert(expected.i == 10);
+ assert(expected.c == 'a');
+ assert_foo_padding(expected, 0);
+ }
+ }
+
+ Foo loaded = a.load();
+ assert(loaded.i == 42);
+ assert(loaded.c == 'b');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_compare_exchange_weak_failure() {
+ std::atomic<Foo> a;
+ Foo stored = make_foo(10, 'a', 0xBB);
+ assert_foo_padding(stored, 0xBB);
+ a.store(stored);
+
+ Foo expected = make_foo(99, 'a', 0xAA);
+ assert_foo_padding(expected, 0xAA);
+ Foo new_value = make_foo(42, 'b', 0xCC);
+ assert_foo_padding(new_value, 0xCC);
+
+ bool r = a.compare_exchange_weak(expected, new_value);
+
+ assert(!r);
+ assert(expected.i == 10);
+ assert(expected.c == 'a');
+ assert_foo_padding(expected, 0);
+ Foo loaded = a.load();
+ assert(loaded.i == 10);
+ assert(loaded.c == 'a');
+ assert_foo_padding(loaded, 0);
+}
+
+void test_no_padding_type() {
+ // Types with unique object representations skip the padding-clearing path.
+ std::atomic<int> a(1);
+ int expected = 1;
+ assert(a.compare_exchange_strong(expected, 2));
+ assert(expected == 1);
+ assert(a.load() == 2);
+
+ expected = 3;
+ assert(!a.compare_exchange_strong(expected, 4));
+ assert(expected == 2);
+ assert(a.load() == 2);
+}
+
+int main(int, char**) {
+ test_default_constructor();
+ test_value_constructor();
+ test_store();
+ test_exchange();
+ test_atomic_init();
+ test_compare_exchange_strong_success_padding_only();
+ test_compare_exchange_strong_failure();
+ test_compare_exchange_weak_success_padding_only();
+ test_compare_exchange_weak_failure();
+ test_no_padding_type();
+
+ return 0;
+}
+
+#else // !__has_builtin(__builtin_clear_padding)
+
+int main(int, char**) { return 0; }
+
+#endif // __has_builtin(__builtin_clear_padding)
More information about the libcxx-commits
mailing list