[libcxx-commits] [libcxx] 73a1383 - [libc++] Allows any types of size 4 and 8 to use native platform ulock_wait (#161086)
via libcxx-commits
libcxx-commits at lists.llvm.org
Sun Dec 7 04:04:15 PST 2025
Author: Hui
Date: 2025-12-07T12:04:11Z
New Revision: 73a13839d3ec4dab9330b3909890105efad9b3bd
URL: https://github.com/llvm/llvm-project/commit/73a13839d3ec4dab9330b3909890105efad9b3bd
DIFF: https://github.com/llvm/llvm-project/commit/73a13839d3ec4dab9330b3909890105efad9b3bd.diff
LOG: [libc++] Allows any types of size 4 and 8 to use native platform ulock_wait (#161086)
This is to address #146145
The issue before was that, for `std::atomic::wait/notify`, we only
support `uint64_t` to go through the native `ulock_wait` directly. Any
other types will go through the global contention table's `atomic`,
increasing the chances of spurious wakeup. This PR tries to allow any
types that are of size 4 or 8 to directly go to the `ulock_wait`.
This PR is just proof of concept. If we like this idea, I can go further
to update the Linux/FreeBSD branch and add ABI macros so the existing
behaviours are reserved under the stable ABI
Here are some benchmark results
```
Benchmark Time CPU Time Old Time New CPU Old CPU New
----------------------------------------------------------------------------------------------------------------------------------------------------
BM_stop_token_single_thread_reg_unreg_callback/1024 -0.1113 -0.1165 51519 45785 51397 45408
BM_stop_token_single_thread_reg_unreg_callback/4096 -0.2727 -0.1447 249685 181608 211865 181203
BM_stop_token_single_thread_reg_unreg_callback/65536 -0.1241 -0.1237 3308930 2898396 3300986 2892608
BM_stop_token_single_thread_reg_unreg_callback/262144 +0.0335 -0.1920 13237682 13681632 13208849 10673254
OVERALL_GEOMEAN -0.1254 -0.1447 0 0 0 0
```
```
Benchmark Time CPU Time Old Time New CPU Old CPU New
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
BM_1_atomic_1_waiter_1_notifier<KeepNotifying, NumHighPrioTasks<0>>/65536 -0.3344 -0.2424 5960741 3967212 5232250 3964085
BM_1_atomic_1_waiter_1_notifier<KeepNotifying, NumHighPrioTasks<0>>/131072 -0.1474 -0.1475 9144356 7796745 9137547 7790193
BM_1_atomic_1_waiter_1_notifier<KeepNotifying, NumHighPrioTasks<0>>/262144 -0.1336 -0.1340 18333441 15883805 18323711 15868500
OVERALL_GEOMEAN -0.2107 -0.1761 0 0 0 0
```
```
Benchmark Time CPU Time Old Time New CPU Old CPU New
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<2>, NumHighPrioTasks<0>>/16384 +0.2321 -0.0081 836618 1030772 833197 826476
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<2>, NumHighPrioTasks<0>>/32768 -0.3034 -0.1329 2182721 1520569 1747211 1515028
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<2>, NumHighPrioTasks<0>>/65536 -0.0924 -0.0924 3389098 3075897 3378486 3066448
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<8>, NumHighPrioTasks<0>>/4096 +0.0464 +0.0474 664233 695080 657736 688892
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<8>, NumHighPrioTasks<0>>/8192 -0.0279 -0.0267 1336041 1298794 1324270 1288953
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<8>, NumHighPrioTasks<0>>/16384 +0.0270 +0.0304 2543004 2611786 2517471 2593975
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<32>, NumHighPrioTasks<0>>/1024 +0.0423 +0.0941 473621 493657 325604 356245
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<32>, NumHighPrioTasks<0>>/2048 +0.0420 +0.0675 906266 944349 636253 679169
BM_1_atomic_multi_waiter_1_notifier<KeepNotifying, NumWaitingThreads<32>, NumHighPrioTasks<0>>/4096 +0.0359 +0.0378 1761584 1824783 1015092 1053439
OVERALL_GEOMEAN -0.0097 -0.0007 0 0 0 0
```
```
Benchmark Time CPU Time Old Time New CPU Old CPU New
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<0>>/4096 -0.0990 -0.1001 371100 334370 369984 332955
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<0>>/8192 -0.0305 -0.0314 698228 676908 696418 674585
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<0>>/16384 -0.0258 -0.0268 1383530 1347894 1380665 1343680
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<8>, NumHighPrioTasks<0>>/1024 +0.0465 +0.4702 937821 981388 472087 694082
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<8>, NumHighPrioTasks<0>>/2048 +0.1596 +0.9140 1704819 1976899 616419 1179852
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<8>, NumHighPrioTasks<0>>/4096 -0.1018 -0.2316 3793976 3407609 1912209 1469331
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<32>, NumHighPrioTasks<0>>/256 +0.0395 +0.5818 30102662 31292982 174650 276270
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<32>, NumHighPrioTasks<0>>/512 -0.0065 +1.2860 33079634 32863968 162150 370680
BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<32>, NumHighPrioTasks<0>>/1024 -0.0325 +0.4683 36581740 35392385 282320 414520
OVERALL_GEOMEAN -0.0084 +0.2878 0 0 0 0
```
---------
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
Added:
libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/lost_wakeup.pass.cpp
Modified:
libcxx/docs/ABIGuarantees.rst
libcxx/docs/ReleaseNotes/22.rst
libcxx/include/__atomic/atomic.h
libcxx/include/__atomic/atomic_flag.h
libcxx/include/__atomic/atomic_ref.h
libcxx/include/__atomic/atomic_sync.h
libcxx/include/__atomic/contention_t.h
libcxx/include/__configuration/abi.h
libcxx/include/__configuration/availability.h
libcxx/lib/abi/CHANGELOG.TXT
libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
libcxx/src/atomic.cpp
libcxx/test/libcxx/atomics/atomics.syn/wait.issue_85107.pass.cpp
Removed:
################################################################################
diff --git a/libcxx/docs/ABIGuarantees.rst b/libcxx/docs/ABIGuarantees.rst
index 4d4674c7756a4..e680f5429880e 100644
--- a/libcxx/docs/ABIGuarantees.rst
+++ b/libcxx/docs/ABIGuarantees.rst
@@ -205,6 +205,16 @@ This flag fixes the implementation of CityHash used for ``hash<fundamental-type>
CityHash has the problem that it drops some bits on the floor. Fixing the implementation changes the hash of values,
resulting in an ABI break.
+``_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE``
+------------------------------------------
+This flag changes the implementation of ``atomic::wait()`` and ``atomic::notify_one()/notify_all()`` to use the
+native atomic wait/notify operations on platforms that support them based on the size of the atomic type, instead
+of the type itself. This means for example that a type with ``sizeof(T) == 4`` on Linux that doesn't have padding
+bytes would be able to use the underlying platform's atomic wait primitive, which is otherwise only used for ``int32_t``.
+Since the whole program must use the same implementation for correctness, changing this is an ABI break since libc++
+supports linking against TUs that were compiled against older versions of the library.
+
+
inline namespaces
=================
Inline namespaces which contain types that are observable by the user need to be kept the same, since they affect
diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst
index 6ea323ced3003..9f1e3d570f254 100644
--- a/libcxx/docs/ReleaseNotes/22.rst
+++ b/libcxx/docs/ReleaseNotes/22.rst
@@ -82,6 +82,8 @@ Improvements and New Features
- The ``std::{generate, generate_n}`` and ``std::ranges::generate_n`` algorithms have been optimized for segmented
iterators, resulting in a performance improvement for ``std::deque<short>`` and
``std::join_view<vector<vector<short>>>`` iterators.
+- ``std::atomic::wait`` has been refactored to accept more types to use platform native wait functions directly.
+ This is guarded behind the ABI Macro ``_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE``.
- The ``num_get::do_get`` integral overloads have been optimized, resulting in a performance improvement of up to 2.8x.
diff --git a/libcxx/include/__atomic/atomic.h b/libcxx/include/__atomic/atomic.h
index 44835820bb06c..554c111d695f2 100644
--- a/libcxx/include/__atomic/atomic.h
+++ b/libcxx/include/__atomic/atomic.h
@@ -206,6 +206,8 @@ struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
template <class _Tp, bool _IsIntegral>
struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
+ using __value_type _LIBCPP_NODEBUG = _Tp;
+
static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
return __a.load(__order);
}
diff --git a/libcxx/include/__atomic/atomic_flag.h b/libcxx/include/__atomic/atomic_flag.h
index 28ed2d53b77da..321a6283ba7ad 100644
--- a/libcxx/include/__atomic/atomic_flag.h
+++ b/libcxx/include/__atomic/atomic_flag.h
@@ -76,6 +76,8 @@ struct atomic_flag {
template <>
struct __atomic_waitable_traits<atomic_flag> {
+ using __value_type _LIBCPP_NODEBUG = _LIBCPP_ATOMIC_FLAG_TYPE;
+
static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE __atomic_load(const atomic_flag& __a, memory_order __order) {
return std::__cxx_atomic_load(&__a.__a_, __order);
}
diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h
index ec5ae2c28c310..9a36aaa3b84fe 100644
--- a/libcxx/include/__atomic/atomic_ref.h
+++ b/libcxx/include/__atomic/atomic_ref.h
@@ -233,6 +233,8 @@ struct __atomic_ref_base {
template <class _Tp>
struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {
+ using __value_type _LIBCPP_NODEBUG = _Tp;
+
static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) {
return __a.load(__order);
}
diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h
index 93953dfbc9b8a..d0119ab5d35ec 100644
--- a/libcxx/include/__atomic/atomic_sync.h
+++ b/libcxx/include/__atomic/atomic_sync.h
@@ -18,7 +18,10 @@
#include <__thread/poll_with_backoff.h>
#include <__type_traits/conjunction.h>
#include <__type_traits/decay.h>
+#include <__type_traits/has_unique_object_representation.h>
#include <__type_traits/invoke.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
#include <__type_traits/void_t.h>
#include <__utility/declval.h>
#include <cstring>
@@ -38,6 +41,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
// The below implementations look ugly to support C++03
template <class _Tp, class = void>
struct __atomic_waitable_traits {
+ using __value_type _LIBCPP_NODEBUG = void;
+
template <class _AtomicWaitable>
static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;
@@ -58,6 +63,9 @@ struct __atomic_waitable< _Tp,
#if _LIBCPP_STD_VER >= 20
# if _LIBCPP_HAS_THREADS
+# if !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+// old dylib interface kept for backwards compatibility
_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT;
_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*) _NOEXCEPT;
@@ -69,6 +77,114 @@ _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
_LIBCPP_EXPORTED_FROM_ABI void
__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t) _NOEXCEPT;
+# endif // !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+// new dylib interface
+
+// return the global contention state's current value for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
+__atomic_monitor_global(void const* __address) _NOEXCEPT;
+
+// wait on the global contention state to be changed from the given value for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_global_table(void const* __address, __cxx_contention_t __monitor_value) _NOEXCEPT;
+
+// notify one waiter waiting on the global contention state for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const*) _NOEXCEPT;
+
+// notify all waiters waiting on the global contention state for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const*) _NOEXCEPT;
+
+// wait on the address directly with the native platform wait
+template <std::size_t _Size>
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_native(void const* __address, void const* __old_value) _NOEXCEPT;
+
+// notify one waiter waiting on the address directly with the native platform wait
+template <std::size_t _Size>
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(const void*) _NOEXCEPT;
+
+// notify all waiters waiting on the address directly with the native platform wait
+template <std::size_t _Size>
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(const void*) _NOEXCEPT;
+
+# ifdef __linux__
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4)
+# elif defined(__APPLE__)
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) \
+ _APPLY(4) \
+ _APPLY(8)
+# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
+# elif defined(_WIN32)
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
+# else
+# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t))
+# endif // __linux__
+
+// concepts defines the types are supported natively by the platform's wait
+
+# if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
+
+_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl(size_t __size) {
+ switch (__size) {
+# define _LIBCPP_MAKE_CASE(n) \
+ case n: \
+ return true;
+ _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE)
+ default:
+ return false;
+# undef _LIBCPP_MAKE_CASE
+ };
+}
+
+template <class _Tp>
+concept __has_native_atomic_wait =
+ has_unique_object_representations_v<_Tp> && is_trivially_copyable_v<_Tp> &&
+ __has_native_atomic_wait_impl(sizeof(_Tp));
+
+# else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+template <class _Tp>
+concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>;
+
+# endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+template <class _AtomicWaitable, class _Poll>
+struct __atomic_wait_backoff_impl {
+ const _AtomicWaitable& __a_;
+ _Poll __poll_;
+ memory_order __order_;
+
+ using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+ using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type;
+
+ _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
+ if (__elapsed > chrono::microseconds(4)) {
+ auto __contention_address = const_cast<const void*>(
+ static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_)));
+
+ if constexpr (__has_native_atomic_wait<__value_type>) {
+ auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
+ if (__poll_(__atomic_value))
+ return true;
+ std::__atomic_wait_native<sizeof(__value_type)>(__contention_address, std::addressof(__atomic_value));
+ } else {
+ __cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address);
+ auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
+ if (__poll_(__atomic_value))
+ return true;
+ std::__atomic_wait_global_table(__contention_address, __monitor_val);
+ }
+ } else {
+ } // poll
+ return false;
+ }
+};
+
+# else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
template <class _AtomicWaitable, class _Poll>
struct __atomic_wait_backoff_impl {
@@ -112,6 +228,8 @@ struct __atomic_wait_backoff_impl {
}
};
+# endif // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
// The semantics of this function are similar to `atomic`'s
// `.wait(T old, std::memory_order order)`, but instead of having a hardcoded
// predicate (is the loaded value unequal to `old`?), the predicate function is
@@ -133,6 +251,38 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memo
/* backoff */ __backoff_fn);
}
+# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+template <class _AtomicWaitable>
+_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
+ static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+ using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type;
+ using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+ auto __contention_address =
+ const_cast<const void*>(static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a)));
+ if constexpr (__has_native_atomic_wait<__value_type>) {
+ std::__atomic_notify_one_native<sizeof(__value_type)>(__contention_address);
+ } else {
+ std::__atomic_notify_one_global_table(__contention_address);
+ }
+}
+
+template <class _AtomicWaitable>
+_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
+ static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+ using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type;
+ using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+ auto __contention_address =
+ const_cast<const void*>(static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a)));
+ if constexpr (__has_native_atomic_wait<__value_type>) {
+ std::__atomic_notify_all_native<sizeof(__value_type)>(__contention_address);
+ } else {
+ std::__atomic_notify_all_global_table(__contention_address);
+ }
+}
+
+# else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
template <class _AtomicWaitable>
_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
@@ -145,6 +295,8 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
}
+# endif
+
# else // _LIBCPP_HAS_THREADS
template <class _AtomicWaitable, class _Poll>
diff --git a/libcxx/include/__atomic/contention_t.h b/libcxx/include/__atomic/contention_t.h
index 5b42a0125f875..b7e370439e67a 100644
--- a/libcxx/include/__atomic/contention_t.h
+++ b/libcxx/include/__atomic/contention_t.h
@@ -19,11 +19,35 @@
_LIBCPP_BEGIN_NAMESPACE_STD
-#if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__))
+// The original definition of `__cxx_contention_t` seemed a bit arbitrary.
+// When we enable the _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE ABI,
+// use definitions that are based on what the underlying platform supports
+// instead.
+#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
+
+# ifdef __linux__
+using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
+# elif defined(__APPLE__)
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+# elif defined(_AIX) && !defined(__64BIT__)
+using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
+# elif defined(_WIN32)
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+# else
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+# endif // __linux__
+
+#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+# if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__))
using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
-#else
+# else
using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
-#endif // __linux__ || (_AIX && !__64BIT__)
+# endif // __linux__ || (_AIX && !__64BIT__)
+
+#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
using __cxx_atomic_contention_t _LIBCPP_NODEBUG = __cxx_atomic_impl<__cxx_contention_t>;
diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h
index 38b85c6ac70de..f30d0a5445b19 100644
--- a/libcxx/include/__configuration/abi.h
+++ b/libcxx/include/__configuration/abi.h
@@ -63,6 +63,7 @@
// These flags are documented in ABIGuarantees.rst
# define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+# define _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
# define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
# define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
# define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
diff --git a/libcxx/include/__configuration/availability.h b/libcxx/include/__configuration/availability.h
index e98d77583c08b..a5bd266b771fd 100644
--- a/libcxx/include/__configuration/availability.h
+++ b/libcxx/include/__configuration/availability.h
@@ -39,6 +39,9 @@
// in all versions of the library are available.
#if !_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS
+# define _LIBCPP_INTRODUCED_IN_LLVM_22 1
+# define _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE /* nothing */
+
# define _LIBCPP_INTRODUCED_IN_LLVM_21 1
# define _LIBCPP_INTRODUCED_IN_LLVM_21_ATTRIBUTE /* nothing */
@@ -67,6 +70,11 @@
// clang-format off
+// LLVM 22
+// TODO: Fill this in
+# define _LIBCPP_INTRODUCED_IN_LLVM_22 0
+# define _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE __attribute__((unavailable))
+
// LLVM 21
// TODO: Fill this in
# define _LIBCPP_INTRODUCED_IN_LLVM_21 0
@@ -197,6 +205,13 @@
#endif
+// This controls the availability of new implementation of std::atomic's
+// wait, notify_one and notify_all. The new implementation uses
+// the native atomic wait/notify operations on platforms that support them
+// based on the size of the atomic type, instead of the type itself.
+#define _LIBCPP_AVAILABILITY_HAS_NEW_SYNC _LIBCPP_INTRODUCED_IN_LLVM_22
+#define _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE
+
// Enable additional explicit instantiations of iostreams components. This
// reduces the number of weak definitions generated in programs that use
// iostreams by providing a single strong definition in the shared library.
diff --git a/libcxx/lib/abi/CHANGELOG.TXT b/libcxx/lib/abi/CHANGELOG.TXT
index 968dc7a22a8c7..990c0a676cb93 100644
--- a/libcxx/lib/abi/CHANGELOG.TXT
+++ b/libcxx/lib/abi/CHANGELOG.TXT
@@ -16,6 +16,20 @@ New entries should be added directly below the "Version" header.
Version 22.0
------------
+* [libc++] Allows any types of size 4 and 8 to use native platform ulock_wait
+
+ This patch added symbols for platform wait functions with the size of the type
+
+ All platforms
+ -------------
+ Symbol added: __ZNSt3__123__atomic_monitor_globalEPKv
+ Symbol added: __ZNSt3__126__atomic_wait_global_tableEPKvx
+ Symbol added: __ZNSt3__126__atomic_notify_one_nativeILm8EEEvPKv
+ Symbol added: __ZNSt3__132__atomic_notify_all_global_tableEPKv
+ Symbol added: __ZNSt3__120__atomic_wait_nativeILm8EEEvPKvS2_
+ Symbol added: __ZNSt3__126__atomic_notify_all_nativeILm8EEEvPKv
+ Symbol added: __ZNSt3__132__atomic_notify_one_global_tableEPKv
+
* [libc++] Remove __time_get_storage::{__analyze,init} from the ABI
These functions have never been used outside the dylib, so there is no point in exporting them.
diff --git a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
index 3a1d8950c2db0..09102437775c7 100644
--- a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -534,6 +534,7 @@
{'is_defined': True, 'name': '__ZNKSt3__115basic_streambufIwNS_11char_traitsIwEEE6getlocEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNKSt3__115basic_stringbufIcNS_11char_traitsIcEENS_9allocatorIcEEE3strEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNKSt3__115error_condition7messageEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNKSt3__117moneypunct_bynameIcLb0EE11do_groupingEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNKSt3__117moneypunct_bynameIcLb0EE13do_neg_formatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNKSt3__117moneypunct_bynameIcLb0EE13do_pos_formatEv', 'type': 'FUNC'}
@@ -1125,6 +1126,7 @@
{'is_defined': True, 'name': '__ZNSt3__112system_errorD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__112system_errorD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__112system_errorD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE11__read_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE12__write_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE4openEPKcj', 'type': 'FUNC'}
@@ -1305,7 +1307,6 @@
{'is_defined': True, 'name': '__ZNSt3__113shared_futureIvED1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__113shared_futureIvED2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__113shared_futureIvEaSERKS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__114__num_get_base10__get_baseERNS_8ios_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__114__num_get_base5__srcE', 'size': 0, 'type': 'OBJECT'}
{'is_defined': True, 'name': '__ZNSt3__114__num_put_base12__format_intEPcPKcbj', 'type': 'FUNC'}
@@ -1508,7 +1509,6 @@
{'is_defined': True, 'name': '__ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1558,6 +1558,7 @@
{'is_defined': True, 'name': '__ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__120__atomic_wait_nativeILm8EEEvPKvS2_', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEEx', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__120__libcpp_atomic_waitEPVKvx', 'type': 'FUNC'}
@@ -1571,6 +1572,7 @@
{'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__123__atomic_monitor_globalEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEE', 'type': 'FUNC'}
@@ -1578,9 +1580,14 @@
{'is_defined': True, 'name': '__ZNSt3__123__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__123__libcpp_atomic_monitorEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__125notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__126__atomic_notify_all_nativeILm8EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__126__atomic_notify_one_nativeILm8EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__126__atomic_wait_global_tableEPKvx', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
{'is_defined': True, 'name': '__ZNSt3__13cinE', 'size': 0, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
index 313de84df20e8..881914d66ec70 100644
--- a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -170,6 +170,7 @@
{'is_defined': True, 'name': '_ZNKSt6__ndk115basic_streambufIwNS_11char_traitsIwEEE6getlocEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk115basic_stringbufIcNS_11char_traitsIcEENS_9allocatorIcEEE3strEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk115error_condition7messageEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt6__ndk117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk117moneypunct_bynameIcLb0EE11do_groupingEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk117moneypunct_bynameIcLb0EE13do_neg_formatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk117moneypunct_bynameIcLb0EE13do_pos_formatEv', 'type': 'FUNC'}
@@ -761,6 +762,7 @@
{'is_defined': True, 'name': '_ZNSt6__ndk112system_errorD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk112system_errorD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk112system_errorD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk113__hash_memoryEPKvj', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113basic_filebufIcNS_11char_traitsIcEEE11__read_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113basic_filebufIcNS_11char_traitsIcEEE12__write_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113basic_filebufIcNS_11char_traitsIcEEE4openEPKcj', 'type': 'FUNC'}
@@ -941,7 +943,6 @@
{'is_defined': True, 'name': '_ZNSt6__ndk113shared_futureIvED1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113shared_futureIvED2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113shared_futureIvEaSERKS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk113__hash_memoryEPKvj', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk114__num_get_base10__get_baseERNS_8ios_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk114__num_get_base5__srcE', 'size': 33, 'type': 'OBJECT'}
{'is_defined': True, 'name': '_ZNSt6__ndk114__num_put_base12__format_intEPcPKcbj', 'type': 'FUNC'}
@@ -1144,7 +1145,6 @@
{'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNKSt6__ndk117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117iostream_categoryEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1194,6 +1194,7 @@
{'is_defined': True, 'name': '_ZNSt6__ndk119__thread_local_dataEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk120__atomic_wait_nativeILj4EEEvPKvS2_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk120__get_collation_nameEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk120__libcpp_atomic_waitEPVKvi', 'type': 'FUNC'}
@@ -1207,6 +1208,7 @@
{'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk123__atomic_monitor_globalEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
@@ -1214,9 +1216,14 @@
{'is_defined': True, 'name': '_ZNSt6__ndk123__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk123__libcpp_atomic_monitorEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk125notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk126__atomic_notify_all_nativeILj4EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk126__atomic_notify_one_nativeILj4EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk126__atomic_wait_global_tableEPKvi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk134__construct_barrier_algorithm_baseERi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk13cinE', 'size': 148, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
index 99cde72885cf2..bce5dab507cad 100644
--- a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -96,6 +96,7 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__115__codecvt_utf16IwLb1EE6do_outERPcPKwS5_RS5_S2_S2_S3_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__115__codecvt_utf16IwLb1EE9do_lengthERPcPKcS5_m', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__115error_condition7messageEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -410,6 +411,7 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__112system_errorD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__112system_errorD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__112system_errorD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113random_deviceC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113random_deviceC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113random_deviceD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -418,7 +420,6 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113shared_futureIvEaSERKS1_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__114__num_get_base10__get_baseERNS_8ios_baseE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__114__num_get_base5__srcE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__114__num_put_base12__format_intEPcPKcbj', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -503,7 +504,6 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -559,6 +559,7 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexC2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__atomic_monitor_globalEPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -566,7 +567,10 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__125notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__126__atomic_wait_global_tableEPKvi', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_all_global_tableEPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__13cinE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
@@ -1705,7 +1709,10 @@
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__120__atomic_wait_nativeILm4EEEvPKvS2_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__122__libcpp_verbose_abortEPKcz', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_all_nativeILm4EEEvPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_one_nativeILm4EEEvPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem16_FilesystemClock9is_steadyE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
index 577d6cf759a77..36eb5b85466e6 100644
--- a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -96,6 +96,7 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__115__codecvt_utf16IwLb1EE6do_outERPcPKwS5_RS5_S2_S2_S3_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__115__codecvt_utf16IwLb1EE9do_lengthERPcPKcS5_m', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__115error_condition7messageEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -410,6 +411,7 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__112system_errorD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__112system_errorD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__112system_errorD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113random_deviceC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113random_deviceC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113random_deviceD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -418,7 +420,6 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113shared_futureIvEaSERKS1_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__114__num_get_base10__get_baseERNS_8ios_baseE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__114__num_get_base5__srcE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__114__num_put_base12__format_intEPcPKcbj', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -503,7 +504,6 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -559,6 +559,7 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexC2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__atomic_monitor_globalEPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -566,7 +567,10 @@
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__125notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__126__atomic_wait_global_tableEPKvl', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_all_global_tableEPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__13cinE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
@@ -1705,7 +1709,10 @@
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__120__atomic_wait_nativeILm8EEEvPKvS2_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__122__libcpp_verbose_abortEPKcz', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_all_nativeILm8EEEvPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_one_nativeILm8EEEvPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem16_FilesystemClock9is_steadyE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
index 1be7d8a2ac20b..154b7cc9b24d9 100644
--- a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -170,6 +170,7 @@
{'is_defined': True, 'name': '_ZNKSt6__ndk115basic_streambufIwNS_11char_traitsIwEEE6getlocEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk115basic_stringbufIcNS_11char_traitsIcEENS_9allocatorIcEEE3strEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk115error_condition7messageEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt6__ndk117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk117moneypunct_bynameIcLb0EE11do_groupingEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk117moneypunct_bynameIcLb0EE13do_neg_formatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt6__ndk117moneypunct_bynameIcLb0EE13do_pos_formatEv', 'type': 'FUNC'}
@@ -761,6 +762,7 @@
{'is_defined': True, 'name': '_ZNSt6__ndk112system_errorD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk112system_errorD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk112system_errorD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113basic_filebufIcNS_11char_traitsIcEEE11__read_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113basic_filebufIcNS_11char_traitsIcEEE12__write_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113basic_filebufIcNS_11char_traitsIcEEE4openEPKcj', 'type': 'FUNC'}
@@ -941,7 +943,6 @@
{'is_defined': True, 'name': '_ZNSt6__ndk113shared_futureIvED1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113shared_futureIvED2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk113shared_futureIvEaSERKS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk114__num_get_base10__get_baseERNS_8ios_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk114__num_get_base5__srcE', 'size': 33, 'type': 'OBJECT'}
{'is_defined': True, 'name': '_ZNSt6__ndk114__num_put_base12__format_intEPcPKcbj', 'type': 'FUNC'}
@@ -1144,7 +1145,6 @@
{'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNKSt6__ndk117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117iostream_categoryEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1194,6 +1194,7 @@
{'is_defined': True, 'name': '_ZNSt6__ndk119__thread_local_dataEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk120__atomic_wait_nativeILm4EEEvPKvS2_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk120__get_collation_nameEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk120__libcpp_atomic_waitEPVKvi', 'type': 'FUNC'}
@@ -1207,6 +1208,7 @@
{'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk123__atomic_monitor_globalEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
@@ -1214,9 +1216,14 @@
{'is_defined': True, 'name': '_ZNSt6__ndk123__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk123__libcpp_atomic_monitorEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk125notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk126__atomic_notify_all_nativeILm4EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk126__atomic_notify_one_nativeILm4EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk126__atomic_wait_global_tableEPKvi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt6__ndk132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt6__ndk13cinE', 'size': 280, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
index 40ae625d3bd69..685d2a45bf9e9 100644
--- a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -227,6 +227,7 @@
{'is_defined': True, 'name': '_ZNKSt3__115basic_streambufIwNS_11char_traitsIwEEE6getlocEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__115basic_stringbufIcNS_11char_traitsIcEENS_9allocatorIcEEE3strEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__115error_condition7messageEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE11do_groupingEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE13do_neg_formatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE13do_pos_formatEv', 'type': 'FUNC'}
@@ -775,6 +776,7 @@
{'is_defined': True, 'name': '_ZNSt3__112system_errorD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__112system_errorD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__112system_errorD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE11__read_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE12__write_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE4openEPKcj', 'type': 'FUNC'}
@@ -955,7 +957,6 @@
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvEaSERKS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__114__num_get_base10__get_baseERNS_8ios_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__114__num_get_base5__srcE', 'size': 33, 'type': 'OBJECT'}
{'is_defined': True, 'name': '_ZNSt3__114__num_put_base12__format_intEPcPKcbj', 'type': 'FUNC'}
@@ -1158,7 +1159,6 @@
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1208,6 +1208,7 @@
{'is_defined': True, 'name': '_ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__120__atomic_wait_nativeILm8EEEvPKvS2_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEEl', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKvl', 'type': 'FUNC'}
@@ -1221,6 +1222,7 @@
{'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__123__atomic_monitor_globalEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'type': 'FUNC'}
@@ -1228,9 +1230,14 @@
{'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__125notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_all_nativeILm8EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_one_nativeILm8EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_wait_global_tableEPKvl', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 400, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
index 90166073b135f..5732ed77f8f67 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -225,6 +225,7 @@
{'is_defined': True, 'name': '_ZNKSt3__115basic_streambufIwNS_11char_traitsIwEEE6getlocEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__115basic_stringbufIcNS_11char_traitsIcEENS_9allocatorIcEEE3strEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__115error_condition7messageEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE11do_groupingEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE13do_neg_formatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE13do_pos_formatEv', 'type': 'FUNC'}
@@ -773,6 +774,7 @@
{'is_defined': True, 'name': '_ZNSt3__112system_errorD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__112system_errorD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__112system_errorD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE11__read_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE12__write_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE4openEPKcj', 'type': 'FUNC'}
@@ -953,7 +955,6 @@
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvEaSERKS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__114__num_get_base10__get_baseERNS_8ios_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__114__num_get_base5__srcE', 'size': 33, 'type': 'OBJECT'}
{'is_defined': True, 'name': '_ZNSt3__114__num_put_base12__format_intEPcPKcbj', 'type': 'FUNC'}
@@ -1156,7 +1157,6 @@
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1206,6 +1206,7 @@
{'is_defined': True, 'name': '_ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__120__atomic_wait_nativeILm4EEEvPKvS2_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKvi', 'type': 'FUNC'}
@@ -1219,6 +1220,7 @@
{'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__123__atomic_monitor_globalEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
@@ -1226,9 +1228,14 @@
{'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__125notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_all_nativeILm4EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_one_nativeILm4EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_wait_global_tableEPKvi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 280, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
index 5855c17cf11ed..c7183c29e3d0f 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
@@ -196,6 +196,7 @@
{'is_defined': True, 'name': '_ZNKSt3__115basic_streambufIwNS_11char_traitsIwEEE6getlocEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__115basic_stringbufIcNS_11char_traitsIcEENS_9allocatorIcEEE3strEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__115error_condition7messageEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE11do_groupingEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE13do_neg_formatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNKSt3__117moneypunct_bynameIcLb0EE13do_pos_formatEv', 'type': 'FUNC'}
@@ -744,6 +745,7 @@
{'is_defined': True, 'name': '_ZNSt3__112system_errorD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__112system_errorD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__112system_errorD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE11__read_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE12__write_modeEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113basic_filebufIcNS_11char_traitsIcEEE4openEPKcj', 'type': 'FUNC'}
@@ -923,7 +925,6 @@
{'is_defined': True, 'name': '_ZNSt3__113random_deviceclEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__113__hash_memoryEPKvm', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__113shared_futureIvEaSERKS1_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__114__num_get_base10__get_baseERNS_8ios_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__114__num_get_base5__srcE', 'size': 33, 'type': 'OBJECT'}
@@ -1127,7 +1128,6 @@
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNKSt3__117bad_function_call4whatEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1177,6 +1177,7 @@
{'is_defined': True, 'name': '_ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__120__atomic_wait_nativeILm4EEEvPKvS2_', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKvi', 'type': 'FUNC'}
@@ -1190,6 +1191,7 @@
{'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__123__atomic_monitor_globalEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
@@ -1197,9 +1199,14 @@
{'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__123__libcpp_atomic_monitorEPVKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__125notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_all_nativeILm4EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_one_nativeILm4EEEvPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__126__atomic_wait_global_tableEPKvi', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__131__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'}
{'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 280, 'type': 'OBJECT'}
diff --git a/libcxx/src/atomic.cpp b/libcxx/src/atomic.cpp
index b9e4aa30bbdcd..2b1ebfa177e1c 100644
--- a/libcxx/src/atomic.cpp
+++ b/libcxx/src/atomic.cpp
@@ -9,8 +9,12 @@
#include <__thread/timed_backoff_policy.h>
#include <atomic>
#include <climits>
+#include <cstddef>
+#include <cstring>
#include <functional>
+#include <new>
#include <thread>
+#include <type_traits>
#include "include/apple_availability.h"
@@ -58,13 +62,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#ifdef __linux__
-static void
-__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
+template <std::size_t _Size>
+static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+ static_assert(_Size == 4, "Can only wait on 4 bytes value");
+ char buffer[_Size];
+ std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
static constexpr timespec __timeout = {2, 0};
- _LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, __val, &__timeout, 0, 0);
+ _LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, *reinterpret_cast<__cxx_contention_t const*>(&buffer), &__timeout, 0, 0);
}
-static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
+template <std::size_t _Size>
+static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
+ static_assert(_Size == 4, "Can only wake up on 4 bytes value");
_LIBCPP_FUTEX(__ptr, FUTEX_WAKE_PRIVATE, __notify_one ? 1 : INT_MAX, 0, 0, 0);
}
@@ -75,19 +84,29 @@ extern "C" int __ulock_wait(
extern "C" int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value);
// https://github.com/apple/darwin-xnu/blob/2ff845c2e033bd0ff64b5b6aa6063a1f8f65aa32/bsd/sys/ulock.h#L82
+# define UL_COMPARE_AND_WAIT 1
# define UL_COMPARE_AND_WAIT64 5
# define ULF_WAKE_ALL 0x00000100
-static void
-__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
- static_assert(sizeof(__cxx_atomic_contention_t) == 8, "Waiting on 8 bytes value");
- __ulock_wait(UL_COMPARE_AND_WAIT64, const_cast<__cxx_atomic_contention_t*>(__ptr), __val, 0);
+template <std::size_t _Size>
+static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+ static_assert(_Size == 8 || _Size == 4, "Can only wait on 8 bytes or 4 bytes value");
+ char buffer[_Size];
+ std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
+ if constexpr (_Size == 4)
+ __ulock_wait(UL_COMPARE_AND_WAIT, const_cast<void*>(__ptr), *reinterpret_cast<uint32_t const*>(&buffer), 0);
+ else
+ __ulock_wait(UL_COMPARE_AND_WAIT64, const_cast<void*>(__ptr), *reinterpret_cast<uint64_t const*>(&buffer), 0);
}
-static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
- static_assert(sizeof(__cxx_atomic_contention_t) == 8, "Waking up on 8 bytes value");
- __ulock_wake(
- UL_COMPARE_AND_WAIT64 | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<__cxx_atomic_contention_t*>(__ptr), 0);
+template <std::size_t _Size>
+static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
+ static_assert(_Size == 8 || _Size == 4, "Can only wake up on 8 bytes or 4 bytes value");
+
+ if constexpr (_Size == 4)
+ __ulock_wake(UL_COMPARE_AND_WAIT | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0);
+ else
+ __ulock_wake(UL_COMPARE_AND_WAIT64 | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0);
}
#elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
@@ -97,13 +116,18 @@ static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const vo
* limit its use to architectures where long and int64_t are synonyms.
*/
-static void
-__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
- _umtx_op(const_cast<__cxx_atomic_contention_t*>(__ptr), UMTX_OP_WAIT, __val, nullptr, nullptr);
+template <std::size_t _Size>
+static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+ static_assert(_Size == 8, "Can only wait on 8 bytes value");
+ char buffer[_Size];
+ std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
+ _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, *reinterpret_cast<__cxx_contention_t*>(&buffer), nullptr, nullptr);
}
-static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
- _umtx_op(const_cast<__cxx_atomic_contention_t*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, nullptr, nullptr);
+template <std::size_t _Size>
+static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
+ static_assert(_Size == 8, "Can only wake up on 8 bytes value");
+ _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, nullptr, nullptr);
}
#elif defined(_WIN32)
@@ -132,27 +156,30 @@ static void* win32_get_synch_api_function(const char* function_name) {
return reinterpret_cast<void*>(GetProcAddress(module_handle, function_name));
}
-static void
-__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
+template <std::size_t _Size>
+static void __platform_wait_on_address(void const* __ptr, void const* __val) {
+ static_assert(_Size == 8, "Can only wait on 8 bytes value");
// WaitOnAddress was added in Windows 8 (build 9200)
- static auto wait_on_address = reinterpret_cast<BOOL(WINAPI*)(volatile void*, PVOID, SIZE_T, DWORD)>(
- win32_get_synch_api_function("WaitOnAddress"));
+ static auto wait_on_address =
+ reinterpret_cast<BOOL(WINAPI*)(void*, PVOID, SIZE_T, DWORD)>(win32_get_synch_api_function("WaitOnAddress"));
if (wait_on_address != nullptr) {
- wait_on_address(const_cast<__cxx_atomic_contention_t*>(__ptr), &__val, sizeof(__val), INFINITE);
+ wait_on_address(const_cast<void*>(__ptr), const_cast<void*>(__val), _Size, INFINITE);
} else {
__libcpp_thread_poll_with_backoff(
- [=]() -> bool { return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__ptr, memory_order_relaxed), __val); },
+ [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
__libcpp_timed_backoff_policy());
}
}
-static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
+template <std::size_t _Size>
+static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
+ static_assert(_Size == 8, "Can only wake up on 8 bytes value");
if (__notify_one) {
// WakeByAddressSingle was added in Windows 8 (build 9200)
static auto wake_by_address_single =
reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressSingle"));
if (wake_by_address_single != nullptr) {
- wake_by_address_single(const_cast<__cxx_atomic_contention_t*>(__ptr));
+ wake_by_address_single(const_cast<void*>(__ptr));
} else {
// The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so
// there's nothing to do here.
@@ -162,7 +189,7 @@ static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const vo
static auto wake_by_address_all =
reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressAll"));
if (wake_by_address_all != nullptr) {
- wake_by_address_all(const_cast<__cxx_atomic_contention_t*>(__ptr));
+ wake_by_address_all(const_cast<void*>(__ptr));
} else {
// The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so
// there's nothing to do here.
@@ -174,106 +201,213 @@ static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const vo
// Baseline is just a timed backoff
-static void
-__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
+template <std::size_t _Size>
+static void __platform_wait_on_address(void const* __ptr, void const* __val) {
__libcpp_thread_poll_with_backoff(
- [=]() -> bool { return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__ptr, memory_order_relaxed), __val); },
+ [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
__libcpp_timed_backoff_policy());
}
-static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile*, bool) {}
+template <std::size_t _Size>
+static void __platform_wake_by_address(void const*, bool) {}
#endif // __linux__
-static constexpr size_t __libcpp_contention_table_size = (1 << 8); /* < there's no magic in this number */
-
-struct alignas(64) /* aim to avoid false sharing */ __libcpp_contention_table_entry {
- __cxx_atomic_contention_t __contention_state;
- __cxx_atomic_contention_t __platform_state;
- inline constexpr __libcpp_contention_table_entry() : __contention_state(0), __platform_state(0) {}
-};
-
-static __libcpp_contention_table_entry __libcpp_contention_table[__libcpp_contention_table_size];
-
-static hash<void const volatile*> __libcpp_contention_hasher;
-
-static __libcpp_contention_table_entry* __libcpp_contention_state(void const volatile* p) {
- return &__libcpp_contention_table[__libcpp_contention_hasher(p) & (__libcpp_contention_table_size - 1)];
-}
+// =============================
+// Local hidden helper functions
+// =============================
/* Given an atomic to track contention and an atomic to actually wait on, which may be
the same atomic, we try to detect contention to avoid spuriously calling the platform. */
-static void __libcpp_contention_notify(__cxx_atomic_contention_t volatile* __contention_state,
- __cxx_atomic_contention_t const volatile* __platform_state,
- bool __notify_one) {
- if (0 != __cxx_atomic_load(__contention_state, memory_order_seq_cst))
+template <std::size_t _Size>
+static void
+__contention_notify(__cxx_atomic_contention_t* __waiter_count, void const* __address_to_notify, bool __notify_one) {
+ if (0 != __cxx_atomic_load(__waiter_count, memory_order_seq_cst))
// We only call 'wake' if we consumed a contention bit here.
- __libcpp_platform_wake_by_address(__platform_state, __notify_one);
-}
-static __cxx_contention_t
-__libcpp_contention_monitor_for_wait(__cxx_atomic_contention_t volatile* /*__contention_state*/,
- __cxx_atomic_contention_t const volatile* __platform_state) {
- // We will monitor this value.
- return __cxx_atomic_load(__platform_state, memory_order_acquire);
+ __platform_wake_by_address<_Size>(__address_to_notify, __notify_one);
}
-static void __libcpp_contention_wait(__cxx_atomic_contention_t volatile* __contention_state,
- __cxx_atomic_contention_t const volatile* __platform_state,
- __cxx_contention_t __old_value) {
- __cxx_atomic_fetch_add(__contention_state, __cxx_contention_t(1), memory_order_relaxed);
+
+template <std::size_t _Size>
+static void
+__contention_wait(__cxx_atomic_contention_t* __waiter_count, void const* __address_to_wait, void const* __old_value) {
+ __cxx_atomic_fetch_add(__waiter_count, __cxx_contention_t(1), memory_order_relaxed);
// https://llvm.org/PR109290
// There are no platform guarantees of a memory barrier in the platform wait implementation
__cxx_atomic_thread_fence(memory_order_seq_cst);
// We sleep as long as the monitored value hasn't changed.
- __libcpp_platform_wait_on_address(__platform_state, __old_value);
- __cxx_atomic_fetch_sub(__contention_state, __cxx_contention_t(1), memory_order_release);
+ __platform_wait_on_address<_Size>(__address_to_wait, __old_value);
+ __cxx_atomic_fetch_sub(__waiter_count, __cxx_contention_t(1), memory_order_release);
+}
+
+#if defined(__APPLE__) && defined(__aarch64__)
+static constexpr size_t __cache_line_size = 128;
+#elif defined(__cpp_lib_hardware_interference_size)
+static constexpr size_t __cache_line_size = std::hardware_constructive_interference_size;
+#else
+static constexpr size_t __cache_line_size = 64;
+#endif
+
+static constexpr size_t __contention_table_size = (1 << 8); /* < there's no magic in this number */
+
+static constexpr hash<void const*> __contention_hasher;
+
+// Waiter count table for all atomics with the correct size that use itself as the wait/notify address.
+
+struct alignas(__cache_line_size) /* aim to avoid false sharing */ __contention_state_native {
+ __cxx_atomic_contention_t __waiter_count;
+ constexpr __contention_state_native() : __waiter_count(0) {}
+};
+
+static __contention_state_native __contention_table_native[__contention_table_size];
+
+static __cxx_atomic_contention_t* __get_native_waiter_count(void const* p) {
+ return &__contention_table_native[__contention_hasher(p) & (__contention_table_size - 1)].__waiter_count;
+}
+
+// Global contention table for all atomics with the wrong size that use the global table's atomic as wait/notify
+// address.
+
+struct alignas(__cache_line_size) /* aim to avoid false sharing */ __contention_state_global {
+ __cxx_atomic_contention_t __waiter_count;
+ __cxx_atomic_contention_t __platform_state;
+ constexpr __contention_state_global() : __waiter_count(0), __platform_state(0) {}
+};
+
+static __contention_state_global __contention_table_global[__contention_table_size];
+
+static __contention_state_global* __get_global_contention_state(void const* p) {
+ return &__contention_table_global[__contention_hasher(p) & (__contention_table_size - 1)];
}
/* When the incoming atomic is the wrong size for the platform wait size, need to
launder the value sequence through an atomic from our table. */
-static void __libcpp_atomic_notify(void const volatile* __location) {
- auto const __entry = __libcpp_contention_state(__location);
+static void __atomic_notify_global_table(void const* __location) {
+ auto const __entry = __get_global_contention_state(__location);
// The value sequence laundering happens on the next line below.
__cxx_atomic_fetch_add(&__entry->__platform_state, __cxx_contention_t(1), memory_order_seq_cst);
- __libcpp_contention_notify(
- &__entry->__contention_state,
- &__entry->__platform_state,
- false /* when laundering, we can't handle notify_one */);
+ __contention_notify<sizeof(__cxx_atomic_contention_t)>(
+ &__entry->__waiter_count, &__entry->__platform_state, false /* when laundering, we can't handle notify_one */);
+}
+
+// =============================
+// New dylib exported symbols
+// =============================
+
+// global
+_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __location) noexcept {
+ auto const __entry = __get_global_contention_state(__location);
+ return __cxx_atomic_load(&__entry->__platform_state, memory_order_acquire);
+}
+
+_LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_global_table(void const* __location, __cxx_contention_t __old_value) noexcept {
+ auto const __entry = __get_global_contention_state(__location);
+ __contention_wait<sizeof(__cxx_atomic_contention_t)>(
+ &__entry->__waiter_count, &__entry->__platform_state, &__old_value);
+}
+
+_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const* __location) noexcept {
+ __atomic_notify_global_table(__location);
+}
+
+_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const* __location) noexcept {
+ __atomic_notify_global_table(__location);
}
+
+// native
+
+template <std::size_t _Size>
+_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native(void const* __address, void const* __old_value) noexcept {
+ __contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value);
+}
+
+template <std::size_t _Size>
+_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(void const* __location) noexcept {
+ __contention_notify<_Size>(__get_native_waiter_count(__location), __location, true);
+}
+
+template <std::size_t _Size>
+_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(void const* __location) noexcept {
+ __contention_notify<_Size>(__get_native_waiter_count(__location), __location, false);
+}
+
+// ==================================================
+// Instantiation of the templates with supported size
+// ==================================================
+
+#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
+
+# define _INSTANTIATE(_SIZE) \
+ template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native<_SIZE>(void const*, void const*) noexcept; \
+ template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native<_SIZE>(void const*) noexcept; \
+ template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native<_SIZE>(void const*) noexcept;
+
+_LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_INSTANTIATE)
+
+# undef _INSTANTIATE
+
+#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+template _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_native<sizeof(__cxx_contention_t)>(void const* __address, void const* __old_value) noexcept;
+
+template _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_notify_one_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
+
+template _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_notify_all_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
+
+#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+// =============================================================
+// Old dylib exported symbols, for backwards compatibility
+// =============================================================
+
_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile* __location) noexcept {
- __libcpp_atomic_notify(__location);
+ __atomic_notify_global_table(const_cast<void const*>(__location));
}
+
_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile* __location) noexcept {
- __libcpp_atomic_notify(__location);
+ __atomic_notify_global_table(const_cast<void const*>(__location));
}
+
_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile* __location) noexcept {
- auto const __entry = __libcpp_contention_state(__location);
- return __libcpp_contention_monitor_for_wait(&__entry->__contention_state, &__entry->__platform_state);
+ auto const __entry = __get_global_contention_state(const_cast<void const*>(__location));
+ return __cxx_atomic_load(&__entry->__platform_state, memory_order_acquire);
}
+
_LIBCPP_EXPORTED_FROM_ABI void
__libcpp_atomic_wait(void const volatile* __location, __cxx_contention_t __old_value) noexcept {
- auto const __entry = __libcpp_contention_state(__location);
- __libcpp_contention_wait(&__entry->__contention_state, &__entry->__platform_state, __old_value);
+ auto const __entry = __get_global_contention_state(const_cast<void const*>(__location));
+ __contention_wait<sizeof(__cxx_atomic_contention_t)>(
+ &__entry->__waiter_count, &__entry->__platform_state, &__old_value);
}
-/* When the incoming atomic happens to be the platform wait size, we still need to use the
- table for the contention detection, but we can use the atomic directly for the wait. */
-
_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile* __location) noexcept {
- __libcpp_contention_notify(&__libcpp_contention_state(__location)->__contention_state, __location, true);
+ auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
+ __contention_notify<sizeof(__cxx_atomic_contention_t)>(
+ __get_native_waiter_count(__location_cast), __location_cast, true);
}
+
_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile* __location) noexcept {
- __libcpp_contention_notify(&__libcpp_contention_state(__location)->__contention_state, __location, false);
-}
-// This function is never used, but still exported for ABI compatibility.
-_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
-__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) noexcept {
- return __libcpp_contention_monitor_for_wait(&__libcpp_contention_state(__location)->__contention_state, __location);
+ auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
+ __contention_notify<sizeof(__cxx_atomic_contention_t)>(
+ __get_native_waiter_count(__location_cast), __location_cast, false);
}
+
_LIBCPP_EXPORTED_FROM_ABI void
__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile* __location, __cxx_contention_t __old_value) noexcept {
- __libcpp_contention_wait(&__libcpp_contention_state(__location)->__contention_state, __location, __old_value);
+ auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
+ __contention_wait<sizeof(__cxx_atomic_contention_t)>(
+ __get_native_waiter_count(__location_cast), __location_cast, &__old_value);
+}
+
+// this function is even unused in the old ABI
+_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
+__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) noexcept {
+ return __cxx_atomic_load(__location, memory_order_acquire);
}
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/libcxx/atomics/atomics.syn/wait.issue_85107.pass.cpp b/libcxx/test/libcxx/atomics/atomics.syn/wait.issue_85107.pass.cpp
index f606e933eeec7..ed2a8b0629120 100644
--- a/libcxx/test/libcxx/atomics/atomics.syn/wait.issue_85107.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.syn/wait.issue_85107.pass.cpp
@@ -41,7 +41,11 @@ int main(int, char**) {
// This would hang forever if the bug is present, but the test will fail in a bounded amount of
// time due to the timeout above.
+#if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+ std::__atomic_wait_native<sizeof(std::__cxx_atomic_contention_t)>(&ct, &old_val);
+#else
std::__libcpp_atomic_wait(&ct, old_val);
+#endif
done = true;
timeout_thread.join();
diff --git a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/lost_wakeup.pass.cpp b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/lost_wakeup.pass.cpp
new file mode 100644
index 0000000000000..17ad72dd74866
--- /dev/null
+++ b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/lost_wakeup.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This is a stress test for std::atomic::wait for lost wake ups.
+
+// <atomic>
+
+#include <atomic>
+#include <functional>
+#include <thread>
+#include <vector>
+
+#include "make_test_thread.h"
+
+constexpr int num_waiters = 8;
+constexpr int num_iterations = 10'000;
+
+int main(int, char**) {
+ for (int run = 0; run < 20; ++run) {
+ std::atomic<int> waiter_ready(0);
+ std::atomic<int> state(0);
+
+ auto wait = [&]() {
+ for (int i = 0; i < num_iterations; ++i) {
+ auto old_state = state.load();
+ waiter_ready.fetch_add(1);
+ state.wait(old_state);
+ }
+ };
+
+ auto notify = [&] {
+ for (int i = 0; i < num_iterations; ++i) {
+ while (waiter_ready.load() < num_waiters) {
+ }
+ waiter_ready.store(0);
+ state.fetch_add(1);
+ state.notify_all();
+ }
+ };
+
+ std::vector<std::jthread> threads;
+ for (int i = 0; i < num_waiters; ++i)
+ threads.push_back(support::make_test_jthread(wait));
+
+ threads.push_back(support::make_test_jthread(notify));
+ }
+
+ return 0;
+}
More information about the libcxx-commits
mailing list