[libcxx-commits] [libcxx] [libc++] Inline fast path for`exception_ptr` copy constructor & destructor (PR #165909)
Adrian Vogelsgesang via libcxx-commits
libcxx-commits at lists.llvm.org
Mon Nov 3 12:35:05 PST 2025
https://github.com/vogelsgesang updated https://github.com/llvm/llvm-project/pull/165909
>From d332adca6c8a7f0aaf8b9796b417f7c5933aef8d Mon Sep 17 00:00:00 2001
From: Adrian Vogelsgesang <avogelsgesang at salesforce.com>
Date: Fri, 31 Oct 2025 16:06:22 +0000
Subject: [PATCH] [libc++] Inline copy constructor & destructor for
`std::exception_ptr`
---
libcxx/include/__exception/exception_ptr.h | 65 +++++++++++++++++--
libcxx/src/exception.cpp | 1 +
.../runtime/exception_pointer_cxxabi.ipp | 15 ++---
.../runtime/exception_pointer_glibcxx.ipp | 27 +++-----
.../exception_pointer_unimplemented.ipp | 9 +--
5 files changed, 77 insertions(+), 40 deletions(-)
diff --git a/libcxx/include/__exception/exception_ptr.h b/libcxx/include/__exception/exception_ptr.h
index e78126ea23852..0f081c6bc8ed0 100644
--- a/libcxx/include/__exception/exception_ptr.h
+++ b/libcxx/include/__exception/exception_ptr.h
@@ -30,6 +30,28 @@ _LIBCPP_PUSH_MACROS
#ifndef _LIBCPP_ABI_MICROSOFT
+// Previously, parts of exception_ptr were defined out-of-line, which prevented
+// useful compiler optimizations. Changing the out-of-line definitions to inline
+// definitions is an ABI break, however. To prevent this, we have to make sure
+// the symbols remain available in the libc++ library, in addition to being
+// defined inline here in this header.
+// To this end, we use _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE macro:
+// The macro is defined as empty for src/exception.cpp, forcing the definitions of
+// the functions to be emitted and included in the library. When users of libc++
+// compile their code, the __gnu_inline__ attribute will suppress generation of
+// these functions while making their definitions available for inlining.
+# ifdef _LIBCPP_EMIT_CODE_FOR_EXCEPTION_PTR
+# define _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE _LIBCPP_EXPORTED_FROM_ABI
+# else
+# if !__has_cpp_attribute(__gnu__::__gnu_inline__)
+# error "GNU inline attribute is not supported"
+# endif
+# define _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE [[__gnu__::__gnu_inline__]] inline
+# endif
+
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wgnu-inline-cpp-without-extern")
+
# if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
namespace __cxxabiv1 {
@@ -67,6 +89,17 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _
class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
void* __ptr_;
+ static void __do_increment_refcount(void* __ptr) _NOEXCEPT;
+ static void __do_decrement_refcount(void* __ptr) _NOEXCEPT;
+ _LIBCPP_HIDE_FROM_ABI static void __increment_refcount(void* __ptr) _NOEXCEPT {
+ if (__ptr)
+ __do_increment_refcount(__ptr);
+ }
+ _LIBCPP_HIDE_FROM_ABI static void __decrement_refcount(void* __ptr) _NOEXCEPT {
+ if (__ptr)
+ __do_decrement_refcount(__ptr);
+ }
+
static exception_ptr __from_native_exception_pointer(void*) _NOEXCEPT;
template <class _Ep>
@@ -81,17 +114,18 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
_LIBCPP_HIDE_FROM_ABI exception_ptr() _NOEXCEPT : __ptr_() {}
_LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {}
- exception_ptr(const exception_ptr&) _NOEXCEPT;
+ _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr(const exception_ptr&) _NOEXCEPT;
_LIBCPP_HIDE_FROM_ABI exception_ptr(exception_ptr&& __other) _NOEXCEPT : __ptr_(__other.__ptr_) {
__other.__ptr_ = nullptr;
}
- exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
+ _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
_LIBCPP_HIDE_FROM_ABI exception_ptr& operator=(exception_ptr&& __other) _NOEXCEPT {
- exception_ptr __tmp(std::move(__other));
- std::swap(__tmp, *this);
+ __decrement_refcount(__ptr_);
+ __ptr_ = __other.__ptr_;
+ __other.__ptr_ = nullptr;
return *this;
}
- ~exception_ptr() _NOEXCEPT;
+ _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE ~exception_ptr() _NOEXCEPT;
_LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __ptr_ != nullptr; }
@@ -109,6 +143,25 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
friend _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
};
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr::exception_ptr(const exception_ptr& __other) _NOEXCEPT
+ : __ptr_(__other.__ptr_) {
+ __increment_refcount(__ptr_);
+}
+
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr& exception_ptr::operator=(const exception_ptr& __other) _NOEXCEPT {
+ if (__ptr_ != __other.__ptr_) {
+ __increment_refcount(__other.__ptr_);
+ __decrement_refcount(__ptr_);
+ __ptr_ = __other.__ptr_;
+ }
+ return *this;
+}
+
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr::~exception_ptr() _NOEXCEPT { __decrement_refcount(__ptr_); }
+
inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT {
std::swap(__x.__ptr_, __y.__ptr_);
}
@@ -222,6 +275,8 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
#endif // _LIBCPP_ABI_MICROSOFT
_LIBCPP_END_UNVERSIONED_NAMESPACE_STD
+_LIBCPP_DIAGNOSTIC_POP
+
_LIBCPP_POP_MACROS
#endif // _LIBCPP___EXCEPTION_EXCEPTION_PTR_H
diff --git a/libcxx/src/exception.cpp b/libcxx/src/exception.cpp
index ac6324cd9fe35..9dd9b0c9938fd 100644
--- a/libcxx/src/exception.cpp
+++ b/libcxx/src/exception.cpp
@@ -8,6 +8,7 @@
#define _LIBCPP_ENABLE_CXX20_REMOVED_UNCAUGHT_EXCEPTION
#define _LIBCPP_DISABLE_DEPRECATION_WARNINGS
+#define _LIBCPP_EMIT_CODE_FOR_EXCEPTION_PTR
#include <exception>
#include <new>
diff --git a/libcxx/src/support/runtime/exception_pointer_cxxabi.ipp b/libcxx/src/support/runtime/exception_pointer_cxxabi.ipp
index 8f5c2060bb06c..e09bf8981263f 100644
--- a/libcxx/src/support/runtime/exception_pointer_cxxabi.ipp
+++ b/libcxx/src/support/runtime/exception_pointer_cxxabi.ipp
@@ -13,19 +13,12 @@
namespace std {
-exception_ptr::~exception_ptr() noexcept { __cxa_decrement_exception_refcount(__ptr_); }
-
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
- __cxa_increment_exception_refcount(__ptr_);
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
+ __cxa_increment_exception_refcount(__ptr);
}
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
- if (__ptr_ != other.__ptr_) {
- __cxa_increment_exception_refcount(other.__ptr_);
- __cxa_decrement_exception_refcount(__ptr_);
- __ptr_ = other.__ptr_;
- }
- return *this;
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
+ __cxa_decrement_exception_refcount(__ptr);
}
exception_ptr exception_ptr::__from_native_exception_pointer(void* __e) noexcept {
diff --git a/libcxx/src/support/runtime/exception_pointer_glibcxx.ipp b/libcxx/src/support/runtime/exception_pointer_glibcxx.ipp
index 174b44ce0e6f7..c7b2e343b5f09 100644
--- a/libcxx/src/support/runtime/exception_pointer_glibcxx.ipp
+++ b/libcxx/src/support/runtime/exception_pointer_glibcxx.ipp
@@ -7,14 +7,14 @@
//
//===----------------------------------------------------------------------===//
+
// libsupc++ does not implement the dependent EH ABI and the functionality
// it uses to implement std::exception_ptr (which it declares as an alias of
// std::__exception_ptr::exception_ptr) is not directly exported to clients. So
// we have little choice but to hijack std::__exception_ptr::exception_ptr's
-// (which fortunately has the same layout as our std::exception_ptr) copy
-// constructor, assignment operator and destructor (which are part of its
-// stable ABI), and its rethrow_exception(std::__exception_ptr::exception_ptr)
-// function.
+// _M_addref and _M_release and its rethrow_exception function. Fortunately,
+// glibcxx's exception_ptr has the same layout as our exception_ptr and we can
+// reinterpret_cast between the two.
namespace std {
@@ -23,27 +23,20 @@ namespace __exception_ptr {
struct exception_ptr {
void* __ptr_;
- explicit exception_ptr(void*) noexcept;
- exception_ptr(const exception_ptr&) noexcept;
- exception_ptr& operator=(const exception_ptr&) noexcept;
- ~exception_ptr() noexcept;
+ void _M_addref() noexcept;
+ void _M_release() noexcept;
};
} // namespace __exception_ptr
[[noreturn]] void rethrow_exception(__exception_ptr::exception_ptr);
-exception_ptr::~exception_ptr() noexcept { reinterpret_cast<__exception_ptr::exception_ptr*>(this)->~exception_ptr(); }
-
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
- new (reinterpret_cast<void*>(this))
- __exception_ptr::exception_ptr(reinterpret_cast<const __exception_ptr::exception_ptr&>(other));
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
+ reinterpret_cast<__exception_ptr::exception_ptr*>(this)->_M_addref();
}
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
- *reinterpret_cast<__exception_ptr::exception_ptr*>(this) =
- reinterpret_cast<const __exception_ptr::exception_ptr&>(other);
- return *this;
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
+ reinterpret_cast<__exception_ptr::exception_ptr*>(this)->_M_release();
}
exception_ptr exception_ptr::__from_native_exception_pointer(void* __e) noexcept {
diff --git a/libcxx/src/support/runtime/exception_pointer_unimplemented.ipp b/libcxx/src/support/runtime/exception_pointer_unimplemented.ipp
index 05a71ce34e5ac..78be16bf95188 100644
--- a/libcxx/src/support/runtime/exception_pointer_unimplemented.ipp
+++ b/libcxx/src/support/runtime/exception_pointer_unimplemented.ipp
@@ -11,17 +11,12 @@
namespace std {
-exception_ptr::~exception_ptr() noexcept {
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
#warning exception_ptr not yet implemented
__libcpp_verbose_abort("exception_ptr not yet implemented\n");
}
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
-#warning exception_ptr not yet implemented
- __libcpp_verbose_abort("exception_ptr not yet implemented\n");
-}
-
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
#warning exception_ptr not yet implemented
__libcpp_verbose_abort("exception_ptr not yet implemented\n");
}
More information about the libcxx-commits
mailing list