[libcxx-commits] [libcxx] [libc++] Use relocation in vector::emplace_back (PR #159365)
Nikolas Klauser via libcxx-commits
libcxx-commits at lists.llvm.org
Tue Sep 23 03:12:02 PDT 2025
https://github.com/philnik777 updated https://github.com/llvm/llvm-project/pull/159365
>From 2355d8003b610c465b2ae3dd6338ed5d1fdd92df Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser at berlin.de>
Date: Mon, 15 Sep 2025 10:37:41 +0200
Subject: [PATCH] [libc++] Use relocation in vector::emplace_back
---
libcxx/include/__utility/exception_guard.h | 5 +-
libcxx/include/__vector/vector.h | 115 ++++++++++++---------
2 files changed, 72 insertions(+), 48 deletions(-)
diff --git a/libcxx/include/__utility/exception_guard.h b/libcxx/include/__utility/exception_guard.h
index 6fa744e8b4f32..8d0da912f9850 100644
--- a/libcxx/include/__utility/exception_guard.h
+++ b/libcxx/include/__utility/exception_guard.h
@@ -80,7 +80,10 @@ struct __exception_guard_exceptions {
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __complete() _NOEXCEPT { __completed_ = true; }
- _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__exception_guard_exceptions() {
+ // __exception_guard is almost always used with a lambda, so the destructor is almost certainly unique to the calling
+ // function. LLVM doesn't know that, so it doesn't inline it due to the destructor being in a cold (exception) path.
+ // Annotate the function with always_inline to work around those problems.
+ _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__exception_guard_exceptions() {
if (!__completed_)
__rollback_();
}
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 27e681aeef22a..f2bca70e68cd7 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -84,6 +84,24 @@ _LIBCPP_PUSH_MACROS
_LIBCPP_BEGIN_NAMESPACE_STD
+// This makes the compiler inline `__else()` if `__cond` is known to be false. Currently LLVM doesn't do that without
+// the `__builtin_constant_p`, since it considers `__else` unlikely even through it's known to be run.
+// See https://llvm.org/PR154292
+template <class _If, class _Else>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __if_likely_else(bool __cond, _If __if, _Else __else) {
+ if (__builtin_constant_p(__cond)) {
+ if (__cond)
+ __if();
+ else
+ __else();
+ } else {
+ if (__cond) [[__likely__]]
+ __if();
+ else
+ __else();
+ }
+}
+
template <class _Tp, class _Allocator /* = allocator<_Tp> */>
class vector {
template <class _Up, class _Alloc>
@@ -469,15 +487,60 @@ class vector {
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x) { emplace_back(std::move(__x)); }
template <class... _Args>
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference __emplace_back(_Args&&... __args) {
+ if constexpr (__libcpp_is_trivially_relocatable<value_type>::value &&
+ __allocator_has_trivial_move_construct_v<allocator_type, value_type> &&
+ __allocator_has_trivial_destroy_v<allocator_type, value_type>) {
+ // This path is written in a way to have the fast path as compact as possible. Specifically,
+ // there is a branch in case there isn't enough capacity left over, which will return to the same location within
+ // the function after growing the vector. This ensures that the relocation code exists only once and the
+ // reallocation path is common across all `vector` instantiations of trivially relocatable types.
+ union _Tmp {
+ _LIBCPP_DIAGNOSTIC_PUSH
+ // Clang complains about __exclude_from_explicit_instantiation__ on a local class member.
+ _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wignored-attributes")
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tmp() {}
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~_Tmp() {}
+ _LIBCPP_DIAGNOSTIC_POP
+ value_type __val_;
+ };
+ _Tmp __tmp;
+
+ __alloc_traits::construct(__alloc_, std::addressof(__tmp.__val_), std::forward<_Args>(__args)...);
+
+ auto __guard =
+ std::__make_exception_guard([&, this] { __alloc_traits::destroy(__alloc_, std::addressof(__tmp.__val_)); });
+ std::__if_likely_else(size() != capacity(), [] {}, [this] { reserve(__recommend(size() + 1)); });
+ __guard.__complete();
+ std::__uninitialized_allocator_relocate(
+ __alloc_, std::addressof(__tmp.__val_), std::addressof(__tmp.__val_) + 1, std::__to_address(__end_));
+ ++__end_;
+ } else {
+ pointer __end = this->__end_;
+ std::__if_likely_else(
+ __end < this->__cap_,
+ [&] {
+ __emplace_back_assume_capacity(std::forward<_Args>(__args)...);
+ ++__end;
+ },
+ [&] { __end = __emplace_back_slow_path(std::forward<_Args>(__args)...); });
+
+ this->__end_ = __end;
+ }
+ return __end_[-1];
+ }
+
#if _LIBCPP_STD_VER >= 17
- reference
- emplace_back(_Args&&... __args);
+ using __emplace_back_result = reference;
#else
- void
- emplace_back(_Args&&... __args);
+ using __emplace_back_result = void;
#endif
+ template <class... _Args>
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __emplace_back_result emplace_back(_Args&&... __args) {
+ return static_cast<__emplace_back_result>(__emplace_back(std::forward<_Args>(__args)...));
+ }
+
template <class... _Args>
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __emplace_back_assume_capacity(_Args&&... __args) {
_LIBCPP_ASSERT_INTERNAL(
@@ -1161,48 +1224,6 @@ vector<_Tp, _Allocator>::__emplace_back_slow_path(_Args&&... __args) {
return this->__end_;
}
-// This makes the compiler inline `__else()` if `__cond` is known to be false. Currently LLVM doesn't do that without
-// the `__builtin_constant_p`, since it considers `__else` unlikely even through it's known to be run.
-// See https://llvm.org/PR154292
-template <class _If, class _Else>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __if_likely_else(bool __cond, _If __if, _Else __else) {
- if (__builtin_constant_p(__cond)) {
- if (__cond)
- __if();
- else
- __else();
- } else {
- if (__cond) [[__likely__]]
- __if();
- else
- __else();
- }
-}
-
-template <class _Tp, class _Allocator>
-template <class... _Args>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline
-#if _LIBCPP_STD_VER >= 17
- typename vector<_Tp, _Allocator>::reference
-#else
- void
-#endif
- vector<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
- pointer __end = this->__end_;
- std::__if_likely_else(
- __end < this->__cap_,
- [&] {
- __emplace_back_assume_capacity(std::forward<_Args>(__args)...);
- ++__end;
- },
- [&] { __end = __emplace_back_slow_path(std::forward<_Args>(__args)...); });
-
- this->__end_ = __end;
-#if _LIBCPP_STD_VER >= 17
- return *(__end - 1);
-#endif
-}
-
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
vector<_Tp, _Allocator>::erase(const_iterator __position) {
More information about the libcxx-commits
mailing list