[libcxx-commits] [libcxx] df361f8 - [libc++] Optimize vector::emplace_back slow path (#222016)

[via libcxx-commits]

Author: Nikolas Klauser
Date: 2026-09-30T13:54:09+02:00
New Revision: df361f8543d1a2fad3f52532ccc6197fc12e3155

URL: https://github.com/llvm/llvm-project/commit/df361f8543d1a2fad3f52532ccc6197fc12e3155
DIFF: https://github.com/llvm/llvm-project/commit/df361f8543d1a2fad3f52532ccc6197fc12e3155.diff

LOG: [libc++] Optimize vector::emplace_back slow path (#222016)

This improves the code size of the slow path by removing some redundant
checks.

Added: 
    

Modified: 
    libcxx/include/__numeric/saturation_arithmetic.h
    libcxx/include/__vector/vector.h

Removed: 
    


################################################################################
diff  --git a/libcxx/include/__numeric/saturation_arithmetic.h b/libcxx/include/__numeric/saturation_arithmetic.h
index 34f784e4a9cf1..f113370c271d0 100644
--- a/libcxx/include/__numeric/saturation_arithmetic.h
+++ b/libcxx/include/__numeric/saturation_arithmetic.h
@@ -26,17 +26,16 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 20
-
-template <__signed_or_unsigned_integer _Tp>
-_LIBCPP_HIDE_FROM_ABI constexpr _Tp __saturating_add(_Tp __x, _Tp __y) noexcept {
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp __saturating_add(_Tp __x, _Tp __y) _NOEXCEPT {
+  static_assert(__is_signed_integer_v<_Tp> || __is_unsigned_integer_v<_Tp>);
 #  if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 2101
   return __builtin_elementwise_add_sat(__x, __y);
 #  else
   if (_Tp __sum; !__builtin_add_overflow(__x, __y, std::addressof(__sum)))
     return __sum;
   // Handle overflow
-  if constexpr (__unsigned_integer<_Tp>) {
+  if _LIBCPP_CONSTEXPR (__is_unsigned_integer_v<_Tp>) {
     return std::numeric_limits<_Tp>::max();
   } else {
     // Signed addition overflow
@@ -50,6 +49,8 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Tp __saturating_add(_Tp __x, _Tp __y) noexcept
 #  endif
 }
 
+#if _LIBCPP_STD_VER >= 20
+
 template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp __saturating_sub(_Tp __x, _Tp __y) noexcept {
 #  if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 2101

diff  --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index a93b890e90de1..aa8441e3f9554 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -41,6 +41,7 @@
 #include <__memory/swap_allocator.h>
 #include <__memory/temp_value.h>
 #include <__memory/uninitialized_algorithms.h>
+#include <__numeric/saturation_arithmetic.h>
 #include <__ranges/access.h>
 #include <__ranges/as_rvalue_view.h>
 #include <__ranges/concepts.h>
@@ -60,12 +61,14 @@
 #include <__type_traits/is_swappable.h>
 #include <__type_traits/remove_const_ref.h>
 #include <__type_traits/type_identity.h>
+#include <__utility/assume.h>
 #include <__utility/declval.h>
 #include <__utility/exception_guard.h>
 #include <__utility/forward.h>
 #include <__utility/is_pointer_in_range.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
+#include <__utility/scope_guard.h>
 #include <__utility/swap.h>
 #include <initializer_list>
 #include <limits>
@@ -882,9 +885,9 @@ vector<_Tp, _Allocator>::__recommend(size_type __new_size) const {
   if (__new_size > __ms)
     this->__throw_length_error();
   const size_type __cap = capacity();
-  if (__cap >= __ms / 2)
-    return __ms;
-  return std::max<size_type>(2 * __cap, __new_size);
+  std::__assume(__cap <= __ms);
+  auto __doubled = std::__saturating_add(__cap, __cap);
+  return std::min(std::max(__new_size, __doubled), __ms);
 }
 
 //  Default constructs __n objects starting at __layout_.__end_ptr()
@@ -1059,13 +1062,26 @@ vector<_Tp, _Alloc>::emplace_back(_Args&&... __args) {
         __self.__emplace_back_assume_capacity(std::forward<_Args>(__largs)...);
       },
       [](vector& __self, _Args&&... __largs) static {
-        _SplitBuffer __v(__self.__recommend(__self.size() + 1), __self.size(), __self.__layout_.__alloc());
-        //    __v.emplace_back(std::forward<_Args>(__args)...);
-        pointer __end = __v.end();
-        __alloc_traits::construct(
-            __self.__layout_.__alloc(), std::__to_address(__end), std::forward<_Args>(__largs)...);
-        __v.__set_sentinel(++__end);
-        __self.__layout_.__relocate(__v);
+        __self.__annotate_delete();
+        auto __asan_guard  = std::__make_scope_guard([&] { __self.__annotate_new(__self.size()); });
+        auto& __alloc      = __self.__layout_.__alloc();
+        auto __size        = __self.size();
+        auto __allocation  = std::__allocate_at_least(__alloc, __self.__recommend(__size + 1));
+        auto __alloc_guard = std::__make_exception_guard([&] {
+          __alloc_traits::deallocate(__alloc, __allocation.ptr, __allocation.count);
+        });
+        auto __ptr         = std::__to_address(__allocation.ptr);
+        auto __old_begin   = __self.__layout_.__begin_ptr();
+        __alloc_traits::construct(__alloc, __ptr + __size, std::forward<_Args>(__largs)...);
+        auto __guard = std::__make_exception_guard([&] { __alloc_traits::destroy(__alloc, __ptr + __size); });
+        std::__uninitialized_allocator_relocate(
+            __alloc, std::__to_address(__old_begin), std::__to_address(__self.__layout_.__end_ptr()), __ptr);
+        __guard.__complete();
+        auto __old_cap = __self.capacity();
+        __self.__layout_.__set_layout(__allocation.ptr, __size + 1, __allocation.count);
+        __alloc_guard.__complete();
+        if ((__is_std_allocator_v<allocator_type> && !__libcpp_is_constant_evaluated()) || __old_begin)
+          __alloc_traits::deallocate(__alloc, __old_begin, __old_cap);
       },
       *this,
       std::forward<_Args>(__args)...);