[libcxx-commits] [PATCH] D72160: Optimize / partially inline basic_string copy constructor

[Martijn Vels via Phabricator via libcxx-commits]

mvels created this revision.
Herald added subscribers: libcxx-commits, christof.
Herald added a project: libc++.
mvels added reviewers: EricWF, mclow.lists.

Splits copy constructor up inlining short initialization, outlining long initialization into __init_long() which is the externally instantiated slow path initialization.

Subsequently changing the copy ctor to be inlined (not externally instantiated) provides significant speed ups for short string initialization.

Generated code given:

void StringCopyCtor(void* mem, const std::string& s) {

  std::string*p = new(mem) std::string{s};

}

asm:

  cmp     byte ptr [rsi + 23], 0
  js      .LBB0_2
  mov     rax, qword ptr [rsi + 16]
  mov     qword ptr [rdi + 16], rax
  movups  xmm0, xmmword ptr [rsi]
  movups  xmmword ptr [rdi], xmm0
  ret

.LBB0_2:

  jmp     std::basic_string::__init_long # TAILCALL

Benchmark:
BM_StringCopy_Empty                                           5.19ns ± 6%             1.50ns ± 8%  -71.02%        (p=0.000 n=10+10)
BM_StringCopy_Small                                           5.14ns ± 8%             1.53ns ± 7%  -70.17%        (p=0.000 n=10+10)
BM_StringCopy_Large                                           18.9ns ± 0%             19.3ns ± 0%   +1.92%        (p=0.000 n=10+10)
BM_StringCopy_Huge                                             309ns ± 1%              316ns ± 5%     ~            (p=0.633 n=8+10)


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D72160

Files:
  libcxx/include/string


Index: libcxx/include/string
===================================================================
--- libcxx/include/string
+++ libcxx/include/string
@@ -796,6 +796,10 @@
         _NOEXCEPT;
 #endif
 
+    // Optimization opportunity: do not externally instantiate the copy
+    // constructor, which inlines short string initialization. Long string
+    // initialization is delegated to the (external) __init_long()method,
+    // which results in a 3X-4X speed up for SSO initialization.
     basic_string(const basic_string& __str);
     basic_string(const basic_string& __str, const allocator_type& __a);
 
@@ -1543,6 +1547,8 @@
     inline
     void __init(size_type __n, value_type __c);
 
+    void __init_long(const basic_string& __str);
+
     template <class _InputIterator>
     inline
     typename enable_if
@@ -1794,6 +1800,20 @@
     traits_type::assign(__p[__sz], value_type());
 }
 
+template <class _CharT, class _Traits, class _Allocator>
+void basic_string<_CharT, _Traits, _Allocator>::__init_long(
+    const basic_string& __str) {
+  const value_type* __s = _VSTD::__to_address(__str.__get_long_pointer());
+  size_type __sz = __str.__get_long_size();
+  size_type __cap = __recommend(__sz);
+  pointer __p = __alloc_traits::allocate(__alloc(), __cap + 1);
+  __set_long_pointer(__p);
+  __set_long_cap(__cap + 1);
+  __set_long_size(__sz);
+  traits_type::copy(_VSTD::__to_address(__p), __s, __sz);
+  traits_type::assign(__p[__sz], value_type());
+}
+
 template <class _CharT, class _Traits, class _Allocator>
 template <class>
 basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const _Allocator& __a)
@@ -1837,7 +1857,7 @@
     if (!__str.__is_long())
         __r_.first().__r = __str.__r_.first().__r;
     else
-        __init(_VSTD::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+        __init_long(__str);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif
@@ -1851,7 +1871,7 @@
     if (!__str.__is_long())
         __r_.first().__r = __str.__r_.first().__r;
     else
-        __init(_VSTD::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+        __init_long(__str);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D72160.236067.patch
Type: text/x-patch
Size: 2264 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/libcxx-commits/attachments/20200103/b172c3dc/attachment-0001.bin>