[libcxx-commits] [libcxx] [WG21] Draft PR for discussion of the P2687 trivial relocation approach (PR #116714)
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Tue Nov 19 07:09:24 PST 2024
https://github.com/ldionne updated https://github.com/llvm/llvm-project/pull/116714
>From 9af42d34c30975ff0592e74f34f9724d5f91796d Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Tue, 19 Nov 2024 00:01:22 +0100
Subject: [PATCH 1/6] TODO: Improve the tests for insert and emplace
---
.../sequences/vector/vector.modifiers/emplace.pass.cpp | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace.pass.cpp
index 4f5e36abb6bb8d..c7d8f7115d5db1 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/emplace.pass.cpp
@@ -168,3 +168,13 @@ int main(int, char**)
#endif
return 0;
}
+
+// TODO:
+// Add emplace test like
+//
+// std::vector<int> v = {1, 2, 3, 4};
+// v.reserve(10000);
+// v.emplace(v.begin() + 1, 999);
+
+// TODO:
+// Add tests for emplacing and inserting from an element in the vector itself
>From 5d55bf7f985b5f0eef201a6be293d0716755f367 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Mon, 18 Nov 2024 23:08:08 +0100
Subject: [PATCH 2/6] Add is_replaceable
---
libcxx/include/CMakeLists.txt | 1 +
libcxx/include/__exception/exception_ptr.h | 1 +
libcxx/include/__expected/expected.h | 2 +
libcxx/include/__memory/shared_ptr.h | 2 +
libcxx/include/__memory/unique_ptr.h | 5 +++
libcxx/include/__split_buffer | 5 +++
libcxx/include/__type_traits/is_replaceable.h | 43 +++++++++++++++++++
libcxx/include/__utility/pair.h | 2 +
libcxx/include/__vector/vector.h | 3 ++
libcxx/include/array | 2 +
libcxx/include/deque | 3 ++
libcxx/include/module.modulemap | 4 ++
libcxx/include/optional | 2 +
libcxx/include/string | 3 ++
libcxx/include/tuple | 2 +
libcxx/include/variant | 2 +
16 files changed, 82 insertions(+)
create mode 100644 libcxx/include/__type_traits/is_replaceable.h
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 83168d153a7f41..27d65bc523d123 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -814,6 +814,7 @@ set(files
__type_traits/is_reference.h
__type_traits/is_reference_wrapper.h
__type_traits/is_referenceable.h
+ __type_traits/is_replaceable.h
__type_traits/is_same.h
__type_traits/is_scalar.h
__type_traits/is_signed.h
diff --git a/libcxx/include/__exception/exception_ptr.h b/libcxx/include/__exception/exception_ptr.h
index c43c005603dd7a..138656ecb2a0ad 100644
--- a/libcxx/include/__exception/exception_ptr.h
+++ b/libcxx/include/__exception/exception_ptr.h
@@ -68,6 +68,7 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
public:
// exception_ptr is basically a COW string.
using __trivially_relocatable = exception_ptr;
+ using __replaceable = exception_ptr;
_LIBCPP_HIDE_FROM_ABI exception_ptr() _NOEXCEPT : __ptr_() {}
_LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {}
diff --git a/libcxx/include/__expected/expected.h b/libcxx/include/__expected/expected.h
index 3d3f11967ee746..4c4db9e789e8cb 100644
--- a/libcxx/include/__expected/expected.h
+++ b/libcxx/include/__expected/expected.h
@@ -29,6 +29,7 @@
#include <__type_traits/is_nothrow_assignable.h>
#include <__type_traits/is_nothrow_constructible.h>
#include <__type_traits/is_reference.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
#include <__type_traits/is_trivially_constructible.h>
@@ -470,6 +471,7 @@ class expected : private __expected_base<_Tp, _Err> {
__conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value && __libcpp_is_trivially_relocatable<_Err>::value,
expected,
void>;
+ using __replaceable = __conditional_t<__is_replaceable<_Tp>::value && __is_replaceable<_Err>::value, expected, void>;
template <class _Up>
using rebind = expected<_Up, error_type>;
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 5c34f2efc5730e..1352be32cb59e8 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -316,6 +316,7 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr {
// A shared_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require
// any bookkeeping, so it's always trivially relocatable.
using __trivially_relocatable = shared_ptr;
+ using __replaceable = shared_ptr;
private:
element_type* __ptr_;
@@ -1211,6 +1212,7 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr {
// A weak_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require
// any bookkeeping, so it's always trivially relocatable.
using __trivially_relocatable = weak_ptr;
+ using __replaceable = weak_ptr;
private:
element_type* __ptr_;
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
index 28c62e13566e24..795db504e8ad0a 100644
--- a/libcxx/include/__memory/unique_ptr.h
+++ b/libcxx/include/__memory/unique_ptr.h
@@ -39,6 +39,7 @@
#include <__type_traits/is_function.h>
#include <__type_traits/is_pointer.h>
#include <__type_traits/is_reference.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
#include <__type_traits/is_trivially_relocatable.h>
@@ -157,6 +158,8 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
__libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<deleter_type>::value,
unique_ptr,
void>;
+ using __replaceable =
+ __conditional_t<__is_replaceable<pointer>::value && __is_replaceable<deleter_type>::value, unique_ptr, void>;
private:
_LIBCPP_COMPRESSED_PAIR(pointer, __ptr_, deleter_type, __deleter_);
@@ -423,6 +426,8 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
__libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<deleter_type>::value,
unique_ptr,
void>;
+ using __replaceable =
+ __conditional_t<__is_replaceable<pointer>::value && __is_replaceable<deleter_type>::value, unique_ptr, void>;
private:
template <class _Up, class _OtherDeleter>
diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer
index a44811c766735a..bdf45fec436cc1 100644
--- a/libcxx/include/__split_buffer
+++ b/libcxx/include/__split_buffer
@@ -30,6 +30,7 @@
#include <__type_traits/integral_constant.h>
#include <__type_traits/is_nothrow_assignable.h>
#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_swappable.h>
#include <__type_traits/is_trivially_destructible.h>
#include <__type_traits/is_trivially_relocatable.h>
@@ -74,6 +75,10 @@ public:
__libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
__split_buffer,
void>;
+ using __replaceable =
+ __conditional_t<__is_replaceable<pointer>::value && __is_replaceable<allocator_type>::value,
+ __split_buffer,
+ void>;
pointer __first_;
pointer __begin_;
diff --git a/libcxx/include/__type_traits/is_replaceable.h b/libcxx/include/__type_traits/is_replaceable.h
new file mode 100644
index 00000000000000..1f265f955c31bc
--- /dev/null
+++ b/libcxx/include/__type_traits/is_replaceable.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_REPLACEABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_REPLACEABLE_H
+
+#include <__config>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// A type is replaceable if `x = std::move(y)` is equivalent to:
+//
+// std::destroy_at(&x)
+// std::construct_at(&x, std::move(y))
+//
+// This allows turning a move-assignment into a sequence of destroy + move-construct, which
+// is often more efficient. This is especially relevant when the move-construct is in fact
+// part of a trivial relocation from somewhere else, in which case there is a huge win.
+//
+// Note that this requires language support in order to be really effective, but we
+// currently emulate the base template with something very conservative.
+template <class _Tp, class = void>
+struct __is_replaceable : is_trivially_copyable<_Tp> {};
+
+template <class _Tp>
+struct __is_replaceable<_Tp, __enable_if_t<is_same<_Tp, typename _Tp::__replaceable>::value> > : true_type {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_REPLACEABLE_H
diff --git a/libcxx/include/__utility/pair.h b/libcxx/include/__utility/pair.h
index f9d0f4e4723113..811c5fb9cf7411 100644
--- a/libcxx/include/__utility/pair.h
+++ b/libcxx/include/__utility/pair.h
@@ -32,6 +32,7 @@
#include <__type_traits/is_implicitly_default_constructible.h>
#include <__type_traits/is_nothrow_assignable.h>
#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
#include <__type_traits/is_trivially_relocatable.h>
@@ -75,6 +76,7 @@ struct _LIBCPP_TEMPLATE_VIS pair
__conditional_t<__libcpp_is_trivially_relocatable<_T1>::value && __libcpp_is_trivially_relocatable<_T2>::value,
pair,
void>;
+ using __replaceable = __conditional_t<__is_replaceable<_T1>::value && __is_replaceable<_T2>::value, pair, void>;
_LIBCPP_HIDE_FROM_ABI pair(pair const&) = default;
_LIBCPP_HIDE_FROM_ABI pair(pair&&) = default;
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index ae3ea1de61de01..0fd9ff55db3bd1 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -51,6 +51,7 @@
#include <__type_traits/is_constructible.h>
#include <__type_traits/is_nothrow_assignable.h>
#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_trivially_relocatable.h>
#include <__type_traits/type_identity.h>
@@ -118,6 +119,8 @@ class _LIBCPP_TEMPLATE_VIS vector {
__libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
vector,
void>;
+ using __replaceable =
+ __conditional_t<__is_replaceable<pointer>::value && __is_replaceable<allocator_type>::value, vector, void>;
static_assert(__check_valid_allocator<allocator_type>::value, "");
static_assert(is_same<typename allocator_type::value_type, value_type>::value,
diff --git a/libcxx/include/array b/libcxx/include/array
index 99b38d049e129d..b92d81cce69a77 100644
--- a/libcxx/include/array
+++ b/libcxx/include/array
@@ -131,6 +131,7 @@ template <size_t I, class T, size_t N> const T&& get(const array<T, N>&&) noexce
#include <__type_traits/is_const.h>
#include <__type_traits/is_constructible.h>
#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
#include <__type_traits/is_trivially_relocatable.h>
@@ -171,6 +172,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
template <class _Tp, size_t _Size>
struct _LIBCPP_TEMPLATE_VIS array {
using __trivially_relocatable = __conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, array, void>;
+ using __replaceable = __conditional_t<__is_replaceable<_Tp>::value, array, void>;
// types:
using __self = array;
diff --git a/libcxx/include/deque b/libcxx/include/deque
index ad667503489741..58ac6e9e0b995a 100644
--- a/libcxx/include/deque
+++ b/libcxx/include/deque
@@ -227,6 +227,7 @@ template <class T, class Allocator, class Predicate>
#include <__type_traits/is_convertible.h>
#include <__type_traits/is_nothrow_assignable.h>
#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
#include <__type_traits/is_trivially_relocatable.h>
@@ -526,6 +527,8 @@ public:
__libcpp_is_trivially_relocatable<__map>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
deque,
void>;
+ using __replaceable =
+ __conditional_t<__is_replaceable<__map>::value && __is_replaceable<allocator_type>::value, deque, void>;
static_assert(is_nothrow_default_constructible<allocator_type>::value ==
is_nothrow_default_constructible<__pointer_allocator>::value,
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 139c0a83666435..dbef9c75d37457 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -274,6 +274,10 @@ module std_core [system] {
header "__type_traits/is_referenceable.h"
export std_core.type_traits.integral_constant
}
+ module is_replaceable {
+ header "__type_traits/is_replaceable.h"
+ export std_core.type_traits.integral_constant
+ }
module is_same {
header "__type_traits/is_same.h"
export std_core.type_traits.integral_constant
diff --git a/libcxx/include/optional b/libcxx/include/optional
index 7ad6a9e116941f..a2425323dbcd49 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -207,6 +207,7 @@ namespace std {
#include <__type_traits/is_nothrow_constructible.h>
#include <__type_traits/is_object.h>
#include <__type_traits/is_reference.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_scalar.h>
#include <__type_traits/is_swappable.h>
@@ -587,6 +588,7 @@ public:
using value_type = _Tp;
using __trivially_relocatable = conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, optional, void>;
+ using __replaceable = conditional_t<__is_replaceable<_Tp>::value, optional, void>;
private:
// Disable the reference extension using this static assert.
diff --git a/libcxx/include/string b/libcxx/include/string
index bf7fc3c37ecd7a..90465615757080 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -627,6 +627,7 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
#include <__type_traits/is_convertible.h>
#include <__type_traits/is_nothrow_assignable.h>
#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_standard_layout.h>
#include <__type_traits/is_trivial.h>
@@ -800,6 +801,8 @@ public:
basic_string,
void>;
#endif
+ using __replaceable =
+ __conditional_t<__is_replaceable<allocator_type>::value && __is_replaceable<pointer>::value, basic_string, void>;
#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __asan_volatile_wrapper(pointer const& __ptr) const {
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index c3f7b8041686d1..1ed88526753989 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -247,6 +247,7 @@ template <class... Types>
#include <__type_traits/is_nothrow_assignable.h>
#include <__type_traits/is_nothrow_constructible.h>
#include <__type_traits/is_reference.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
#include <__type_traits/is_trivially_relocatable.h>
@@ -550,6 +551,7 @@ class _LIBCPP_TEMPLATE_VIS tuple {
public:
using __trivially_relocatable = __conditional_t<_And<__libcpp_is_trivially_relocatable<_Tp>...>::value, tuple, void>;
+ using __replaceable = __conditional_t<_And<__is_replaceable<_Tp>...>::value, tuple, void>;
// [tuple.cnstr]
diff --git a/libcxx/include/variant b/libcxx/include/variant
index f604527cd22569..7457ffa26fba81 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -243,6 +243,7 @@ namespace std {
#include <__type_traits/is_nothrow_assignable.h>
#include <__type_traits/is_nothrow_constructible.h>
#include <__type_traits/is_reference.h>
+#include <__type_traits/is_replaceable.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
#include <__type_traits/is_trivially_assignable.h>
@@ -1173,6 +1174,7 @@ class _LIBCPP_TEMPLATE_VIS _LIBCPP_DECLSPEC_EMPTY_BASES variant
public:
using __trivially_relocatable =
conditional_t<_And<__libcpp_is_trivially_relocatable<_Types>...>::value, variant, void>;
+ using __replaceable = conditional_t<_And<__is_replaceable<_Types>...>::value, variant, void>;
template <bool _Dummy = true,
enable_if_t<__dependent_type<is_default_constructible<__first_type>, _Dummy>::value, int> = 0>
>From 49ff3550b42d4dfeadf1107f71d32287e2aedfb2 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Mon, 18 Nov 2024 14:15:02 +0100
Subject: [PATCH 3/6] Add relocation facilities that P2687 could provide
---
libcxx/include/CMakeLists.txt | 3 +
.../is_trivially_allocator_relocatable.h | 57 +++++++++++
libcxx/include/__memory/relocate_at.h | 70 ++++++++++++++
.../__memory/uninitialized_algorithms.h | 21 +---
.../include/__memory/uninitialized_relocate.h | 95 +++++++++++++++++++
.../__type_traits/is_trivially_relocatable.h | 5 +
libcxx/include/__vector/vector.h | 6 +-
libcxx/include/module.modulemap | 5 +
8 files changed, 242 insertions(+), 20 deletions(-)
create mode 100644 libcxx/include/__memory/is_trivially_allocator_relocatable.h
create mode 100644 libcxx/include/__memory/relocate_at.h
create mode 100644 libcxx/include/__memory/uninitialized_relocate.h
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 27d65bc523d123..e884d2dcfd27f9 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -552,18 +552,21 @@ set(files
__memory/construct_at.h
__memory/destruct_n.h
__memory/inout_ptr.h
+ __memory/is_trivially_allocator_relocatable.h
__memory/noexcept_move_assign_container.h
__memory/out_ptr.h
__memory/pointer_traits.h
__memory/ranges_construct_at.h
__memory/ranges_uninitialized_algorithms.h
__memory/raw_storage_iterator.h
+ __memory/relocate_at.h
__memory/shared_count.h
__memory/shared_ptr.h
__memory/swap_allocator.h
__memory/temp_value.h
__memory/temporary_buffer.h
__memory/uninitialized_algorithms.h
+ __memory/uninitialized_relocate.h
__memory/unique_ptr.h
__memory/unique_temporary_buffer.h
__memory/uses_allocator.h
diff --git a/libcxx/include/__memory/is_trivially_allocator_relocatable.h b/libcxx/include/__memory/is_trivially_allocator_relocatable.h
new file mode 100644
index 00000000000000..64563b3d7a92df
--- /dev/null
+++ b/libcxx/include/__memory/is_trivially_allocator_relocatable.h
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_IS_TRIVIALLY_ALLOCATOR_RELOCATABLE_H
+#define _LIBCPP___MEMORY_IS_TRIVIALLY_ALLOCATOR_RELOCATABLE_H
+
+#include <__config>
+#include <__memory/allocator_traits.h>
+#include <__type_traits/conjunction.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/negation.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// A type is trivially allocator relocatable if the allocator's move construction and destruction
+// don't do anything beyond calling the type's move constructor and destructor, and if the type
+// itself is trivially relocatable.
+
+template <class _Alloc, class _Type>
+struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {};
+
+template <class _Type>
+struct __allocator_has_trivial_move_construct<allocator<_Type>, _Type> : true_type {};
+
+template <class _Alloc, class _Tp>
+struct __allocator_has_trivial_destroy : _Not<__has_destroy<_Alloc, _Tp*> > {};
+
+template <class _Tp, class _Up>
+struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
+
+template <class _Alloc, class _Tp>
+struct __is_trivially_allocator_relocatable
+ : integral_constant<bool,
+ __allocator_has_trivial_move_construct<_Alloc, _Tp>::value &&
+ __allocator_has_trivial_destroy<_Alloc, _Tp>::value &&
+ __libcpp_is_trivially_relocatable<_Tp>::value> {};
+
+template <class _Alloc, class _Tp>
+struct __is_nothrow_allocator_relocatable
+ : _Or<_And<__allocator_has_trivial_move_construct<_Alloc, _Tp>, is_nothrow_move_constructible<_Tp>>,
+ __is_trivially_allocator_relocatable<_Alloc, _Tp>> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_IS_TRIVIALLY_ALLOCATOR_RELOCATABLE_H
diff --git a/libcxx/include/__memory/relocate_at.h b/libcxx/include/__memory/relocate_at.h
new file mode 100644
index 00000000000000..4ec47f04d9990b
--- /dev/null
+++ b/libcxx/include/__memory/relocate_at.h
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RELOCATE_AT_H
+#define _LIBCPP___MEMORY_RELOCATE_AT_H
+
+#include <__memory/allocator_traits.h>
+#include <__memory/construct_at.h>
+#include <__memory/is_trivially_allocator_relocatable.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__utility/move.h>
+#include <__utility/scope_guard.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __destroy_object {
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 void operator()() const { std::__destroy_at(__obj_); }
+ _Tp* __obj_;
+};
+
+template <class _Tp, __enable_if_t<!__libcpp_is_trivially_relocatable<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __relocate_at(_Tp* __source, _Tp* __dest) {
+ __scope_guard<__destroy_object<_Tp> > __guard(__destroy_object<_Tp>{__source});
+ return std::__construct_at(__dest, std::move(*__source));
+}
+
+template <class _Tp, __enable_if_t<__libcpp_is_trivially_relocatable<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __relocate_at(_Tp* __source, _Tp* __dest) {
+ if (__libcpp_is_constant_evaluated()) {
+ std::__construct_at(__dest, std::move(*__source));
+ } else {
+ // Casting to void* to suppress clang complaining that this is technically UB.
+ __builtin_memcpy(static_cast<void*>(__dest), __source, sizeof(_Tp));
+ }
+ return __dest;
+}
+
+template <class _Alloc, class _Tp, __enable_if_t<!__is_trivially_allocator_relocatable<_Alloc, _Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp*
+__allocator_relocate_at(_Alloc& __alloc, _Tp* __source, _Tp* __dest) {
+ allocator_traits<_Alloc>::construct(__alloc, __dest, std::move(*__source));
+ allocator_traits<_Alloc>::destroy(__alloc, __source);
+ return __dest;
+}
+
+template <class _Alloc, class _Tp, __enable_if_t<__is_trivially_allocator_relocatable<_Alloc, _Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __allocator_relocate_at(_Alloc&, _Tp* __source, _Tp* __dest) {
+ return std::__relocate_at(__source, __dest);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_RELOCATE_AT_H
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index 71c7ed94fec13e..1462643075a593 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -20,6 +20,7 @@
#include <__memory/addressof.h>
#include <__memory/allocator_traits.h>
#include <__memory/construct_at.h>
+#include <__memory/is_trivially_allocator_relocatable.h>
#include <__memory/pointer_traits.h>
#include <__type_traits/enable_if.h>
#include <__type_traits/extent.h>
@@ -591,19 +592,7 @@ __uninitialized_allocator_copy(_Alloc& __alloc, _Iter1 __first1, _Sent1 __last1,
return std::__rewrap_iter(__first2, __result);
}
-template <class _Alloc, class _Type>
-struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {};
-
-template <class _Type>
-struct __allocator_has_trivial_move_construct<allocator<_Type>, _Type> : true_type {};
-
-template <class _Alloc, class _Tp>
-struct __allocator_has_trivial_destroy : _Not<__has_destroy<_Alloc, _Tp*> > {};
-
-template <class _Tp, class _Up>
-struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
-
-// __uninitialized_allocator_relocate relocates the objects in [__first, __last) into __result.
+// __uninitialized_allocator_relocate_for_vector relocates the objects in [__first, __last) into __result.
// Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy,
// except that the move constructor and destructor may never be called if they are known to be equivalent to a memcpy.
//
@@ -616,15 +605,13 @@ struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
// - is_copy_constructible<_ValueType>
// - __libcpp_is_trivially_relocatable<_ValueType>
template <class _Alloc, class _ContiguousIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __uninitialized_allocator_relocate(
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __uninitialized_allocator_relocate_for_vector(
_Alloc& __alloc, _ContiguousIterator __first, _ContiguousIterator __last, _ContiguousIterator __result) {
static_assert(__libcpp_is_contiguous_iterator<_ContiguousIterator>::value, "");
using _ValueType = typename iterator_traits<_ContiguousIterator>::value_type;
static_assert(__is_cpp17_move_insertable<_Alloc>::value,
"The specified type does not meet the requirements of Cpp17MoveInsertable");
- if (__libcpp_is_constant_evaluated() || !__libcpp_is_trivially_relocatable<_ValueType>::value ||
- !__allocator_has_trivial_move_construct<_Alloc, _ValueType>::value ||
- !__allocator_has_trivial_destroy<_Alloc, _ValueType>::value) {
+ if (__libcpp_is_constant_evaluated() || !__is_trivially_allocator_relocatable<_Alloc, _ValueType>::value) {
auto __destruct_first = __result;
auto __guard = std::__make_exception_guard(
_AllocatorDestroyRangeReverse<_Alloc, _ContiguousIterator>(__alloc, __destruct_first, __result));
diff --git a/libcxx/include/__memory/uninitialized_relocate.h b/libcxx/include/__memory/uninitialized_relocate.h
new file mode 100644
index 00000000000000..0e445f1cbe0beb
--- /dev/null
+++ b/libcxx/include/__memory/uninitialized_relocate.h
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H
+#define _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/is_trivially_allocator_relocatable.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/relocate_at.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __uninitialized_relocate relocates the objects in [__first, __last) into __result.
+//
+// Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy,
+// except that the move constructor and destructor may never be called if they are known to be equivalent to a memcpy.
+//
+// This algorithm works even if part of the resulting range overlaps with [__first, __last), as long as __result itself
+// is not in [__first, last).
+//
+// This algorithm doesn't throw any exceptions. However, it requires the types in the range to be nothrow
+// move-constructible and the iterator operations not to throw any exceptions.
+//
+// Preconditions:
+// - __result doesn't contain any objects and [__first, __last) contains objects
+// - __result is not in [__first, __last)
+// Postconditions:
+// - __result contains the objects from [__first, __last)
+// - [__first, __last) doesn't contain any objects
+template <class _ContiguousIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ContiguousIterator __uninitialized_relocate(
+ _ContiguousIterator __first, _ContiguousIterator __last, _ContiguousIterator __result) _NOEXCEPT {
+ using _ValueType = typename iterator_traits<_ContiguousIterator>::value_type;
+ static_assert(__libcpp_is_contiguous_iterator<_ContiguousIterator>::value, "");
+ static_assert(__is_nothrow_relocatable<_ValueType>::value, "");
+ if (!__libcpp_is_constant_evaluated() && __libcpp_is_trivially_relocatable<_ValueType>::value) {
+ auto const __n = __last - __first;
+ // Casting to void* to suppress clang complaining that this is technically UB.
+ __builtin_memmove(
+ static_cast<void*>(std::__to_address(__result)), std::__to_address(__first), sizeof(_ValueType) * __n);
+ return __result + __n;
+ } else {
+ while (__first != __last) {
+ std::__relocate_at(std::__to_address(__first), std::__to_address(__result));
+ ++__first;
+ ++__result;
+ }
+ return __result;
+ }
+}
+
+// Equivalent to __uninitialized_relocate, but uses the provided allocator's construct() and destroy() methods.
+template <class _Alloc, class _ContiguousIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ContiguousIterator __uninitialized_allocator_relocate(
+ _Alloc& __alloc, _ContiguousIterator __first, _ContiguousIterator __last, _ContiguousIterator __result) _NOEXCEPT {
+ using _ValueType = typename iterator_traits<_ContiguousIterator>::value_type;
+ static_assert(__libcpp_is_contiguous_iterator<_ContiguousIterator>::value, "");
+ static_assert(__is_nothrow_relocatable<_ValueType>::value, "");
+ if (!__libcpp_is_constant_evaluated() && __is_trivially_allocator_relocatable<_Alloc, _ValueType>::value) {
+ (void)__alloc; // ignore the allocator
+ return std::__uninitialized_relocate(std::move(__first), std::move(__last), std::move(__result));
+ } else {
+ while (__first != __last) {
+ std::__allocator_relocate_at(__alloc, std::__to_address(__first), std::__to_address(__result));
+ ++__first;
+ ++__result;
+ }
+ return __result;
+ }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H
diff --git a/libcxx/include/__type_traits/is_trivially_relocatable.h b/libcxx/include/__type_traits/is_trivially_relocatable.h
index c0871731cc0016..4c3cc3749adc1b 100644
--- a/libcxx/include/__type_traits/is_trivially_relocatable.h
+++ b/libcxx/include/__type_traits/is_trivially_relocatable.h
@@ -10,8 +10,10 @@
#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
#include <__config>
+#include <__type_traits/disjunction.h>
#include <__type_traits/enable_if.h>
#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_nothrow_constructible.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_trivially_copyable.h>
@@ -37,6 +39,9 @@ struct __libcpp_is_trivially_relocatable<_Tp,
__enable_if_t<is_same<_Tp, typename _Tp::__trivially_relocatable>::value> >
: true_type {};
+template <class _Tp>
+struct __is_nothrow_relocatable : _Or<is_nothrow_move_constructible<_Tp>, __libcpp_is_trivially_relocatable<_Tp>> {};
+
_LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 0fd9ff55db3bd1..76c931508f673d 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -809,7 +809,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void
vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v) {
__annotate_delete();
auto __new_begin = __v.__begin_ - (__end_ - __begin_);
- std::__uninitialized_allocator_relocate(
+ std::__uninitialized_allocator_relocate_for_vector(
this->__alloc_, std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin));
__v.__begin_ = __new_begin;
__end_ = __begin_; // All the objects have been destroyed by relocating them.
@@ -832,13 +832,13 @@ vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, a
// Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_)
// in case something in [__begin_, __p) throws.
- std::__uninitialized_allocator_relocate(
+ std::__uninitialized_allocator_relocate_for_vector(
this->__alloc_, std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_));
__v.__end_ += (__end_ - __p);
__end_ = __p; // The objects in [__p, __end_) have been destroyed by relocating them.
auto __new_begin = __v.__begin_ - (__p - __begin_);
- std::__uninitialized_allocator_relocate(
+ std::__uninitialized_allocator_relocate_for_vector(
this->__alloc_, std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin));
__v.__begin_ = __new_begin;
__end_ = __begin_; // All the objects have been destroyed by relocating them.
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index dbef9c75d37457..8bb549e8e87879 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1539,6 +1539,7 @@ module std [system] {
module destruct_n { header "__memory/destruct_n.h" }
module fwd { header "__fwd/memory.h" }
module inout_ptr { header "__memory/inout_ptr.h" }
+ module is_trivially_allocator_relocatable { header "__memory/is_trivially_allocator_relocatable.h" }
module noexcept_move_assign_container { header "__memory/noexcept_move_assign_container.h" }
module out_ptr { header "__memory/out_ptr.h" }
module pointer_traits { header "__memory/pointer_traits.h" }
@@ -1548,6 +1549,7 @@ module std [system] {
export std.algorithm.in_out_result
}
module raw_storage_iterator { header "__memory/raw_storage_iterator.h" }
+ module relocate_at { header "__memory/relocate_at.h" }
module shared_count { header "__memory/shared_count.h" }
module shared_ptr { header "__memory/shared_ptr.h" }
module swap_allocator { header "__memory/swap_allocator.h" }
@@ -1560,6 +1562,9 @@ module std [system] {
header "__memory/uninitialized_algorithms.h"
export std.utility.pair
}
+ module uninitialized_relocate {
+ header "__memory/uninitialized_relocate.h"
+ }
module unique_ptr {
header "__memory/unique_ptr.h"
}
>From 21f29c4d180c333396008d1b3e69b246fd05868f Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Thu, 14 Nov 2024 13:54:18 +0100
Subject: [PATCH 4/6] Take advantage of trivial relocation in
std::vector::erase
---
libcxx/docs/ReleaseNotes/20.rst | 3 ++
libcxx/include/__vector/vector.h | 61 +++++++++++++++++++-------------
2 files changed, 40 insertions(+), 24 deletions(-)
diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index 9039c6f046445b..fdb862aa69da9f 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -52,6 +52,9 @@ Improvements and New Features
- The ``lexicographical_compare`` and ``ranges::lexicographical_compare`` algorithms have been optimized for trivially
equality comparable types, resulting in a performance improvement of up to 40x.
+- The ``std::vector::erase`` function has been optimized for types that can be relocated trivially (such as ``std::string``),
+ yielding speed ups witnessed to be around 2x for these types (but subject to the use case).
+
- The ``_LIBCPP_ENABLE_CXX20_REMOVED_TEMPORARY_BUFFER`` macro has been added to make ``std::get_temporary_buffer`` and
``std::return_temporary_buffer`` available.
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 76c931508f673d..9ceefedeadd674 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -34,11 +34,13 @@
#include <__memory/allocator.h>
#include <__memory/allocator_traits.h>
#include <__memory/compressed_pair.h>
+#include <__memory/is_trivially_allocator_relocatable.h>
#include <__memory/noexcept_move_assign_container.h>
#include <__memory/pointer_traits.h>
#include <__memory/swap_allocator.h>
#include <__memory/temp_value.h>
#include <__memory/uninitialized_algorithms.h>
+#include <__memory/uninitialized_relocate.h>
#include <__ranges/access.h>
#include <__ranges/concepts.h>
#include <__ranges/container_compatible_range.h>
@@ -518,8 +520,41 @@ class _LIBCPP_TEMPLATE_VIS vector {
}
#endif
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position);
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last);
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position) {
+ _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+ __position != end(), "vector::erase(iterator) called with a non-dereferenceable iterator");
+ return erase(__position, __position + 1);
+ }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __cfirst, const_iterator __clast) {
+ _LIBCPP_ASSERT_VALID_INPUT_RANGE(__cfirst <= __clast, "vector::erase(first, last) called with invalid range");
+
+ iterator __first = begin() + std::distance(cbegin(), __cfirst);
+ iterator __last = begin() + std::distance(cbegin(), __clast);
+ if (__first == __last)
+ return __last;
+
+ auto const __n = std::distance(__first, __last);
+
+ // If the value_type is nothrow relocatable, we destroy the range being erased and we relocate the tail
+ // of the vector into the created gap. This is especially efficient if the elements are trivially relocatable.
+ // Otherwise, we use the standard technique with move-assignments.
+ //
+ // Note that we also require __is_replaceable here for backwards compatibility, because we used
+ // to perform move-assignments unconditionally. If we didn't enforce that, we would no longer call
+ // the assignment operator of types that have a funky operator= and expect it to be called in
+ // vector::erase.
+ if constexpr (__is_replaceable<value_type>::value && __is_nothrow_allocator_relocatable<_Allocator, value_type>::value) {
+ std::__allocator_destroy(this->__alloc_, __first, __last);
+ std::__uninitialized_allocator_relocate(this->__alloc_, __last, end(), __first);
+ } else {
+ auto __new_end = std::move(__last, end(), __first);
+ std::__allocator_destroy(this->__alloc_, __new_end, end());
+ }
+
+ this->__end_ -= __n;
+ __annotate_shrink(size() + __n);
+ return __first;
+ }
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT {
size_type __old_size = size();
@@ -1113,28 +1148,6 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 inline
#endif
}
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::erase(const_iterator __position) {
- _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
- __position != end(), "vector::erase(iterator) called with a non-dereferenceable iterator");
- difference_type __ps = __position - cbegin();
- pointer __p = this->__begin_ + __ps;
- this->__destruct_at_end(std::move(__p + 1, this->__end_, __p));
- return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::erase(const_iterator __first, const_iterator __last) {
- _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "vector::erase(first, last) called with invalid range");
- pointer __p = this->__begin_ + (__first - begin());
- if (__first != __last) {
- this->__destruct_at_end(std::move(__p + (__last - __first), this->__end_, __p));
- }
- return __make_iter(__p);
-}
-
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void
vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointer __to) {
>From 757933e65460904db1fddb3c1e5a7b9e9faaf159 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Mon, 18 Nov 2024 17:36:19 +0100
Subject: [PATCH 5/6] Formulate vector::emplace and insert in terms of
relocation
---
libcxx/include/__memory/temp_value.h | 18 ++++
libcxx/include/__vector/vector.h | 134 ++++++++++++++-------------
2 files changed, 87 insertions(+), 65 deletions(-)
diff --git a/libcxx/include/__memory/temp_value.h b/libcxx/include/__memory/temp_value.h
index 4a133b3fbcf6c0..bd23de3aef8320 100644
--- a/libcxx/include/__memory/temp_value.h
+++ b/libcxx/include/__memory/temp_value.h
@@ -21,6 +21,24 @@
_LIBCPP_BEGIN_NAMESPACE_STD
+template <class _Tp>
+struct __temporary_emplace_value {
+ union {
+ _Tp __value_;
+ };
+
+ template <class _Allocator, class... _Args>
+ _LIBCPP_HIDE_FROM_ABI
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __temporary_emplace_value(_Allocator& __alloc, _Args&&... __args) {
+ allocator_traits<_Allocator>::construct(__alloc, std::addressof(__value_), std::forward<_Args>(__args)...);
+ }
+
+ // Don't destroy anything, since we assume that the value gets relocated by whoever uses this type
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__temporary_emplace_value() {}
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __address() { return std::addressof(__value_); }
+};
+
template <class _Tp, class _Alloc>
struct __temp_value {
typedef allocator_traits<_Alloc> _Traits;
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 9ceefedeadd674..3ed5ad5bafaf93 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -472,11 +472,77 @@ class _LIBCPP_TEMPLATE_VIS vector {
this->__destruct_at_end(this->__end_ - 1);
}
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x);
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x) {
+ return emplace(std::move(__position), __x);
+ }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x);
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x) {
+ return emplace(std::move(__position), std::move(__x));
+ }
template <class... _Args>
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __position, _Args&&... __args);
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __cposition, _Args&&... __args) {
+ iterator __position = begin() + (__cposition - cbegin());
+ if (this->__end_ < this->__cap_) {
+ if (__position == end()) {
+ allocator_traits<_Allocator>::construct(
+ this->__alloc_, std::__to_address(__position), std::forward<_Args>(__args)...);
+ ++this->__end_;
+ } else {
+ // Construct a temporary value on the stack, so that in case this throws we haven't modified
+ // the vector yet. This also takes care of the corner case where we'd be trying to insert
+ // from an element located in the vector itself, in which case we'd otherwise have to be
+ // careful about reference invalidation if we didn't make a temporary value.
+ __temporary_emplace_value<value_type> __tmp(this->__alloc_, std::forward<_Args>(__args)...);
+
+ // If the elements are nothrow relocatable, we can relocate them without risking an exception.
+ // So open up a gap inside the vector, relocate everything to the right and insert the new
+ // element into the right spot.
+ //
+ // Otherwise, we have no choice but to shift everything to the right using move-assignments
+ // and to move-assign the new element into its final location, to ensure that everything gets
+ // properly destroyed in case of an exception.
+ //
+ // Note that we also require __is_replaceable here for backwards compatibility, because we used
+ // to perform move-assignments unconditionally. If we didn't enforce that, we would no longer call
+ // the assignment operator of types that have a funky operator= and expect it to be called in
+ // vector::insert.
+ if constexpr (__is_replaceable<value_type>::value && __is_nothrow_allocator_relocatable<_Allocator, value_type>::value) {
+ // Relocate all the elements in the vector to open up a gap.
+ std::__uninitialized_allocator_relocate(this->__alloc_, __position, end(), __position + 1);
+
+ // Finally, relocate the temporary value into its final location. We don't need to destroy
+ // the temporary value since it has been relocated.
+ std::__allocator_relocate_at(this->__alloc_, __tmp.__address(), std::__to_address(__position));
+ ++this->__end_;
+ } else {
+ // Ensure the destruction of the temporary no matter what happens exception-wise.
+ auto __guard = std::__make_scope_guard([&] {
+ allocator_traits<_Allocator>::destroy(this->__alloc_, __tmp.__address());
+ });
+
+ // Move-construct the (new) last element. There is no object at this location, so
+ // we must use construction.
+ allocator_traits<_Allocator>::construct(this->__alloc_, std::__to_address(end()), std::move(*(end() - 1)));
+ ++this->__end_;
+
+ // We now have a moved-from object at `end() - 1`. Shift the rest of the range to the right,
+ // opening up a gap containing a moved-from object at the insert position.
+ std::move_backward(__position, end() - 1, end());
+
+ // Finally, move-assign the new element into its insert position.
+ *__position = std::move(*__tmp.__address());
+ }
+ }
+ __annotate_increase(1);
+ return __position;
+ } else {
+ __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __position - begin(), this->__alloc_);
+ __v.emplace_back(std::forward<_Args>(__args)...);
+ pointer __p = this->__begin_ + (__position - begin());
+ __p = __swap_out_circular_buffer(__v, __p);
+ return __make_iter(__p);
+ }
+ }
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
insert(const_iterator __position, size_type __n, const_reference __x);
@@ -1163,68 +1229,6 @@ vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointe
std::move_backward(__from_s, __from_s + __n, __old_last);
}
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x) {
- pointer __p = this->__begin_ + (__position - begin());
- if (this->__end_ < this->__cap_) {
- if (__p == this->__end_) {
- __construct_one_at_end(__x);
- } else {
- __move_range(__p, this->__end_, __p + 1);
- const_pointer __xr = pointer_traits<const_pointer>::pointer_to(__x);
- if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end_), std::addressof(__x)))
- ++__xr;
- *__p = *__xr;
- }
- } else {
- __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
- __v.emplace_back(__x);
- __p = __swap_out_circular_buffer(__v, __p);
- }
- return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, value_type&& __x) {
- pointer __p = this->__begin_ + (__position - begin());
- if (this->__end_ < this->__cap_) {
- if (__p == this->__end_) {
- __construct_one_at_end(std::move(__x));
- } else {
- __move_range(__p, this->__end_, __p + 1);
- *__p = std::move(__x);
- }
- } else {
- __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
- __v.emplace_back(std::move(__x));
- __p = __swap_out_circular_buffer(__v, __p);
- }
- return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-template <class... _Args>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) {
- pointer __p = this->__begin_ + (__position - begin());
- if (this->__end_ < this->__cap_) {
- if (__p == this->__end_) {
- __construct_one_at_end(std::forward<_Args>(__args)...);
- } else {
- __temp_value<value_type, _Allocator> __tmp(this->__alloc_, std::forward<_Args>(__args)...);
- __move_range(__p, this->__end_, __p + 1);
- *__p = std::move(__tmp.get());
- }
- } else {
- __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
- __v.emplace_back(std::forward<_Args>(__args)...);
- __p = __swap_out_circular_buffer(__v, __p);
- }
- return __make_iter(__p);
-}
-
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
vector<_Tp, _Allocator>::insert(const_iterator __position, size_type __n, const_reference __x) {
>From 7d8338366f56c2a12f27c7d19918e27dcd21a855 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Tue, 19 Nov 2024 00:31:21 +0100
Subject: [PATCH 6/6] OPTIONAL: Remove the requirement of replaceability
This is an optional relaxation in case we are OK with changing
the behavior for some users that had questionable assignment
operators.
---
libcxx/include/__vector/vector.h | 14 ++--------
.../vector/vector.modifiers/common.h | 22 ----------------
.../vector.modifiers/erase_iter.pass.cpp | 26 -------------------
.../vector.modifiers/erase_iter_iter.pass.cpp | 26 -------------------
4 files changed, 2 insertions(+), 86 deletions(-)
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 3ed5ad5bafaf93..8175039977218a 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -501,12 +501,7 @@ class _LIBCPP_TEMPLATE_VIS vector {
// Otherwise, we have no choice but to shift everything to the right using move-assignments
// and to move-assign the new element into its final location, to ensure that everything gets
// properly destroyed in case of an exception.
- //
- // Note that we also require __is_replaceable here for backwards compatibility, because we used
- // to perform move-assignments unconditionally. If we didn't enforce that, we would no longer call
- // the assignment operator of types that have a funky operator= and expect it to be called in
- // vector::insert.
- if constexpr (__is_replaceable<value_type>::value && __is_nothrow_allocator_relocatable<_Allocator, value_type>::value) {
+ if constexpr (__is_nothrow_allocator_relocatable<_Allocator, value_type>::value) {
// Relocate all the elements in the vector to open up a gap.
std::__uninitialized_allocator_relocate(this->__alloc_, __position, end(), __position + 1);
@@ -604,12 +599,7 @@ class _LIBCPP_TEMPLATE_VIS vector {
// If the value_type is nothrow relocatable, we destroy the range being erased and we relocate the tail
// of the vector into the created gap. This is especially efficient if the elements are trivially relocatable.
// Otherwise, we use the standard technique with move-assignments.
- //
- // Note that we also require __is_replaceable here for backwards compatibility, because we used
- // to perform move-assignments unconditionally. If we didn't enforce that, we would no longer call
- // the assignment operator of types that have a funky operator= and expect it to be called in
- // vector::erase.
- if constexpr (__is_replaceable<value_type>::value && __is_nothrow_allocator_relocatable<_Allocator, value_type>::value) {
+ if constexpr (__is_nothrow_allocator_relocatable<_Allocator, value_type>::value) {
std::__allocator_destroy(this->__alloc_, __first, __last);
std::__uninitialized_allocator_relocate(this->__alloc_, __last, end(), __first);
} else {
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/common.h b/libcxx/test/std/containers/sequences/vector/vector.modifiers/common.h
index 72cd47a50b2c0d..24627d9fe5f450 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/common.h
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/common.h
@@ -40,28 +40,6 @@ struct Throws {
bool Throws::sThrows = false;
#endif
-struct Tracker {
- int copy_assignments = 0;
- int move_assignments = 0;
-};
-
-struct TrackedAssignment {
- Tracker* tracker_;
- TEST_CONSTEXPR_CXX14 explicit TrackedAssignment(Tracker* tracker) : tracker_(tracker) {}
-
- TrackedAssignment(TrackedAssignment const&) = default;
- TrackedAssignment(TrackedAssignment&&) = default;
-
- TEST_CONSTEXPR_CXX14 TrackedAssignment& operator=(TrackedAssignment const&) {
- tracker_->copy_assignments++;
- return *this;
- }
- TEST_CONSTEXPR_CXX14 TrackedAssignment& operator=(TrackedAssignment&&) {
- tracker_->move_assignments++;
- return *this;
- }
-};
-
struct NonTriviallyRelocatable {
int value_;
TEST_CONSTEXPR NonTriviallyRelocatable() : value_(0) {}
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp
index f0157eb74b90f3..b2948ba5a46cd1 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp
@@ -107,31 +107,5 @@ int main(int, char**) {
}
#endif
- // Make sure we satisfy the complexity requirement in terms of the number of times the assignment
- // operator is called.
- //
- // There is currently ambiguity as to whether this is truly mandated by the Standard, so we only
- // test it for libc++.
-#ifdef _LIBCPP_VERSION
- {
- Tracker tracker;
- std::vector<TrackedAssignment> v;
-
- // Set up the vector with 5 elements.
- for (int i = 0; i != 5; ++i) {
- v.emplace_back(&tracker);
- }
- assert(tracker.copy_assignments == 0);
- assert(tracker.move_assignments == 0);
-
- // Erase element [1] from it. Elements [2] [3] [4] should be shifted, so we should
- // see 3 move assignments (and nothing else).
- v.erase(v.begin() + 1);
- assert(v.size() == 4);
- assert(tracker.copy_assignments == 0);
- assert(tracker.move_assignments == 3);
- }
-#endif
-
return 0;
}
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter_iter.pass.cpp
index 104dfb4cb07d4f..fc00a720ece2f7 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter_iter.pass.cpp
@@ -196,31 +196,5 @@ int main(int, char**) {
assert(it == v.begin() + 2);
}
- // Make sure we satisfy the complexity requirement in terms of the number of times the assignment
- // operator is called.
- //
- // There is currently ambiguity as to whether this is truly mandated by the Standard, so we only
- // test it for libc++.
-#ifdef _LIBCPP_VERSION
- {
- Tracker tracker;
- std::vector<TrackedAssignment> v;
-
- // Set up the vector with 5 elements.
- for (int i = 0; i != 5; ++i) {
- v.emplace_back(&tracker);
- }
- assert(tracker.copy_assignments == 0);
- assert(tracker.move_assignments == 0);
-
- // Erase elements [1] and [2] from it. Elements [3] [4] should be shifted, so we should
- // see 2 move assignments (and nothing else).
- v.erase(v.begin() + 1, v.begin() + 3);
- assert(v.size() == 3);
- assert(tracker.copy_assignments == 0);
- assert(tracker.move_assignments == 2);
- }
-#endif
-
return 0;
}
More information about the libcxx-commits
mailing list