[libcxx-commits] [libcxx] [libc++] Introduce preliminary relocation primitives (PR #156903)
Louis Dionne via libcxx-commits
libcxx-commits at lists.llvm.org
Thu Sep 4 08:01:30 PDT 2025
https://github.com/ldionne created https://github.com/llvm/llvm-project/pull/156903
This patch introduces the following utilities around relocation:
- __relocate_at, which relocates a single object
- a __is_nothrow_relocatable trait
- __uninitialized_relocate and __uninitialized_relocate_backward
- allocator-aware variants of all of those
These facilities are preliminary work towards implementing https://wg21.link/p3516, which has been design-approved but not yet voted into the Standard. This patch is careful not to introduce any new public names officially, since the TR APIs are still expected to undergo changes in the C++26 cycle.
The reason for wanting to land this patch is that it will allow reworking the implementation of vector to take advantage of relocation in more cases, while being consistent with the ongoing and future direction of trivial relocation in the Standard.
>From 306d3f366977d68157a75f531b4da0b3f648813d Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Fri, 21 Mar 2025 14:18:34 -0400
Subject: [PATCH] [libc++] Introduce preliminary relocation primitives
This patch introduces the following utilities around relocation:
- __relocate_at, which relocates a single object
- a __is_nothrow_relocatable trait
- __uninitialized_relocate and __uninitialized_relocate_backward
- allocator-aware variants of all of those
These facilities are preliminary work towards implementing
https://wg21.link/p3516, which has been design-approved but
not yet voted into the Standard. This patch is careful not
to introduce any new public names officially, since the TR
APIs are still expected to undergo changes in the C++26 cycle.
The reason for wanting to land this patch is that it will allow
reworking the implementation of vector to take advantage of
relocation in more cases, while being consistent with the ongoing
and future direction of trivial relocation in the Standard.
---
libcxx/include/CMakeLists.txt | 4 +
.../include/__memory/allocator_relocation.h | 196 ++++++++++++
libcxx/include/__memory/relocate_at.h | 95 ++++++
.../__memory/uninitialized_algorithms.h | 9 +-
.../include/__memory/uninitialized_relocate.h | 147 +++++++++
.../__type_traits/is_nothrow_relocatable.h | 33 ++
libcxx/include/__vector/vector.h | 10 +-
libcxx/include/module.modulemap.in | 9 +
.../uninitialized_allocator_relocate.pass.cpp | 284 ++++++++++++++++++
9 files changed, 780 insertions(+), 7 deletions(-)
create mode 100644 libcxx/include/__memory/allocator_relocation.h
create mode 100644 libcxx/include/__memory/relocate_at.h
create mode 100644 libcxx/include/__memory/uninitialized_relocate.h
create mode 100644 libcxx/include/__type_traits/is_nothrow_relocatable.h
create mode 100644 libcxx/test/libcxx/memory/uninitialized_allocator_relocate.pass.cpp
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index f2f1ec50d5b75..e16bc87e79885 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -571,6 +571,7 @@ set(files
__memory/allocator.h
__memory/allocator_arg_t.h
__memory/allocator_destructor.h
+ __memory/allocator_relocation.h
__memory/allocator_traits.h
__memory/array_cookie.h
__memory/assume_aligned.h
@@ -589,12 +590,14 @@ set(files
__memory/ranges_destroy.h
__memory/ranges_uninitialized_algorithms.h
__memory/raw_storage_iterator.h
+ __memory/relocate_at.h
__memory/shared_count.h
__memory/shared_ptr.h
__memory/swap_allocator.h
__memory/temp_value.h
__memory/temporary_buffer.h
__memory/uninitialized_algorithms.h
+ __memory/uninitialized_relocate.h
__memory/unique_ptr.h
__memory/unique_temporary_buffer.h
__memory/uses_allocator.h
@@ -846,6 +849,7 @@ set(files
__type_traits/is_nothrow_assignable.h
__type_traits/is_nothrow_constructible.h
__type_traits/is_nothrow_destructible.h
+ __type_traits/is_nothrow_relocatable.h
__type_traits/is_null_pointer.h
__type_traits/is_object.h
__type_traits/is_pod.h
diff --git a/libcxx/include/__memory/allocator_relocation.h b/libcxx/include/__memory/allocator_relocation.h
new file mode 100644
index 0000000000000..62dfa817bf2e5
--- /dev/null
+++ b/libcxx/include/__memory/allocator_relocation.h
@@ -0,0 +1,196 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ALLOCATOR_RELOCATION_H
+#define _LIBCPP___MEMORY_ALLOCATOR_RELOCATION_H
+
+#include <__assert>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/destroy.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/relocate_at.h>
+#include <__memory/uninitialized_relocate.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_nothrow_destructible.h>
+#include <__type_traits/is_nothrow_relocatable.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/negation.h>
+#include <__utility/move.h>
+#include <__utility/scope_guard.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+//
+// This file implements allocator-aware companions to the various relocation facilities. Those are
+// necessary to use from allocator-aware containers like std::vector.
+//
+
+template <class _Alloc, class _Type>
+struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {};
+
+template <class _Type>
+struct __allocator_has_trivial_move_construct<allocator<_Type>, _Type> : true_type {};
+
+template <class _Alloc, class _Tp>
+struct __allocator_has_trivial_destroy : _Not<__has_destroy<_Alloc, _Tp*> > {};
+
+template <class _Tp, class _Up>
+struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
+
+// __is_trivially_allocator_relocatable:
+//
+// A type is trivially allocator-relocatable if the allocator's move construction and destruction
+// don't do anything beyond calling the type's move constructor and destructor, and if the type
+// itself is trivially relocatable.
+template <class _Alloc, class _Tp>
+struct __is_trivially_allocator_relocatable
+ : integral_constant<bool,
+ __allocator_has_trivial_move_construct<_Alloc, _Tp>::value &&
+ __allocator_has_trivial_destroy<_Alloc, _Tp>::value &&
+ __libcpp_is_trivially_relocatable<_Tp>::value> {};
+
+// __is_nothrow_allocator_relocatable:
+//
+// A type is nothrow allocator-relocatable if the allocator operations are trivial and the type is
+// nothrow relocatable, or if it is trivially allocator-relocatable.
+template <class _Alloc, class _Tp>
+struct __is_nothrow_allocator_relocatable
+ : _BoolConstant<(__allocator_has_trivial_move_construct<_Alloc, _Tp>::value &&
+ __allocator_has_trivial_destroy<_Alloc, _Tp>::value && __is_nothrow_relocatable<_Tp>::value) ||
+ __is_trivially_allocator_relocatable<_Alloc, _Tp>::value> {};
+
+// __allocator_relocate_at:
+//
+// Either perform relocation using the allocator's non-trivial operations, or forward to allocator
+// unaware relocation (which may then be trivial or not).
+template <class _Alloc,
+ class _Tp,
+ __enable_if_t<__allocator_has_trivial_move_construct<_Alloc, _Tp>::value &&
+ __allocator_has_trivial_destroy<_Alloc, _Tp>::value,
+ int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp*
+__allocator_relocate_at(_Alloc& __alloc, _Tp* __dest, _Tp* __source)
+ _NOEXCEPT_(__is_nothrow_allocator_relocatable<_Alloc, _Tp>::value) {
+ (void)__alloc; // ignore the allocator since its operations are trivial anyway
+ return std::__relocate_at(__dest, __source);
+}
+
+template <class _Alloc, class _Tp>
+struct __allocator_destroy_object {
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __allocator_destroy_object(_Alloc& __alloc, _Tp* __obj)
+ : __alloc_(__alloc), __obj_(__obj) {}
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() const {
+ allocator_traits<_Alloc>::destroy(__alloc_, __obj_);
+ }
+ _Alloc& __alloc_;
+ _Tp* __obj_;
+};
+
+template <class _Alloc,
+ class _Tp,
+ __enable_if_t<!(__allocator_has_trivial_move_construct<_Alloc, _Tp>::value &&
+ __allocator_has_trivial_destroy<_Alloc, _Tp>::value),
+ int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp*
+__allocator_relocate_at(_Alloc& __alloc, _Tp* __dest, _Tp* __source)
+ _NOEXCEPT_(__is_nothrow_allocator_relocatable<_Alloc, _Tp>::value) {
+ auto __guard = std::__make_scope_guard(__allocator_destroy_object<_Alloc, _Tp>(__alloc, __source));
+ allocator_traits<_Alloc>::construct(__alloc, __dest, std::move(*__source));
+ return __dest;
+}
+
+// __uninitialized_allocator_relocate:
+//
+// Equivalent to __uninitialized_relocate, but uses the provided allocator's construct() and destroy() methods.
+template <class _Alloc, class _ForwardIter, class _NothrowForwardIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _NothrowForwardIter __uninitialized_allocator_relocate(
+ _Alloc& __alloc, _ForwardIter __first, _ForwardIter __last, _NothrowForwardIter __result) {
+#ifndef _LIBCPP_CXX03_LANG
+ if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value &&
+ __libcpp_is_contiguous_iterator<_NothrowForwardIter>::value) {
+ _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(
+ !std::__is_pointer_in_range(std::__to_address(__first), std::__to_address(__last), std::__to_address(__last)),
+ "uninitialized_allocator_relocate requires the result not to overlap with the input range");
+ }
+#endif
+
+ using _ValueType = typename iterator_traits<_ForwardIter>::value_type;
+ if _LIBCPP_CONSTEXPR (__allocator_has_trivial_move_construct<_Alloc, _ValueType>::value &&
+ __allocator_has_trivial_destroy<_Alloc, _ValueType>::value) {
+ (void)__alloc; // ignore the allocator
+ return std::__uninitialized_relocate(std::move(__first), std::move(__last), std::move(__result));
+ } else {
+ auto const __first_result = __result;
+ try {
+ while (__first != __last) {
+ std::__allocator_relocate_at(__alloc, std::addressof(*__result), std::addressof(*__first));
+ ++__first;
+ ++__result;
+ }
+ } catch (...) {
+ std::__allocator_destroy(__alloc, ++__first, __last);
+ std::__allocator_destroy(__alloc, __first_result, __result);
+ throw;
+ }
+ return __result;
+ }
+}
+
+// __uninitialized_allocator_relocate_backward:
+//
+// Equivalent to __uninitialized_relocate_backward, but uses the provided allocator's construct() and destroy() methods.
+template <class _Alloc, class _BidirectionalIter, class _NothrowBidirectionalIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _NothrowBidirectionalIter
+__uninitialized_allocator_relocate_backward(
+ _Alloc& __alloc, _BidirectionalIter __first, _BidirectionalIter __last, _NothrowBidirectionalIter __result_last) {
+#ifndef _LIBCPP_CXX03_LANG
+ if constexpr (__libcpp_is_contiguous_iterator<_BidirectionalIter>::value &&
+ __libcpp_is_contiguous_iterator<_NothrowBidirectionalIter>::value) {
+ _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(
+ !std::__is_pointer_in_range(
+ std::__to_address(__first), std::__to_address(__last), std::__to_address(__result_last) - 1),
+ "uninitialized_allocator_relocate_backward requires the end of the result not to overlap with the input range");
+ }
+#endif
+
+ using _ValueType = typename iterator_traits<_BidirectionalIter>::value_type;
+ if _LIBCPP_CONSTEXPR (__allocator_has_trivial_move_construct<_Alloc, _ValueType>::value &&
+ __allocator_has_trivial_destroy<_Alloc, _ValueType>::value) {
+ (void)__alloc; // ignore the allocator
+ return std::__uninitialized_relocate_backward(std::move(__first), std::move(__last), std::move(__result_last));
+ } else {
+ auto __result = __result_last;
+ try {
+ while (__last != __first) {
+ --__last;
+ --__result;
+ std::__allocator_relocate_at(__alloc, std::addressof(*__result), std::addressof(*__last));
+ }
+ } catch (...) {
+ std::__allocator_destroy(__alloc, __first, __last);
+ std::__allocator_destroy(__alloc, __result, __result_last);
+ throw;
+ }
+
+ return __result;
+ }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ALLOCATOR_RELOCATION_H
diff --git a/libcxx/include/__memory/relocate_at.h b/libcxx/include/__memory/relocate_at.h
new file mode 100644
index 0000000000000..0783b429f03a1
--- /dev/null
+++ b/libcxx/include/__memory/relocate_at.h
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RELOCATE_AT_H
+#define _LIBCPP___MEMORY_RELOCATE_AT_H
+
+#include <__memory/allocator_traits.h>
+#include <__memory/construct_at.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_nothrow_relocatable.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__utility/move.h>
+#include <__utility/scope_guard.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Source, class _Dest>
+struct __relocatable_from
+ : _BoolConstant<_IsSame<_Source, _Dest>::value &&
+ (is_move_constructible<_Dest>::value || __libcpp_is_trivially_relocatable<_Dest>::value)> {};
+
+template <class _Tp>
+struct __destroy_object {
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __destroy_object(_Tp* __obj) : __obj_(__obj) {}
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() const { std::__destroy_at(__obj_); }
+ _Tp* __obj_;
+};
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp* __libcpp_builtin_trivially_relocate_at(_Tp* __source, _Tp* __dest) _NOEXCEPT {
+ static_assert(__libcpp_is_trivially_relocatable<_Tp>::value, "");
+ // Casting to void* to suppress clang complaining that this is technically UB.
+ __builtin_memcpy(static_cast<void*>(__dest), __source, sizeof(_Tp));
+ return __dest;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp* __libcpp_builtin_trivially_relocate_at(_Tp* __first, _Tp* __last, _Tp* __dest) _NOEXCEPT {
+ static_assert(__libcpp_is_trivially_relocatable<_Tp>::value, "");
+ // Casting to void* to suppress clang complaining that this is technically UB.
+ __builtin_memmove(static_cast<void*>(__dest), __first, (__last - __first) * sizeof(_Tp));
+ return __dest;
+}
+
+template <class _Tp,
+ __enable_if_t<__relocatable_from<_Tp, _Tp>::value, int> = 0,
+ __enable_if_t<__libcpp_is_trivially_relocatable<_Tp>::value && is_move_constructible<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __relocate_at(_Tp* __dest, _Tp* __source)
+ _NOEXCEPT_(__is_nothrow_relocatable<_Tp>::value) {
+ if (__libcpp_is_constant_evaluated()) {
+ auto __guard = std::__make_scope_guard(__destroy_object<_Tp>(__source));
+ return std::__construct_at(__dest, std::move(*__source));
+ } else {
+ return std::__libcpp_builtin_trivially_relocate_at(__source, __dest);
+ }
+}
+
+template <class _Tp,
+ __enable_if_t<__relocatable_from<_Tp, _Tp>::value, int> = 0,
+ __enable_if_t<__libcpp_is_trivially_relocatable<_Tp>::value && !is_move_constructible<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __relocate_at(_Tp* __dest, _Tp* __source)
+ _NOEXCEPT_(__is_nothrow_relocatable<_Tp>::value) {
+ return std::__libcpp_builtin_trivially_relocate_at(__source, __dest);
+}
+
+template <class _Tp,
+ __enable_if_t<__relocatable_from<_Tp, _Tp>::value, int> = 0,
+ __enable_if_t<!__libcpp_is_trivially_relocatable<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __relocate_at(_Tp* __dest, _Tp* __source)
+ _NOEXCEPT_(__is_nothrow_relocatable<_Tp>::value) {
+ auto __guard = std::__make_scope_guard(__destroy_object<_Tp>(__source));
+ return std::__construct_at(__dest, std::move(*__source));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_RELOCATE_AT_H
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index e80236640052c..072c5d4ad890b 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -20,6 +20,7 @@
#include <__iterator/iterator_traits.h>
#include <__iterator/reverse_iterator.h>
#include <__memory/addressof.h>
+#include <__memory/allocator_relocation.h>
#include <__memory/allocator_traits.h>
#include <__memory/construct_at.h>
#include <__memory/destroy.h>
@@ -596,7 +597,7 @@ inline const bool __allocator_has_trivial_destroy_v = !__has_destroy_v<_Alloc, _
template <class _Tp, class _Up>
inline const bool __allocator_has_trivial_destroy_v<allocator<_Tp>, _Up> = true;
-// __uninitialized_allocator_relocate relocates the objects in [__first, __last) into __result.
+// __uninitialized_allocator_relocate_strong relocates the objects in [__first, __last) into __result.
// Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy,
// except that the move constructor and destructor may never be called if they are known to be equivalent to a memcpy.
//
@@ -608,8 +609,12 @@ inline const bool __allocator_has_trivial_destroy_v<allocator<_Tp>, _Up> = true;
// - is_nothrow_move_constructible<_ValueType>
// - is_copy_constructible<_ValueType>
// - __libcpp_is_trivially_relocatable<_ValueType>
+//
+// TODO: Replace this function by appropriate uses of __uninitialized_allocator_relocate from std::vector by
+// handling the strong exception safety guarantee from outside the algorithm. The strong exception safety
+// is a property of the data structure operation, not of this algorithm, so it doesn't belong here.
template <class _Alloc, class _ContiguousIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __uninitialized_allocator_relocate(
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __uninitialized_allocator_relocate_strong(
_Alloc& __alloc, _ContiguousIterator __first, _ContiguousIterator __last, _ContiguousIterator __result) {
static_assert(__libcpp_is_contiguous_iterator<_ContiguousIterator>::value, "");
using _ValueType = typename iterator_traits<_ContiguousIterator>::value_type;
diff --git a/libcxx/include/__memory/uninitialized_relocate.h b/libcxx/include/__memory/uninitialized_relocate.h
new file mode 100644
index 0000000000000..97d2b1666ec02
--- /dev/null
+++ b/libcxx/include/__memory/uninitialized_relocate.h
@@ -0,0 +1,147 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H
+#define _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__memory/destroy.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/relocate_at.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__utility/is_pointer_in_range.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __uninitialized_relocate relocates the objects in [__first, __last) into __result.
+//
+// Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy,
+// except that the move constructor and destructor may never be called if they are known to be trivially relocatable.
+//
+// This algorithm works even if part of the resulting range overlaps with [__first, __last), as long as __result itself
+// is not in [__first, last).
+//
+// If an exception is thrown, all the elements in the input range and in the output range are destroyed.
+//
+// Preconditions:
+// - __result doesn't contain any objects and [__first, __last) contains objects
+// - __result is not in [__first, __last)
+// Postconditions:
+// - __result contains the objects from [__first, __last)
+// - [__first, __last) doesn't contain any objects
+template <class _ForwardIter, class _NothrowForwardIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _NothrowForwardIter
+__uninitialized_relocate(_ForwardIter __first, _ForwardIter __last, _NothrowForwardIter __result) {
+#ifndef _LIBCPP_CXX03_LANG
+ if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value &&
+ __libcpp_is_contiguous_iterator<_NothrowForwardIter>::value) {
+ _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(
+ !std::__is_pointer_in_range(std::__to_address(__first), std::__to_address(__last), std::__to_address(__result)),
+ "uninitialized_relocate requires the start of the result not to overlap with the input range");
+ }
+
+ // If we have contiguous iterators over a trivially relocatable type, use the builtin that is
+ // roughly equivalent to memmove.
+ using _ValueType = typename iterator_traits<_ForwardIter>::value_type;
+ if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value &&
+ __libcpp_is_contiguous_iterator<_NothrowForwardIter>::value &&
+ __libcpp_is_trivially_relocatable<_ValueType>::value) {
+ if (!__libcpp_is_constant_evaluated()) {
+ // TODO: We might be able to memcpy if we don't overlap at all?
+ std::__libcpp_builtin_trivially_relocate_at(
+ std::__to_address(__first), std::__to_address(__last), std::__to_address(__result));
+ return __result + (__last - __first);
+ }
+ }
+#endif
+
+ // Otherwise, relocate elements one by one.
+ //
+ // If this throws an exception, we destroy the rest of the range we were relocating, and
+ // we also destroy everything we had relocated up to now.
+ auto const __first_result = __result;
+ try {
+ while (__first != __last) {
+ std::__relocate_at(std::addressof(*__result), std::addressof(*__first));
+ ++__first;
+ ++__result;
+ }
+ } catch (...) {
+ std::__destroy(++__first, __last);
+ std::__destroy(__first_result, __result);
+ throw;
+ }
+ return __result;
+}
+
+// __uninitialized_relocate_backward is like __uninitialized_relocate, but it relocates the elements in
+// the range [first, last) to another range ending at __result_last. The elements are relocated in reverse
+// order, but their relative order is preserved.
+template <class _BidirectionalIter, class _NothrowBidirectionalIter>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _NothrowBidirectionalIter __uninitialized_relocate_backward(
+ _BidirectionalIter __first, _BidirectionalIter __last, _NothrowBidirectionalIter __result_last) {
+#ifndef _LIBCPP_CXX03_LANG
+ if constexpr (__libcpp_is_contiguous_iterator<_BidirectionalIter>::value &&
+ __libcpp_is_contiguous_iterator<_NothrowBidirectionalIter>::value) {
+ _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(
+ !std::__is_pointer_in_range(
+ std::__to_address(__first), std::__to_address(__last), std::__to_address(__result_last) - 1),
+ "uninitialized_relocate_backward requires the end of the result not to overlap with the input range");
+ }
+
+ // If we have contiguous iterators over a trivially relocatable type, use the builtin that is
+ // roughly equivalent to memmove.
+ using _ValueType = typename iterator_traits<_BidirectionalIter>::value_type;
+ if constexpr (__libcpp_is_contiguous_iterator<_BidirectionalIter>::value &&
+ __libcpp_is_contiguous_iterator<_NothrowBidirectionalIter>::value &&
+ __libcpp_is_trivially_relocatable<_ValueType>::value) {
+ if (!__libcpp_is_constant_evaluated()) {
+ auto __result = __result_last - (__last - __first);
+ // TODO: We might be able to memcpy if we don't overlap at all?
+ std::__libcpp_builtin_trivially_relocate_at(
+ std::__to_address(__first), std::__to_address(__last), std::__to_address(__result));
+ return __result;
+ }
+ }
+#endif
+
+ // Otherwise, relocate elements one by one, starting from the end.
+ //
+ // If this throws an exception, we destroy the rest of the range we were relocating, and
+ // we also destroy everything we had relocated up to now.
+ auto __result = __result_last;
+ try {
+ while (__last != __first) {
+ --__last;
+ --__result;
+ std::__relocate_at(std::addressof(*__result), std::addressof(*__last));
+ }
+ } catch (...) {
+ std::__destroy(__first, __last);
+ std::__destroy(__result, __result_last);
+ throw;
+ }
+ return __result;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H
diff --git a/libcxx/include/__type_traits/is_nothrow_relocatable.h b/libcxx/include/__type_traits/is_nothrow_relocatable.h
new file mode 100644
index 0000000000000..b1055fdfe88a1
--- /dev/null
+++ b/libcxx/include/__type_traits/is_nothrow_relocatable.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_RELOCATABLE_H
+#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_RELOCATABLE_H
+
+#include <__config>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_nothrow_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/remove_all_extents.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __is_nothrow_relocatable
+ : integral_constant<bool,
+ __libcpp_is_trivially_relocatable<_Tp>::value ||
+ (is_nothrow_move_constructible<__remove_all_extents_t<_Tp> >::value &&
+ is_nothrow_destructible<__remove_all_extents_t<_Tp> >::value)> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_RELOCATABLE_H
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 4307e78f6ddbc..1b6e60507f0f9 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -851,7 +851,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void
vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v) {
__annotate_delete();
auto __new_begin = __v.__begin_ - (__end_ - __begin_);
- std::__uninitialized_allocator_relocate(
+ std::__uninitialized_allocator_relocate_strong(
this->__alloc_, std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin));
__v.__begin_ = __new_begin;
__end_ = __begin_; // All the objects have been destroyed by relocating them.
@@ -874,13 +874,13 @@ vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, a
// Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_)
// in case something in [__begin_, __p) throws.
- std::__uninitialized_allocator_relocate(
+ std::__uninitialized_allocator_relocate_strong(
this->__alloc_, std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_));
__v.__end_ += (__end_ - __p);
__end_ = __p; // The objects in [__p, __end_) have been destroyed by relocating them.
auto __new_begin = __v.__begin_ - (__p - __begin_);
- std::__uninitialized_allocator_relocate(
+ std::__uninitialized_allocator_relocate_strong(
this->__alloc_, std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin));
__v.__begin_ = __new_begin;
__end_ = __begin_; // All the objects have been destroyed by relocating them.
@@ -1311,12 +1311,12 @@ vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _Inpu
__v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last));
__split_buffer<value_type, allocator_type&> __merged(
__recommend(size() + __v.size()), __off, __alloc_); // has `__off` positions available at the front
- std::__uninitialized_allocator_relocate(
+ std::__uninitialized_allocator_relocate_strong(
__alloc_, std::__to_address(__old_last), std::__to_address(this->__end_), std::__to_address(__merged.__end_));
__guard.__complete(); // Release the guard once objects in [__old_last_, __end_) have been successfully relocated.
__merged.__end_ += this->__end_ - __old_last;
this->__end_ = __old_last;
- std::__uninitialized_allocator_relocate(
+ std::__uninitialized_allocator_relocate_strong(
__alloc_, std::__to_address(__v.__begin_), std::__to_address(__v.__end_), std::__to_address(__merged.__end_));
__merged.__end_ += __v.size();
__v.__end_ = __v.__begin_;
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 1d2d51275704b..fc42c19b5b345 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -235,6 +235,10 @@ module std_core [system] {
header "__type_traits/is_nothrow_destructible.h"
export std_core.type_traits.integral_constant
}
+ module is_nothrow_relocatable {
+ header "__type_traits/is_nothrow_relocatable.h"
+ export std_core.type_traits.integral_constant
+ }
module is_null_pointer {
header "__type_traits/is_null_pointer.h"
export std_core.type_traits.integral_constant
@@ -1641,6 +1645,7 @@ module std [system] {
}
module allocator_arg_t { header "__memory/allocator_arg_t.h" }
module allocator_destructor { header "__memory/allocator_destructor.h" }
+ module allocator_relocation { header "__memory/allocator_relocation.h" }
module allocator_traits { header "__memory/allocator_traits.h" }
module array_cookie { header "__memory/array_cookie.h" }
module assume_aligned { header "__memory/assume_aligned.h" }
@@ -1663,6 +1668,7 @@ module std [system] {
export std.algorithm.in_out_result
}
module raw_storage_iterator { header "__memory/raw_storage_iterator.h" }
+ module relocate_at { header "__memory/relocate_at.h" }
module shared_count { header "__memory/shared_count.h" }
module shared_ptr { header "__memory/shared_ptr.h" }
module swap_allocator { header "__memory/swap_allocator.h" }
@@ -1675,6 +1681,9 @@ module std [system] {
header "__memory/uninitialized_algorithms.h"
export std.utility.pair
}
+ module uninitialized_relocate {
+ header "__memory/uninitialized_relocate.h"
+ }
module unique_ptr {
header "__memory/unique_ptr.h"
}
diff --git a/libcxx/test/libcxx/memory/uninitialized_allocator_relocate.pass.cpp b/libcxx/test/libcxx/memory/uninitialized_allocator_relocate.pass.cpp
new file mode 100644
index 0000000000000..2c4a15ad0436e
--- /dev/null
+++ b/libcxx/test/libcxx/memory/uninitialized_allocator_relocate.pass.cpp
@@ -0,0 +1,284 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Test the std::__uninitialized_allocator_relocate internal algorithm.
+
+// This test is impossible to write without std::to_address and std::construct_at
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <__memory/allocator_relocation.h>
+#include <cassert>
+#include <cstddef>
+#include <memory>
+
+#include "min_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+#include "type_algorithms.h"
+
+template <class Allocator>
+struct Fixture {
+ using Traits = std::allocator_traits<Allocator>;
+ using ValueType = typename Traits::value_type;
+ using Pointer = typename Traits::pointer;
+
+ constexpr Fixture(std::size_t n) : size(n) {
+ source = allocator.allocate(n);
+ for (std::size_t i = 0; i != n; ++i) {
+ Traits::construct(allocator, std::to_address(source + i), ValueType(i));
+ }
+
+ dest = allocator.allocate(n);
+ }
+
+ constexpr void relocated(std::size_t n) { relocated_ = n; }
+
+ constexpr ~Fixture() {
+ for (std::size_t i = 0; i != relocated_; ++i) {
+ Traits::destroy(allocator, std::to_address(dest + i));
+ }
+ allocator.deallocate(dest, size);
+
+ for (std::size_t i = relocated_; i != size; ++i) {
+ Traits::destroy(allocator, std::to_address(source + i));
+ }
+ allocator.deallocate(source, size);
+ }
+
+ Allocator allocator;
+ std::size_t size;
+ std::size_t relocated_ = 0;
+ Pointer source;
+ Pointer dest;
+};
+
+struct AliveTracker {
+ explicit AliveTracker(int v, bool do_throw, bool* alive) : value(v), do_throw_(do_throw), alive_(alive) {
+ if (alive_ != nullptr)
+ *alive_ = true;
+ }
+ AliveTracker(AliveTracker&& other) : value(other.value), do_throw_(other.do_throw_), alive_(other.alive_) {
+ other.alive_ = nullptr;
+ if (do_throw_) {
+ if (alive_ != nullptr)
+ *alive_ = false; // we failed to be constructed
+ throw 42;
+ }
+ }
+ ~AliveTracker() {
+ if (alive_ != nullptr) {
+ assert(*alive_ == true && "double destroy?");
+ *alive_ = false;
+ }
+ }
+ int value;
+ bool do_throw_;
+ bool* alive_;
+};
+
+template <template <class...> class Alloc,
+ template <class...> class Iter,
+ template <class...> class OutputIter,
+ class T>
+constexpr void test() {
+ using Allocator = Alloc<T>;
+ using Iterator = Iter<T*>;
+ using OutputIterator = OutputIter<T*>;
+
+ // Relocate an empty range
+ {
+ Fixture<Allocator> t(10);
+
+ OutputIterator res = std::__uninitialized_allocator_relocate(
+ t.allocator,
+ Iterator(std::to_address(t.source)),
+ Iterator(std::to_address(t.source)),
+ OutputIterator(std::to_address(t.dest)));
+ assert(res == OutputIterator(std::to_address(t.dest)));
+ t.relocated(0);
+
+ for (int i = 0; i != 10; ++i) {
+ assert(t.source[i] == T(i));
+ }
+ }
+
+ // Range of size 1
+ {
+ Fixture<Allocator> t(10);
+
+ OutputIterator res = std::__uninitialized_allocator_relocate(
+ t.allocator,
+ Iterator(std::to_address(t.source)),
+ Iterator(std::to_address(t.source + 1)),
+ OutputIterator(std::to_address(t.dest)));
+ assert(res == OutputIterator(std::to_address(t.dest + 1)));
+ t.relocated(1);
+
+ assert(t.dest[0] == T(0));
+ assert(t.source[1] == T(1));
+ assert(t.source[2] == T(2));
+ // ...
+ }
+
+ // Range of normal size
+ {
+ Fixture<Allocator> t(10);
+
+ OutputIterator res = std::__uninitialized_allocator_relocate(
+ t.allocator,
+ Iterator(std::to_address(t.source)),
+ Iterator(std::to_address(t.source + 10)),
+ OutputIterator(std::to_address(t.dest)));
+ assert(res == OutputIterator(std::to_address(t.dest + 10)));
+ t.relocated(10);
+
+ for (int i = 0; i != 10; ++i) {
+ assert(t.dest[i] == T(i));
+ }
+ }
+
+ // Relocate with some overlap between the input and the output range
+ {
+ Allocator allocator;
+ auto buff = allocator.allocate(10);
+ // x x x x x 5 6 7 8 9
+ for (std::size_t i = 5; i != 10; ++i) {
+ std::allocator_traits<Allocator>::construct(allocator, std::to_address(buff + i), T(i));
+ }
+
+ OutputIterator res = std::__uninitialized_allocator_relocate(
+ allocator,
+ Iterator(std::to_address(buff + 5)),
+ Iterator(std::to_address(buff + 10)),
+ OutputIterator(std::to_address(buff)));
+ assert(res == OutputIterator(std::to_address(buff + 5)));
+
+ for (int i = 0; i != 5; ++i) {
+ assert(buff[i] == T(i + 5));
+ std::allocator_traits<Allocator>::destroy(allocator, std::to_address(buff + i));
+ }
+ allocator.deallocate(buff, 10);
+ }
+
+ // Test throwing an exception while we're relocating
+#ifndef TEST_HAS_NO_EXCEPTIONS
+ if (!TEST_IS_CONSTANT_EVALUATED) {
+ {
+ using A = Alloc<AliveTracker>;
+ using It = Iter<AliveTracker*>;
+ using OutIt = OutputIter<AliveTracker*>;
+ // With some overlap between the input and the output
+ {
+ // Note: this is just storage for the bools and doesn't actually represent which elements are
+ // alive in the right order when they are moved around.
+ bool alive[10] = {false, false, false, false, false, false, false, false, false, false};
+ bool throw_on_move[10] = {false, false, false, false, false, false, true, false, false, false};
+ A allocator;
+ auto buff = allocator.allocate(10);
+ // x x x 3 4 5 6 7 8 9
+ for (std::size_t i = 3; i != 10; ++i) {
+ std::allocator_traits<A>::construct(allocator, std::to_address(buff + i), i, throw_on_move[i], &alive[i]);
+ }
+
+ // test the test
+ for (std::size_t i = 3; i != 10; ++i)
+ assert(alive[i]);
+
+ try {
+ std::__uninitialized_allocator_relocate(
+ allocator, It(std::to_address(buff + 3)), It(std::to_address(buff + 10)), OutIt(std::to_address(buff)));
+ assert(false && "there should have been an exception");
+ } catch (...) {
+ }
+
+ for (std::size_t i = 0; i != 10; ++i)
+ assert(!alive[i]);
+
+ allocator.deallocate(buff, 10);
+ }
+
+ // Without overlap
+ {
+ bool alive[10] = {false, false, false, false, false, false, false, false, false, false};
+ bool throw_on_move[10] = {false, false, false, false, false, false, false, true, false, false};
+ A allocator;
+ auto buff = allocator.allocate(10);
+ // x x x x x 5 6 7 8 9
+ for (std::size_t i = 5; i != 10; ++i) {
+ std::allocator_traits<A>::construct(allocator, std::to_address(buff + i), i, throw_on_move[i], &alive[i]);
+ }
+
+ // test the test
+ for (std::size_t i = 5; i != 10; ++i)
+ assert(alive[i]);
+
+ try {
+ std::__uninitialized_allocator_relocate(
+ allocator, It(std::to_address(buff + 5)), It(std::to_address(buff + 10)), OutIt(std::to_address(buff)));
+ assert(false && "there should have been an exception");
+ } catch (...) {
+ }
+
+ for (std::size_t i = 0; i != 10; ++i)
+ assert(!alive[i]);
+
+ allocator.deallocate(buff, 10);
+ }
+ }
+ }
+#endif // TEST_HAS_NO_EXCEPTIONS
+}
+
+struct NotTriviallyRelocatable {
+ constexpr explicit NotTriviallyRelocatable(int i) : value_(i) {}
+ constexpr NotTriviallyRelocatable(NotTriviallyRelocatable&& other) : value_(other.value_) { other.value_ = -1; }
+ constexpr friend bool operator==(NotTriviallyRelocatable const& a, NotTriviallyRelocatable const& b) {
+ return a.value_ == b.value_;
+ }
+
+ int value_;
+};
+static_assert(!std::__libcpp_is_trivially_relocatable<NotTriviallyRelocatable>::value, "");
+
+template <class T>
+struct ConstructAllocator : std::allocator<T> {
+ template < class... Args>
+ constexpr void construct(T* loc, Args&&... args) {
+ std::construct_at(loc, std::forward<Args>(args)...);
+ }
+};
+
+template <class T>
+struct DestroyAllocator : std::allocator<T> {
+ constexpr void destroy(T* loc) { std::destroy_at(loc); }
+};
+
+constexpr bool tests() {
+ types::for_each(types::type_list<int, NotTriviallyRelocatable>{}, []<class T> {
+ test<std::allocator, forward_iterator, forward_iterator, T>();
+ test<std::allocator, contiguous_iterator, contiguous_iterator, T>();
+
+ test<min_allocator, forward_iterator, forward_iterator, T>();
+ test<min_allocator, contiguous_iterator, contiguous_iterator, T>();
+
+ test<ConstructAllocator, forward_iterator, forward_iterator, T>();
+ test<ConstructAllocator, contiguous_iterator, contiguous_iterator, T>();
+
+ test<DestroyAllocator, forward_iterator, forward_iterator, T>();
+ test<DestroyAllocator, contiguous_iterator, contiguous_iterator, T>();
+ });
+ return true;
+}
+
+int main(int, char**) {
+ tests();
+#if TEST_STD_VER >= 20
+ static_assert(tests(), "");
+#endif
+ return 0;
+}
More information about the libcxx-commits
mailing list