[libcxx-commits] [libcxx] [libc++] Add tombstone traits and use them in optional (PR #98498)
Nikolas Klauser via libcxx-commits
libcxx-commits at lists.llvm.org
Fri Nov 8 04:36:48 PST 2024
https://github.com/philnik777 updated https://github.com/llvm/llvm-project/pull/98498
>From f67cde64e20daa23a0eaa9a058c4d4511814e740 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser at berlin.de>
Date: Sun, 9 Jun 2024 10:35:22 +0200
Subject: [PATCH] [libc++] Add tombstone traits to use in optional, variant
---
libcxx/include/CMakeLists.txt | 1 +
libcxx/include/__configuration/abi.h | 5 +
.../include/__functional/reference_wrapper.h | 11 +-
libcxx/include/__locale | 10 +
libcxx/include/__memory/shared_ptr.h | 24 +-
libcxx/include/__memory/tombstone_traits.h | 282 ++++++++++++++++++
libcxx/include/__memory/unique_ptr.h | 21 ++
libcxx/include/__type_traits/enable_if.h | 3 +
libcxx/include/__utility/pair.h | 23 +-
libcxx/include/__vector/vector.h | 12 +
libcxx/include/module.modulemap | 4 +
libcxx/include/optional | 68 ++++-
libcxx/include/string | 16 +
libcxx/include/string_view | 8 +
.../optional.object/optional_size.pass.cpp | 30 ++
.../optional/tombstone_types.pass.cpp | 68 +++++
16 files changed, 579 insertions(+), 7 deletions(-)
create mode 100644 libcxx/include/__memory/tombstone_traits.h
create mode 100644 libcxx/test/std/utilities/optional/tombstone_types.pass.cpp
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index ae2e8bcb32aaa4..aca8ea182ec511 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -560,6 +560,7 @@ set(files
__memory/swap_allocator.h
__memory/temp_value.h
__memory/temporary_buffer.h
+ __memory/tombstone_traits.h
__memory/uninitialized_algorithms.h
__memory/unique_ptr.h
__memory/unique_temporary_buffer.h
diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h
index c6ef6fdcdf96e6..124e349e09024b 100644
--- a/libcxx/include/__configuration/abi.h
+++ b/libcxx/include/__configuration/abi.h
@@ -124,6 +124,8 @@
// This setting disables the addition of such artificial padding, leading to a more optimal
// representation for several types.
# define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
+// Use __tombstone_traits to optimize the memory layout of std::optional.
+# define _LIBCPP_ABI_OPTIONAL_USE_TOMBSTONE_TRAITS
#elif _LIBCPP_ABI_VERSION == 1
# if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
// Enable compiling copies of now inline methods into the dylib to support
@@ -141,6 +143,9 @@
# if defined(__FreeBSD__) && __FreeBSD__ < 14
# define _LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR
# endif
+
+// TODO: This shouldn't be in the final commit - this just to test the changes across all the different configurations
+# define _LIBCPP_ABI_OPTIONAL_USE_TOMBSTONE_TRAITS
#endif
// We had some bugs where we use [[no_unique_address]] together with construct_at,
diff --git a/libcxx/include/__functional/reference_wrapper.h b/libcxx/include/__functional/reference_wrapper.h
index a4a66a50cf84ca..20f148f9e70700 100644
--- a/libcxx/include/__functional/reference_wrapper.h
+++ b/libcxx/include/__functional/reference_wrapper.h
@@ -13,8 +13,10 @@
#include <__compare/synth_three_way.h>
#include <__concepts/boolean_testable.h>
#include <__config>
+#include <__cstddef/size_t.h>
#include <__functional/weak_result_type.h>
#include <__memory/addressof.h>
+#include <__memory/tombstone_traits.h>
#include <__type_traits/enable_if.h>
#include <__type_traits/invoke.h>
#include <__type_traits/is_const.h>
@@ -22,6 +24,7 @@
#include <__type_traits/void_t.h>
#include <__utility/declval.h>
#include <__utility/forward.h>
+#include <cstdint>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -120,7 +123,13 @@ class _LIBCPP_TEMPLATE_VIS reference_wrapper : public __weak_result_type<_Tp> {
#if _LIBCPP_STD_VER >= 17
template <class _Tp>
reference_wrapper(_Tp&) -> reference_wrapper<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<reference_wrapper<_Tp>> {
+ static constexpr uintptr_t __disengaged_value_ = __builtin_offsetof(reference_wrapper<_Tp>, __f_);
+ static constexpr size_t __is_disengaged_offset_ = 0;
+};
+#endif // _LIBCPP_STD_VER >= 17
template <class _Tp>
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> ref(_Tp& __t) _NOEXCEPT {
diff --git a/libcxx/include/__locale b/libcxx/include/__locale
index b07b9f3329f42c..be74f7072ee373 100644
--- a/libcxx/include/__locale
+++ b/libcxx/include/__locale
@@ -13,6 +13,7 @@
#include <__config>
#include <__locale_dir/locale_base_api.h>
#include <__memory/shared_ptr.h> // __shared_count
+#include <__memory/tombstone_traits.h>
#include <__mutex/once_flag.h>
#include <__type_traits/make_unsigned.h>
#include <__utility/no_destroy.h>
@@ -112,8 +113,17 @@ private:
friend bool has_facet(const locale&) _NOEXCEPT;
template <class _Facet>
friend const _Facet& use_facet(const locale&);
+
+ friend struct __tombstone_traits<locale>;
};
+#if _LIBCPP_STD_VER >= 17
+template <>
+struct __tombstone_traits<locale> : __tombstone_traits_assume_aligned_pointer {
+ static_assert(__builtin_offsetof(locale, __locale_) == 0);
+};
+#endif
+
class _LIBCPP_EXPORTED_FROM_ABI locale::facet : public __shared_count {
protected:
_LIBCPP_HIDE_FROM_ABI explicit facet(size_t __refs = 0) : __shared_count(static_cast<long>(__refs) - 1) {}
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 5a84c2ce9bfe17..18b0f30d84f397 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -30,6 +30,7 @@
#include <__memory/compressed_pair.h>
#include <__memory/construct_at.h>
#include <__memory/pointer_traits.h>
+#include <__memory/tombstone_traits.h>
#include <__memory/uninitialized_algorithms.h>
#include <__memory/unique_ptr.h>
#include <__type_traits/add_lvalue_reference.h>
@@ -824,6 +825,8 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr {
friend class _LIBCPP_TEMPLATE_VIS shared_ptr;
template <class _Up>
friend class _LIBCPP_TEMPLATE_VIS weak_ptr;
+
+ friend struct __tombstone_traits<shared_ptr<_Tp>>;
};
#if _LIBCPP_STD_VER >= 17
@@ -831,7 +834,15 @@ template <class _Tp>
shared_ptr(weak_ptr<_Tp>) -> shared_ptr<_Tp>;
template <class _Tp, class _Dp>
shared_ptr(unique_ptr<_Tp, _Dp>) -> shared_ptr<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<shared_ptr<_Tp>> {
+ static constexpr auto __disengaged_value_ = __tombstone_traits_assume_aligned_pointer::__disengaged_value_;
+ static constexpr size_t __is_disengaged_offset_ =
+ __builtin_offsetof(shared_ptr<_Tp>, __cntrl_) +
+ __tombstone_traits_assume_aligned_pointer::__is_disengaged_offset_;
+};
+#endif // _LIBCPP_STD_VER >= 17
//
// std::allocate_shared and std::make_shared
@@ -1376,12 +1387,21 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr {
friend class _LIBCPP_TEMPLATE_VIS weak_ptr;
template <class _Up>
friend class _LIBCPP_TEMPLATE_VIS shared_ptr;
+
+ friend struct __tombstone_traits<weak_ptr<_Tp>>;
};
#if _LIBCPP_STD_VER >= 17
template <class _Tp>
weak_ptr(shared_ptr<_Tp>) -> weak_ptr<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<weak_ptr<_Tp>> {
+ static constexpr auto __disengaged_value_ = __tombstone_traits_assume_aligned_pointer::__disengaged_value_;
+ static constexpr size_t __is_disengaged_offset_ =
+ __builtin_offsetof(weak_ptr<_Tp>, __cntrl_) + __tombstone_traits_assume_aligned_pointer::__is_disengaged_offset_;
+};
+#endif // _LIBCPP_STD_VER >= 17
template <class _Tp>
inline _LIBCPP_CONSTEXPR weak_ptr<_Tp>::weak_ptr() _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {}
diff --git a/libcxx/include/__memory/tombstone_traits.h b/libcxx/include/__memory/tombstone_traits.h
new file mode 100644
index 00000000000000..6f9ef31f706e09
--- /dev/null
+++ b/libcxx/include/__memory/tombstone_traits.h
@@ -0,0 +1,282 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H
+#define _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H
+
+#include <__assert>
+#include <__config>
+#include <__cstddef/size_t.h>
+#include <__memory/construct_at.h>
+#include <__type_traits/datasizeof.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_fundamental.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_trivial.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/void_t.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/piecewise_construct.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class>
+struct __tombstone_traits;
+
+// bools have always exactly one bit set. If there is more than one set it's disengaged.
+template <>
+struct __tombstone_traits<bool> {
+ static constexpr uint8_t __disengaged_value_ = 3;
+ static constexpr size_t __is_disengaged_offset_ = 0;
+};
+
+struct __tombstone_traits_assume_aligned_pointer {
+ static constexpr uint8_t __disengaged_value_ = 1;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+ static constexpr size_t __is_disengaged_offset_ = 0;
+#else
+ static constexpr size_t __is_disengaged_offset_ = sizeof(void*) - 1;
+#endif
+};
+
+// TODO: Look into
+// - filesystem::directory_iterator
+// - vector<T> with alignof(T) == 1
+// - string_view (basic_string_view<T> works with alignof(T) >= 2)
+
+// This is constrained on fundamental types because we might not always know the alignment of a user-defined type.
+// For example, in one TU there may only be a forward declaration and in another there is already the definition
+// available. If we made this optimization conditional on the completeness of the type this would result in a non-benign
+// ODR violation.
+template <class _Tp>
+struct __tombstone_traits<__enable_specialization_if<is_fundamental_v<_Tp> && alignof(_Tp) >= 2, _Tp*>>
+ : __tombstone_traits_assume_aligned_pointer {};
+
+template <class _Tp>
+struct __tombstone_traits<_Tp**> : __tombstone_traits_assume_aligned_pointer {
+ static_assert(alignof(_Tp*) >= 2, "alignment of a pointer isn't at least 2!?");
+};
+
+inline constexpr struct __init_engaged_t {
+} __init_engaged;
+inline constexpr struct __init_disengaged_t {
+} __init_disengaged;
+
+template <class _Tp, class _Payload, bool = __tombstone_traits<_Tp>::__is_disengaged_offset_ == 0>
+struct __tombstone_is_disengaged {
+ using _TombstoneLayout = __tombstone_traits<_Tp>;
+ using _IsDisengagedT = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+ char __padding_[_TombstoneLayout::__is_disengaged_offset_];
+ _IsDisengagedT __is_disengaged_;
+};
+
+template <class _Tp, class _Payload>
+struct __tombstone_is_disengaged<_Tp, _Payload, true> {
+ using _TombstoneLayout = __tombstone_traits<_Tp>;
+ using _IsDisengagedT = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+ _IsDisengagedT __is_disengaged_;
+};
+
+template <class _Tp, class _Payload, bool = __tombstone_traits<_Tp>::__is_disengaged_offset_ == 0>
+struct __tombstone_data {
+ using _TombstoneLayout = __tombstone_traits<_Tp>;
+ using _IsDisengagedT = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+ static_assert(is_trivial<_IsDisengagedT>::value, "disengaged type has to be trivial!");
+ static_assert(_TombstoneLayout::__is_disengaged_offset_ >= __datasizeof_v<_Payload>);
+
+ _LIBCPP_NO_UNIQUE_ADDRESS _Payload __payload_;
+ char __padding_[_TombstoneLayout::__is_disengaged_offset_ - __datasizeof_v<_Payload>];
+ _IsDisengagedT __is_disengaged_;
+
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI constexpr __tombstone_data(_Args&&... __args)
+ : __payload_(std::forward<_Args>(__args)...), __is_disengaged_(_TombstoneLayout::__disengaged_value_) {}
+};
+
+template <class _Tp, class _Payload>
+struct __tombstone_data<_Tp, _Payload, true> {
+ using _TombstoneLayout = __tombstone_traits<_Tp>;
+ using _IsDisengagedT = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+ _IsDisengagedT __is_disengaged_;
+ _LIBCPP_NO_UNIQUE_ADDRESS _Payload __payload_;
+
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI constexpr __tombstone_data(_Args&&... __args)
+ : __is_disengaged_(_TombstoneLayout::__disengaged_value_), __payload_(std::forward<_Args>(__args)...) {}
+};
+
+template <class _Tp, class _Payload>
+struct __tombstoned_value final {
+ using _TombstoneLayout = __tombstone_traits<_Tp>;
+ using _TombstoneData = __tombstone_data<_Tp, _Payload>;
+
+ template <bool = is_trivially_destructible_v<_Tp> && is_trivially_destructible_v<_Payload>>
+ union _MaybeTombstone {
+ _Tp __value_;
+ _TombstoneData __tombstone_;
+
+ template <class... _Args>
+ constexpr _MaybeTombstone(__init_disengaged_t, _Args&&... __args) : __tombstone_(std::forward<_Args>(__args)...) {}
+
+ template <class... _Args>
+ constexpr _MaybeTombstone(__init_engaged_t, _Args&&... __args) : __value_(std::forward<_Args>(__args)...) {}
+
+ _MaybeTombstone(const _MaybeTombstone&) = default;
+ _MaybeTombstone(_MaybeTombstone&&) = default;
+ _MaybeTombstone& operator=(const _MaybeTombstone&) = default;
+ _MaybeTombstone& operator=(_MaybeTombstone&&) = default;
+
+ _LIBCPP_HIDE_FROM_ABI constexpr bool __is_engaged() const noexcept {
+ if (__libcpp_is_constant_evaluated())
+ return !__builtin_constant_p(__tombstone_.__is_disengaged_ == _TombstoneLayout::__disengaged_value_);
+ __tombstone_is_disengaged<_Tp, _Payload> __is_disengaged;
+ static_assert(sizeof(__tombstone_is_disengaged<_Tp, _Payload>) <= sizeof(_MaybeTombstone));
+ __builtin_memcpy(&__is_disengaged, this, sizeof(__tombstone_is_disengaged<_Tp, _Payload>));
+
+ return __is_disengaged.__is_disengaged_ != _TombstoneLayout::__disengaged_value_;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~_MaybeTombstone() {
+ if (__is_engaged()) {
+ std::destroy_at(&__value_);
+ } else {
+ std::destroy_at(&__tombstone_);
+ }
+ }
+ };
+
+ template <>
+ union _MaybeTombstone<true> {
+ _Tp __value_;
+ _TombstoneData __tombstone_;
+
+ template <class... _Args>
+ constexpr _MaybeTombstone(__init_disengaged_t, _Args&&... __args) : __tombstone_(std::forward<_Args>(__args)...) {}
+
+ template <class... _Args>
+ constexpr _MaybeTombstone(__init_engaged_t, _Args&&... __args) : __value_(std::forward<_Args>(__args)...) {}
+
+ _MaybeTombstone(const _MaybeTombstone&) = default;
+ _MaybeTombstone(_MaybeTombstone&&) = default;
+ _MaybeTombstone& operator=(const _MaybeTombstone&) = default;
+ _MaybeTombstone& operator=(_MaybeTombstone&&) = default;
+
+ _LIBCPP_HIDE_FROM_ABI constexpr bool __is_engaged() const noexcept {
+ if (__libcpp_is_constant_evaluated())
+ return !__builtin_constant_p(__tombstone_.__is_disengaged_ == _TombstoneLayout::__disengaged_value_);
+ __tombstone_is_disengaged<_Tp, _Payload> __is_disengaged;
+ static_assert(sizeof(__tombstone_is_disengaged<_Tp, _Payload>) <= sizeof(_MaybeTombstone));
+ __builtin_memcpy(&__is_disengaged, this, sizeof(__tombstone_is_disengaged<_Tp, _Payload>));
+
+ return __is_disengaged.__is_disengaged_ != _TombstoneLayout::__disengaged_value_;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 ~_MaybeTombstone() = default;
+ };
+
+ _MaybeTombstone<> __data_;
+
+ static_assert(sizeof(__tombstone_data<_Tp, _Payload>) <= sizeof(_Tp));
+ static_assert(is_integral_v<decltype(_TombstoneLayout::__disengaged_value_)>);
+ static_assert(__builtin_offsetof(_TombstoneData, __is_disengaged_) == _TombstoneLayout::__is_disengaged_offset_);
+
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI constexpr __tombstoned_value(__init_disengaged_t, _Args&&... __args)
+ : __data_(__init_disengaged, std::forward<_Args>(__args)...) {}
+
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI constexpr __tombstoned_value(__init_engaged_t, _Args&&... __args)
+ : __data_(__init_engaged, std::forward<_Args>(__args)...) {}
+
+ _LIBCPP_HIDE_FROM_ABI constexpr bool __is_engaged() const noexcept { return __data_.__is_engaged(); }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value() & noexcept {
+ _LIBCPP_ASSERT_INTERNAL(__is_engaged(), "Trying to get the value of a disenagaged tombstoned value");
+ return __data_.__value_;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value() const & noexcept {
+ _LIBCPP_ASSERT_INTERNAL(__is_engaged(), "Trying to get the value of a disenagaged tombstoned value");
+ return __data_.__value_;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value() && noexcept {
+ _LIBCPP_ASSERT_INTERNAL(__is_engaged(), "Trying to get the value of a disenagaged tombstoned value");
+ return std::move(__data_.__value_);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value() const && noexcept {
+ _LIBCPP_ASSERT_INTERNAL(__is_engaged(), "Trying to get the value of a disenagaged tombstoned value");
+ return std::move(__data_.__value_);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload() & noexcept {
+ _LIBCPP_ASSERT_INTERNAL(!__is_engaged(), "Trying to get the payload of an enagaged tombstoned value");
+ return __data_.__tombstone_.__payload_;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload() const & noexcept {
+ _LIBCPP_ASSERT_INTERNAL(!__is_engaged(), "Trying to get the payload of an enagaged tombstoned value");
+ return __data_.__tombstone_.__payload_;
+ }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload() && noexcept {
+ _LIBCPP_ASSERT_INTERNAL(!__is_engaged(), "Trying to get the payload of an enagaged tombstoned value");
+ return std::move(__data_.__tombstone_.__payload_);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload() const && noexcept {
+ _LIBCPP_ASSERT_INTERNAL(!__is_engaged(), "Trying to get the payload of an enagaged tombstoned value");
+ return std::move(__data_.__tombstone_.__payload_);
+ }
+
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI constexpr void __engage(piecewise_construct_t, _Args&&... __args) {
+ _LIBCPP_ASSERT_INTERNAL(!__is_engaged(), "Trying to enage a already engaged tombstoned value");
+ std::destroy_at(&__data_.__tombstone_);
+ std::__construct_at(&__data_.__value_, std::forward<_Args>(__args)...);
+ }
+
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI constexpr void __disengage(piecewise_construct_t, _Args&&... __args) {
+ _LIBCPP_ASSERT_INTERNAL(!__is_engaged(), "Trying to disenage a disengaged tombstoned value");
+ std::destroy_at(&__data_.__value_);
+ std::__construct_at(&__data_.__tombstone_, std::forward<_Args>(__args)...);
+ }
+};
+
+template <class _Tp, class = void>
+inline constexpr bool __has_tombstone_v = false;
+
+template <class _Tp>
+inline constexpr bool __has_tombstone_v<_Tp, void_t<decltype(sizeof(__tombstone_traits<_Tp>))>> = true;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
index 28c62e13566e24..37d3e7e195c099 100644
--- a/libcxx/include/__memory/unique_ptr.h
+++ b/libcxx/include/__memory/unique_ptr.h
@@ -24,6 +24,7 @@
#include <__memory/auto_ptr.h>
#include <__memory/compressed_pair.h>
#include <__memory/pointer_traits.h>
+#include <__memory/tombstone_traits.h>
#include <__type_traits/add_lvalue_reference.h>
#include <__type_traits/common_type.h>
#include <__type_traits/conditional.h>
@@ -305,8 +306,18 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
swap(__ptr_, __u.__ptr_);
swap(__deleter_, __u.__deleter_);
}
+
+ friend struct __tombstone_traits<unique_ptr>;
};
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+struct __tombstone_traits<__enable_specialization_if<__has_tombstone_v<_Tp*>, unique_ptr<_Tp>>>
+ : __tombstone_traits<_Tp*> {
+ static_assert(__builtin_offsetof(unique_ptr<_Tp>, __ptr_) == 0, "Unexpected pointer offset");
+};
+#endif // _LIBCPP_STD_VER >= 17
+
// Bounds checking in unique_ptr<T[]>
// ==================================
//
@@ -630,8 +641,18 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
swap(__deleter_, __u.__deleter_);
swap(__checker_, __u.__checker_);
}
+
+ friend struct __tombstone_traits<unique_ptr>;
};
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+struct __tombstone_traits<__enable_specialization_if<__has_tombstone_v<_Tp*>, unique_ptr<_Tp[]>>>
+ : __tombstone_traits<_Tp*> {
+ static_assert(__builtin_offsetof(unique_ptr<_Tp[]>, __ptr_) == 0, "Unexpected pointer offset");
+};
+#endif
+
template <class _Tp, class _Dp, __enable_if_t<__is_swappable_v<_Dp>, int> = 0>
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
swap(unique_ptr<_Tp, _Dp>& __x, unique_ptr<_Tp, _Dp>& __y) _NOEXCEPT {
diff --git a/libcxx/include/__type_traits/enable_if.h b/libcxx/include/__type_traits/enable_if.h
index 77da9622ca28fc..67c3f7ffa24c14 100644
--- a/libcxx/include/__type_traits/enable_if.h
+++ b/libcxx/include/__type_traits/enable_if.h
@@ -32,6 +32,9 @@ template <bool _Bp, class _Tp = void>
using enable_if_t = typename enable_if<_Bp, _Tp>::type;
#endif
+template <bool _Bp, class _Tp, class = __enable_if_t<_Bp> >
+using __enable_specialization_if = _Tp;
+
_LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
diff --git a/libcxx/include/__utility/pair.h b/libcxx/include/__utility/pair.h
index f9d0f4e4723113..a3cb83eb40f2b9 100644
--- a/libcxx/include/__utility/pair.h
+++ b/libcxx/include/__utility/pair.h
@@ -17,6 +17,7 @@
#include <__fwd/array.h>
#include <__fwd/pair.h>
#include <__fwd/tuple.h>
+#include <__memory/tombstone_traits.h>
#include <__tuple/tuple_indices.h>
#include <__tuple/tuple_like_no_subrange.h>
#include <__tuple/tuple_size.h>
@@ -444,7 +445,27 @@ struct _LIBCPP_TEMPLATE_VIS pair
#if _LIBCPP_STD_VER >= 17
template <class _T1, class _T2>
pair(_T1, _T2) -> pair<_T1, _T2>;
-#endif
+
+template <class _Tp, class _Up, bool __first_tombstone>
+struct __tombstone_traits_pair;
+
+template <class _Tp, class _Up>
+struct __tombstone_traits_pair<_Tp, _Up, true> : __tombstone_traits<_Tp> {
+ static_assert(__builtin_offsetof(pair<_Tp, _Up>, first) == 0);
+};
+
+template <class _Tp, class _Up>
+struct __tombstone_traits_pair<_Tp, _Up, false> {
+ static constexpr auto __disengaged_value_ = __tombstone_traits<_Up>::__disengaged_value_;
+ static constexpr size_t __is_disengaged_offset_ =
+ __builtin_offsetof(pair<_Tp, _Up>, second) + __tombstone_traits<_Up>::__is_disengaged_offset_;
+};
+
+template <class _Tp, class _Up>
+struct __tombstone_traits<__enable_specialization_if<__has_tombstone_v<_Tp> || __has_tombstone_v<_Up>, pair<_Tp, _Up>>>
+ : __tombstone_traits_pair<_Tp, _Up, __has_tombstone_v<_Tp>> {};
+
+#endif // _LIBCPP_STD_VER >= 17
// [pairs.spec], specialized algorithms
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 6db202efb279b3..759370a42be109 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -38,6 +38,7 @@
#include <__memory/pointer_traits.h>
#include <__memory/swap_allocator.h>
#include <__memory/temp_value.h>
+#include <__memory/tombstone_traits.h>
#include <__memory/uninitialized_algorithms.h>
#include <__ranges/access.h>
#include <__ranges/concepts.h>
@@ -785,6 +786,8 @@ class _LIBCPP_TEMPLATE_VIS vector {
}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector&, false_type) _NOEXCEPT {}
+
+ friend struct __tombstone_traits<vector<_Tp, _Allocator>>;
};
#if _LIBCPP_STD_VER >= 17
@@ -808,6 +811,15 @@ template <ranges::input_range _Range,
vector(from_range_t, _Range&&, _Alloc = _Alloc()) -> vector<ranges::range_value_t<_Range>, _Alloc>;
#endif
+#if _LIBCPP_STD_VER >= 17
+template <class _Tp, class _Allocator>
+struct __tombstone_traits<
+ __enable_specialization_if<__has_tombstone_v<typename vector<_Tp, _Allocator>::pointer>, vector<_Tp, _Allocator>>>
+ : __tombstone_traits<typename vector<_Tp>::pointer> {
+ static_assert(__builtin_offsetof(vector<_Tp, _Allocator>, __begin_) == 0);
+};
+#endif
+
// __swap_out_circular_buffer relocates the objects in [__begin_, __end_) into the front of __v and swaps the buffers of
// *this and __v. It is assumed that __v provides space for exactly (__end_ - __begin_) objects in the front. This
// function has a strong exception guarantee.
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 6b0cc07fca0787..57d4c49e375dab 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1542,6 +1542,10 @@ module std [system] {
header "__memory/temporary_buffer.h"
export std.utility.pair // return type of std::get_temporary_buffer()
}
+ module tombstone_traits {
+ header "__memory/tombstone_traits.h"
+ export std.utility.piecewise_construct
+ }
module uninitialized_algorithms {
header "__memory/uninitialized_algorithms.h"
export std.utility.pair
diff --git a/libcxx/include/optional b/libcxx/include/optional
index 7ad6a9e116941f..04034e6cf83c44 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -190,6 +190,7 @@ namespace std {
#include <__fwd/functional.h>
#include <__memory/addressof.h>
#include <__memory/construct_at.h>
+#include <__memory/tombstone_traits.h>
#include <__tuple/sfinae_helpers.h>
#include <__type_traits/add_pointer.h>
#include <__type_traits/conditional.h>
@@ -220,6 +221,7 @@ namespace std {
#include <__type_traits/remove_cvref.h>
#include <__type_traits/remove_reference.h>
#include <__utility/declval.h>
+#include <__utility/empty.h>
#include <__utility/forward.h>
#include <__utility/in_place.h>
#include <__utility/move.h>
@@ -349,8 +351,19 @@ struct __optional_destruct_base<_Tp, true> {
}
};
-template <class _Tp, bool = is_reference<_Tp>::value>
-struct __optional_storage_base : __optional_destruct_base<_Tp> {
+template <class _Tp,
+ bool = is_reference<_Tp>::value,
+ bool =
+# ifdef _LIBCPP_ABI_OPTIONAL_USE_TOMBSTONE_TRAITS
+ __has_tombstone_v<_Tp>
+# else
+ false
+# endif
+ >
+struct __optional_storage_base;
+
+template <class _Tp>
+struct __optional_storage_base<_Tp, false, false> : __optional_destruct_base<_Tp> {
using __base = __optional_destruct_base<_Tp>;
using value_type = _Tp;
using __base::__base;
@@ -393,7 +406,7 @@ struct __optional_storage_base : __optional_destruct_base<_Tp> {
// be allowed in the future. For this reason, it has already been implemented
// to ensure we can make the change in an ABI-compatible manner.
template <class _Tp>
-struct __optional_storage_base<_Tp, true> {
+struct __optional_storage_base<_Tp, true, false> {
using value_type = _Tp;
using __raw_type = remove_reference_t<_Tp>;
__raw_type* __value_;
@@ -461,6 +474,55 @@ struct __optional_storage_base<_Tp, true> {
}
};
+template <class _Tp, bool _Bp>
+struct __optional_storage_base<_Tp, _Bp, true> {
+ using value_type = _Tp;
+ __tombstoned_value<_Tp, __empty> __storage_;
+
+ _LIBCPP_HIDE_FROM_ABI constexpr __optional_storage_base() noexcept : __storage_(__init_disengaged) {}
+
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_storage_base(in_place_t, _Args&&... __args)
+ : __storage_(__init_engaged, std::forward<_Args>(__args)...) {}
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reset() noexcept {
+ if (__storage_.__is_engaged())
+ __storage_.__disengage(piecewise_construct);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return __storage_.__is_engaged(); }
+
+ _LIBCPP_HIDE_FROM_ABI constexpr _Tp& __get() & { return __storage_.__get_value(); }
+ _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& __get() && { return std::move(__storage_.__get_value()); }
+ _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& __get() const& { return __storage_.__get_value(); }
+ _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& __get() const&& { return std::move(__storage_.__get_value()); }
+
+ template <class... _Args>
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct(_Args&&... __args) {
+ _LIBCPP_ASSERT_INTERNAL(!has_value(), "__construct called for engaged __optional_storage_base");
+ __storage_.__engage(piecewise_construct, std::forward<_Args>(__args)...);
+ }
+
+ template <class _Other>
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct_from(_Other&& __opt) {
+ if (__opt.has_value())
+ __construct(std::forward<_Other>(__opt).__get());
+ }
+
+ template <class _Other>
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign_from(_Other&& __opt) {
+ if (has_value() == __opt.has_value()) {
+ if (has_value())
+ __storage_.__get_value() = std::forward<_Other>(__opt).__get();
+ } else {
+ if (has_value())
+ __storage_.__disengage(piecewise_construct);
+ else
+ __construct(std::forward<_Other>(__opt).__get());
+ }
+ }
+};
+
template <class _Tp, bool = is_trivially_copy_constructible<_Tp>::value>
struct __optional_copy_base : __optional_storage_base<_Tp> {
using __optional_storage_base<_Tp>::__optional_storage_base;
diff --git a/libcxx/include/string b/libcxx/include/string
index b1cedbe68f7956..3b585aaac1fe46 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -612,6 +612,7 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
#include <__memory/noexcept_move_assign_container.h>
#include <__memory/pointer_traits.h>
#include <__memory/swap_allocator.h>
+#include <__memory/tombstone_traits.h>
#include <__memory_resource/polymorphic_allocator.h>
#include <__ranges/access.h>
#include <__ranges/concepts.h>
@@ -641,6 +642,7 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
#include <__utility/swap.h>
#include <__utility/unreachable.h>
#include <climits>
+#include <cstdint>
#include <cstdio> // EOF
#include <cstring>
#include <limits>
@@ -2320,6 +2322,20 @@ basic_string(from_range_t, _Range&&, _Allocator = _Allocator())
-> basic_string<ranges::range_value_t<_Range>, char_traits<ranges::range_value_t<_Range>>, _Allocator>;
#endif
+#if _LIBCPP_STD_VER >= 17
+template <class _CharT, class _Traits, class _Allocator>
+struct __tombstone_traits<basic_string<_CharT, _Traits, _Allocator>> {
+ static constexpr uint8_t __disengaged_value_ = 65;
+#ifdef _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+ static constexpr size_t __is_disengaged_offset_ = -1;
+ static_assert(__is_disengaged_offset_ == 0,
+ "The disengaged offset for the alternate string layout isn't correctly set");
+# else
+ static constexpr size_t __is_disengaged_offset_ = 0;
+# endif
+};
+#endif // _LIBCPP_STD_VER >= 17
+
template <class _CharT, class _Traits, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void
basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_type __sz, size_type __reserve) {
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index 6e05844b72cadf..a127d2677b97f5 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -222,12 +222,14 @@ namespace std {
#include <__iterator/reverse_iterator.h>
#include <__iterator/wrap_iter.h>
#include <__memory/pointer_traits.h>
+#include <__memory/tombstone_traits.h>
#include <__ranges/concepts.h>
#include <__ranges/data.h>
#include <__ranges/enable_borrowed_range.h>
#include <__ranges/enable_view.h>
#include <__ranges/size.h>
#include <__string/char_traits.h>
+#include <__type_traits/enable_if.h>
#include <__type_traits/is_array.h>
#include <__type_traits/is_convertible.h>
#include <__type_traits/is_same.h>
@@ -706,6 +708,12 @@ template <class _CharT, class _Traits>
inline constexpr bool ranges::enable_borrowed_range<basic_string_view<_CharT, _Traits> > = true;
#endif // _LIBCPP_STD_VER >= 20
+#if _LIBCPP_STD_VER >= 17
+template <class _CharT, class _Traits>
+struct __tombstone_traits<__enable_specialization_if<alignof(_CharT) >= 2, basic_string_view<_CharT, _Traits>>>
+ : __tombstone_traits_assume_aligned_pointer {};
+#endif
+
// [string.view.deduct]
#if _LIBCPP_STD_VER >= 20
diff --git a/libcxx/test/libcxx/utilities/optional/optional.object/optional_size.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.object/optional_size.pass.cpp
index b928ed7432791e..baf0db31fb0a97 100644
--- a/libcxx/test/libcxx/utilities/optional/optional.object/optional_size.pass.cpp
+++ b/libcxx/test/libcxx/utilities/optional/optional.object/optional_size.pass.cpp
@@ -12,7 +12,12 @@
// template <class T> class optional;
+#include <functional>
+#include <locale>
+#include <memory>
#include <optional>
+#include <string>
+#include <vector>
template <class T>
struct type_with_bool {
@@ -20,6 +25,8 @@ struct type_with_bool {
bool has_value;
};
+struct alignas(void*) aligned_s {};
+
int main(int, char**) {
// Test that std::optional achieves the expected size. See https://llvm.org/PR61095.
static_assert(sizeof(std::optional<char>) == sizeof(type_with_bool<char>));
@@ -27,5 +34,28 @@ int main(int, char**) {
static_assert(sizeof(std::optional<long>) == sizeof(type_with_bool<long>));
static_assert(sizeof(std::optional<std::size_t>) == sizeof(type_with_bool<std::size_t>));
+ // Check that the optionals using a tombstone have the expected size
+ static_assert(sizeof(std::optional<bool>) == sizeof(bool));
+ static_assert(sizeof(std::optional<std::string>) == sizeof(std::string));
+ static_assert(sizeof(std::optional<int*>) == sizeof(void*));
+ static_assert(sizeof(std::optional<short*>) == sizeof(void*));
+ static_assert(sizeof(std::optional<char*>) == sizeof(void*) * 2);
+ static_assert(sizeof(std::optional<aligned_s*>) == sizeof(void*) * 2);
+ static_assert(sizeof(std::optional<std::unique_ptr<int>>) == sizeof(void*));
+ static_assert(sizeof(std::optional<std::unique_ptr<int[]>>) == sizeof(void*));
+ static_assert(sizeof(std::optional<std::unique_ptr<aligned_s>>) == sizeof(void*) * 2);
+ static_assert(sizeof(std::optional<std::reference_wrapper<char*>>) == sizeof(void*));
+ static_assert(sizeof(std::optional<std::reference_wrapper<aligned_s>>) == sizeof(void*));
+ static_assert(sizeof(std::optional<std::locale>) == sizeof(void*));
+ static_assert(sizeof(std::optional<std::shared_ptr<char>>) == sizeof(void*) * 2);
+ static_assert(sizeof(std::optional<std::shared_ptr<int>>) == sizeof(void*) * 2);
+ static_assert(sizeof(std::optional<std::weak_ptr<char>>) == sizeof(void*) * 2);
+ static_assert(sizeof(std::optional<std::weak_ptr<int>>) == sizeof(void*) * 2);
+ static_assert(sizeof(std::optional<std::pair<int, int>>) == sizeof(int) * 3);
+ static_assert(sizeof(std::optional<std::pair<std::string, char*>>) == sizeof(void*) * 4);
+ static_assert(sizeof(std::optional<std::pair<char*, std::string>>) == sizeof(void*) * 4);
+ static_assert(sizeof(std::optional<std::vector<int>>) == sizeof(void*) * 3);
+ static_assert(sizeof(std::optional<std::vector<char>>) == sizeof(void*) * 4);
+
return 0;
}
diff --git a/libcxx/test/std/utilities/optional/tombstone_types.pass.cpp b/libcxx/test/std/utilities/optional/tombstone_types.pass.cpp
new file mode 100644
index 00000000000000..43cab57d250713
--- /dev/null
+++ b/libcxx/test/std/utilities/optional/tombstone_types.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+// <optional>
+
+#include <cassert>
+#include <optional>
+#include <string>
+
+#include "test_macros.h"
+
+TEST_CONSTEXPR_CXX20 bool test() {
+ {
+ std::optional<bool> opt;
+ assert(!opt);
+ opt = true;
+ assert(opt);
+ assert(*opt);
+ opt = false;
+ assert(opt);
+ assert(!*opt);
+ opt.reset();
+ assert(!opt);
+ }
+
+ {
+ std::optional<std::string> opt;
+ assert(!opt);
+ opt = "";
+ assert(opt);
+ assert(*opt == "");
+ opt = "23 letter string to SSO";
+ assert(opt);
+ assert(*opt == "23 letter string to SSO");
+ opt.reset();
+ assert(!opt);
+ }
+
+ {
+ int i;
+ int j;
+ std::optional<int*> opt;
+ assert(!opt);
+ opt = &i;
+ assert(opt);
+ assert(*opt == &i);
+ opt = &j;
+ assert(opt);
+ assert(*opt == &j);
+ opt.reset();
+ assert(!opt);
+ }
+
+ return true;
+}
+
+int main(int, char**) {
+ test();
+#if TEST_STD_VER >= 20
+ static_assert(test());
+#endif
+}
More information about the libcxx-commits
mailing list