[llvm] [ADT][NFC] Simplify llvm::unique_function (PR #208251)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 09:15:36 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-adt
Author: Alexis Engelke (aengelke)
<details>
<summary>Changes</summary>
Refactor the core of UniqueFunctionBase to move away from static
function pointer tables, which consume ~4kiB in .data.rel.ro. Instead,
store two function pointers on construction of the UniqueFunctionBase,
one for call and one that combines move and destruction. Retain the
memcpy path for trivial inline structs. (I haven't verified whether this
is actually beneficial.)
Also, greatly simplify the class and implement the callbacks using
lambdas.
---
Full diff: https://github.com/llvm/llvm-project/pull/208251.diff
1 Files Affected:
- (modified) llvm/include/llvm/ADT/FunctionExtras.h (+71-180)
``````````diff
diff --git a/llvm/include/llvm/ADT/FunctionExtras.h b/llvm/include/llvm/ADT/FunctionExtras.h
index f853bbf7d976d..1c0fe8a523635 100644
--- a/llvm/include/llvm/ADT/FunctionExtras.h
+++ b/llvm/include/llvm/ADT/FunctionExtras.h
@@ -108,139 +108,29 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
// The type of the erased function pointer we use as a callback to dispatch to
// the stored callable when it is trivial to move and destroy.
- using CallPtrT = ReturnT (*)(void *CallableAddr,
+ using CallPtrT = ReturnT (*)(const UniqueFunctionBase *Self,
AdjustedParamT<ParamTs>... Params);
- using MovePtrT = void (*)(void *LHSCallableAddr, void *RHSCallableAddr);
- using DestroyPtrT = void (*)(void *CallableAddr);
-
- /// A struct to hold a single trivial callback with sufficient alignment for
- /// our bitpacking.
- struct alignas(8) TrivialCallback {
- CallPtrT CallPtr;
- };
-
- /// A struct we use to aggregate three callbacks when we need full set of
- /// operations.
- struct alignas(8) NonTrivialCallbacks {
- CallPtrT CallPtr;
- MovePtrT MovePtr;
- DestroyPtrT DestroyPtr;
- };
-
- // Create a pointer union between either a pointer to a static trivial call
- // pointer in a struct or a pointer to a static struct of the call, move, and
- // destroy pointers.
- using CallbackPointerUnionT =
- PointerUnion<TrivialCallback *, NonTrivialCallbacks *>;
+ using DestroyMovePtrT = void (*)(UniqueFunctionBase *LHS,
+ UniqueFunctionBase *RHS);
// The main storage buffer. This will either have a pointer to out-of-line
// storage or an inline buffer storing the callable.
- union StorageUnionT {
- // For out-of-line storage we keep a pointer to the underlying storage and
- // the size. This is enough to deallocate the memory.
- struct OutOfLineStorageT {
- void *StoragePtr;
- size_t Size;
- size_t Alignment;
- } OutOfLineStorage;
+ union StorageT {
+ // For out-of-line storage we keep a pointer to the underlying storage.
+ void *OutOfLine;
static_assert(
- sizeof(OutOfLineStorageT) <= InlineStorageSize,
+ sizeof(OutOfLine) <= InlineStorageSize,
"Should always use all of the out-of-line storage for inline storage!");
// For in-line storage, we just provide an aligned character buffer. We
// provide three pointers worth of storage here.
// This is mutable as an inlined `const unique_function<void() const>` may
// still modify its own mutable members.
- alignas(InlineStorageAlign) mutable std::byte
- InlineStorage[InlineStorageSize];
- } StorageUnion;
-
- // A compressed pointer to either our dispatching callback or our table of
- // dispatching callbacks and the flag for whether the callable itself is
- // stored inline or not.
- PointerIntPair<CallbackPointerUnionT, 1, bool> CallbackAndInlineFlag;
-
- bool isInlineStorage() const { return CallbackAndInlineFlag.getInt(); }
-
- bool isTrivialCallback() const {
- return isa<TrivialCallback *>(CallbackAndInlineFlag.getPointer());
- }
-
- CallPtrT getTrivialCallback() const {
- return cast<TrivialCallback *>(CallbackAndInlineFlag.getPointer())->CallPtr;
- }
-
- NonTrivialCallbacks *getNonTrivialCallbacks() const {
- return cast<NonTrivialCallbacks *>(CallbackAndInlineFlag.getPointer());
- }
-
- CallPtrT getCallPtr() const {
- return isTrivialCallback() ? getTrivialCallback()
- : getNonTrivialCallbacks()->CallPtr;
- }
-
- // These three functions are only const in the narrow sense. They return
- // mutable pointers to function state.
- // This allows unique_function<T const>::operator() to be const, even if the
- // underlying functor may be internally mutable.
- //
- // const callers must ensure they're only used in const-correct ways.
- void *getCalleePtr() const {
- return isInlineStorage() ? getInlineStorage() : getOutOfLineStorage();
- }
- void *getInlineStorage() const { return &StorageUnion.InlineStorage; }
- void *getOutOfLineStorage() const {
- return StorageUnion.OutOfLineStorage.StoragePtr;
- }
-
- size_t getOutOfLineStorageSize() const {
- return StorageUnion.OutOfLineStorage.Size;
- }
- size_t getOutOfLineStorageAlignment() const {
- return StorageUnion.OutOfLineStorage.Alignment;
- }
+ alignas(InlineStorageAlign) mutable std::byte Inline[InlineStorageSize];
+ } Storage;
- void setOutOfLineStorage(void *Ptr, size_t Size, size_t Alignment) {
- StorageUnion.OutOfLineStorage = {Ptr, Size, Alignment};
- }
-
- template <typename CalledAsT>
- static ReturnT CallImpl(void *CallableAddr,
- AdjustedParamT<ParamTs>... Params) {
- auto &Func = *reinterpret_cast<CalledAsT *>(CallableAddr);
- return Func(std::forward<ParamTs>(Params)...);
- }
-
- template <typename CallableT>
- static void MoveImpl(void *LHSCallableAddr, void *RHSCallableAddr) noexcept {
- new (LHSCallableAddr)
- CallableT(std::move(*reinterpret_cast<CallableT *>(RHSCallableAddr)));
- }
-
- template <typename CallableT>
- static void DestroyImpl(void *CallableAddr) noexcept {
- reinterpret_cast<CallableT *>(CallableAddr)->~CallableT();
- }
-
- // The pointers to call/move/destroy functions are determined for each
- // callable type (and called-as type, which determines the overload chosen).
-
- // By default, we need an object that contains all the different
- // type erased behaviors needed. Create a static instance of the struct type
- // here and each instance will contain a pointer to it.
- // Wrap in a struct to avoid https://gcc.gnu.org/PR71954
- template <typename CallableT, typename CalledAs> struct CallbacksHolder {
- inline static auto Callbacks = []() constexpr {
- // For trivial callables, we don't need to store move and destroy
- // callbacks.
- if constexpr (std::is_trivially_move_constructible_v<CallableT> &&
- std::is_trivially_destructible_v<CallableT>)
- return TrivialCallback{&CallImpl<CalledAs>};
- else
- return NonTrivialCallbacks{&CallImpl<CalledAs>, &MoveImpl<CallableT>,
- &DestroyImpl<CallableT>};
- }();
- };
+ CallPtrT CallPtr = nullptr;
+ DestroyMovePtrT DestroyMovePtr = nullptr;
// A simple tag type so the call-as type to be passed to the constructor.
template <typename T> struct CalledAs {};
@@ -250,68 +140,75 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
// (We always store a T, even if the call will use a pointer to const T).
template <typename CallableT, typename CalledAsT>
UniqueFunctionBase(CallableT Callable, CalledAs<CalledAsT>) {
- bool IsInlineStorage = true;
- void *CallableAddr = getInlineStorage();
- if (sizeof(CallableT) > InlineStorageSize ||
- alignof(CallableT) > InlineStorageAlign) {
- IsInlineStorage = false;
- // Allocate out-of-line storage. FIXME: Use an explicit alignment
- // parameter in C++17 mode.
- auto Size = sizeof(CallableT);
- auto Alignment = alignof(CallableT);
- CallableAddr = allocate_buffer(Size, Alignment);
- setOutOfLineStorage(CallableAddr, Size, Alignment);
+ constexpr bool UsesInlineStorage = sizeof(CallableT) <= InlineStorageSize &&
+ alignof(CallableT) <= InlineStorageAlign;
+ void *CallableAddr = &Storage.Inline;
+ if constexpr (!UsesInlineStorage) {
+ CallableAddr = allocate_buffer(sizeof(CallableT), alignof(CallableT));
+ Storage.OutOfLine = CallableAddr;
}
// Now move into the storage.
new (CallableAddr) CallableT(std::move(Callable));
- CallbackAndInlineFlag.setPointerAndInt(
- &CallbacksHolder<CallableT, CalledAsT>::Callbacks, IsInlineStorage);
+
+ CallPtr = [](const UniqueFunctionBase *Self,
+ AdjustedParamT<ParamTs>... Params) -> ReturnT {
+ void *CallableAddr =
+ UsesInlineStorage ? &Self->Storage.Inline : Self->Storage.OutOfLine;
+ auto &Func = *reinterpret_cast<CalledAsT *>(CallableAddr);
+ return Func(std::forward<ParamTs>(Params)...);
+ };
+
+ // For trivial inline storage, we don't need to do anything on move/destroy.
+ if constexpr (!std::is_trivially_move_constructible_v<CallableT> ||
+ !std::is_trivially_destructible_v<CallableT> ||
+ !UsesInlineStorage) {
+ // If LHS is set, move LHS callable to RHS, then destroy RHS callable.
+ DestroyMovePtr = [](UniqueFunctionBase *LHS, UniqueFunctionBase *RHS) {
+ if constexpr (!UsesInlineStorage) {
+ if (LHS) {
+ // Out-of-line move: just move the pointer.
+ LHS->Storage.OutOfLine = RHS->Storage.OutOfLine;
+ } else {
+ // Out-of-line destroy.
+ void *RHSCallableAddr = RHS->Storage.OutOfLine;
+ reinterpret_cast<CallableT *>(RHSCallableAddr)->~CallableT();
+ deallocate_buffer(RHSCallableAddr, sizeof(CallableT),
+ alignof(CallableT));
+ }
+ } else {
+ auto *RHSCallable =
+ reinterpret_cast<CallableT *>(&RHS->Storage.Inline);
+ if (LHS) {
+ // Inline move: move-construct first...
+ auto *LHSCallable =
+ reinterpret_cast<CallableT *>(&LHS->Storage.Inline);
+ new (LHSCallable) CallableT(std::move(*RHSCallable));
+ }
+ // ... destroy RHS in any case.
+ RHSCallable->~CallableT();
+ }
+ };
+ }
}
~UniqueFunctionBase() {
- if (!CallbackAndInlineFlag.getPointer())
- return;
-
- // Cache this value so we don't re-check it after type-erased operations.
- bool IsInlineStorage = isInlineStorage();
-
- if (!isTrivialCallback())
- getNonTrivialCallbacks()->DestroyPtr(
- IsInlineStorage ? getInlineStorage() : getOutOfLineStorage());
-
- if (!IsInlineStorage)
- deallocate_buffer(getOutOfLineStorage(), getOutOfLineStorageSize(),
- getOutOfLineStorageAlignment());
+ if (DestroyMovePtr)
+ DestroyMovePtr(nullptr, this);
}
UniqueFunctionBase(UniqueFunctionBase &&RHS) noexcept {
- // Copy the callback and inline flag.
- CallbackAndInlineFlag = RHS.CallbackAndInlineFlag;
-
- // If the RHS is empty, just copying the above is sufficient.
- if (!RHS)
- return;
-
- if (!isInlineStorage()) {
- // The out-of-line case is easiest to move.
- StorageUnion.OutOfLineStorage = RHS.StorageUnion.OutOfLineStorage;
- } else if (isTrivialCallback()) {
- // Move is trivial, just memcpy the bytes across.
- memcpy(getInlineStorage(), RHS.getInlineStorage(), InlineStorageSize);
- } else {
- // Non-trivial move, so dispatch to a type-erased implementation.
- getNonTrivialCallbacks()->MovePtr(getInlineStorage(),
- RHS.getInlineStorage());
- getNonTrivialCallbacks()->DestroyPtr(RHS.getInlineStorage());
- }
-
- // Clear the old callback and inline flag to get back to as-if-null.
- RHS.CallbackAndInlineFlag = {};
-
+ CallPtr = RHS.CallPtr;
+ DestroyMovePtr = RHS.DestroyMovePtr;
+ if (DestroyMovePtr)
+ DestroyMovePtr(this, &RHS);
+ else // Trivial callable stored inline => memcpy.
+ memcpy(&Storage.Inline, &RHS.Storage.Inline, InlineStorageSize);
+
+ RHS.DestroyMovePtr = nullptr; // Moved everything out of RHS.
#ifndef NDEBUG
// In debug builds, we also scribble across the rest of the storage.
- memset(RHS.getInlineStorage(), 0xAD, InlineStorageSize);
+ memset(RHS.Storage.Inline, 0xAD, InlineStorageSize);
#endif
}
@@ -330,9 +227,7 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
UniqueFunctionBase() = default;
public:
- explicit operator bool() const {
- return (bool)CallbackAndInlineFlag.getPointer();
- }
+ explicit operator bool() const { return (bool)CallPtr; }
};
} // namespace detail
@@ -357,9 +252,7 @@ class unique_function<R(P...)> : public detail::UniqueFunctionBase<R, P...> {
: Base(std::forward<CallableT>(Callable),
typename Base::template CalledAs<CallableT>{}) {}
- R operator()(P... Params) {
- return this->getCallPtr()(this->getCalleePtr(), Params...);
- }
+ R operator()(P... Params) { return this->CallPtr(this, Params...); }
};
template <typename R, typename... P>
@@ -383,9 +276,7 @@ class unique_function<R(P...) const>
: Base(std::forward<CallableT>(Callable),
typename Base::template CalledAs<const CallableT>{}) {}
- R operator()(P... Params) const {
- return this->getCallPtr()(this->getCalleePtr(), Params...);
- }
+ R operator()(P... Params) const { return this->CallPtr(this, Params...); }
};
} // end namespace llvm
``````````
</details>
https://github.com/llvm/llvm-project/pull/208251
More information about the llvm-commits
mailing list