[libcxx] r348812 - Add a version of std::function that includes a few optimizations in ABI V2.
Eric Fiselier
eric at efcs.ca
Mon Dec 10 16:14:35 PST 2018
Author: ericwf
Date: Mon Dec 10 16:14:34 2018
New Revision: 348812
URL: http://llvm.org/viewvc/llvm-project?rev=348812&view=rev
Log:
Add a version of std::function that includes a few optimizations in ABI V2.
Patch by Jordan Soyke (jsoyke at google.com)
Reviewed as D55045
The result of running the benchmarks and comparing them can be found
here: https://gist.github.com/EricWF/a77fd42ec87fc98da8039e26d0349498
Modified:
libcxx/trunk/include/__config
libcxx/trunk/include/functional
Modified: libcxx/trunk/include/__config
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/__config?rev=348812&r1=348811&r2=348812&view=diff
==============================================================================
--- libcxx/trunk/include/__config (original)
+++ libcxx/trunk/include/__config Mon Dec 10 16:14:34 2018
@@ -95,6 +95,8 @@
// Use the smallest possible integer type to represent the index of the variant.
// Previously libc++ used "unsigned int" exclusivly.
# define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
+// Unstable attempt to provide a more optimized std::function
+# define _LIBCPP_ABI_OPTIMIZED_FUNCTION
#elif _LIBCPP_ABI_VERSION == 1
# if !defined(_LIBCPP_OBJECT_FORMAT_COFF)
// Enable compiling copies of now inline methods into the dylib to support
Modified: libcxx/trunk/include/functional
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/functional?rev=348812&r1=348811&r2=348812&view=diff
==============================================================================
--- libcxx/trunk/include/functional (original)
+++ libcxx/trunk/include/functional Mon Dec 10 16:14:34 2018
@@ -1859,6 +1859,305 @@ template <class _Rp, class... _ArgTypes>
#endif // _LIBCPP_NO_RTTI
};
+// Storage for a functor object, to be used with __policy to manage copy and
+// destruction.
+union __policy_storage
+{
+ mutable char __small[sizeof(void*) * 2];
+ void* __large;
+};
+
+// True if _Fun can safely be held in __policy_storage.__small.
+template <typename _Fun>
+struct __use_small_storage
+ : public _VSTD::integral_constant<
+ bool, sizeof(_Fun) <= sizeof(__policy_storage) &&
+ alignof(_Fun) <= alignof(__policy_storage) &&
+ _VSTD::is_trivially_copy_constructible<_Fun>::value &&
+ _VSTD::is_trivially_destructible<_Fun>::value> {};
+
+// Policy contains information about how to copy, destroy, and move the
+// underlying functor. You can think of it as a vtable of sorts.
+struct __policy
+{
+ // Used to copy or destroy __large values. null for trivial objects.
+ void* (*const __clone)(const void*);
+ void (*const __destroy)(void*);
+
+ // True if this is the null policy (no value).
+ const bool __is_null;
+
+ // The target type. May be null if RTTI is disabled.
+ const std::type_info* const __type_info;
+
+ // Returns a pointer to a static policy object suitable for the functor
+ // type.
+ template <typename _Fun>
+ _LIBCPP_INLINE_VISIBILITY static const __policy* __create()
+ {
+ return __choose_policy<_Fun>(__use_small_storage<_Fun>());
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ static const __policy* __create_empty()
+ {
+ static const _LIBCPP_CONSTEXPR __policy __policy_ = {nullptr, nullptr,
+ true,
+#ifndef _LIBCPP_NO_RTTI
+ &typeid(void)
+#else
+ nullptr
+#endif
+ };
+ return &__policy_;
+ }
+
+ private:
+ template <typename _Fun> static void* __large_clone(const void* __s)
+ {
+ const _Fun* __f = static_cast<const _Fun*>(__s);
+ return __f->__clone();
+ }
+
+ template <typename _Fun> static void __large_destroy(void* __s)
+ {
+ typedef allocator_traits<typename _Fun::_Alloc> __alloc_traits;
+ typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type
+ _FunAlloc;
+ _Fun* __f = static_cast<_Fun*>(__s);
+ _FunAlloc __a(__f->__allocator());
+ __f->destroy();
+ __a.deallocate(__f, 1);
+ }
+
+ template <typename _Fun>
+ _LIBCPP_INLINE_VISIBILITY static const __policy*
+ __choose_policy(/* is_small = */ false_type)
+ {
+ static const _LIBCPP_CONSTEXPR __policy __policy_ = {
+ &__large_clone<_Fun>, &__large_destroy<_Fun>, false,
+#ifndef _LIBCPP_NO_RTTI
+ &typeid(typename _Fun::_Target)
+#else
+ nullptr
+#endif
+ };
+ return &__policy_;
+ }
+
+ template <typename _Fun>
+ _LIBCPP_INLINE_VISIBILITY static const __policy*
+ __choose_policy(/* is_small = */ true_type)
+ {
+ static const _LIBCPP_CONSTEXPR __policy __policy_ = {
+ nullptr, nullptr, false,
+#ifndef _LIBCPP_NO_RTTI
+ &typeid(typename _Fun::_Target)
+#else
+ nullptr
+#endif
+ };
+ return &__policy_;
+ }
+};
+
+// Used to choose between perfect forwarding or pass-by-value. Pass-by-value is
+// faster for types that can be passed in registers.
+template <typename _Tp>
+using __fast_forward =
+ typename _VSTD::conditional<_VSTD::is_scalar<_Tp>::value, _Tp, _Tp&&>::type;
+
+// __policy_invoker calls an instance of __alloc_func held in __policy_storage.
+
+template <class _Fp> struct __policy_invoker;
+
+template <class _Rp, class... _ArgTypes>
+struct __policy_invoker<_Rp(_ArgTypes...)>
+{
+ typedef _Rp (*__Call)(const __policy_storage*,
+ __fast_forward<_ArgTypes>...);
+
+ __Call __call_;
+
+ // Creates an invoker that throws bad_function_call.
+ _LIBCPP_INLINE_VISIBILITY
+ __policy_invoker() : __call_(&__call_empty) {}
+
+ // Creates an invoker that calls the given instance of __func.
+ template <typename _Fun>
+ _LIBCPP_INLINE_VISIBILITY static __policy_invoker __create()
+ {
+ return __policy_invoker(&__call_impl<_Fun>);
+ }
+
+ private:
+ _LIBCPP_INLINE_VISIBILITY
+ explicit __policy_invoker(__Call __c) : __call_(__c) {}
+
+ static _Rp __call_empty(const __policy_storage*,
+ __fast_forward<_ArgTypes>...)
+ {
+ __throw_bad_function_call();
+ }
+
+ template <typename _Fun>
+ static _Rp __call_impl(const __policy_storage* __buf,
+ __fast_forward<_ArgTypes>... __args)
+ {
+ _Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value
+ ? &__buf->__small
+ : __buf->__large);
+ return (*__f)(_VSTD::forward<_ArgTypes>(__args)...);
+ }
+};
+
+// __policy_func uses a __policy and __policy_invoker to create a type-erased,
+// copyable functor.
+
+template <class _Fp> class __policy_func;
+
+template <class _Rp, class... _ArgTypes> class __policy_func<_Rp(_ArgTypes...)>
+{
+ // Inline storage for small objects.
+ __policy_storage __buf_;
+
+ // Calls the value stored in __buf_. This could technically be part of
+ // policy, but storing it here eliminates a level of indirection inside
+ // operator().
+ typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker;
+ __invoker __invoker_;
+
+ // The policy that describes how to move / copy / destroy __buf_. Never
+ // null, even if the function is empty.
+ const __policy* __policy_;
+
+ public:
+ _LIBCPP_INLINE_VISIBILITY
+ __policy_func() : __policy_(__policy::__create_empty()) {}
+
+ template <class _Fp, class _Alloc>
+ _LIBCPP_INLINE_VISIBILITY __policy_func(_Fp&& __f, const _Alloc& __a)
+ : __policy_(__policy::__create_empty())
+ {
+ typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun;
+ typedef allocator_traits<_Alloc> __alloc_traits;
+ typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type
+ _FunAlloc;
+
+ if (__function::__not_null(__f))
+ {
+ __invoker_ = __invoker::template __create<_Fun>();
+ __policy_ = __policy::__create<_Fun>();
+
+ _FunAlloc __af(__a);
+ if (__use_small_storage<_Fun>())
+ {
+ ::new ((void*)&__buf_.__small)
+ _Fun(_VSTD::move(__f), _Alloc(__af));
+ }
+ else
+ {
+ typedef __allocator_destructor<_FunAlloc> _Dp;
+ unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1));
+ ::new ((void*)__hold.get())
+ _Fun(_VSTD::move(__f), _Alloc(__af));
+ __buf_.__large = __hold.release();
+ }
+ }
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ __policy_func(const __policy_func& __f)
+ : __buf_(__f.__buf_), __invoker_(__f.__invoker_),
+ __policy_(__f.__policy_)
+ {
+ if (__policy_->__clone)
+ __buf_.__large = __policy_->__clone(__f.__buf_.__large);
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ __policy_func(__policy_func&& __f)
+ : __buf_(__f.__buf_), __invoker_(__f.__invoker_),
+ __policy_(__f.__policy_)
+ {
+ if (__policy_->__destroy)
+ {
+ __f.__policy_ = __policy::__create_empty();
+ __f.__invoker_ = __invoker();
+ }
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ ~__policy_func()
+ {
+ if (__policy_->__destroy)
+ __policy_->__destroy(__buf_.__large);
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ __policy_func& operator=(__policy_func&& __f)
+ {
+ *this = nullptr;
+ __buf_ = __f.__buf_;
+ __invoker_ = __f.__invoker_;
+ __policy_ = __f.__policy_;
+ __f.__policy_ = __policy::__create_empty();
+ __f.__invoker_ = __invoker();
+ return *this;
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ __policy_func& operator=(nullptr_t)
+ {
+ const __policy* __p = __policy_;
+ __policy_ = __policy::__create_empty();
+ __invoker_ = __invoker();
+ if (__p->__destroy)
+ __p->__destroy(__buf_.__large);
+ return *this;
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ _Rp operator()(_ArgTypes&&... __args) const
+ {
+ return __invoker_.__call_(_VSTD::addressof(__buf_),
+ _VSTD::forward<_ArgTypes>(__args)...);
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ void swap(__policy_func& __f)
+ {
+ _VSTD::swap(__invoker_, __f.__invoker_);
+ _VSTD::swap(__policy_, __f.__policy_);
+ _VSTD::swap(__buf_, __f.__buf_);
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ explicit operator bool() const _NOEXCEPT
+ {
+ return !__policy_->__is_null;
+ }
+
+#ifndef _LIBCPP_NO_RTTI
+ _LIBCPP_INLINE_VISIBILITY
+ const std::type_info& target_type() const _NOEXCEPT
+ {
+ return *__policy_->__type_info;
+ }
+
+ template <typename _Tp>
+ _LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT
+ {
+ if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info)
+ return nullptr;
+ if (__policy_->__clone) // Out of line storage.
+ return reinterpret_cast<const _Tp*>(__buf_.__large);
+ else
+ return reinterpret_cast<const _Tp*>(&__buf_.__small);
+ }
+#endif // _LIBCPP_NO_RTTI
+};
+
} // __function
template<class _Rp, class ..._ArgTypes>
@@ -1866,7 +2165,11 @@ class _LIBCPP_TEMPLATE_VIS function<_Rp(
: public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>,
public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)>
{
+#ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION
typedef __function::__value_func<_Rp(_ArgTypes...)> __func;
+#else
+ typedef __function::__policy_func<_Rp(_ArgTypes...)> __func;
+#endif
__func __f_;
More information about the libcxx-commits
mailing list