ceph/src/rocksdb/third-party/folly/folly/synchronization/detail/InlineFunctionRef.h

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 #pragma once
   7
   8 #include <cstdint>
   9 #include <type_traits>
  10
  11 #include <folly/Traits.h>
  12 #include <folly/Utility.h>
  13 #include <folly/functional/Invoke.h>
  14 #include <folly/lang/Launder.h>
  15
  16 namespace folly {
  17 namespace detail {
  18
  19 /**
  20  * InlineFunctionRef is similar to folly::FunctionRef but has the additional
  21  * benefit of being able to store the function it was instantiated with inline
  22  * in a buffer of the given capacity.  Inline storage is only used if the
  23  * function object and a pointer (for type-erasure) are small enough to fit in
  24  * the templated size.  If there is not enough in-situ capacity for the
  25  * callable, this just stores a reference to the function object like
  26  * FunctionRef.
  27  *
  28  * This helps give a perf boost in the case where the data gets separated from
  29  * the point of invocation.  If, for example, at the point of invocation, the
  30  * InlineFunctionRef object is not cached, a remote memory/cache read might be
  31  * required to invoke the original callable.  Customizable inline storage
  32  * helps tune storage so we can store a type-erased callable with better
  33  * performance and locality.  A real-life example of this might be a
  34  * folly::FunctionRef with a function pointer.  The folly::FunctionRef would
  35  * point to the function pointer object in a remote location.  This causes a
  36  * double-indirection at the point of invocation, and if that memory is dirty,
  37  * or not cached, it would cause additional cache misses.  On the other hand
  38  * with InlineFunctionRef, inline storage would store the value of the
  39  * function pointer, avoiding the need to do a remote lookup to fetch the
  40  * value of the function pointer.
  41  *
  42  * To prevent misuse, InlineFunctionRef disallows construction from an lvalue
  43  * callable.  This is to prevent usage where a user relies on the callable's
  44  * state after invocation through InlineFunctionRef.  This has the potential
  45  * to copy the callable into inline storage when the callable is small, so we
  46  * might not use the same function when invoking, but rather a copy of it.
  47  *
  48  * Also note that InlineFunctionRef will always invoke the const qualified
  49  * version of the call operator for any callable that is passed.  Regardless
  50  * of whether it has a non-const version.  This is done to enforce the logical
  51  * constraint of function state being immutable.
  52  *
  53  * This class is always trivially-copyable (and therefore
  54  * trivially-destructible), making it suitable for use in a union without
  55  * requiring manual destruction.
  56  */
  57 template <typename FunctionType, std::size_t Size>
  58 class InlineFunctionRef;
  59
  60 template <typename ReturnType, typename... Args, std::size_t Size>
  61 class InlineFunctionRef<ReturnType(Args...), Size> {
  62   using Storage =
  63       _t<std::aligned_storage<Size - sizeof(uintptr_t), sizeof(uintptr_t)>>;
  64   using Call = ReturnType (*)(const Storage&, Args&&...);
  65
  66   struct InSituTag {};
  67   struct RefTag {};
  68
  69   static_assert(
  70       (Size % sizeof(uintptr_t)) == 0,
  71       "Size has to be a multiple of sizeof(uintptr_t)");
  72   static_assert(Size >= 2 * sizeof(uintptr_t), "This doesn't work");
  73   static_assert(alignof(Call) == alignof(Storage), "Mismatching alignments");
  74
  75   // This defines a mode tag that is used in the construction of
  76   // InlineFunctionRef to determine the storage and indirection method for the
  77   // passed callable.
  78   //
  79   // This requires that the we pass in a type that is not ref-qualified.
  80   template <typename Func>
  81   using ConstructMode = _t<std::conditional<
  82       folly::is_trivially_copyable<Func>{} &&
  83           (sizeof(Func) <= sizeof(Storage)) &&
  84           (alignof(Func) <= alignof(Storage)),
  85       InSituTag,
  86       RefTag>>;
  87
  88  public:
  89   /**
  90    * InlineFunctionRef can be constructed from a nullptr, callable or another
  91    * InlineFunctionRef with the same size.  These are the constructors that
  92    * don't take a callable.
  93    *
  94    * InlineFunctionRef is meant to be trivially copyable so we default the
  95    * constructors and assignment operators.
  96    */
  97   InlineFunctionRef(std::nullptr_t) : call_{nullptr} {}
  98   InlineFunctionRef() : call_{nullptr} {}
  99   InlineFunctionRef(const InlineFunctionRef& other) = default;
 100   InlineFunctionRef(InlineFunctionRef&&) = default;
 101   InlineFunctionRef& operator=(const InlineFunctionRef&) = default;
 102   InlineFunctionRef& operator=(InlineFunctionRef&&) = default;
 103
 104   /**
 105    * Constructors from callables.
 106    *
 107    * If all of the following conditions are satisfied, then we store the
 108    * callable in the inline storage:
 109    *
 110    *  1) The function has been passed as an rvalue, meaning that there is no
 111    *     use of the original in the user's code after it has been passed to
 112    *     us.
 113    *  2) Size of the callable is less than the size of the inline storage
 114    *     buffer.
 115    *  3) The callable is trivially constructible and destructible.
 116    *
 117    * If any one of the above conditions is not satisfied, we fall back to
 118    * reference semantics and store the function as a pointer, and add a level
 119    * of indirection through type erasure.
 120    */
 121   template <
 122       typename Func,
 123       _t<std::enable_if<
 124           !std::is_same<_t<std::decay<Func>>, InlineFunctionRef>{} &&
 125           !std::is_reference<Func>{} &&
 126           std::is_convertible<
 127               decltype(std::declval<Func&&>()(std::declval<Args&&>()...)),
 128               ReturnType>{}>>* = nullptr>
 129   InlineFunctionRef(Func&& func) {
 130     // We disallow construction from lvalues, so assert that this is not a
 131     // reference type.  When invoked with an lvalue, Func is a lvalue
 132     // reference type, when invoked with an rvalue, Func is not ref-qualified.
 133     static_assert(
 134         !std::is_reference<Func>{},
 135         "InlineFunctionRef cannot be used with lvalues");
 136     static_assert(std::is_rvalue_reference<Func&&>{}, "");
 137     construct(ConstructMode<Func>{}, folly::as_const(func));
 138   }
 139
 140   /**
 141    * The call operator uses the function pointer and a reference to the
 142    * storage to do the dispatch.  The function pointer takes care of the
 143    * appropriate casting.
 144    */
 145   ReturnType operator()(Args... args) const {
 146     return call_(storage_, static_cast<Args&&>(args)...);
 147   }
 148
 149   /**
 150    * We have a function engaged if the call function points to anything other
 151    * than null.
 152    */
 153   operator bool() const noexcept {
 154     return call_;
 155   }
 156
 157  private:
 158   friend class InlineFunctionRefTest;
 159
 160   /**
 161    * Inline storage constructor implementation.
 162    */
 163   template <typename Func>
 164   void construct(InSituTag, Func& func) {
 165     using Value = _t<std::remove_reference<Func>>;
 166
 167     // Assert that the following two assumptions are valid
 168     //    1) fit in the storage space we have and match alignments, and
 169     //    2) be invocable in a const context, it does not make sense to copy a
 170     //       callable into inline storage if it makes state local
 171     //       modifications.
 172     static_assert(alignof(Value) <= alignof(Storage), "");
 173     static_assert(is_invocable<const _t<std::decay<Func>>, Args&&...>{}, "");
 174     static_assert(folly::is_trivially_copyable<Value>{}, "");
 175
 176     new (&storage_) Value{func};
 177     call_ = &callInline<Value>;
 178   }
 179
 180   /**
 181    * Ref storage constructor implementation.  This is identical to
 182    * folly::FunctionRef.
 183    */
 184   template <typename Func>
 185   void construct(RefTag, Func& func) {
 186     // store a pointer to the function
 187     using Pointer = _t<std::add_pointer<_t<std::remove_reference<Func>>>>;
 188     new (&storage_) Pointer{&func};
 189     call_ = &callPointer<Pointer>;
 190   }
 191
 192   template <typename Func>
 193   static ReturnType callInline(const Storage& object, Args&&... args) {
 194     // The only type of pointer allowed is a function pointer, no other
 195     // pointer types are invocable.
 196     static_assert(
 197         !std::is_pointer<Func>::value ||
 198             std::is_function<_t<std::remove_pointer<Func>>>::value,
 199         "");
 200     return (*folly::launder(reinterpret_cast<const Func*>(&object)))(
 201         static_cast<Args&&>(args)...);
 202   }
 203
 204   template <typename Func>
 205   static ReturnType callPointer(const Storage& object, Args&&... args) {
 206     // When the function we were instantiated with was not trivial, the given
 207     // pointer points to a pointer, which pointers to the callable.  So we
 208     // cast to a pointer and then to the pointee.
 209     static_assert(std::is_pointer<Func>::value, "");
 210     return (**folly::launder(reinterpret_cast<const Func*>(&object)))(
 211         static_cast<Args&&>(args)...);
 212   }
 213
 214   Call call_;
 215   Storage storage_;
 216 };
 217
 218 } // namespace detail
 219 } // namespace folly