ceph/src/boost/boost/fiber/context.hpp

   1
   2 //          Copyright Oliver Kowalke 2013.
   3 // Distributed under the Boost Software License, Version 1.0.
   4 //    (See accompanying file LICENSE_1_0.txt or copy at
   5 //          http://www.boost.org/LICENSE_1_0.txt)
   6
   7 #ifndef BOOST_FIBERS_CONTEXT_H
   8 #define BOOST_FIBERS_CONTEXT_H
   9
  10 #include <atomic>
  11 #include <chrono>
  12 #include <cstdint>
  13 #include <exception>
  14 #include <functional>
  15 #include <iostream>
  16 #include <map>
  17 #include <memory>
  18 #include <tuple>
  19 #include <type_traits>
  20 #include <utility>
  21
  22 #include <boost/assert.hpp>
  23 #include <boost/config.hpp>
  24 #include <boost/core/ignore_unused.hpp>
  25 #if defined(BOOST_NO_CXX17_STD_APPLY)
  26 #include <boost/context/detail/apply.hpp>
  27 #endif
  28 #include <boost/context/fiber.hpp>
  29 #include <boost/context/stack_context.hpp>
  30 #include <boost/intrusive/list.hpp>
  31 #include <boost/intrusive/parent_from_member.hpp>
  32 #include <boost/intrusive_ptr.hpp>
  33 #include <boost/intrusive/set.hpp>
  34 #include <boost/intrusive/slist.hpp>
  35
  36 #include <boost/fiber/detail/config.hpp>
  37 #include <boost/fiber/detail/data.hpp>
  38 #include <boost/fiber/detail/decay_copy.hpp>
  39 #include <boost/fiber/detail/fss.hpp>
  40 #include <boost/fiber/detail/spinlock.hpp>
  41 #include <boost/fiber/exceptions.hpp>
  42 #include <boost/fiber/fixedsize_stack.hpp>
  43 #include <boost/fiber/policy.hpp>
  44 #include <boost/fiber/properties.hpp>
  45 #include <boost/fiber/segmented_stack.hpp>
  46 #include <boost/fiber/type.hpp>
  47 #include <boost/fiber/waker.hpp>
  48
  49 #ifdef BOOST_HAS_ABI_HEADERS
  50 #  include BOOST_ABI_PREFIX
  51 #endif
  52
  53 #ifdef _MSC_VER
  54 # pragma warning(push)
  55 # pragma warning(disable:4251)
  56 #endif
  57
  58 namespace boost {
  59 namespace fibers {
  60
  61 class context;
  62 class fiber;
  63 class scheduler;
  64
  65 namespace detail {
  66
  67 struct ready_tag;
  68 typedef intrusive::list_member_hook<
  69     intrusive::tag< ready_tag >,
  70     intrusive::link_mode<
  71         intrusive::auto_unlink
  72     >
  73 >                                       ready_hook;
  74
  75 struct sleep_tag;
  76 typedef intrusive::set_member_hook<
  77     intrusive::tag< sleep_tag >,
  78     intrusive::link_mode<
  79         intrusive::auto_unlink
  80     >
  81 >                                       sleep_hook;
  82
  83 struct worker_tag;
  84 typedef intrusive::list_member_hook<
  85     intrusive::tag< worker_tag >,
  86     intrusive::link_mode<
  87         intrusive::auto_unlink
  88     >
  89 >                                       worker_hook;
  90
  91 struct terminated_tag;
  92 typedef intrusive::slist_member_hook<
  93     intrusive::tag< terminated_tag >,
  94     intrusive::link_mode<
  95         intrusive::safe_link
  96     >
  97 >                                       terminated_hook;
  98
  99 struct remote_ready_tag;
 100 typedef intrusive::slist_member_hook<
 101     intrusive::tag< remote_ready_tag >,
 102     intrusive::link_mode<
 103         intrusive::safe_link
 104     >
 105 >                                       remote_ready_hook;
 106
 107 }
 108
 109 class BOOST_FIBERS_DECL context {
 110 private:
 111     friend class dispatcher_context;
 112     friend class main_context;
 113     template< typename Fn, typename ... Arg > friend class worker_context;
 114     friend class scheduler;
 115
 116     struct fss_data {
 117         void                                *   vp{ nullptr };
 118         detail::fss_cleanup_function::ptr_t     cleanup_function{};
 119
 120         fss_data() noexcept = default;
 121
 122         fss_data( void * vp_,
 123                   detail::fss_cleanup_function::ptr_t fn) noexcept :
 124             vp( vp_),
 125             cleanup_function(std::move( fn)) {
 126             BOOST_ASSERT( cleanup_function);
 127         }
 128
 129         void do_cleanup() {
 130             ( * cleanup_function)( vp);
 131         }
 132     };
 133
 134     typedef std::map< uintptr_t, fss_data >             fss_data_t;
 135
 136 #if ! defined(BOOST_FIBERS_NO_ATOMICS)
 137     std::atomic< std::size_t >                          use_count_;
 138 #else
 139     std::size_t                                         use_count_;
 140 #endif
 141 #if ! defined(BOOST_FIBERS_NO_ATOMICS)
 142     detail::remote_ready_hook                           remote_ready_hook_{};
 143 #endif
 144     detail::spinlock                                    splk_{};
 145     bool                                                terminated_{ false };
 146     wait_queue                                          wait_queue_{};
 147 public:
 148 #if ! defined(BOOST_FIBERS_NO_ATOMICS)
 149     std::atomic<size_t>                                 waker_epoch_{ 0 };
 150 #endif
 151 private:
 152     scheduler                                       *   scheduler_{ nullptr };
 153     fss_data_t                                          fss_data_{};
 154     detail::sleep_hook                                  sleep_hook_{};
 155     waker                                               sleep_waker_{};
 156     detail::ready_hook                                  ready_hook_{};
 157     detail::terminated_hook                             terminated_hook_{};
 158     detail::worker_hook                                 worker_hook_{};
 159     fiber_properties                                *   properties_{ nullptr };
 160     boost::context::fiber                               c_{};
 161     std::chrono::steady_clock::time_point               tp_;
 162     type                                                type_;
 163     launch                                              policy_;
 164
 165     context( std::size_t initial_count, type t, launch policy) noexcept :
 166         use_count_{ initial_count },
 167         tp_{ (std::chrono::steady_clock::time_point::max)() },
 168         type_{ t },
 169         policy_{ policy } {
 170     }
 171
 172 public:
 173     class id {
 174     private:
 175         context  *   impl_{ nullptr };
 176
 177     public:
 178         id() = default;
 179
 180         explicit id( context * impl) noexcept :
 181             impl_{ impl } {
 182         }
 183
 184         bool operator==( id const& other) const noexcept {
 185             return impl_ == other.impl_;
 186         }
 187
 188         bool operator!=( id const& other) const noexcept {
 189             return impl_ != other.impl_;
 190         }
 191
 192         bool operator<( id const& other) const noexcept {
 193             return impl_ < other.impl_;
 194         }
 195
 196         bool operator>( id const& other) const noexcept {
 197             return other.impl_ < impl_;
 198         }
 199
 200         bool operator<=( id const& other) const noexcept {
 201             return ! ( * this > other);
 202         }
 203
 204         bool operator>=( id const& other) const noexcept {
 205             return ! ( * this < other);
 206         }
 207
 208         template< typename charT, class traitsT >
 209         friend std::basic_ostream< charT, traitsT > &
 210         operator<<( std::basic_ostream< charT, traitsT > & os, id const& other) {
 211             if ( nullptr != other.impl_) {
 212                 return os << other.impl_;
 213             }
 214             return os << "{not-valid}";
 215         }
 216
 217         explicit operator bool() const noexcept {
 218             return nullptr != impl_;
 219         }
 220
 221         bool operator!() const noexcept {
 222             return nullptr == impl_;
 223         }
 224     };
 225
 226     static context * active() noexcept;
 227
 228     static void reset_active() noexcept;
 229
 230     context( context const&) = delete;
 231     context( context &&) = delete;
 232     context & operator=( context const&) = delete;
 233     context & operator=( context &&) = delete;
 234
 235     #if !defined(BOOST_EMBTC)
 236
 237     friend bool
 238     operator==( context const& lhs, context const& rhs) noexcept {
 239         return & lhs == & rhs;
 240     }
 241
 242     #else
 243
 244     friend bool
 245     operator==( context const& lhs, context const& rhs) noexcept;
 246
 247     #endif
 248
 249     virtual ~context();
 250
 251     scheduler * get_scheduler() const noexcept {
 252         return scheduler_;
 253     }
 254
 255     id get_id() const noexcept;
 256
 257     bool is_resumable() const noexcept {
 258         return static_cast<bool>(c_);
 259     }
 260
 261     void resume() noexcept;
 262     void resume( detail::spinlock_lock &) noexcept;
 263     void resume( context *) noexcept;
 264
 265     void suspend() noexcept;
 266     void suspend( detail::spinlock_lock &) noexcept;
 267
 268     boost::context::fiber suspend_with_cc() noexcept;
 269     boost::context::fiber terminate() noexcept;
 270
 271     void join();
 272
 273     void yield() noexcept;
 274
 275     bool wait_until( std::chrono::steady_clock::time_point const&) noexcept;
 276     bool wait_until( std::chrono::steady_clock::time_point const&,
 277                      detail::spinlock_lock &,
 278                      waker &&) noexcept;
 279
 280     bool wake(const size_t) noexcept;
 281
 282     waker create_waker() noexcept {
 283         // this operation makes all previously created wakers to be outdated
 284         return { this, ++waker_epoch_ };
 285     }
 286
 287     void schedule( context *) noexcept;
 288
 289     bool is_context( type t) const noexcept {
 290         return type::none != ( type_ & t);
 291     }
 292
 293     void * get_fss_data( void const * vp) const;
 294
 295     void set_fss_data(
 296         void const * vp,
 297         detail::fss_cleanup_function::ptr_t const& cleanup_fn,
 298         void * data,
 299         bool cleanup_existing);
 300
 301     void set_properties( fiber_properties * props) noexcept;
 302
 303     fiber_properties * get_properties() const noexcept {
 304         return properties_;
 305     }
 306
 307     launch get_policy() const noexcept {
 308         return policy_;
 309     }
 310
 311     bool worker_is_linked() const noexcept;
 312
 313     bool ready_is_linked() const noexcept;
 314
 315     bool remote_ready_is_linked() const noexcept;
 316
 317     bool sleep_is_linked() const noexcept;
 318
 319     bool terminated_is_linked() const noexcept;
 320
 321     template< typename List >
 322     void worker_link( List & lst) noexcept {
 323         static_assert( std::is_same< typename List::value_traits::hook_type, detail::worker_hook >::value, "not a worker-queue");
 324         BOOST_ASSERT( ! worker_is_linked() );
 325         lst.push_back( * this);
 326     }
 327
 328     template< typename List >
 329     void ready_link( List & lst) noexcept {
 330         static_assert( std::is_same< typename List::value_traits::hook_type, detail::ready_hook >::value, "not a ready-queue");
 331         BOOST_ASSERT( ! ready_is_linked() );
 332         lst.push_back( * this);
 333     }
 334
 335     template< typename List >
 336     void remote_ready_link( List & lst) noexcept {
 337         static_assert( std::is_same< typename List::value_traits::hook_type, detail::remote_ready_hook >::value, "not a remote-ready-queue");
 338         BOOST_ASSERT( ! remote_ready_is_linked() );
 339         lst.push_back( * this);
 340     }
 341
 342     template< typename Set >
 343     void sleep_link( Set & set) noexcept {
 344         static_assert( std::is_same< typename Set::value_traits::hook_type,detail::sleep_hook >::value, "not a sleep-queue");
 345         BOOST_ASSERT( ! sleep_is_linked() );
 346         set.insert( * this);
 347     }
 348
 349     template< typename List >
 350     void terminated_link( List & lst) noexcept {
 351         static_assert( std::is_same< typename List::value_traits::hook_type, detail::terminated_hook >::value, "not a terminated-queue");
 352         BOOST_ASSERT( ! terminated_is_linked() );
 353         lst.push_back( * this);
 354     }
 355
 356     void worker_unlink() noexcept;
 357
 358     void ready_unlink() noexcept;
 359
 360     void sleep_unlink() noexcept;
 361
 362     void detach() noexcept;
 363
 364     void attach( context *) noexcept;
 365
 366     #if !defined(BOOST_EMBTC)
 367
 368     friend void intrusive_ptr_add_ref( context * ctx) noexcept {
 369         BOOST_ASSERT( nullptr != ctx);
 370         ctx->use_count_.fetch_add( 1, std::memory_order_relaxed);
 371     }
 372
 373     friend void intrusive_ptr_release( context * ctx) noexcept {
 374         BOOST_ASSERT( nullptr != ctx);
 375         if ( 1 == ctx->use_count_.fetch_sub( 1, std::memory_order_release) ) {
 376             std::atomic_thread_fence( std::memory_order_acquire);
 377             boost::context::fiber c = std::move( ctx->c_);
 378             // destruct context
 379             ctx->~context();
 380             // deallocated stack
 381             std::move( c).resume();
 382         }
 383     }
 384
 385     #else
 386
 387     friend void intrusive_ptr_add_ref( context * ctx) noexcept;
 388     friend void intrusive_ptr_release( context * ctx) noexcept;
 389
 390     #endif
 391
 392 };
 393
 394 #if defined(BOOST_EMBTC)
 395
 396     inline bool
 397     operator==( context const& lhs, context const& rhs) noexcept {
 398         return & lhs == & rhs;
 399     }
 400
 401     inline void intrusive_ptr_add_ref( context * ctx) noexcept {
 402         BOOST_ASSERT( nullptr != ctx);
 403         ctx->use_count_.fetch_add( 1, std::memory_order_relaxed);
 404     }
 405
 406     inline void intrusive_ptr_release( context * ctx) noexcept {
 407         BOOST_ASSERT( nullptr != ctx);
 408         if ( 1 == ctx->use_count_.fetch_sub( 1, std::memory_order_release) ) {
 409             std::atomic_thread_fence( std::memory_order_acquire);
 410             boost::context::fiber c = std::move( ctx->c_);
 411             // destruct context
 412             ctx->~context();
 413             // deallocated stack
 414             std::move( c).resume();
 415         }
 416     }
 417
 418 #endif
 419
 420 inline
 421 bool operator<( context const& l, context const& r) noexcept {
 422     return l.get_id() < r.get_id();
 423 }
 424
 425 template< typename Fn, typename ... Arg >
 426 class worker_context final : public context {
 427 private:
 428     typename std::decay< Fn >::type                     fn_;
 429     std::tuple< Arg ... >                               arg_;
 430
 431     boost::context::fiber
 432     run_( boost::context::fiber && c) {
 433         {
 434             // fn and tpl must be destroyed before calling terminate()
 435             auto fn = std::move( fn_);
 436             auto arg = std::move( arg_);
 437 #if (defined(BOOST_USE_UCONTEXT)||defined(BOOST_USE_WINFIB))
 438             std::move( c).resume();
 439 #else
 440             boost::ignore_unused(c);
 441 #endif
 442 #if defined(BOOST_NO_CXX17_STD_APPLY)
 443            boost::context::detail::apply( std::move( fn), std::move( arg) );
 444 #else
 445            std::apply( std::move( fn), std::move( arg) );
 446 #endif
 447         }
 448         // terminate context
 449         return terminate();
 450     }
 451
 452 public:
 453     template< typename StackAlloc >
 454     worker_context( launch policy,
 455                     boost::context::preallocated const& palloc, StackAlloc && salloc,
 456                     Fn && fn, Arg ... arg) :
 457             context{ 1, type::worker_context, policy },
 458             fn_( std::forward< Fn >( fn) ),
 459             arg_( std::forward< Arg >( arg) ... ) {
 460         c_ = boost::context::fiber{ std::allocator_arg, palloc, std::forward< StackAlloc >( salloc),
 461                                     std::bind( & worker_context::run_, this, std::placeholders::_1) };
 462 #if (defined(BOOST_USE_UCONTEXT)||defined(BOOST_USE_WINFIB))
 463         c_ = std::move( c_).resume();
 464 #endif
 465     }
 466 };
 467
 468
 469 template< typename StackAlloc, typename Fn, typename ... Arg >
 470 static intrusive_ptr< context > make_worker_context( launch policy,
 471                                                      StackAlloc && salloc,
 472                                                      Fn && fn, Arg ... arg) {
 473     typedef worker_context< Fn, Arg ... >   context_t;
 474
 475     auto sctx = salloc.allocate();
 476     // reserve space for control structure
 477     void * storage = reinterpret_cast< void * >(
 478             ( reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sizeof( context_t) ) )
 479             & ~ static_cast< uintptr_t >( 0xff) );
 480     void * stack_bottom = reinterpret_cast< void * >(
 481             reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sctx.size) );
 482     const std::size_t size = reinterpret_cast< uintptr_t >( storage) - reinterpret_cast< uintptr_t >( stack_bottom);
 483     // placement new of context on top of fiber's stack
 484     return intrusive_ptr< context >{
 485             new ( storage) context_t{
 486                 policy,
 487                 boost::context::preallocated{ storage, size, sctx },
 488                 std::forward< StackAlloc >( salloc),
 489                 std::forward< Fn >( fn),
 490                 std::forward< Arg >( arg) ... } };
 491 }
 492
 493 }}
 494
 495 #ifdef _MSC_VER
 496 # pragma warning(pop)
 497 #endif
 498
 499 #ifdef BOOST_HAS_ABI_HEADERS
 500 #  include BOOST_ABI_SUFFIX
 501 #endif
 502
 503 #endif // BOOST_FIBERS_CONTEXT_H