ceph/src/boost/boost/fiber/context.hpp

   1
   2 //          Copyright Oliver Kowalke 2013.
   3 // Distributed under the Boost Software License, Version 1.0.
   4 //    (See accompanying file LICENSE_1_0.txt or copy at
   5 //          http://www.boost.org/LICENSE_1_0.txt)
   6
   7 #ifndef BOOST_FIBERS_CONTEXT_H
   8 #define BOOST_FIBERS_CONTEXT_H
   9
  10 #include <atomic>
  11 #include <chrono>
  12 #include <cstdint>
  13 #include <exception>
  14 #include <functional>
  15 #include <iostream>
  16 #include <map>
  17 #include <memory>
  18 #include <tuple>
  19 #include <type_traits>
  20
  21 #include <boost/assert.hpp>
  22 #include <boost/config.hpp>
  23 #if defined(BOOST_NO_CXX17_STD_APPLY)
  24 #include <boost/context/detail/apply.hpp>
  25 #endif
  26 #include <boost/context/continuation.hpp>
  27 #include <boost/context/stack_context.hpp>
  28 #include <boost/intrusive/list.hpp>
  29 #include <boost/intrusive/parent_from_member.hpp>
  30 #include <boost/intrusive_ptr.hpp>
  31 #include <boost/intrusive/set.hpp>
  32 #include <boost/intrusive/slist.hpp>
  33
  34 #include <boost/fiber/detail/config.hpp>
  35 #include <boost/fiber/detail/data.hpp>
  36 #include <boost/fiber/detail/decay_copy.hpp>
  37 #include <boost/fiber/detail/fss.hpp>
  38 #include <boost/fiber/detail/spinlock.hpp>
  39 #include <boost/fiber/exceptions.hpp>
  40 #include <boost/fiber/fixedsize_stack.hpp>
  41 #include <boost/fiber/policy.hpp>
  42 #include <boost/fiber/properties.hpp>
  43 #include <boost/fiber/segmented_stack.hpp>
  44 #include <boost/fiber/type.hpp>
  45
  46 #ifdef BOOST_HAS_ABI_HEADERS
  47 #  include BOOST_ABI_PREFIX
  48 #endif
  49
  50 #ifdef _MSC_VER
  51 # pragma warning(push)
  52 # pragma warning(disable:4251)
  53 #endif
  54
  55 namespace boost {
  56 namespace fibers {
  57
  58 class context;
  59 class fiber;
  60 class scheduler;
  61
  62 namespace detail {
  63
  64 struct wait_tag;
  65 typedef intrusive::list_member_hook<
  66     intrusive::tag< wait_tag >,
  67     intrusive::link_mode<
  68         intrusive::auto_unlink
  69     >
  70 >                                 wait_hook;
  71 // declaration of the functor that converts between
  72 // the context class and the wait-hook
  73 struct wait_functor {
  74     // required types
  75     typedef wait_hook               hook_type;
  76     typedef hook_type           *   hook_ptr;
  77     typedef const hook_type     *   const_hook_ptr;
  78     typedef context                 value_type;
  79     typedef value_type          *   pointer;
  80     typedef const value_type    *   const_pointer;
  81
  82     // required static functions
  83     static hook_ptr to_hook_ptr( value_type &value);
  84     static const_hook_ptr to_hook_ptr( value_type const& value);
  85     static pointer to_value_ptr( hook_ptr n);
  86     static const_pointer to_value_ptr( const_hook_ptr n);
  87 };
  88
  89 struct ready_tag;
  90 typedef intrusive::list_member_hook<
  91     intrusive::tag< ready_tag >,
  92     intrusive::link_mode<
  93         intrusive::auto_unlink
  94     >
  95 >                                       ready_hook;
  96
  97 struct sleep_tag;
  98 typedef intrusive::set_member_hook<
  99     intrusive::tag< sleep_tag >,
 100     intrusive::link_mode<
 101         intrusive::auto_unlink
 102     >
 103 >                                       sleep_hook;
 104
 105 struct worker_tag;
 106 typedef intrusive::list_member_hook<
 107     intrusive::tag< worker_tag >,
 108     intrusive::link_mode<
 109         intrusive::auto_unlink
 110     >
 111 >                                       worker_hook;
 112
 113 struct terminated_tag;
 114 typedef intrusive::slist_member_hook<
 115     intrusive::tag< terminated_tag >,
 116     intrusive::link_mode<
 117         intrusive::safe_link
 118     >
 119 >                                       terminated_hook;
 120
 121 struct remote_ready_tag;
 122 typedef intrusive::slist_member_hook<
 123     intrusive::tag< remote_ready_tag >,
 124     intrusive::link_mode<
 125         intrusive::safe_link
 126     >
 127 >                                       remote_ready_hook;
 128
 129 }
 130
 131 class BOOST_FIBERS_DECL context {
 132 public:
 133     typedef intrusive::list<
 134                 context,
 135                 intrusive::function_hook< detail::wait_functor >,
 136                 intrusive::constant_time_size< false >
 137             >   wait_queue_t;
 138
 139 private:
 140     friend class dispatcher_context;
 141     friend class main_context;
 142     template< typename Fn, typename ... Arg > friend class worker_context;
 143     friend class scheduler;
 144
 145     struct fss_data {
 146         void                                *   vp{ nullptr };
 147         detail::fss_cleanup_function::ptr_t     cleanup_function{};
 148
 149         fss_data() noexcept {
 150         }
 151
 152         fss_data( void * vp_,
 153                   detail::fss_cleanup_function::ptr_t const& fn) noexcept :
 154             vp( vp_),
 155             cleanup_function( fn) {
 156             BOOST_ASSERT( cleanup_function);
 157         }
 158
 159         void do_cleanup() {
 160             ( * cleanup_function)( vp);
 161         }
 162     };
 163
 164     typedef std::map< uintptr_t, fss_data >             fss_data_t;
 165
 166 #if ! defined(BOOST_FIBERS_NO_ATOMICS)
 167     std::atomic< std::size_t >                          use_count_;
 168 #else
 169     std::size_t                                         use_count_;
 170 #endif
 171 #if ! defined(BOOST_FIBERS_NO_ATOMICS)
 172     detail::remote_ready_hook                           remote_ready_hook_{};
 173 #endif
 174     detail::spinlock                                    splk_{};
 175     bool                                                terminated_{ false };
 176     wait_queue_t                                        wait_queue_{};
 177 public:
 178     detail::wait_hook                                   wait_hook_{};
 179 #if ! defined(BOOST_FIBERS_NO_ATOMICS)
 180     std::atomic< std::intptr_t >                        twstatus{ 0 };
 181 #endif
 182 private:
 183     scheduler                                       *   scheduler_{ nullptr };
 184     fss_data_t                                          fss_data_{};
 185     detail::sleep_hook                                  sleep_hook_{};
 186     detail::ready_hook                                  ready_hook_{};
 187     detail::terminated_hook                             terminated_hook_{};
 188     detail::worker_hook                                 worker_hook_{};
 189     fiber_properties                                *   properties_{ nullptr };
 190     boost::context::continuation                        c_{};
 191     std::chrono::steady_clock::time_point               tp_;
 192     type                                                type_;
 193     launch                                              policy_;
 194
 195     context( std::size_t initial_count, type t, launch policy) noexcept :
 196         use_count_{ initial_count },
 197         tp_{ (std::chrono::steady_clock::time_point::max)() },
 198         type_{ t },
 199         policy_{ policy } {
 200     }
 201
 202 public:
 203     class id {
 204     private:
 205         context  *   impl_{ nullptr };
 206
 207     public:
 208         id() = default;
 209
 210         explicit id( context * impl) noexcept :
 211             impl_{ impl } {
 212         }
 213
 214         bool operator==( id const& other) const noexcept {
 215             return impl_ == other.impl_;
 216         }
 217
 218         bool operator!=( id const& other) const noexcept {
 219             return impl_ != other.impl_;
 220         }
 221
 222         bool operator<( id const& other) const noexcept {
 223             return impl_ < other.impl_;
 224         }
 225
 226         bool operator>( id const& other) const noexcept {
 227             return other.impl_ < impl_;
 228         }
 229
 230         bool operator<=( id const& other) const noexcept {
 231             return ! ( * this > other);
 232         }
 233
 234         bool operator>=( id const& other) const noexcept {
 235             return ! ( * this < other);
 236         }
 237
 238         template< typename charT, class traitsT >
 239         friend std::basic_ostream< charT, traitsT > &
 240         operator<<( std::basic_ostream< charT, traitsT > & os, id const& other) {
 241             if ( nullptr != other.impl_) {
 242                 return os << other.impl_;
 243             } else {
 244                 return os << "{not-valid}";
 245             }
 246         }
 247
 248         explicit operator bool() const noexcept {
 249             return nullptr != impl_;
 250         }
 251
 252         bool operator!() const noexcept {
 253             return nullptr == impl_;
 254         }
 255     };
 256
 257     static context * active() noexcept;
 258
 259     static void reset_active() noexcept;
 260
 261     context( context const&) = delete;
 262     context & operator=( context const&) = delete;
 263
 264     friend bool
 265     operator==( context const& lhs, context const& rhs) noexcept {
 266         return & lhs == & rhs;
 267     }
 268
 269     virtual ~context();
 270
 271     scheduler * get_scheduler() const noexcept {
 272         return scheduler_;
 273     }
 274
 275     id get_id() const noexcept;
 276
 277     bool is_resumable() const noexcept {
 278         if ( c_) return true;
 279         else return false;
 280     }
 281
 282     void resume() noexcept;
 283     void resume( detail::spinlock_lock &) noexcept;
 284     void resume( context *) noexcept;
 285
 286     void suspend() noexcept;
 287     void suspend( detail::spinlock_lock &) noexcept;
 288
 289     boost::context::continuation suspend_with_cc() noexcept;
 290     boost::context::continuation terminate() noexcept;
 291
 292     void join();
 293
 294     void yield() noexcept;
 295
 296     bool wait_until( std::chrono::steady_clock::time_point const&) noexcept;
 297     bool wait_until( std::chrono::steady_clock::time_point const&,
 298                      detail::spinlock_lock &) noexcept;
 299
 300     void schedule( context *) noexcept;
 301
 302     bool is_context( type t) const noexcept {
 303         return type::none != ( type_ & t);
 304     }
 305
 306     void * get_fss_data( void const * vp) const;
 307
 308     void set_fss_data(
 309         void const * vp,
 310         detail::fss_cleanup_function::ptr_t const& cleanup_fn,
 311         void * data,
 312         bool cleanup_existing);
 313
 314     void set_properties( fiber_properties * props) noexcept;
 315
 316     fiber_properties * get_properties() const noexcept {
 317         return properties_;
 318     }
 319
 320     launch get_policy() const noexcept {
 321         return policy_;
 322     }
 323
 324     bool worker_is_linked() const noexcept;
 325
 326     bool ready_is_linked() const noexcept;
 327
 328     bool remote_ready_is_linked() const noexcept;
 329
 330     bool sleep_is_linked() const noexcept;
 331
 332     bool terminated_is_linked() const noexcept;
 333
 334     bool wait_is_linked() const noexcept;
 335
 336     template< typename List >
 337     void worker_link( List & lst) noexcept {
 338         static_assert( std::is_same< typename List::value_traits::hook_type, detail::worker_hook >::value, "not a worker-queue");
 339         BOOST_ASSERT( ! worker_is_linked() );
 340         lst.push_back( * this);
 341     }
 342
 343     template< typename List >
 344     void ready_link( List & lst) noexcept {
 345         static_assert( std::is_same< typename List::value_traits::hook_type, detail::ready_hook >::value, "not a ready-queue");
 346         BOOST_ASSERT( ! ready_is_linked() );
 347         lst.push_back( * this);
 348     }
 349
 350     template< typename List >
 351     void remote_ready_link( List & lst) noexcept {
 352         static_assert( std::is_same< typename List::value_traits::hook_type, detail::remote_ready_hook >::value, "not a remote-ready-queue");
 353         BOOST_ASSERT( ! remote_ready_is_linked() );
 354         lst.push_back( * this);
 355     }
 356
 357     template< typename Set >
 358     void sleep_link( Set & set) noexcept {
 359         static_assert( std::is_same< typename Set::value_traits::hook_type,detail::sleep_hook >::value, "not a sleep-queue");
 360         BOOST_ASSERT( ! sleep_is_linked() );
 361         set.insert( * this);
 362     }
 363
 364     template< typename List >
 365     void terminated_link( List & lst) noexcept {
 366         static_assert( std::is_same< typename List::value_traits::hook_type, detail::terminated_hook >::value, "not a terminated-queue");
 367         BOOST_ASSERT( ! terminated_is_linked() );
 368         lst.push_back( * this);
 369     }
 370
 371     template< typename List >
 372     void wait_link( List & lst) noexcept {
 373         static_assert( std::is_same< typename List::value_traits::hook_type, detail::wait_hook >::value, "not a wait-queue");
 374         BOOST_ASSERT( ! wait_is_linked() );
 375         lst.push_back( * this);
 376     }
 377
 378     void worker_unlink() noexcept;
 379
 380     void ready_unlink() noexcept;
 381
 382     void sleep_unlink() noexcept;
 383
 384     void wait_unlink() noexcept;
 385
 386     void detach() noexcept;
 387
 388     void attach( context *) noexcept;
 389
 390     friend void intrusive_ptr_add_ref( context * ctx) noexcept {
 391         BOOST_ASSERT( nullptr != ctx);
 392         ctx->use_count_.fetch_add( 1, std::memory_order_relaxed);
 393     }
 394
 395     friend void intrusive_ptr_release( context * ctx) noexcept {
 396         BOOST_ASSERT( nullptr != ctx);
 397         if ( 1 == ctx->use_count_.fetch_sub( 1, std::memory_order_release) ) {
 398             std::atomic_thread_fence( std::memory_order_acquire);
 399             boost::context::continuation c = std::move( ctx->c_);
 400             // destruct context
 401             ctx->~context();
 402             // deallocated stack
 403             c.resume();
 404         }
 405     }
 406 };
 407
 408 inline
 409 bool operator<( context const& l, context const& r) noexcept {
 410     return l.get_id() < r.get_id();
 411 }
 412
 413 template< typename Fn, typename ... Arg >
 414 class worker_context final : public context {
 415 private:
 416     typename std::decay< Fn >::type                     fn_;
 417     std::tuple< Arg ... >                               arg_;
 418
 419     boost::context::continuation
 420     run_( boost::context::continuation && c) {
 421         {
 422             // fn and tpl must be destroyed before calling terminate()
 423             auto fn = std::move( fn_);
 424             auto arg = std::move( arg_);
 425             c.resume();
 426 #if defined(BOOST_NO_CXX17_STD_APPLY)
 427            boost::context::detail::apply( std::move( fn), std::move( arg) );
 428 #else
 429            std::apply( std::move( fn), std::move( arg) );
 430 #endif
 431         }
 432         // terminate context
 433         return terminate();
 434     }
 435
 436 public:
 437     template< typename StackAlloc >
 438     worker_context( launch policy,
 439                     boost::context::preallocated const& palloc, StackAlloc const& salloc,
 440                     Fn && fn, Arg ... arg) :
 441             context{ 1, type::worker_context, policy },
 442             fn_( std::forward< Fn >( fn) ),
 443             arg_( std::forward< Arg >( arg) ... ) {
 444         c_ = boost::context::callcc(
 445                 std::allocator_arg, palloc, salloc,
 446                 std::bind( & worker_context::run_, this, std::placeholders::_1) );
 447     }
 448 };
 449
 450
 451 template< typename StackAlloc, typename Fn, typename ... Arg >
 452 static intrusive_ptr< context > make_worker_context( launch policy,
 453                                                      StackAlloc salloc,
 454                                                      Fn && fn, Arg ... arg) {
 455     typedef worker_context< Fn, Arg ... >   context_t;
 456
 457     auto sctx = salloc.allocate();
 458     // reserve space for control structure
 459     void * storage = reinterpret_cast< void * >(
 460             ( reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sizeof( context_t) ) )
 461             & ~ static_cast< uintptr_t >( 0xff) );
 462     void * stack_bottom = reinterpret_cast< void * >(
 463             reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sctx.size) );
 464     const std::size_t size = reinterpret_cast< uintptr_t >( storage) - reinterpret_cast< uintptr_t >( stack_bottom);
 465     // placement new of context on top of fiber's stack
 466     return intrusive_ptr< context >{
 467             new ( storage) context_t{
 468                 policy,
 469                 boost::context::preallocated{ storage, size, sctx },
 470                 salloc,
 471                 std::forward< Fn >( fn),
 472                 std::forward< Arg >( arg) ... } };
 473 }
 474
 475 namespace detail {
 476
 477 inline
 478 wait_functor::hook_ptr wait_functor::to_hook_ptr( wait_functor::value_type & value) {
 479     return & value.wait_hook_;
 480 }
 481
 482 inline
 483 wait_functor::const_hook_ptr wait_functor::to_hook_ptr( wait_functor::value_type const& value) {
 484     return & value.wait_hook_;
 485 }
 486
 487 inline
 488 wait_functor::pointer wait_functor::to_value_ptr( wait_functor::hook_ptr n) {
 489     return intrusive::get_parent_from_member< context >( n, & context::wait_hook_);
 490 }
 491
 492 inline
 493 wait_functor::const_pointer wait_functor::to_value_ptr( wait_functor::const_hook_ptr n) {
 494     return intrusive::get_parent_from_member< context >( n, & context::wait_hook_);
 495 }
 496
 497 }}}
 498
 499 #ifdef _MSC_VER
 500 # pragma warning(pop)
 501 #endif
 502
 503 #ifdef BOOST_HAS_ABI_HEADERS
 504 #  include BOOST_ABI_SUFFIX
 505 #endif
 506
 507 #endif // BOOST_FIBERS_CONTEXT_H