ceph/src/boost/libs/fiber/include/boost/fiber/context.hpp

   1
   2 //          Copyright Oliver Kowalke 2013.
   3 // Distributed under the Boost Software License, Version 1.0.
   4 //    (See accompanying file LICENSE_1_0.txt or copy at
   5 //          http://www.boost.org/LICENSE_1_0.txt)
   6
   7 #ifndef BOOST_FIBERS_CONTEXT_H
   8 #define BOOST_FIBERS_CONTEXT_H
   9
  10 #include <atomic>
  11 #include <chrono>
  12 #include <exception>
  13 #include <functional>
  14 #include <map>
  15 #include <memory>
  16 #include <type_traits>
  17
  18 #include <boost/assert.hpp>
  19 #include <boost/config.hpp>
  20 #include <boost/context/detail/apply.hpp>
  21 #include <boost/context/execution_context.hpp>
  22 #include <boost/context/stack_context.hpp>
  23 #include <boost/intrusive/list.hpp>
  24 #include <boost/intrusive/parent_from_member.hpp>
  25 #include <boost/intrusive_ptr.hpp>
  26 #include <boost/intrusive/set.hpp>
  27
  28 #include <boost/fiber/detail/config.hpp>
  29 #include <boost/fiber/detail/data.hpp>
  30 #include <boost/fiber/detail/decay_copy.hpp>
  31 #include <boost/fiber/detail/fss.hpp>
  32 #include <boost/fiber/detail/spinlock.hpp>
  33 #include <boost/fiber/detail/wrap.hpp>
  34 #include <boost/fiber/exceptions.hpp>
  35 #include <boost/fiber/fixedsize_stack.hpp>
  36 #include <boost/fiber/policy.hpp>
  37 #include <boost/fiber/properties.hpp>
  38 #include <boost/fiber/segmented_stack.hpp>
  39 #include <boost/fiber/type.hpp>
  40
  41 #ifdef BOOST_HAS_ABI_HEADERS
  42 #  include BOOST_ABI_PREFIX
  43 #endif
  44
  45 #ifdef _MSC_VER
  46 # pragma warning(push)
  47 # pragma warning(disable:4251)
  48 #endif
  49
  50 namespace boost {
  51 namespace fibers {
  52
  53 class context;
  54 class fiber;
  55 class scheduler;
  56
  57 namespace detail {
  58
  59 struct wait_tag;
  60 typedef intrusive::list_member_hook<
  61     intrusive::tag< wait_tag >,
  62     intrusive::link_mode<
  63         intrusive::auto_unlink
  64     >
  65 >                                 wait_hook;
  66 // declaration of the functor that converts between
  67 // the context class and the wait-hook
  68 struct wait_functor {
  69     // required types
  70     typedef wait_hook               hook_type;
  71     typedef hook_type           *   hook_ptr;
  72     typedef const hook_type     *   const_hook_ptr;
  73     typedef context                 value_type;
  74     typedef value_type          *   pointer;
  75     typedef const value_type    *   const_pointer;
  76
  77     // required static functions
  78     static hook_ptr to_hook_ptr( value_type &value);
  79     static const_hook_ptr to_hook_ptr( value_type const& value);
  80     static pointer to_value_ptr( hook_ptr n);
  81     static const_pointer to_value_ptr( const_hook_ptr n);
  82 };
  83
  84 struct ready_tag;
  85 typedef intrusive::list_member_hook<
  86     intrusive::tag< ready_tag >,
  87     intrusive::link_mode<
  88         intrusive::auto_unlink
  89     >
  90 >                                       ready_hook;
  91
  92 struct sleep_tag;
  93 typedef intrusive::set_member_hook<
  94     intrusive::tag< sleep_tag >,
  95     intrusive::link_mode<
  96         intrusive::auto_unlink
  97     >
  98 >                                       sleep_hook;
  99
 100 struct terminated_tag;
 101 typedef intrusive::list_member_hook<
 102     intrusive::tag< terminated_tag >,
 103     intrusive::link_mode<
 104         intrusive::auto_unlink
 105     >
 106 >                                       terminated_hook;
 107
 108 struct worker_tag;
 109 typedef intrusive::list_member_hook<
 110     intrusive::tag< worker_tag >,
 111     intrusive::link_mode<
 112         intrusive::auto_unlink
 113     >
 114 >                                       worker_hook;
 115
 116 }
 117
 118 struct main_context_t {};
 119 const main_context_t main_context{};
 120
 121 struct dispatcher_context_t {};
 122 const dispatcher_context_t dispatcher_context{};
 123
 124 struct worker_context_t {};
 125 const worker_context_t worker_context{};
 126
 127 class BOOST_FIBERS_DECL context {
 128 private:
 129     friend class scheduler;
 130
 131     enum flag_t {
 132         flag_terminated = 1 << 1
 133     };
 134
 135     struct fss_data {
 136         void                                *   vp{ nullptr };
 137         detail::fss_cleanup_function::ptr_t     cleanup_function{};
 138
 139         fss_data() noexcept {
 140         }
 141
 142         fss_data( void * vp_,
 143                   detail::fss_cleanup_function::ptr_t const& fn) noexcept :
 144             vp( vp_),
 145             cleanup_function( fn) {
 146             BOOST_ASSERT( cleanup_function);
 147         }
 148
 149         void do_cleanup() {
 150             ( * cleanup_function)( vp);
 151         }
 152     };
 153
 154     typedef std::map< uintptr_t, fss_data >     fss_data_t;
 155
 156 #if ! defined(BOOST_FIBERS_NO_ATOMICS)
 157     std::atomic< std::size_t >                      use_count_{ 0 };
 158     std::atomic< unsigned int >                     flags_;
 159     std::atomic< type >                             type_;
 160 #else
 161     std::size_t                                     use_count_{ 0 };
 162     unsigned int                                    flags_;
 163     type                                            type_;
 164 #endif
 165     launch                                          policy_{ launch::post };
 166     scheduler                                   *   scheduler_{ nullptr };
 167 #if (BOOST_EXECUTION_CONTEXT==1)
 168     boost::context::execution_context               ctx_;
 169 #else
 170     boost::context::execution_context< detail::data_t * >   ctx_;
 171 #endif
 172
 173     void resume_( detail::data_t &) noexcept;
 174     void set_ready_( context *) noexcept;
 175
 176 #if (BOOST_EXECUTION_CONTEXT==1)
 177     template< typename Fn, typename Tpl >
 178     void run_( Fn && fn_, Tpl && tpl_, detail::data_t * dp) noexcept {
 179         {
 180             // fn and tpl must be destroyed before calling set_terminated()
 181             typename std::decay< Fn >::type fn = std::forward< Fn >( fn_);
 182             typename std::decay< Tpl >::type tpl = std::forward< Tpl >( tpl_);
 183             if ( nullptr != dp->lk) {
 184                 dp->lk->unlock();
 185             } else if ( nullptr != dp->ctx) {
 186                 active()->set_ready_( dp->ctx);
 187             }
 188             boost::context::detail::apply( std::move( fn), std::move( tpl) );
 189         }
 190         // terminate context
 191         set_terminated();
 192         BOOST_ASSERT_MSG( false, "fiber already terminated");
 193     }
 194 #else
 195     template< typename Fn, typename Tpl >
 196     boost::context::execution_context< detail::data_t * >
 197     run_( boost::context::execution_context< detail::data_t * > ctx, Fn && fn_, Tpl && tpl_, detail::data_t * dp) noexcept {
 198         {
 199             // fn and tpl must be destroyed before calling set_terminated()
 200             typename std::decay< Fn >::type fn = std::forward< Fn >( fn_);
 201             typename std::decay< Tpl >::type tpl = std::forward< Tpl >( tpl_);
 202             // update execution_context of calling fiber
 203             dp->from->ctx_ = std::move( ctx);
 204             if ( nullptr != dp->lk) {
 205                 dp->lk->unlock();
 206             } else if ( nullptr != dp->ctx) {
 207                 active()->set_ready_( dp->ctx);
 208             }
 209             boost::context::detail::apply( std::move( fn), std::move( tpl) );
 210         }
 211         // terminate context
 212         return set_terminated();
 213     }
 214 #endif
 215
 216 public:
 217     detail::ready_hook                      ready_hook_{};
 218     detail::sleep_hook                      sleep_hook_{};
 219     detail::terminated_hook                 terminated_hook_{};
 220     detail::wait_hook                       wait_hook_{};
 221     detail::worker_hook                     worker_hook_{};
 222     std::chrono::steady_clock::time_point   tp_{ (std::chrono::steady_clock::time_point::max)() };
 223
 224     typedef intrusive::list<
 225         context,
 226         intrusive::function_hook< detail::wait_functor >,
 227         intrusive::constant_time_size< false > >   wait_queue_t;
 228
 229 private:
 230     fss_data_t                              fss_data_{};
 231     wait_queue_t                            wait_queue_{};
 232     detail::spinlock                        splk_{};
 233     fiber_properties                    *   properties_{ nullptr };
 234
 235 public:
 236     class id {
 237     private:
 238         context  *   impl_{ nullptr };
 239
 240     public:
 241         id() noexcept {
 242         }
 243
 244         explicit id( context * impl) noexcept :
 245             impl_( impl) {
 246         }
 247
 248         bool operator==( id const& other) const noexcept {
 249             return impl_ == other.impl_;
 250         }
 251
 252         bool operator!=( id const& other) const noexcept {
 253             return impl_ != other.impl_;
 254         }
 255
 256         bool operator<( id const& other) const noexcept {
 257             return impl_ < other.impl_;
 258         }
 259
 260         bool operator>( id const& other) const noexcept {
 261             return other.impl_ < impl_;
 262         }
 263
 264         bool operator<=( id const& other) const noexcept {
 265             return ! ( * this > other);
 266         }
 267
 268         bool operator>=( id const& other) const noexcept {
 269             return ! ( * this < other);
 270         }
 271
 272         template< typename charT, class traitsT >
 273         friend std::basic_ostream< charT, traitsT > &
 274         operator<<( std::basic_ostream< charT, traitsT > & os, id const& other) {
 275             if ( nullptr != other.impl_) {
 276                 return os << other.impl_;
 277             } else {
 278                 return os << "{not-valid}";
 279             }
 280         }
 281
 282         explicit operator bool() const noexcept {
 283             return nullptr != impl_;
 284         }
 285
 286         bool operator!() const noexcept {
 287             return nullptr == impl_;
 288         }
 289     };
 290
 291     static context * active() noexcept;
 292
 293     static void reset_active() noexcept;
 294
 295     // main fiber context
 296     explicit context( main_context_t) noexcept;
 297
 298     // dispatcher fiber context
 299     context( dispatcher_context_t, boost::context::preallocated const&,
 300              default_stack const&, scheduler *);
 301
 302     // worker fiber context
 303     template< typename StackAlloc,
 304               typename Fn,
 305               typename Tpl
 306     >
 307     context( worker_context_t,
 308              launch policy,
 309              boost::context::preallocated palloc, StackAlloc salloc,
 310              Fn && fn, Tpl && tpl) :
 311         use_count_{ 1 }, // fiber instance or scheduler owner
 312         flags_{ 0 },
 313         type_{ type::worker_context },
 314         policy_{ policy },
 315 #if (BOOST_EXECUTION_CONTEXT==1)
 316 # if defined(BOOST_NO_CXX14_GENERIC_LAMBDAS)
 317         ctx_{ std::allocator_arg, palloc, salloc,
 318               detail::wrap(
 319                   [this]( typename std::decay< Fn >::type & fn, typename std::decay< Tpl >::type & tpl,
 320                           boost::context::execution_context & ctx, void * vp) mutable noexcept {
 321                         run_( std::move( fn), std::move( tpl), static_cast< detail::data_t * >( vp) );
 322                   },
 323                   std::forward< Fn >( fn),
 324                   std::forward< Tpl >( tpl),
 325                   boost::context::execution_context::current() )
 326               }
 327 # else
 328         ctx_{ std::allocator_arg, palloc, salloc,
 329               [this,fn=detail::decay_copy( std::forward< Fn >( fn) ),tpl=std::forward< Tpl >( tpl),
 330                ctx=boost::context::execution_context::current()] (void * vp) mutable noexcept {
 331                     run_( std::move( fn), std::move( tpl), static_cast< detail::data_t * >( vp) );
 332               }}
 333 # endif
 334 #else
 335 # if defined(BOOST_NO_CXX14_GENERIC_LAMBDAS)
 336         ctx_{ std::allocator_arg, palloc, salloc,
 337               detail::wrap(
 338                   [this]( typename std::decay< Fn >::type & fn, typename std::decay< Tpl >::type & tpl,
 339                           boost::context::execution_context< detail::data_t * > ctx, detail::data_t * dp) mutable noexcept {
 340                         return run_( std::move( ctx), std::move( fn), std::move( tpl), dp);
 341                   },
 342                   std::forward< Fn >( fn),
 343                   std::forward< Tpl >( tpl) )}
 344
 345 # else
 346         ctx_{ std::allocator_arg, palloc, salloc,
 347               [this,fn=detail::decay_copy( std::forward< Fn >( fn) ),tpl=std::forward< Tpl >( tpl)]
 348                (boost::context::execution_context< detail::data_t * > ctx, detail::data_t * dp) mutable noexcept {
 349                     return run_( std::move( ctx), std::move( fn), std::move( tpl), dp);
 350               }}
 351 # endif
 352 #endif
 353     {}
 354
 355     context( context const&) = delete;
 356     context & operator=( context const&) = delete;
 357
 358     virtual ~context();
 359
 360     scheduler * get_scheduler() const noexcept;
 361
 362     id get_id() const noexcept;
 363
 364     void resume() noexcept;
 365     void resume( detail::spinlock_lock &) noexcept;
 366     void resume( context *) noexcept;
 367
 368     void suspend() noexcept;
 369     void suspend( detail::spinlock_lock &) noexcept;
 370
 371 #if (BOOST_EXECUTION_CONTEXT==1)
 372     void set_terminated() noexcept;
 373 #else
 374     boost::context::execution_context< detail::data_t * > suspend_with_cc() noexcept;
 375     boost::context::execution_context< detail::data_t * > set_terminated() noexcept;
 376 #endif
 377     void join();
 378
 379     void yield() noexcept;
 380
 381     bool wait_until( std::chrono::steady_clock::time_point const&) noexcept;
 382     bool wait_until( std::chrono::steady_clock::time_point const&,
 383                      detail::spinlock_lock &) noexcept;
 384
 385     void set_ready( context *) noexcept;
 386
 387     bool is_context( type t) const noexcept {
 388         return type::none != ( type_ & t);
 389     }
 390
 391     bool is_terminated() const noexcept {
 392         return 0 != ( flags_ & flag_terminated);
 393     }
 394
 395     void * get_fss_data( void const * vp) const;
 396
 397     void set_fss_data(
 398         void const * vp,
 399         detail::fss_cleanup_function::ptr_t const& cleanup_fn,
 400         void * data,
 401         bool cleanup_existing);
 402
 403     void set_properties( fiber_properties * props) noexcept;
 404
 405     fiber_properties * get_properties() const noexcept {
 406         return properties_;
 407     }
 408
 409     launch get_policy() const noexcept {
 410         return policy_;
 411     }
 412
 413     bool ready_is_linked() const noexcept;
 414
 415     bool sleep_is_linked() const noexcept;
 416
 417     bool terminated_is_linked() const noexcept;
 418
 419     bool wait_is_linked() const noexcept;
 420
 421     bool worker_is_linked() const noexcept;
 422
 423     template< typename List >
 424     void ready_link( List & lst) noexcept {
 425         static_assert( std::is_same< typename List::value_traits::hook_type, detail::ready_hook >::value, "not a ready-queue");
 426         lst.push_back( * this);
 427     }
 428
 429     template< typename Set >
 430     void sleep_link( Set & set) noexcept {
 431         static_assert( std::is_same< typename Set::value_traits::hook_type,detail::sleep_hook >::value, "not a sleep-queue");
 432         set.insert( * this);
 433     }
 434
 435     template< typename List >
 436     void terminated_link( List & lst) noexcept {
 437         static_assert( std::is_same< typename List::value_traits::hook_type, detail::terminated_hook >::value, "not a terminated-queue");
 438         lst.push_back( * this);
 439     }
 440
 441     template< typename List >
 442     void wait_link( List & lst) noexcept {
 443         static_assert( std::is_same< typename List::value_traits::hook_type, detail::wait_hook >::value, "not a wait-queue");
 444         lst.push_back( * this);
 445     }
 446
 447     template< typename List >
 448     void worker_link( List & lst) noexcept {
 449         static_assert( std::is_same< typename List::value_traits::hook_type, detail::worker_hook >::value, "not a worker-queue");
 450         lst.push_back( * this);
 451     }
 452
 453     void ready_unlink() noexcept;
 454
 455     void sleep_unlink() noexcept;
 456
 457     void wait_unlink() noexcept;
 458
 459     void worker_unlink() noexcept;
 460
 461     void detach() noexcept;
 462
 463     void attach( context *) noexcept;
 464
 465     friend void intrusive_ptr_add_ref( context * ctx) noexcept {
 466         BOOST_ASSERT( nullptr != ctx);
 467         ++ctx->use_count_;
 468     }
 469
 470     friend void intrusive_ptr_release( context * ctx) noexcept {
 471         BOOST_ASSERT( nullptr != ctx);
 472         if ( 0 == --ctx->use_count_) {
 473 #if (BOOST_EXECUTION_CONTEXT==1)
 474             boost::context::execution_context ec( ctx->ctx_);
 475             // destruct context
 476             // deallocates stack (execution_context is ref counted)
 477             ctx->~context();
 478 #else
 479             boost::context::execution_context< detail::data_t * > cc( std::move( ctx->ctx_) );
 480             // destruct context
 481             ctx->~context();
 482             // deallocated stack
 483             cc( nullptr);
 484 #endif
 485         }
 486     }
 487 };
 488
 489 inline
 490 bool operator<( context const& l, context const& r) noexcept {
 491     return l.get_id() < r.get_id();
 492 }
 493
 494 template< typename StackAlloc, typename Fn, typename ... Args >
 495 static intrusive_ptr< context > make_worker_context( launch policy,
 496                                                      StackAlloc salloc,
 497                                                      Fn && fn, Args && ... args) {
 498     boost::context::stack_context sctx = salloc.allocate();
 499 #if defined(BOOST_NO_CXX14_CONSTEXPR) || defined(BOOST_NO_CXX11_STD_ALIGN)
 500     // reserve space for control structure
 501     const std::size_t size = sctx.size - sizeof( context);
 502     void * sp = static_cast< char * >( sctx.sp) - sizeof( context);
 503 #else
 504     constexpr std::size_t func_alignment = 64; // alignof( context);
 505     constexpr std::size_t func_size = sizeof( context);
 506     // reserve space on stack
 507     void * sp = static_cast< char * >( sctx.sp) - func_size - func_alignment;
 508     // align sp pointer
 509     std::size_t space = func_size + func_alignment;
 510     sp = std::align( func_alignment, func_size, sp, space);
 511     BOOST_ASSERT( nullptr != sp);
 512     // calculate remaining size
 513     const std::size_t size = sctx.size - ( static_cast< char * >( sctx.sp) - static_cast< char * >( sp) );
 514 #endif
 515     // placement new of context on top of fiber's stack
 516     return intrusive_ptr< context >(
 517             ::new ( sp) context(
 518                 worker_context,
 519                 policy,
 520                 boost::context::preallocated( sp, size, sctx),
 521                 salloc,
 522                 std::forward< Fn >( fn),
 523                 std::make_tuple( std::forward< Args >( args) ... ) ) );
 524 }
 525
 526 namespace detail {
 527
 528 inline
 529 wait_functor::hook_ptr wait_functor::to_hook_ptr( wait_functor::value_type & value) {
 530     return & value.wait_hook_;
 531 }
 532
 533 inline
 534 wait_functor::const_hook_ptr wait_functor::to_hook_ptr( wait_functor::value_type const& value) {
 535     return & value.wait_hook_;
 536 }
 537
 538 inline
 539 wait_functor::pointer wait_functor::to_value_ptr( wait_functor::hook_ptr n) {
 540     return intrusive::get_parent_from_member< context >( n, & context::wait_hook_);
 541 }
 542
 543 inline
 544 wait_functor::const_pointer wait_functor::to_value_ptr( wait_functor::const_hook_ptr n) {
 545     return intrusive::get_parent_from_member< context >( n, & context::wait_hook_);
 546 }
 547
 548 }}}
 549
 550 #ifdef _MSC_VER
 551 # pragma warning(pop)
 552 #endif
 553
 554 #ifdef BOOST_HAS_ABI_HEADERS
 555 #  include BOOST_ABI_SUFFIX
 556 #endif
 557
 558 #endif // BOOST_FIBERS_CONTEXT_H