ceph/src/boost/boost/context/fiber_ucontext.hpp

   1
   2 //          Copyright Oliver Kowalke 2017.
   3 // Distributed under the Boost Software License, Version 1.0.
   4 //    (See accompanying file LICENSE_1_0.txt or copy at
   5 //          http://www.boost.org/LICENSE_1_0.txt)
   6
   7 #ifndef BOOST_CONTEXT_FIBER_H
   8 #define BOOST_CONTEXT_FIBER_H
   9
  10 #include <boost/predef.h>
  11 #if BOOST_OS_MACOS
  12 #define _XOPEN_SOURCE 600
  13 #endif
  14
  15 extern "C" {
  16 #include <ucontext.h>
  17 }
  18
  19 #include <boost/context/detail/config.hpp>
  20
  21 #include <algorithm>
  22 #include <cstddef>
  23 #include <cstdint>
  24 #include <cstdlib>
  25 #include <cstring>
  26 #include <functional>
  27 #include <memory>
  28 #include <ostream>
  29 #include <system_error>
  30 #include <tuple>
  31 #include <utility>
  32
  33 #include <boost/assert.hpp>
  34 #include <boost/config.hpp>
  35 #include <boost/predef.h>
  36
  37 #include <boost/context/detail/disable_overload.hpp>
  38 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
  39 #include <boost/context/detail/exchange.hpp>
  40 #endif
  41 #include <boost/context/detail/externc.hpp>
  42 #if defined(BOOST_NO_CXX17_STD_INVOKE)
  43 #include <boost/context/detail/invoke.hpp>
  44 #endif
  45 #include <boost/context/fixedsize_stack.hpp>
  46 #include <boost/context/flags.hpp>
  47 #include <boost/context/preallocated.hpp>
  48 #if defined(BOOST_USE_SEGMENTED_STACKS)
  49 #include <boost/context/segmented_stack.hpp>
  50 #endif
  51 #include <boost/context/stack_context.hpp>
  52
  53 #ifdef BOOST_HAS_ABI_HEADERS
  54 # include BOOST_ABI_PREFIX
  55 #endif
  56
  57 #ifdef BOOST_USE_TSAN
  58 #include <sanitizer/tsan_interface.h>
  59 #endif
  60
  61 namespace boost {
  62 namespace context {
  63 namespace detail {
  64
  65 // tampoline function
  66 // entered if the execution context
  67 // is resumed for the first time
  68 template< typename Record >
  69 static void fiber_entry_func( void * data) noexcept {
  70     Record * record = static_cast< Record * >( data);
  71     BOOST_ASSERT( nullptr != record);
  72     // start execution of toplevel context-function
  73     record->run();
  74 }
  75
  76 struct BOOST_CONTEXT_DECL fiber_activation_record {
  77     ucontext_t                                                  uctx{};
  78     stack_context                                               sctx{};
  79     bool                                                        main_ctx{ true };
  80         fiber_activation_record                                       * from{ nullptr };
  81     std::function< fiber_activation_record*(fiber_activation_record*&) >    ontop{};
  82     bool                                                        terminated{ false };
  83     bool                                                        force_unwind{ false };
  84 #if defined(BOOST_USE_ASAN)
  85     void                                                    *   fake_stack{ nullptr };
  86     void                                                    *   stack_bottom{ nullptr };
  87     std::size_t                                                 stack_size{ 0 };
  88 #endif
  89
  90 #if defined(BOOST_USE_TSAN)
  91     void * tsan_fiber{ nullptr };
  92     bool destroy_tsan_fiber{ true };
  93 #endif
  94
  95     static fiber_activation_record *& current() noexcept;
  96
  97     // used for toplevel-context
  98     // (e.g. main context, thread-entry context)
  99     fiber_activation_record() {
 100         if ( BOOST_UNLIKELY( 0 != ::getcontext( & uctx) ) ) {
 101             throw std::system_error(
 102                     std::error_code( errno, std::system_category() ),
 103                     "getcontext() failed");
 104         }
 105
 106 #if defined(BOOST_USE_TSAN)
 107         tsan_fiber = __tsan_get_current_fiber();
 108         destroy_tsan_fiber = false;
 109 #endif
 110     }
 111
 112     fiber_activation_record( stack_context sctx_) noexcept :
 113         sctx( sctx_ ),
 114         main_ctx( false ) {
 115     }
 116
 117     virtual ~fiber_activation_record() {
 118 #if defined(BOOST_USE_TSAN)
 119         if (destroy_tsan_fiber)
 120             __tsan_destroy_fiber(tsan_fiber);
 121 #endif
 122         }
 123
 124     fiber_activation_record( fiber_activation_record const&) = delete;
 125     fiber_activation_record & operator=( fiber_activation_record const&) = delete;
 126
 127     bool is_main_context() const noexcept {
 128         return main_ctx;
 129     }
 130
 131     fiber_activation_record * resume() {
 132                 from = current();
 133         // store `this` in static, thread local pointer
 134         // `this` will become the active (running) context
 135         current() = this;
 136 #if defined(BOOST_USE_SEGMENTED_STACKS)
 137         // adjust segmented stack properties
 138         __splitstack_getcontext( from->sctx.segments_ctx);
 139         __splitstack_setcontext( sctx.segments_ctx);
 140 #endif
 141 #if defined(BOOST_USE_ASAN)
 142         if ( terminated) {
 143             __sanitizer_start_switch_fiber( nullptr, stack_bottom, stack_size);
 144         } else {
 145             __sanitizer_start_switch_fiber( & from->fake_stack, stack_bottom, stack_size);
 146         }
 147 #endif
 148 #if defined (BOOST_USE_TSAN)
 149         __tsan_switch_to_fiber(tsan_fiber, 0);
 150 #endif
 151         // context switch from parent context to `this`-context
 152         ::swapcontext( & from->uctx, & uctx);
 153 #if defined(BOOST_USE_ASAN)
 154         __sanitizer_finish_switch_fiber( current()->fake_stack,
 155                                          (const void **) & current()->from->stack_bottom,
 156                                          & current()->from->stack_size);
 157 #endif
 158 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 159         return exchange( current()->from, nullptr);
 160 #else
 161         return std::exchange( current()->from, nullptr);
 162 #endif
 163     }
 164
 165     template< typename Ctx, typename Fn >
 166     fiber_activation_record * resume_with( Fn && fn) {
 167                 from = current();
 168         // store `this` in static, thread local pointer
 169         // `this` will become the active (running) context
 170         // returned by fiber::current()
 171         current() = this;
 172 #if defined(BOOST_NO_CXX14_GENERIC_LAMBDAS)
 173         current()->ontop = std::bind(
 174                 [](typename std::decay< Fn >::type & fn, fiber_activation_record *& ptr){
 175                     Ctx c{ ptr };
 176                     c = fn( std::move( c) );
 177                     if ( ! c) {
 178                         ptr = nullptr;
 179                     }
 180 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 181                     return exchange( c.ptr_, nullptr);
 182 #else
 183                     return std::exchange( c.ptr_, nullptr);
 184 #endif
 185                 },
 186                 std::forward< Fn >( fn),
 187                 std::placeholders::_1);
 188 #else
 189         current()->ontop = [fn=std::forward<Fn>(fn)](fiber_activation_record *& ptr){
 190             Ctx c{ ptr };
 191             c = fn( std::move( c) );
 192             if ( ! c) {
 193                 ptr = nullptr;
 194             }
 195 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 196             return exchange( c.ptr_, nullptr);
 197 #else
 198             return std::exchange( c.ptr_, nullptr);
 199 #endif
 200         };
 201 #endif
 202 #if defined(BOOST_USE_SEGMENTED_STACKS)
 203         // adjust segmented stack properties
 204         __splitstack_getcontext( from->sctx.segments_ctx);
 205         __splitstack_setcontext( sctx.segments_ctx);
 206 #endif
 207 #if defined(BOOST_USE_ASAN)
 208         __sanitizer_start_switch_fiber( & from->fake_stack, stack_bottom, stack_size);
 209 #endif
 210 #if defined (BOOST_USE_TSAN)
 211         __tsan_switch_to_fiber(tsan_fiber, 0);
 212 #endif
 213         // context switch from parent context to `this`-context
 214         ::swapcontext( & from->uctx, & uctx);
 215 #if defined(BOOST_USE_ASAN)
 216         __sanitizer_finish_switch_fiber( current()->fake_stack,
 217                                          (const void **) & current()->from->stack_bottom,
 218                                          & current()->from->stack_size);
 219 #endif
 220 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 221         return exchange( current()->from, nullptr);
 222 #else
 223         return std::exchange( current()->from, nullptr);
 224 #endif
 225     }
 226
 227     virtual void deallocate() noexcept {
 228     }
 229 };
 230
 231 struct BOOST_CONTEXT_DECL fiber_activation_record_initializer {
 232     fiber_activation_record_initializer() noexcept;
 233     ~fiber_activation_record_initializer();
 234 };
 235
 236 struct forced_unwind {
 237     fiber_activation_record  *  from{ nullptr };
 238
 239     forced_unwind( fiber_activation_record * from_) noexcept :
 240         from{ from_ } {
 241     }
 242 };
 243
 244 template< typename Ctx, typename StackAlloc, typename Fn >
 245 class fiber_capture_record : public fiber_activation_record {
 246 private:
 247     typename std::decay< StackAlloc >::type             salloc_;
 248     typename std::decay< Fn >::type                     fn_;
 249
 250     static void destroy( fiber_capture_record * p) noexcept {
 251         typename std::decay< StackAlloc >::type salloc = std::move( p->salloc_);
 252         stack_context sctx = p->sctx;
 253         // deallocate activation record
 254         p->~fiber_capture_record();
 255         // destroy stack with stack allocator
 256         salloc.deallocate( sctx);
 257     }
 258
 259 public:
 260     fiber_capture_record( stack_context sctx, StackAlloc && salloc, Fn && fn) noexcept :
 261         fiber_activation_record{ sctx },
 262         salloc_{ std::forward< StackAlloc >( salloc) },
 263         fn_( std::forward< Fn >( fn) ) {
 264     }
 265
 266     void deallocate() noexcept override final {
 267         BOOST_ASSERT( main_ctx || ( ! main_ctx && terminated) );
 268         destroy( this);
 269     }
 270
 271     void run() {
 272 #if defined(BOOST_USE_ASAN)
 273         __sanitizer_finish_switch_fiber( fake_stack,
 274                                          (const void **) & from->stack_bottom,
 275                                          & from->stack_size);
 276 #endif
 277         Ctx c{ from };
 278         try {
 279             // invoke context-function
 280 #if defined(BOOST_NO_CXX17_STD_INVOKE)
 281             c = boost::context::detail::invoke( fn_, std::move( c) );
 282 #else
 283             c = std::invoke( fn_, std::move( c) );
 284 #endif
 285         } catch ( forced_unwind const& ex) {
 286             c = Ctx{ ex.from };
 287         }
 288         // this context has finished its task
 289                 from = nullptr;
 290         ontop = nullptr;
 291         terminated = true;
 292         force_unwind = false;
 293         std::move( c).resume();
 294         BOOST_ASSERT_MSG( false, "fiber already terminated");
 295     }
 296 };
 297
 298 template< typename Ctx, typename StackAlloc, typename Fn >
 299 static fiber_activation_record * create_fiber1( StackAlloc && salloc, Fn && fn) {
 300     typedef fiber_capture_record< Ctx, StackAlloc, Fn >  capture_t;
 301
 302     auto sctx = salloc.allocate();
 303     // reserve space for control structure
 304     void * storage = reinterpret_cast< void * >(
 305             ( reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sizeof( capture_t) ) )
 306             & ~ static_cast< uintptr_t >( 0xff) );
 307     // placment new for control structure on context stack
 308     capture_t * record = new ( storage) capture_t{
 309             sctx, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) };
 310     // stack bottom
 311     void * stack_bottom = reinterpret_cast< void * >(
 312             reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sctx.size) );
 313     // create user-context
 314     if ( BOOST_UNLIKELY( 0 != ::getcontext( & record->uctx) ) ) {
 315         record->~capture_t();
 316         salloc.deallocate( sctx);
 317         throw std::system_error(
 318                 std::error_code( errno, std::system_category() ),
 319                 "getcontext() failed");
 320     }
 321 #if BOOST_OS_BSD_FREE
 322     // because FreeBSD defines stack_t::ss_sp as char *
 323     record->uctx.uc_stack.ss_sp = static_cast< char * >( stack_bottom);
 324 #else
 325     record->uctx.uc_stack.ss_sp = stack_bottom;
 326 #endif
 327     // 64byte gap between control structure and stack top
 328     record->uctx.uc_stack.ss_size = reinterpret_cast< uintptr_t >( storage) -
 329             reinterpret_cast< uintptr_t >( stack_bottom) - static_cast< uintptr_t >( 64);
 330     record->uctx.uc_link = nullptr;
 331     ::makecontext( & record->uctx, ( void (*)() ) & fiber_entry_func< capture_t >, 1, record);
 332 #if defined(BOOST_USE_ASAN)
 333     record->stack_bottom = record->uctx.uc_stack.ss_sp;
 334     record->stack_size = record->uctx.uc_stack.ss_size;
 335 #endif
 336 #if defined (BOOST_USE_TSAN)
 337     record->tsan_fiber = __tsan_create_fiber(0);
 338 #endif
 339     return record;
 340 }
 341
 342 template< typename Ctx, typename StackAlloc, typename Fn >
 343 static fiber_activation_record * create_fiber2( preallocated palloc, StackAlloc && salloc, Fn && fn) {
 344     typedef fiber_capture_record< Ctx, StackAlloc, Fn >  capture_t;
 345
 346     // reserve space for control structure
 347     void * storage = reinterpret_cast< void * >(
 348             ( reinterpret_cast< uintptr_t >( palloc.sp) - static_cast< uintptr_t >( sizeof( capture_t) ) )
 349             & ~ static_cast< uintptr_t >( 0xff) );
 350     // placment new for control structure on context stack
 351     capture_t * record = new ( storage) capture_t{
 352             palloc.sctx, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) };
 353     // stack bottom
 354     void * stack_bottom = reinterpret_cast< void * >(
 355             reinterpret_cast< uintptr_t >( palloc.sctx.sp) - static_cast< uintptr_t >( palloc.sctx.size) );
 356     // create user-context
 357     if ( BOOST_UNLIKELY( 0 != ::getcontext( & record->uctx) ) ) {
 358         record->~capture_t();
 359         salloc.deallocate( palloc.sctx);
 360         throw std::system_error(
 361                 std::error_code( errno, std::system_category() ),
 362                 "getcontext() failed");
 363     }
 364 #if BOOST_OS_BSD_FREE
 365     // because FreeBSD defines stack_t::ss_sp as char *
 366     record->uctx.uc_stack.ss_sp = static_cast< char * >( stack_bottom);
 367 #else
 368     record->uctx.uc_stack.ss_sp = stack_bottom;
 369 #endif
 370     // 64byte gap between control structure and stack top
 371     record->uctx.uc_stack.ss_size = reinterpret_cast< uintptr_t >( storage) -
 372             reinterpret_cast< uintptr_t >( stack_bottom) - static_cast< uintptr_t >( 64);
 373     record->uctx.uc_link = nullptr;
 374     ::makecontext( & record->uctx,  ( void (*)() ) & fiber_entry_func< capture_t >, 1, record);
 375 #if defined(BOOST_USE_ASAN)
 376     record->stack_bottom = record->uctx.uc_stack.ss_sp;
 377     record->stack_size = record->uctx.uc_stack.ss_size;
 378 #endif
 379 #if defined (BOOST_USE_TSAN)
 380     record->tsan_fiber = __tsan_create_fiber(0);
 381 #endif
 382     return record;
 383 }
 384
 385 }
 386
 387 class BOOST_CONTEXT_DECL fiber {
 388 private:
 389     friend struct detail::fiber_activation_record;
 390
 391     template< typename Ctx, typename StackAlloc, typename Fn >
 392     friend class detail::fiber_capture_record;
 393
 394         template< typename Ctx, typename StackAlloc, typename Fn >
 395         friend detail::fiber_activation_record * detail::create_fiber1( StackAlloc &&, Fn &&);
 396
 397         template< typename Ctx, typename StackAlloc, typename Fn >
 398         friend detail::fiber_activation_record * detail::create_fiber2( preallocated, StackAlloc &&, Fn &&);
 399
 400     template< typename StackAlloc, typename Fn >
 401     friend fiber
 402     callcc( std::allocator_arg_t, StackAlloc &&, Fn &&);
 403
 404     template< typename StackAlloc, typename Fn >
 405     friend fiber
 406     callcc( std::allocator_arg_t, preallocated, StackAlloc &&, Fn &&);
 407
 408     detail::fiber_activation_record   *   ptr_{ nullptr };
 409
 410     fiber( detail::fiber_activation_record * ptr) noexcept :
 411         ptr_{ ptr } {
 412     }
 413
 414 public:
 415     fiber() = default;
 416
 417     template< typename Fn, typename = detail::disable_overload< fiber, Fn > >
 418     fiber( Fn && fn) :
 419         fiber{
 420                 std::allocator_arg,
 421 #if defined(BOOST_USE_SEGMENTED_STACKS)
 422                 segmented_stack(),
 423 #else
 424                 fixedsize_stack(),
 425 #endif
 426                 std::forward< Fn >( fn) } {
 427     }
 428
 429     template< typename StackAlloc, typename Fn >
 430     fiber( std::allocator_arg_t, StackAlloc && salloc, Fn && fn) :
 431         ptr_{ detail::create_fiber1< fiber >(
 432                 std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) ) } {
 433     }
 434
 435     template< typename StackAlloc, typename Fn >
 436     fiber( std::allocator_arg_t, preallocated palloc, StackAlloc && salloc, Fn && fn) :
 437         ptr_{ detail::create_fiber2< fiber >(
 438                 palloc, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) ) } {
 439     }
 440
 441     ~fiber() {
 442         if ( BOOST_UNLIKELY( nullptr != ptr_) && ! ptr_->main_ctx) {
 443             if ( BOOST_LIKELY( ! ptr_->terminated) ) {
 444                 ptr_->force_unwind = true;
 445                 ptr_->resume();
 446                 BOOST_ASSERT( ptr_->terminated);
 447             }
 448             ptr_->deallocate();
 449         }
 450     }
 451
 452     fiber( fiber const&) = delete;
 453     fiber & operator=( fiber const&) = delete;
 454
 455     fiber( fiber && other) noexcept {
 456         swap( other);
 457     }
 458
 459     fiber & operator=( fiber && other) noexcept {
 460         if ( BOOST_LIKELY( this != & other) ) {
 461             fiber tmp = std::move( other);
 462             swap( tmp);
 463         }
 464         return * this;
 465     }
 466
 467     fiber resume() && {
 468         BOOST_ASSERT( nullptr != ptr_);
 469 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 470         detail::fiber_activation_record * ptr = detail::exchange( ptr_, nullptr)->resume();
 471 #else
 472         detail::fiber_activation_record * ptr = std::exchange( ptr_, nullptr)->resume();
 473 #endif
 474         if ( BOOST_UNLIKELY( detail::fiber_activation_record::current()->force_unwind) ) {
 475             throw detail::forced_unwind{ ptr};
 476         } else if ( BOOST_UNLIKELY( nullptr != detail::fiber_activation_record::current()->ontop) ) {
 477             ptr = detail::fiber_activation_record::current()->ontop( ptr);
 478             detail::fiber_activation_record::current()->ontop = nullptr;
 479         }
 480         return { ptr };
 481     }
 482
 483     template< typename Fn >
 484     fiber resume_with( Fn && fn) && {
 485         BOOST_ASSERT( nullptr != ptr_);
 486 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 487         detail::fiber_activation_record * ptr =
 488             detail::exchange( ptr_, nullptr)->resume_with< fiber >( std::forward< Fn >( fn) );
 489 #else
 490         detail::fiber_activation_record * ptr =
 491             std::exchange( ptr_, nullptr)->resume_with< fiber >( std::forward< Fn >( fn) );
 492 #endif
 493         if ( BOOST_UNLIKELY( detail::fiber_activation_record::current()->force_unwind) ) {
 494             throw detail::forced_unwind{ ptr};
 495         } else if ( BOOST_UNLIKELY( nullptr != detail::fiber_activation_record::current()->ontop) ) {
 496             ptr = detail::fiber_activation_record::current()->ontop( ptr);
 497             detail::fiber_activation_record::current()->ontop = nullptr;
 498         }
 499         return { ptr };
 500     }
 501
 502     explicit operator bool() const noexcept {
 503         return nullptr != ptr_ && ! ptr_->terminated;
 504     }
 505
 506     bool operator!() const noexcept {
 507         return nullptr == ptr_ || ptr_->terminated;
 508     }
 509
 510     bool operator<( fiber const& other) const noexcept {
 511         return ptr_ < other.ptr_;
 512     }
 513
 514     #if !defined(BOOST_EMBTC)
 515
 516     template< typename charT, class traitsT >
 517     friend std::basic_ostream< charT, traitsT > &
 518     operator<<( std::basic_ostream< charT, traitsT > & os, fiber const& other) {
 519         if ( nullptr != other.ptr_) {
 520             return os << other.ptr_;
 521         } else {
 522             return os << "{not-a-context}";
 523         }
 524     }
 525
 526     #else
 527
 528     template< typename charT, class traitsT >
 529     friend std::basic_ostream< charT, traitsT > &
 530     operator<<( std::basic_ostream< charT, traitsT > & os, fiber const& other);
 531
 532     #endif
 533
 534     void swap( fiber & other) noexcept {
 535         std::swap( ptr_, other.ptr_);
 536     }
 537 };
 538
 539 #if defined(BOOST_EMBTC)
 540
 541     template< typename charT, class traitsT >
 542     inline std::basic_ostream< charT, traitsT > &
 543     operator<<( std::basic_ostream< charT, traitsT > & os, fiber const& other) {
 544         if ( nullptr != other.ptr_) {
 545             return os << other.ptr_;
 546         } else {
 547             return os << "{not-a-context}";
 548         }
 549     }
 550
 551 #endif
 552
 553 inline
 554 void swap( fiber & l, fiber & r) noexcept {
 555     l.swap( r);
 556 }
 557
 558 typedef fiber fiber_context;
 559
 560 }}
 561
 562 #ifdef BOOST_HAS_ABI_HEADERS
 563 # include BOOST_ABI_SUFFIX
 564 #endif
 565
 566 #endif // BOOST_CONTEXT_FIBER_H