ceph/src/boost/boost/context/continuation_ucontext.hpp

   1
   2 //          Copyright Oliver Kowalke 2017.
   3 // Distributed under the Boost Software License, Version 1.0.
   4 //    (See accompanying file LICENSE_1_0.txt or copy at
   5 //          http://www.boost.org/LICENSE_1_0.txt)
   6
   7 #ifndef BOOST_CONTEXT_CONTINUATION_H
   8 #define BOOST_CONTEXT_CONTINUATION_H
   9
  10 #include <boost/predef.h>
  11 #if BOOST_OS_MACOS
  12 #define _XOPEN_SOURCE 600
  13 #endif
  14
  15 extern "C" {
  16 #include <ucontext.h>
  17 }
  18
  19 #include <boost/context/detail/config.hpp>
  20
  21 #include <algorithm>
  22 #include <cstddef>
  23 #include <cstdint>
  24 #include <cstdlib>
  25 #include <cstring>
  26 #include <functional>
  27 #include <memory>
  28 #include <ostream>
  29 #include <system_error>
  30 #include <tuple>
  31 #include <utility>
  32
  33 #include <boost/assert.hpp>
  34 #include <boost/config.hpp>
  35
  36 #include <boost/context/detail/disable_overload.hpp>
  37 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
  38 #include <boost/context/detail/exchange.hpp>
  39 #endif
  40 #if defined(BOOST_NO_CXX17_STD_INVOKE)
  41 #include <boost/context/detail/invoke.hpp>
  42 #endif
  43 #include <boost/context/fixedsize_stack.hpp>
  44 #include <boost/context/flags.hpp>
  45 #include <boost/context/preallocated.hpp>
  46 #if defined(BOOST_USE_SEGMENTED_STACKS)
  47 #include <boost/context/segmented_stack.hpp>
  48 #endif
  49 #include <boost/context/stack_context.hpp>
  50
  51 #ifdef BOOST_HAS_ABI_HEADERS
  52 # include BOOST_ABI_PREFIX
  53 #endif
  54
  55 #if defined(BOOST_USE_ASAN)
  56 extern "C" {
  57 void __sanitizer_start_switch_fiber( void **, const void *, size_t);
  58 void __sanitizer_finish_switch_fiber( void *, const void **, size_t *);
  59 }
  60 #endif
  61
  62 #if defined(BOOST_USE_SEGMENTED_STACKS)
  63 extern "C" {
  64 void __splitstack_getcontext( void * [BOOST_CONTEXT_SEGMENTS]);
  65 void __splitstack_setcontext( void * [BOOST_CONTEXT_SEGMENTS]);
  66 }
  67 #endif
  68
  69 namespace boost {
  70 namespace context {
  71 namespace detail {
  72
  73 // tampoline function
  74 // entered if the execution context
  75 // is resumed for the first time
  76 template< typename Record >
  77 static void entry_func( void * data) noexcept {
  78     Record * record = static_cast< Record * >( data);
  79     BOOST_ASSERT( nullptr != record);
  80     // start execution of toplevel context-function
  81     record->run();
  82 }
  83
  84 struct BOOST_CONTEXT_DECL activation_record {
  85     ucontext_t                                                  uctx{};
  86     stack_context                                               sctx{};
  87     bool                                                        main_ctx{ true };
  88         activation_record                                       *       from{ nullptr };
  89     std::function< activation_record*(activation_record*&) >    ontop{};
  90     bool                                                        terminated{ false };
  91     bool                                                        force_unwind{ false };
  92 #if defined(BOOST_USE_ASAN)
  93     void                                                    *   fake_stack{ nullptr };
  94     void                                                    *   stack_bottom{ nullptr };
  95     std::size_t                                                 stack_size{ 0 };
  96 #endif
  97
  98     static activation_record *& current() noexcept;
  99
 100     // used for toplevel-context
 101     // (e.g. main context, thread-entry context)
 102     activation_record() {
 103         if ( BOOST_UNLIKELY( 0 != ::getcontext( & uctx) ) ) {
 104             throw std::system_error(
 105                     std::error_code( errno, std::system_category() ),
 106                     "getcontext() failed");
 107         }
 108     }
 109
 110     activation_record( stack_context sctx_) noexcept :
 111         sctx( sctx_ ),
 112         main_ctx( false ) {
 113     }
 114
 115     virtual ~activation_record() {
 116         }
 117
 118     activation_record( activation_record const&) = delete;
 119     activation_record & operator=( activation_record const&) = delete;
 120
 121     bool is_main_context() const noexcept {
 122         return main_ctx;
 123     }
 124
 125     activation_record * resume() {
 126                 from = current();
 127         // store `this` in static, thread local pointer
 128         // `this` will become the active (running) context
 129         current() = this;
 130 #if defined(BOOST_USE_SEGMENTED_STACKS)
 131         // adjust segmented stack properties
 132         __splitstack_getcontext( from->sctx.segments_ctx);
 133         __splitstack_setcontext( sctx.segments_ctx);
 134 #endif
 135 #if defined(BOOST_USE_ASAN)
 136         if ( terminated) {
 137             __sanitizer_start_switch_fiber( nullptr, stack_bottom, stack_size);
 138         } else {
 139             __sanitizer_start_switch_fiber( & from->fake_stack, stack_bottom, stack_size);
 140         }
 141 #endif
 142         // context switch from parent context to `this`-context
 143         ::swapcontext( & from->uctx, & uctx);
 144 #if defined(BOOST_USE_ASAN)
 145         __sanitizer_finish_switch_fiber( current()->fake_stack,
 146                                          (const void **) & current()->from->stack_bottom,
 147                                          & current()->from->stack_size);
 148 #endif
 149 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 150         return exchange( current()->from, nullptr);
 151 #else
 152         return std::exchange( current()->from, nullptr);
 153 #endif
 154     }
 155
 156     template< typename Ctx, typename Fn >
 157     activation_record * resume_with( Fn && fn) {
 158                 from = current();
 159         // store `this` in static, thread local pointer
 160         // `this` will become the active (running) context
 161         // returned by continuation::current()
 162         current() = this;
 163 #if defined(BOOST_NO_CXX14_GENERIC_LAMBDAS)
 164         current()->ontop = std::bind(
 165                 [](typename std::decay< Fn >::type & fn, activation_record *& ptr){
 166                     Ctx c{ ptr };
 167                     c = fn( std::move( c) );
 168                     if ( ! c) {
 169                         ptr = nullptr;
 170                     }
 171 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 172                     return exchange( c.ptr_, nullptr);
 173 #else
 174                     return std::exchange( c.ptr_, nullptr);
 175 #endif
 176                 },
 177                 std::forward< Fn >( fn),
 178                 std::placeholders::_1);
 179 #else
 180         current()->ontop = [fn=std::forward<Fn>(fn)](activation_record *& ptr){
 181             Ctx c{ ptr };
 182             c = fn( std::move( c) );
 183             if ( ! c) {
 184                 ptr = nullptr;
 185             }
 186 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 187             return exchange( c.ptr_, nullptr);
 188 #else
 189             return std::exchange( c.ptr_, nullptr);
 190 #endif
 191         };
 192 #endif
 193 #if defined(BOOST_USE_SEGMENTED_STACKS)
 194         // adjust segmented stack properties
 195         __splitstack_getcontext( from->sctx.segments_ctx);
 196         __splitstack_setcontext( sctx.segments_ctx);
 197 #endif
 198 #if defined(BOOST_USE_ASAN)
 199         __sanitizer_start_switch_fiber( & from->fake_stack, stack_bottom, stack_size);
 200 #endif
 201         // context switch from parent context to `this`-context
 202         ::swapcontext( & from->uctx, & uctx);
 203 #if defined(BOOST_USE_ASAN)
 204         __sanitizer_finish_switch_fiber( current()->fake_stack,
 205                                          (const void **) & current()->from->stack_bottom,
 206                                          & current()->from->stack_size);
 207 #endif
 208 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 209         return exchange( current()->from, nullptr);
 210 #else
 211         return std::exchange( current()->from, nullptr);
 212 #endif
 213     }
 214
 215     virtual void deallocate() noexcept {
 216     }
 217 };
 218
 219 struct BOOST_CONTEXT_DECL activation_record_initializer {
 220     activation_record_initializer() noexcept;
 221     ~activation_record_initializer();
 222 };
 223
 224 struct forced_unwind {
 225     activation_record  *   from{ nullptr };
 226
 227     forced_unwind( activation_record * from_) noexcept :
 228         from{ from_ } {
 229     }
 230 };
 231
 232 template< typename Ctx, typename StackAlloc, typename Fn >
 233 class capture_record : public activation_record {
 234 private:
 235     typename std::decay< StackAlloc >::type             salloc_;
 236     typename std::decay< Fn >::type                     fn_;
 237
 238     static void destroy( capture_record * p) noexcept {
 239         typename std::decay< StackAlloc >::type salloc = std::move( p->salloc_);
 240         stack_context sctx = p->sctx;
 241         // deallocate activation record
 242         p->~capture_record();
 243         // destroy stack with stack allocator
 244         salloc.deallocate( sctx);
 245     }
 246
 247 public:
 248     capture_record( stack_context sctx, StackAlloc && salloc, Fn && fn) noexcept :
 249         activation_record{ sctx },
 250         salloc_{ std::forward< StackAlloc >( salloc) },
 251         fn_( std::forward< Fn >( fn) ) {
 252     }
 253
 254     void deallocate() noexcept override final {
 255         BOOST_ASSERT( main_ctx || ( ! main_ctx && terminated) );
 256         destroy( this);
 257     }
 258
 259     void run() {
 260 #if defined(BOOST_USE_ASAN)
 261         __sanitizer_finish_switch_fiber( fake_stack,
 262                                          (const void **) & from->stack_bottom,
 263                                          & from->stack_size);
 264 #endif
 265         Ctx c{ from };
 266         try {
 267             // invoke context-function
 268 #if defined(BOOST_NO_CXX17_STD_INVOKE)
 269             c = invoke( fn_, std::move( c) );
 270 #else
 271             c = std::invoke( fn_, std::move( c) );
 272 #endif
 273         } catch ( forced_unwind const& ex) {
 274             c = Ctx{ ex.from };
 275         }
 276         // this context has finished its task
 277                 from = nullptr;
 278         ontop = nullptr;
 279         terminated = true;
 280         force_unwind = false;
 281         c.resume();
 282         BOOST_ASSERT_MSG( false, "continuation already terminated");
 283     }
 284 };
 285
 286 template< typename Ctx, typename StackAlloc, typename Fn >
 287 static activation_record * create_context1( StackAlloc && salloc, Fn && fn) {
 288     typedef capture_record< Ctx, StackAlloc, Fn >  capture_t;
 289
 290     auto sctx = salloc.allocate();
 291     // reserve space for control structure
 292     void * storage = reinterpret_cast< void * >(
 293             ( reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sizeof( capture_t) ) )
 294             & ~ static_cast< uintptr_t >( 0xff) );
 295     // placment new for control structure on context stack
 296     capture_t * record = new ( storage) capture_t{
 297             sctx, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) };
 298     // stack bottom
 299     void * stack_bottom = reinterpret_cast< void * >(
 300             reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sctx.size) );
 301     // create user-context
 302     if ( BOOST_UNLIKELY( 0 != ::getcontext( & record->uctx) ) ) {
 303         throw std::system_error(
 304                 std::error_code( errno, std::system_category() ),
 305                 "getcontext() failed");
 306     }
 307     record->uctx.uc_stack.ss_sp = stack_bottom;
 308     // 64byte gap between control structure and stack top
 309     record->uctx.uc_stack.ss_size = reinterpret_cast< uintptr_t >( storage) -
 310             reinterpret_cast< uintptr_t >( stack_bottom) - static_cast< uintptr_t >( 64);
 311     record->uctx.uc_link = nullptr;
 312     ::makecontext( & record->uctx, ( void (*)() ) & entry_func< capture_t >, 1, record);
 313 #if defined(BOOST_USE_ASAN)
 314     record->stack_bottom = record->uctx.uc_stack.ss_sp;
 315     record->stack_size = record->uctx.uc_stack.ss_size;
 316 #endif
 317     return record;
 318 }
 319
 320 template< typename Ctx, typename StackAlloc, typename Fn >
 321 static activation_record * create_context2( preallocated palloc, StackAlloc && salloc, Fn && fn) {
 322     typedef capture_record< Ctx, StackAlloc, Fn >  capture_t;
 323
 324     // reserve space for control structure
 325     void * storage = reinterpret_cast< void * >(
 326             ( reinterpret_cast< uintptr_t >( palloc.sp) - static_cast< uintptr_t >( sizeof( capture_t) ) )
 327             & ~ static_cast< uintptr_t >( 0xff) );
 328     // placment new for control structure on context stack
 329     capture_t * record = new ( storage) capture_t{
 330             palloc.sctx, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) };
 331     // stack bottom
 332     void * stack_bottom = reinterpret_cast< void * >(
 333             reinterpret_cast< uintptr_t >( palloc.sctx.sp) - static_cast< uintptr_t >( palloc.sctx.size) );
 334     // create user-context
 335     if ( BOOST_UNLIKELY( 0 != ::getcontext( & record->uctx) ) ) {
 336         throw std::system_error(
 337                 std::error_code( errno, std::system_category() ),
 338                 "getcontext() failed");
 339     }
 340     record->uctx.uc_stack.ss_sp = stack_bottom;
 341     // 64byte gap between control structure and stack top
 342     record->uctx.uc_stack.ss_size = reinterpret_cast< uintptr_t >( storage) -
 343             reinterpret_cast< uintptr_t >( stack_bottom) - static_cast< uintptr_t >( 64);
 344     record->uctx.uc_link = nullptr;
 345     ::makecontext( & record->uctx,  ( void (*)() ) & entry_func< capture_t >, 1, record);
 346 #if defined(BOOST_USE_ASAN)
 347     record->stack_bottom = record->uctx.uc_stack.ss_sp;
 348     record->stack_size = record->uctx.uc_stack.ss_size;
 349 #endif
 350     return record;
 351 }
 352
 353 }
 354
 355 class BOOST_CONTEXT_DECL continuation {
 356 private:
 357     friend struct detail::activation_record;
 358
 359     template< typename Ctx, typename StackAlloc, typename Fn >
 360     friend class detail::capture_record;
 361
 362         template< typename Ctx, typename StackAlloc, typename Fn >
 363         friend detail::activation_record * detail::create_context1( StackAlloc &&, Fn &&);
 364
 365         template< typename Ctx, typename StackAlloc, typename Fn >
 366         friend detail::activation_record * detail::create_context2( preallocated, StackAlloc &&, Fn &&);
 367
 368     template< typename StackAlloc, typename Fn >
 369     friend continuation
 370     callcc( std::allocator_arg_t, StackAlloc &&, Fn &&);
 371
 372     template< typename StackAlloc, typename Fn >
 373     friend continuation
 374     callcc( std::allocator_arg_t, preallocated, StackAlloc &&, Fn &&);
 375
 376     detail::activation_record   *   ptr_{ nullptr };
 377
 378     continuation( detail::activation_record * ptr) noexcept :
 379         ptr_{ ptr } {
 380     }
 381
 382 public:
 383     continuation() = default;
 384
 385     ~continuation() {
 386         if ( BOOST_UNLIKELY( nullptr != ptr_) && ! ptr_->main_ctx) {
 387             if ( BOOST_LIKELY( ! ptr_->terminated) ) {
 388                 ptr_->force_unwind = true;
 389                 ptr_->resume();
 390                 BOOST_ASSERT( ptr_->terminated);
 391             }
 392             ptr_->deallocate();
 393         }
 394     }
 395
 396     continuation( continuation const&) = delete;
 397     continuation & operator=( continuation const&) = delete;
 398
 399     continuation( continuation && other) noexcept {
 400         swap( other);
 401     }
 402
 403     continuation & operator=( continuation && other) noexcept {
 404         if ( BOOST_LIKELY( this != & other) ) {
 405             continuation tmp = std::move( other);
 406             swap( tmp);
 407         }
 408         return * this;
 409     }
 410
 411     continuation resume() {
 412 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 413         detail::activation_record * ptr = detail::exchange( ptr_, nullptr)->resume();
 414 #else
 415         detail::activation_record * ptr = std::exchange( ptr_, nullptr)->resume();
 416 #endif
 417         if ( BOOST_UNLIKELY( detail::activation_record::current()->force_unwind) ) {
 418             throw detail::forced_unwind{ ptr};
 419         } else if ( BOOST_UNLIKELY( nullptr != detail::activation_record::current()->ontop) ) {
 420             ptr = detail::activation_record::current()->ontop( ptr);
 421             detail::activation_record::current()->ontop = nullptr;
 422         }
 423         return continuation{ ptr };
 424     }
 425
 426     template< typename Fn >
 427     continuation resume_with( Fn && fn) {
 428 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
 429         detail::activation_record * ptr =
 430             detail::exchange( ptr_, nullptr)->resume_with< continuation >( std::forward< Fn >( fn) );
 431 #else
 432         detail::activation_record * ptr =
 433             std::exchange( ptr_, nullptr)->resume_with< continuation >( std::forward< Fn >( fn) );
 434 #endif
 435         if ( BOOST_UNLIKELY( detail::activation_record::current()->force_unwind) ) {
 436             throw detail::forced_unwind{ ptr};
 437         } else if ( BOOST_UNLIKELY( nullptr != detail::activation_record::current()->ontop) ) {
 438             ptr = detail::activation_record::current()->ontop( ptr);
 439             detail::activation_record::current()->ontop = nullptr;
 440         }
 441         return continuation{ ptr };
 442     }
 443
 444     explicit operator bool() const noexcept {
 445         return nullptr != ptr_ && ! ptr_->terminated;
 446     }
 447
 448     bool operator!() const noexcept {
 449         return nullptr == ptr_ || ptr_->terminated;
 450     }
 451
 452     bool operator==( continuation const& other) const noexcept {
 453         return ptr_ == other.ptr_;
 454     }
 455
 456     bool operator!=( continuation const& other) const noexcept {
 457         return ptr_ != other.ptr_;
 458     }
 459
 460     bool operator<( continuation const& other) const noexcept {
 461         return ptr_ < other.ptr_;
 462     }
 463
 464     bool operator>( continuation const& other) const noexcept {
 465         return other.ptr_ < ptr_;
 466     }
 467
 468     bool operator<=( continuation const& other) const noexcept {
 469         return ! ( * this > other);
 470     }
 471
 472     bool operator>=( continuation const& other) const noexcept {
 473         return ! ( * this < other);
 474     }
 475
 476     template< typename charT, class traitsT >
 477     friend std::basic_ostream< charT, traitsT > &
 478     operator<<( std::basic_ostream< charT, traitsT > & os, continuation const& other) {
 479         if ( nullptr != other.ptr_) {
 480             return os << other.ptr_;
 481         } else {
 482             return os << "{not-a-context}";
 483         }
 484     }
 485
 486     void swap( continuation & other) noexcept {
 487         std::swap( ptr_, other.ptr_);
 488     }
 489 };
 490
 491 template<
 492     typename Fn,
 493     typename = detail::disable_overload< continuation, Fn >
 494 >
 495 continuation
 496 callcc( Fn && fn) {
 497         return callcc(
 498                         std::allocator_arg,
 499 #if defined(BOOST_USE_SEGMENTED_STACKS)
 500                         segmented_stack(),
 501 #else
 502                         fixedsize_stack(),
 503 #endif
 504                         std::forward< Fn >( fn) );
 505 }
 506
 507 template< typename StackAlloc, typename Fn >
 508 continuation
 509 callcc( std::allocator_arg_t, StackAlloc && salloc, Fn && fn) {
 510         return continuation{
 511                 detail::create_context1< continuation >(
 512                                 std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) ) }.resume();
 513 }
 514
 515 template< typename StackAlloc, typename Fn >
 516 continuation
 517 callcc( std::allocator_arg_t, preallocated palloc, StackAlloc && salloc, Fn && fn) {
 518         return continuation{
 519                 detail::create_context2< continuation >(
 520                                 palloc, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) ) }.resume();
 521 }
 522
 523 inline
 524 void swap( continuation & l, continuation & r) noexcept {
 525     l.swap( r);
 526 }
 527
 528 }}
 529
 530 #ifdef BOOST_HAS_ABI_HEADERS
 531 # include BOOST_ABI_SUFFIX
 532 #endif
 533
 534 #endif // BOOST_CONTEXT_CONTINUATION_H