1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
4 #ifndef CEPH_RGW_COROUTINE_H
5 #define CEPH_RGW_COROUTINE_H
9 #pragma push_macro("_ASSERT_H")
12 #include <boost/asio.hpp>
13 #include <boost/intrusive_ptr.hpp>
16 #pragma pop_macro("_ASSERT_H")
19 #include "include/utime.h"
20 #include "common/RefCountedObj.h"
21 #include "common/debug.h"
22 #include "common/Timer.h"
23 #include "common/admin_socket.h"
24 #include "common/RWLock.h"
26 #include "rgw_common.h"
27 #include "rgw_http_client_types.h"
29 #include <boost/asio/coroutine.hpp>
33 #define RGW_ASYNC_OPS_MGR_WINDOW 100
35 class RGWCoroutinesStack
;
36 class RGWCoroutinesManager
;
37 class RGWAioCompletionNotifier
;
39 class RGWCompletionManager
: public RefCountedObject
{
40 friend class RGWCoroutinesManager
;
44 struct io_completion
{
48 list
<io_completion
> complete_reqs
;
49 set
<rgw_io_id
> complete_reqs_set
;
50 using NotifierRef
= boost::intrusive_ptr
<RGWAioCompletionNotifier
>;
53 ceph::mutex lock
= ceph::make_mutex("RGWCompletionManager::lock");
54 ceph::condition_variable cond
;
58 std::atomic
<bool> going_down
= { false };
60 map
<void *, void *> waiters
;
65 void _wakeup(void *opaque
);
66 void _complete(RGWAioCompletionNotifier
*cn
, const rgw_io_id
& io_id
, void *user_info
);
68 explicit RGWCompletionManager(CephContext
*_cct
);
69 ~RGWCompletionManager() override
;
71 void complete(RGWAioCompletionNotifier
*cn
, const rgw_io_id
& io_id
, void *user_info
);
72 int get_next(io_completion
*io
);
73 bool try_get_next(io_completion
*io
);
78 * wait for interval length to complete user_info
80 void wait_interval(void *opaque
, const utime_t
& interval
, void *user_info
);
81 void wakeup(void *opaque
);
83 void register_completion_notifier(RGWAioCompletionNotifier
*cn
);
84 void unregister_completion_notifier(RGWAioCompletionNotifier
*cn
);
87 /* a single use librados aio completion notifier that hooks into the RGWCompletionManager */
88 class RGWAioCompletionNotifier
: public RefCountedObject
{
89 librados::AioCompletion
*c
;
90 RGWCompletionManager
*completion_mgr
;
93 ceph::mutex lock
= ceph::make_mutex("RGWAioCompletionNotifier");
97 RGWAioCompletionNotifier(RGWCompletionManager
*_mgr
, const rgw_io_id
& _io_id
, void *_user_data
);
98 ~RGWAioCompletionNotifier() override
{
101 bool need_unregister
= registered
;
103 completion_mgr
->get();
107 if (need_unregister
) {
108 completion_mgr
->unregister_completion_notifier(this);
109 completion_mgr
->put();
113 librados::AioCompletion
*completion() {
118 std::lock_guard l
{lock
};
132 completion_mgr
->get();
135 completion_mgr
->complete(this, io_id
, user_data
);
136 completion_mgr
->put();
141 // completion notifier with opaque payload (ie a reference-counted pointer)
142 template <typename T
>
143 class RGWAioCompletionNotifierWith
: public RGWAioCompletionNotifier
{
146 RGWAioCompletionNotifierWith(RGWCompletionManager
*mgr
,
147 const rgw_io_id
& io_id
, void *user_data
,
149 : RGWAioCompletionNotifier(mgr
, io_id
, user_data
), value(std::move(value
))
153 struct RGWCoroutinesEnv
{
154 uint64_t run_context
;
155 RGWCoroutinesManager
*manager
;
156 list
<RGWCoroutinesStack
*> *scheduled_stacks
;
157 RGWCoroutinesStack
*stack
;
159 RGWCoroutinesEnv() : run_context(0), manager(NULL
), scheduled_stacks(NULL
), stack(NULL
) {}
162 enum RGWCoroutineState
{
163 RGWCoroutine_Error
= -2,
164 RGWCoroutine_Done
= -1,
165 RGWCoroutine_Run
= 0,
168 struct rgw_spawned_stacks
{
169 vector
<RGWCoroutinesStack
*> entries
;
171 rgw_spawned_stacks() {}
173 void add_pending(RGWCoroutinesStack
*s
) {
174 entries
.push_back(s
);
177 void inherit(rgw_spawned_stacks
*source
) {
178 for (vector
<RGWCoroutinesStack
*>::iterator iter
= source
->entries
.begin();
179 iter
!= source
->entries
.end(); ++iter
) {
182 source
->entries
.clear();
188 class RGWCoroutine
: public RefCountedObject
, public boost::asio::coroutine
{
189 friend class RGWCoroutinesStack
;
195 StatusItem(utime_t
& t
, const string
& s
) : timestamp(t
), status(s
) {}
197 void dump(Formatter
*f
) const;
200 #define MAX_COROUTINE_HISTORY 10
204 ceph::shared_mutex lock
=
205 ceph::make_shared_mutex("RGWCoroutine::Status::lock");
211 explicit Status(CephContext
*_cct
) : cct(_cct
), max_history(MAX_COROUTINE_HISTORY
) {}
213 deque
<StatusItem
> history
;
215 stringstream
& set_status();
218 stringstream description
;
224 boost::asio::coroutine cr
;
225 bool should_exit
{false};
229 cr
= boost::asio::coroutine();
237 RGWCoroutinesStack
*stack
;
241 rgw_spawned_stacks spawned
;
243 stringstream error_stream
;
245 int set_state(int s
, int ret
= 0) {
250 int set_cr_error(int ret
) {
251 return set_state(RGWCoroutine_Error
, ret
);
254 return set_state(RGWCoroutine_Done
, 0);
256 void set_io_blocked(bool flag
);
258 void reset_description() {
259 description
.str(string());
262 stringstream
& set_description() {
265 stringstream
& set_status() {
266 return status
.set_status();
269 stringstream
& set_status(const string
& s
) {
270 stringstream
& status
= set_status();
275 virtual int operate_wrapper() {
279 RGWCoroutine(CephContext
*_cct
) : status(_cct
), _yield_ret(false), cct(_cct
), stack(NULL
), retcode(0), state(RGWCoroutine_Run
) {}
280 ~RGWCoroutine() override
;
282 virtual int operate() = 0;
284 bool is_done() { return (state
== RGWCoroutine_Done
|| state
== RGWCoroutine_Error
); }
285 bool is_error() { return (state
== RGWCoroutine_Error
); }
287 stringstream
& log_error() { return error_stream
; }
289 return error_stream
.str();
292 void set_retcode(int r
) {
296 int get_ret_status() {
300 void call(RGWCoroutine
*op
); /* call at the same stack we're in */
301 RGWCoroutinesStack
*spawn(RGWCoroutine
*op
, bool wait
); /* execute on a different stack */
302 bool collect(int *ret
, RGWCoroutinesStack
*skip_stack
, uint64_t *stack_id
= nullptr); /* returns true if needs to be called again */
303 bool collect_next(int *ret
, RGWCoroutinesStack
**collected_stack
= NULL
); /* returns true if found a stack to collect */
305 RGWCoroutinesStack
*prealloc_stack(); /* prepare a stack that will be used in the next spawn operation */
306 uint64_t prealloc_stack_id(); /* prepare a stack that will be used in the next spawn operation, return its id */
308 int wait(const utime_t
& interval
);
309 bool drain_children(int num_cr_left
,
310 RGWCoroutinesStack
*skip_stack
= nullptr,
311 std::optional
<std::function
<void(uint64_t stack_id
, int ret
)> > cb
= std::nullopt
); /* returns true if needed to be called again,
312 cb will be called on completion of every
314 bool drain_children(int num_cr_left
,
315 std::optional
<std::function
<int(uint64_t stack_id
, int ret
)> > cb
); /* returns true if needed to be called again,
316 cb will be called on every completion, can filter errors.
317 A negative return value from cb means that current cr
320 void set_sleeping(bool flag
); /* put in sleep, or wakeup from sleep */
322 size_t num_spawned() {
323 return spawned
.entries
.size();
326 void wait_for_child();
328 virtual string
to_str() const;
330 RGWCoroutinesStack
*get_stack() const {
334 RGWCoroutinesEnv
*get_env() const;
336 void dump(Formatter
*f
) const;
338 void init_new_io(RGWIOProvider
*io_provider
); /* only links the default io id */
340 int io_block(int ret
= 0) {
341 return io_block(ret
, -1);
343 int io_block(int ret
, int64_t io_id
);
344 int io_block(int ret
, const rgw_io_id
& io_id
);
346 io_complete(rgw_io_id
{});
348 void io_complete(const rgw_io_id
& io_id
);
351 ostream
& operator<<(ostream
& out
, const RGWCoroutine
& cr
);
353 #define yield_until_true(x) \
356 yield _yield_ret = x; \
357 } while (!_yield_ret); \
358 _yield_ret = false; \
361 #define drain_all() \
362 drain_status.init(); \
363 yield_until_true(drain_children(0))
365 #define drain_all_but(n) \
366 drain_status.init(); \
367 yield_until_true(drain_children(n))
369 #define drain_all_but_stack(stack) \
370 drain_status.init(); \
371 yield_until_true(drain_children(1, stack))
373 #define drain_all_but_stack_cb(stack, cb) \
374 drain_status.init(); \
375 yield_until_true(drain_children(1, stack, cb))
377 #define drain_with_cb(n, cb) \
378 drain_status.init(); \
379 yield_until_true(drain_children(n, cb)); \
380 if (drain_status.should_exit) { \
381 return set_cr_error(drain_status.ret); \
384 #define drain_all_cb(cb) \
387 #define yield_spawn_window(cr, n, cb) \
390 drain_with_cb(n, cb); /* this is guaranteed to yield */ \
396 class RGWConsumerCR
: public RGWCoroutine
{
400 explicit RGWConsumerCR(CephContext
*_cct
) : RGWCoroutine(_cct
) {}
403 return !product
.empty();
406 void wait_for_product() {
407 if (!has_product()) {
413 if (product
.empty()) {
416 *p
= product
.front();
421 void receive(const T
& p
, bool wakeup
= true);
422 void receive(list
<T
>& l
, bool wakeup
= true);
425 class RGWCoroutinesStack
: public RefCountedObject
{
426 friend class RGWCoroutine
;
427 friend class RGWCoroutinesManager
;
433 RGWCoroutinesManager
*ops_mgr
;
435 list
<RGWCoroutine
*> ops
;
436 list
<RGWCoroutine
*>::iterator pos
;
438 rgw_spawned_stacks spawned
;
440 RGWCoroutinesStack
*preallocated_stack
{nullptr};
442 set
<RGWCoroutinesStack
*> blocked_by_stack
;
443 set
<RGWCoroutinesStack
*> blocking_stacks
;
445 map
<int64_t, rgw_io_id
> io_finish_ids
;
446 rgw_io_id io_blocked_id
;
452 bool interval_wait_flag
;
456 bool is_waiting_for_child
;
463 RGWCoroutinesEnv
*env
;
464 RGWCoroutinesStack
*parent
;
466 RGWCoroutinesStack
*spawn(RGWCoroutine
*source_op
, RGWCoroutine
*next_op
, bool wait
);
467 bool collect(RGWCoroutine
*op
, int *ret
, RGWCoroutinesStack
*skip_stack
, uint64_t *stack_id
); /* returns true if needs to be called again */
468 bool collect_next(RGWCoroutine
*op
, int *ret
, RGWCoroutinesStack
**collected_stack
); /* returns true if found a stack to collect */
470 RGWCoroutinesStack(CephContext
*_cct
, RGWCoroutinesManager
*_ops_mgr
, RGWCoroutine
*start
= NULL
);
471 ~RGWCoroutinesStack() override
;
473 int64_t get_id() const {
477 int operate(RGWCoroutinesEnv
*env
);
485 bool is_blocked_by_stack() {
486 return !blocked_by_stack
.empty();
488 void set_io_blocked(bool flag
) {
491 void set_io_blocked_id(const rgw_io_id
& io_id
) {
492 io_blocked_id
= io_id
;
494 bool is_io_blocked() {
495 return blocked_flag
&& !done_flag
;
497 bool can_io_unblock(const rgw_io_id
& io_id
) {
498 return ((io_blocked_id
.id
< 0) ||
499 io_blocked_id
.intersects(io_id
));
501 bool try_io_unblock(const rgw_io_id
& io_id
);
502 bool consume_io_finish(const rgw_io_id
& io_id
);
503 void set_interval_wait(bool flag
) {
504 interval_wait_flag
= flag
;
506 bool is_interval_waiting() {
507 return interval_wait_flag
;
509 void set_sleeping(bool flag
) {
510 bool wakeup
= sleep_flag
& !flag
;
519 void set_is_scheduled(bool flag
) {
524 return is_blocked_by_stack() || is_sleeping() ||
525 is_io_blocked() || waiting_for_child() ;
531 int get_ret_status() {
537 void call(RGWCoroutine
*next_op
);
538 RGWCoroutinesStack
*spawn(RGWCoroutine
*next_op
, bool wait
);
539 RGWCoroutinesStack
*prealloc_stack();
540 int unwind(int retcode
);
542 int wait(const utime_t
& interval
);
545 io_complete(rgw_io_id
{});
547 void io_complete(const rgw_io_id
& io_id
);
549 bool collect(int *ret
, RGWCoroutinesStack
*skip_stack
, uint64_t *stack_id
); /* returns true if needs to be called again */
553 RGWAioCompletionNotifier
*create_completion_notifier();
554 template <typename T
>
555 RGWAioCompletionNotifier
*create_completion_notifier(T value
);
556 RGWCompletionManager
*get_completion_mgr();
558 void set_blocked_by(RGWCoroutinesStack
*s
) {
559 blocked_by_stack
.insert(s
);
560 s
->blocking_stacks
.insert(this);
563 void set_wait_for_child(bool flag
) {
564 is_waiting_for_child
= flag
;
567 bool waiting_for_child() {
568 return is_waiting_for_child
;
571 bool unblock_stack(RGWCoroutinesStack
**s
);
573 RGWCoroutinesEnv
*get_env() const { return env
; }
575 void dump(Formatter
*f
) const;
577 void init_new_io(RGWIOProvider
*io_provider
);
581 void RGWConsumerCR
<T
>::receive(list
<T
>& l
, bool wakeup
)
583 product
.splice(product
.end(), l
);
591 void RGWConsumerCR
<T
>::receive(const T
& p
, bool wakeup
)
593 product
.push_back(p
);
599 class RGWCoroutinesManagerRegistry
: public RefCountedObject
, public AdminSocketHook
{
602 set
<RGWCoroutinesManager
*> managers
;
603 ceph::shared_mutex lock
=
604 ceph::make_shared_mutex("RGWCoroutinesRegistry::lock");
606 string admin_command
;
609 explicit RGWCoroutinesManagerRegistry(CephContext
*_cct
) : cct(_cct
) {}
610 ~RGWCoroutinesManagerRegistry() override
;
612 void add(RGWCoroutinesManager
*mgr
);
613 void remove(RGWCoroutinesManager
*mgr
);
615 int hook_to_admin_command(const string
& command
);
616 int call(std::string_view command
, const cmdmap_t
& cmdmap
,
619 bufferlist
& out
) override
;
621 void dump(Formatter
*f
) const;
624 class RGWCoroutinesManager
{
626 std::atomic
<bool> going_down
= { false };
628 std::atomic
<int64_t> run_context_count
= { 0 };
629 map
<uint64_t, set
<RGWCoroutinesStack
*> > run_contexts
;
631 std::atomic
<int64_t> max_io_id
= { 0 };
632 std::atomic
<uint64_t> max_stack_id
= { 0 };
634 mutable ceph::shared_mutex lock
=
635 ceph::make_shared_mutex("RGWCoroutinesManager::lock");
637 RGWIOIDProvider io_id_provider
;
639 void handle_unblocked_stack(set
<RGWCoroutinesStack
*>& context_stacks
, list
<RGWCoroutinesStack
*>& scheduled_stacks
,
640 RGWCompletionManager::io_completion
& io
, int *waiting_count
);
642 RGWCompletionManager
*completion_mgr
;
643 RGWCoroutinesManagerRegistry
*cr_registry
;
649 void put_completion_notifier(RGWAioCompletionNotifier
*cn
);
651 RGWCoroutinesManager(CephContext
*_cct
, RGWCoroutinesManagerRegistry
*_cr_registry
) : cct(_cct
),
652 cr_registry(_cr_registry
), ops_window(RGW_ASYNC_OPS_MGR_WINDOW
) {
653 completion_mgr
= new RGWCompletionManager(cct
);
655 cr_registry
->add(this);
658 virtual ~RGWCoroutinesManager() {
660 completion_mgr
->put();
662 cr_registry
->remove(this);
666 int run(list
<RGWCoroutinesStack
*>& ops
);
667 int run(RGWCoroutine
*op
);
669 bool expected
= false;
670 if (going_down
.compare_exchange_strong(expected
, true)) {
671 completion_mgr
->go_down();
675 virtual void report_error(RGWCoroutinesStack
*op
);
677 RGWAioCompletionNotifier
*create_completion_notifier(RGWCoroutinesStack
*stack
);
678 template <typename T
>
679 RGWAioCompletionNotifier
*create_completion_notifier(RGWCoroutinesStack
*stack
, T value
);
680 RGWCompletionManager
*get_completion_mgr() { return completion_mgr
; }
682 void schedule(RGWCoroutinesEnv
*env
, RGWCoroutinesStack
*stack
);
683 void _schedule(RGWCoroutinesEnv
*env
, RGWCoroutinesStack
*stack
);
684 RGWCoroutinesStack
*allocate_stack();
686 int64_t get_next_io_id();
687 uint64_t get_next_stack_id();
689 void set_sleeping(RGWCoroutine
*cr
, bool flag
);
690 void io_complete(RGWCoroutine
*cr
, const rgw_io_id
& io_id
);
692 virtual string
get_id();
693 void dump(Formatter
*f
) const;
695 RGWIOIDProvider
& get_io_id_provider() {
696 return io_id_provider
;
700 template <typename T
>
701 RGWAioCompletionNotifier
*RGWCoroutinesManager::create_completion_notifier(RGWCoroutinesStack
*stack
, T value
)
703 rgw_io_id io_id
{get_next_io_id(), -1};
704 RGWAioCompletionNotifier
*cn
= new RGWAioCompletionNotifierWith
<T
>(completion_mgr
, io_id
, (void *)stack
, std::move(value
));
705 completion_mgr
->register_completion_notifier(cn
);
709 template <typename T
>
710 RGWAioCompletionNotifier
*RGWCoroutinesStack::create_completion_notifier(T value
)
712 return ops_mgr
->create_completion_notifier(this, std::move(value
));
715 class RGWSimpleCoroutine
: public RGWCoroutine
{
718 int operate() override
;
721 int state_send_request();
722 int state_request_complete();
723 int state_all_complete();
728 RGWSimpleCoroutine(CephContext
*_cct
) : RGWCoroutine(_cct
), called_cleanup(false) {}
729 ~RGWSimpleCoroutine() override
;
731 virtual int init() { return 0; }
732 virtual int send_request() = 0;
733 virtual int request_complete() = 0;
734 virtual int finish() { return 0; }
735 virtual void request_cleanup() {}