1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "librbd/ImageState.h"
5 #include "include/rbd/librbd.hpp"
6 #include "common/dout.h"
7 #include "common/errno.h"
8 #include "common/Cond.h"
9 #include "common/WorkQueue.h"
10 #include "librbd/ImageCtx.h"
11 #include "librbd/Utils.h"
12 #include "librbd/image/CloseRequest.h"
13 #include "librbd/image/OpenRequest.h"
14 #include "librbd/image/RefreshRequest.h"
15 #include "librbd/image/SetSnapRequest.h"
17 #define dout_subsys ceph_subsys_rbd
19 #define dout_prefix *_dout << "librbd::ImageState: " << this << " "
23 using util::create_async_context_callback
;
24 using util::create_context_callback
;
26 class ImageUpdateWatchers
{
29 explicit ImageUpdateWatchers(CephContext
*cct
) : m_cct(cct
),
30 m_lock(ceph::make_mutex(util::unique_lock_name("librbd::ImageUpdateWatchers::m_lock", this))) {
33 ~ImageUpdateWatchers() {
34 ceph_assert(m_watchers
.empty());
35 ceph_assert(m_in_flight
.empty());
36 ceph_assert(m_pending_unregister
.empty());
37 ceph_assert(m_on_shut_down_finish
== nullptr);
42 void flush(Context
*on_finish
) {
43 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
<< dendl
;
45 std::lock_guard locker
{m_lock
};
46 if (!m_in_flight
.empty()) {
47 Context
*ctx
= new LambdaContext(
48 [this, on_finish
](int r
) {
49 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
50 << ": completing flush" << dendl
;
51 on_finish
->complete(r
);
53 m_work_queue
->queue(ctx
, 0);
57 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
58 << ": completing flush" << dendl
;
59 on_finish
->complete(0);
62 void shut_down(Context
*on_finish
) {
63 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
<< dendl
;
65 std::lock_guard locker
{m_lock
};
66 ceph_assert(m_on_shut_down_finish
== nullptr);
68 if (!m_in_flight
.empty()) {
69 m_on_shut_down_finish
= on_finish
;
73 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
74 << ": completing shut down" << dendl
;
75 on_finish
->complete(0);
78 void register_watcher(UpdateWatchCtx
*watcher
, uint64_t *handle
) {
79 ldout(m_cct
, 20) << __func__
<< ": watcher=" << watcher
<< dendl
;
81 std::lock_guard locker
{m_lock
};
82 ceph_assert(m_on_shut_down_finish
== nullptr);
86 *handle
= m_next_handle
++;
87 m_watchers
.insert(std::make_pair(*handle
, watcher
));
90 void unregister_watcher(uint64_t handle
, Context
*on_finish
) {
91 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
<< ": handle="
95 std::lock_guard locker
{m_lock
};
96 auto it
= m_watchers
.find(handle
);
97 if (it
== m_watchers
.end()) {
100 if (m_in_flight
.find(handle
) != m_in_flight
.end()) {
101 ceph_assert(m_pending_unregister
.find(handle
) == m_pending_unregister
.end());
102 m_pending_unregister
[handle
] = on_finish
;
105 m_watchers
.erase(it
);
110 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
111 << ": completing unregister" << dendl
;
112 on_finish
->complete(r
);
117 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
<< dendl
;
119 std::lock_guard locker
{m_lock
};
120 for (auto it
: m_watchers
) {
121 send_notify(it
.first
, it
.second
);
125 void send_notify(uint64_t handle
, UpdateWatchCtx
*watcher
) {
126 ceph_assert(ceph_mutex_is_locked(m_lock
));
128 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
<< ": handle="
129 << handle
<< ", watcher=" << watcher
<< dendl
;
131 m_in_flight
.insert(handle
);
133 Context
*ctx
= new LambdaContext(
134 [this, handle
, watcher
](int r
) {
135 handle_notify(handle
, watcher
);
138 m_work_queue
->queue(ctx
, 0);
141 void handle_notify(uint64_t handle
, UpdateWatchCtx
*watcher
) {
143 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
<< ": handle="
144 << handle
<< ", watcher=" << watcher
<< dendl
;
146 watcher
->handle_notify();
148 Context
*on_unregister_finish
= nullptr;
149 Context
*on_shut_down_finish
= nullptr;
152 std::lock_guard locker
{m_lock
};
154 auto in_flight_it
= m_in_flight
.find(handle
);
155 ceph_assert(in_flight_it
!= m_in_flight
.end());
156 m_in_flight
.erase(in_flight_it
);
158 // If there is no more in flight notifications for this watcher
159 // and it is pending unregister, complete it now.
160 if (m_in_flight
.find(handle
) == m_in_flight
.end()) {
161 auto it
= m_pending_unregister
.find(handle
);
162 if (it
!= m_pending_unregister
.end()) {
163 on_unregister_finish
= it
->second
;
164 m_pending_unregister
.erase(it
);
168 if (m_in_flight
.empty()) {
169 ceph_assert(m_pending_unregister
.empty());
170 if (m_on_shut_down_finish
!= nullptr) {
171 std::swap(m_on_shut_down_finish
, on_shut_down_finish
);
176 if (on_unregister_finish
!= nullptr) {
177 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
178 << ": completing unregister" << dendl
;
179 on_unregister_finish
->complete(0);
182 if (on_shut_down_finish
!= nullptr) {
183 ldout(m_cct
, 20) << "ImageUpdateWatchers::" << __func__
184 << ": completing shut down" << dendl
;
185 on_shut_down_finish
->complete(0);
190 class ThreadPoolSingleton
: public ThreadPool
{
192 explicit ThreadPoolSingleton(CephContext
*cct
)
193 : ThreadPool(cct
, "librbd::ImageUpdateWatchers::thread_pool", "tp_librbd",
197 ~ThreadPoolSingleton() override
{
204 ContextWQ
*m_work_queue
= nullptr;
205 std::map
<uint64_t, UpdateWatchCtx
*> m_watchers
;
206 uint64_t m_next_handle
= 0;
207 std::multiset
<uint64_t> m_in_flight
;
208 std::map
<uint64_t, Context
*> m_pending_unregister
;
209 Context
*m_on_shut_down_finish
= nullptr;
211 void create_work_queue() {
212 if (m_work_queue
!= nullptr) {
215 auto& thread_pool
= m_cct
->lookup_or_create_singleton_object
<
216 ThreadPoolSingleton
>("librbd::ImageUpdateWatchers::thread_pool",
218 m_work_queue
= new ContextWQ("librbd::ImageUpdateWatchers::op_work_queue",
219 m_cct
->_conf
.get_val
<uint64_t>("rbd_op_thread_timeout"),
223 void destroy_work_queue() {
224 if (m_work_queue
== nullptr) {
227 m_work_queue
->drain();
232 template <typename I
>
233 ImageState
<I
>::ImageState(I
*image_ctx
)
234 : m_image_ctx(image_ctx
), m_state(STATE_UNINITIALIZED
),
235 m_lock(ceph::make_mutex(util::unique_lock_name("librbd::ImageState::m_lock", this))),
236 m_last_refresh(0), m_refresh_seq(0),
237 m_update_watchers(new ImageUpdateWatchers(image_ctx
->cct
)) {
240 template <typename I
>
241 ImageState
<I
>::~ImageState() {
242 ceph_assert(m_state
== STATE_UNINITIALIZED
|| m_state
== STATE_CLOSED
);
243 delete m_update_watchers
;
246 template <typename I
>
247 int ImageState
<I
>::open(uint64_t flags
) {
258 template <typename I
>
259 void ImageState
<I
>::open(uint64_t flags
, Context
*on_finish
) {
260 CephContext
*cct
= m_image_ctx
->cct
;
261 ldout(cct
, 20) << __func__
<< dendl
;
264 ceph_assert(m_state
== STATE_UNINITIALIZED
);
265 m_open_flags
= flags
;
267 Action
action(ACTION_TYPE_OPEN
);
268 action
.refresh_seq
= m_refresh_seq
;
270 execute_action_unlock(action
, on_finish
);
273 template <typename I
>
274 int ImageState
<I
>::close() {
283 template <typename I
>
284 void ImageState
<I
>::close(Context
*on_finish
) {
285 CephContext
*cct
= m_image_ctx
->cct
;
286 ldout(cct
, 20) << __func__
<< dendl
;
289 ceph_assert(!is_closed());
291 Action
action(ACTION_TYPE_CLOSE
);
292 action
.refresh_seq
= m_refresh_seq
;
293 execute_action_unlock(action
, on_finish
);
296 template <typename I
>
297 void ImageState
<I
>::handle_update_notification() {
298 std::lock_guard locker
{m_lock
};
301 CephContext
*cct
= m_image_ctx
->cct
;
302 ldout(cct
, 20) << __func__
<< ": refresh_seq = " << m_refresh_seq
<< ", "
303 << "last_refresh = " << m_last_refresh
<< dendl
;
306 case STATE_UNINITIALIZED
:
310 ldout(cct
, 5) << "dropping update notification to watchers" << dendl
;
316 m_update_watchers
->notify();
319 template <typename I
>
320 bool ImageState
<I
>::is_refresh_required() const {
321 std::lock_guard locker
{m_lock
};
322 return (m_last_refresh
!= m_refresh_seq
|| find_pending_refresh() != nullptr);
325 template <typename I
>
326 int ImageState
<I
>::refresh() {
327 C_SaferCond refresh_ctx
;
328 refresh(&refresh_ctx
);
329 return refresh_ctx
.wait();
332 template <typename I
>
333 void ImageState
<I
>::refresh(Context
*on_finish
) {
334 CephContext
*cct
= m_image_ctx
->cct
;
335 ldout(cct
, 20) << __func__
<< dendl
;
340 on_finish
->complete(-ESHUTDOWN
);
344 Action
action(ACTION_TYPE_REFRESH
);
345 action
.refresh_seq
= m_refresh_seq
;
346 execute_action_unlock(action
, on_finish
);
349 template <typename I
>
350 int ImageState
<I
>::refresh_if_required() {
354 Action
action(ACTION_TYPE_REFRESH
);
355 action
.refresh_seq
= m_refresh_seq
;
357 auto refresh_action
= find_pending_refresh();
358 if (refresh_action
!= nullptr) {
359 // if a refresh is in-flight, delay until it is finished
360 action
= *refresh_action
;
361 } else if (m_last_refresh
== m_refresh_seq
) {
364 } else if (is_closed()) {
369 execute_action_unlock(action
, &ctx
);
375 template <typename I
>
376 const typename ImageState
<I
>::Action
*
377 ImageState
<I
>::find_pending_refresh() const {
378 ceph_assert(ceph_mutex_is_locked(m_lock
));
380 auto it
= std::find_if(m_actions_contexts
.rbegin(),
381 m_actions_contexts
.rend(),
382 [](const ActionContexts
& action_contexts
) {
383 return (action_contexts
.first
== ACTION_TYPE_REFRESH
);
385 if (it
!= m_actions_contexts
.rend()) {
391 template <typename I
>
392 void ImageState
<I
>::snap_set(uint64_t snap_id
, Context
*on_finish
) {
393 CephContext
*cct
= m_image_ctx
->cct
;
394 ldout(cct
, 20) << __func__
<< ": snap_id=" << snap_id
<< dendl
;
396 Action
action(ACTION_TYPE_SET_SNAP
);
397 action
.snap_id
= snap_id
;
400 execute_action_unlock(action
, on_finish
);
403 template <typename I
>
404 void ImageState
<I
>::prepare_lock(Context
*on_ready
) {
405 CephContext
*cct
= m_image_ctx
->cct
;
406 ldout(cct
, 10) << __func__
<< dendl
;
411 on_ready
->complete(-ESHUTDOWN
);
415 Action
action(ACTION_TYPE_LOCK
);
416 action
.on_ready
= on_ready
;
417 execute_action_unlock(action
, nullptr);
420 template <typename I
>
421 void ImageState
<I
>::handle_prepare_lock_complete() {
422 CephContext
*cct
= m_image_ctx
->cct
;
423 ldout(cct
, 10) << __func__
<< dendl
;
426 if (m_state
!= STATE_PREPARING_LOCK
) {
431 complete_action_unlock(STATE_OPEN
, 0);
434 template <typename I
>
435 int ImageState
<I
>::register_update_watcher(UpdateWatchCtx
*watcher
,
437 CephContext
*cct
= m_image_ctx
->cct
;
438 ldout(cct
, 20) << __func__
<< dendl
;
440 m_update_watchers
->register_watcher(watcher
, handle
);
442 ldout(cct
, 20) << __func__
<< ": handle=" << *handle
<< dendl
;
446 template <typename I
>
447 void ImageState
<I
>::unregister_update_watcher(uint64_t handle
,
448 Context
*on_finish
) {
449 CephContext
*cct
= m_image_ctx
->cct
;
450 ldout(cct
, 20) << __func__
<< ": handle=" << handle
<< dendl
;
452 m_update_watchers
->unregister_watcher(handle
, on_finish
);
455 template <typename I
>
456 int ImageState
<I
>::unregister_update_watcher(uint64_t handle
) {
458 unregister_update_watcher(handle
, &ctx
);
462 template <typename I
>
463 void ImageState
<I
>::flush_update_watchers(Context
*on_finish
) {
464 CephContext
*cct
= m_image_ctx
->cct
;
465 ldout(cct
, 20) << __func__
<< dendl
;
467 m_update_watchers
->flush(on_finish
);
470 template <typename I
>
471 void ImageState
<I
>::shut_down_update_watchers(Context
*on_finish
) {
472 CephContext
*cct
= m_image_ctx
->cct
;
473 ldout(cct
, 20) << __func__
<< dendl
;
475 m_update_watchers
->shut_down(on_finish
);
478 template <typename I
>
479 bool ImageState
<I
>::is_transition_state() const {
481 case STATE_UNINITIALIZED
:
487 case STATE_REFRESHING
:
488 case STATE_SETTING_SNAP
:
489 case STATE_PREPARING_LOCK
:
495 template <typename I
>
496 bool ImageState
<I
>::is_closed() const {
497 ceph_assert(ceph_mutex_is_locked(m_lock
));
499 return ((m_state
== STATE_CLOSED
) ||
500 (!m_actions_contexts
.empty() &&
501 m_actions_contexts
.back().first
.action_type
== ACTION_TYPE_CLOSE
));
504 template <typename I
>
505 void ImageState
<I
>::append_context(const Action
&action
, Context
*context
) {
506 ceph_assert(ceph_mutex_is_locked(m_lock
));
508 ActionContexts
*action_contexts
= nullptr;
509 for (auto &action_ctxs
: m_actions_contexts
) {
510 if (action
== action_ctxs
.first
) {
511 action_contexts
= &action_ctxs
;
516 if (action_contexts
== nullptr) {
517 m_actions_contexts
.push_back({action
, {}});
518 action_contexts
= &m_actions_contexts
.back();
521 if (context
!= nullptr) {
522 action_contexts
->second
.push_back(context
);
526 template <typename I
>
527 void ImageState
<I
>::execute_next_action_unlock() {
528 ceph_assert(ceph_mutex_is_locked(m_lock
));
529 ceph_assert(!m_actions_contexts
.empty());
530 switch (m_actions_contexts
.front().first
.action_type
) {
531 case ACTION_TYPE_OPEN
:
534 case ACTION_TYPE_CLOSE
:
537 case ACTION_TYPE_REFRESH
:
538 send_refresh_unlock();
540 case ACTION_TYPE_SET_SNAP
:
541 send_set_snap_unlock();
543 case ACTION_TYPE_LOCK
:
544 send_prepare_lock_unlock();
550 template <typename I
>
551 void ImageState
<I
>::execute_action_unlock(const Action
&action
,
552 Context
*on_finish
) {
553 ceph_assert(ceph_mutex_is_locked(m_lock
));
555 append_context(action
, on_finish
);
556 if (!is_transition_state()) {
557 execute_next_action_unlock();
563 template <typename I
>
564 void ImageState
<I
>::complete_action_unlock(State next_state
, int r
) {
565 ceph_assert(ceph_mutex_is_locked(m_lock
));
566 ceph_assert(!m_actions_contexts
.empty());
568 ActionContexts
action_contexts(std::move(m_actions_contexts
.front()));
569 m_actions_contexts
.pop_front();
571 m_state
= next_state
;
574 for (auto ctx
: action_contexts
.second
) {
578 if (next_state
!= STATE_UNINITIALIZED
&& next_state
!= STATE_CLOSED
) {
580 if (!is_transition_state() && !m_actions_contexts
.empty()) {
581 execute_next_action_unlock();
588 template <typename I
>
589 void ImageState
<I
>::send_open_unlock() {
590 ceph_assert(ceph_mutex_is_locked(m_lock
));
591 CephContext
*cct
= m_image_ctx
->cct
;
592 ldout(cct
, 10) << this << " " << __func__
<< dendl
;
594 m_state
= STATE_OPENING
;
596 Context
*ctx
= create_async_context_callback(
597 *m_image_ctx
, create_context_callback
<
598 ImageState
<I
>, &ImageState
<I
>::handle_open
>(this));
599 image::OpenRequest
<I
> *req
= image::OpenRequest
<I
>::create(
600 m_image_ctx
, m_open_flags
, ctx
);
606 template <typename I
>
607 void ImageState
<I
>::handle_open(int r
) {
608 CephContext
*cct
= m_image_ctx
->cct
;
609 ldout(cct
, 10) << this << " " << __func__
<< ": r=" << r
<< dendl
;
611 if (r
< 0 && r
!= -ENOENT
) {
612 lderr(cct
) << "failed to open image: " << cpp_strerror(r
) << dendl
;
616 complete_action_unlock(r
< 0 ? STATE_UNINITIALIZED
: STATE_OPEN
, r
);
619 template <typename I
>
620 void ImageState
<I
>::send_close_unlock() {
621 ceph_assert(ceph_mutex_is_locked(m_lock
));
622 CephContext
*cct
= m_image_ctx
->cct
;
623 ldout(cct
, 10) << this << " " << __func__
<< dendl
;
625 m_state
= STATE_CLOSING
;
627 Context
*ctx
= create_context_callback
<
628 ImageState
<I
>, &ImageState
<I
>::handle_close
>(this);
629 image::CloseRequest
<I
> *req
= image::CloseRequest
<I
>::create(
636 template <typename I
>
637 void ImageState
<I
>::handle_close(int r
) {
638 CephContext
*cct
= m_image_ctx
->cct
;
639 ldout(cct
, 10) << this << " " << __func__
<< ": r=" << r
<< dendl
;
642 lderr(cct
) << "error occurred while closing image: " << cpp_strerror(r
)
647 complete_action_unlock(STATE_CLOSED
, r
);
650 template <typename I
>
651 void ImageState
<I
>::send_refresh_unlock() {
652 ceph_assert(ceph_mutex_is_locked(m_lock
));
653 CephContext
*cct
= m_image_ctx
->cct
;
654 ldout(cct
, 10) << this << " " << __func__
<< dendl
;
656 m_state
= STATE_REFRESHING
;
657 ceph_assert(!m_actions_contexts
.empty());
658 auto &action_context
= m_actions_contexts
.front().first
;
659 ceph_assert(action_context
.action_type
== ACTION_TYPE_REFRESH
);
661 Context
*ctx
= create_async_context_callback(
662 *m_image_ctx
, create_context_callback
<
663 ImageState
<I
>, &ImageState
<I
>::handle_refresh
>(this));
664 image::RefreshRequest
<I
> *req
= image::RefreshRequest
<I
>::create(
665 *m_image_ctx
, false, false, ctx
);
671 template <typename I
>
672 void ImageState
<I
>::handle_refresh(int r
) {
673 CephContext
*cct
= m_image_ctx
->cct
;
674 ldout(cct
, 10) << this << " " << __func__
<< ": r=" << r
<< dendl
;
677 ceph_assert(!m_actions_contexts
.empty());
679 ActionContexts
&action_contexts(m_actions_contexts
.front());
680 ceph_assert(action_contexts
.first
.action_type
== ACTION_TYPE_REFRESH
);
681 ceph_assert(m_last_refresh
<= action_contexts
.first
.refresh_seq
);
683 if (r
== -ERESTART
) {
684 ldout(cct
, 5) << "incomplete refresh: not updating sequence" << dendl
;
687 m_last_refresh
= action_contexts
.first
.refresh_seq
;
690 complete_action_unlock(STATE_OPEN
, r
);
693 template <typename I
>
694 void ImageState
<I
>::send_set_snap_unlock() {
695 ceph_assert(ceph_mutex_is_locked(m_lock
));
697 m_state
= STATE_SETTING_SNAP
;
699 ceph_assert(!m_actions_contexts
.empty());
700 ActionContexts
&action_contexts(m_actions_contexts
.front());
701 ceph_assert(action_contexts
.first
.action_type
== ACTION_TYPE_SET_SNAP
);
703 CephContext
*cct
= m_image_ctx
->cct
;
704 ldout(cct
, 10) << this << " " << __func__
<< ": "
705 << "snap_id=" << action_contexts
.first
.snap_id
<< dendl
;
707 Context
*ctx
= create_async_context_callback(
708 *m_image_ctx
, create_context_callback
<
709 ImageState
<I
>, &ImageState
<I
>::handle_set_snap
>(this));
710 image::SetSnapRequest
<I
> *req
= image::SetSnapRequest
<I
>::create(
711 *m_image_ctx
, action_contexts
.first
.snap_id
, ctx
);
717 template <typename I
>
718 void ImageState
<I
>::handle_set_snap(int r
) {
719 CephContext
*cct
= m_image_ctx
->cct
;
720 ldout(cct
, 10) << this << " " << __func__
<< " r=" << r
<< dendl
;
722 if (r
< 0 && r
!= -ENOENT
) {
723 lderr(cct
) << "failed to set snapshot: " << cpp_strerror(r
) << dendl
;
727 complete_action_unlock(STATE_OPEN
, r
);
730 template <typename I
>
731 void ImageState
<I
>::send_prepare_lock_unlock() {
732 CephContext
*cct
= m_image_ctx
->cct
;
733 ldout(cct
, 10) << this << " " << __func__
<< dendl
;
735 ceph_assert(ceph_mutex_is_locked(m_lock
));
736 m_state
= STATE_PREPARING_LOCK
;
738 ceph_assert(!m_actions_contexts
.empty());
739 ActionContexts
&action_contexts(m_actions_contexts
.front());
740 ceph_assert(action_contexts
.first
.action_type
== ACTION_TYPE_LOCK
);
742 Context
*on_ready
= action_contexts
.first
.on_ready
;
745 if (on_ready
== nullptr) {
746 complete_action_unlock(STATE_OPEN
, 0);
750 // wake up the lock handler now that its safe to proceed
751 on_ready
->complete(0);
754 } // namespace librbd
756 template class librbd::ImageState
<librbd::ImageCtx
>;