1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #ifndef CEPH_RBD_MIRROR_LEADER_WATCHER_H
5 #define CEPH_RBD_MIRROR_LEADER_WATCHER_H
11 #include "common/AsyncOpTracker.h"
12 #include "librbd/ManagedLock.h"
13 #include "librbd/Watcher.h"
14 #include "librbd/managed_lock/Types.h"
15 #include "librbd/watcher/Types.h"
16 #include "Instances.h"
17 #include "MirrorStatusWatcher.h"
18 #include "tools/rbd_mirror/leader_watcher/Types.h"
20 namespace librbd
{ class ImageCtx
; }
25 template <typename
> struct Threads
;
27 template <typename ImageCtxT
= librbd::ImageCtx
>
28 class LeaderWatcher
: protected librbd::Watcher
{
34 virtual void post_acquire_handler(Context
*on_finish
) = 0;
35 virtual void pre_release_handler(Context
*on_finish
) = 0;
37 virtual void update_leader_handler(
38 const std::string
&leader_instance_id
) = 0;
41 LeaderWatcher(Threads
<ImageCtxT
> *threads
, librados::IoCtx
&io_ctx
,
43 ~LeaderWatcher() override
;
48 void init(Context
*on_finish
);
49 void shut_down(Context
*on_finish
);
51 bool is_leader() const;
52 bool is_releasing_leader() const;
53 bool get_leader_instance_id(std::string
*instance_id
) const;
54 void release_leader();
55 void list_instances(std::vector
<std::string
> *instance_ids
);
57 std::string
get_instance_id();
63 * <uninitialized> <------------------------------ WAIT_FOR_TASKS
66 * CREATE_OBJECT * * (error) UNREGISTER_WATCH
69 * REGISTER_WATCH * * SHUT_DOWN_LEADER_LOCK
71 * | (no leader heartbeat and acquire failed) |
72 * | BREAK_LOCK <-------------------------------------\ |
73 * | | (no leader heartbeat) | | (shut down)
74 * | | /----------------------------------------\ | |
75 * | | | (lock_released received) | |
76 * | | | /-------------------------------------\ | |
77 * | | | | (lock_acquired or | | |
78 * | | | | heartbeat received) | | |
79 * | | | | (ENOENT) /-----------\ | | |
80 * | | | | * * * * * * * * * * | | | | |
81 * v v v v v (error) * v | | | |
82 * ACQUIRE_LEADER_LOCK * * * * *> GET_LOCKER ---> <secondary>
84 * ....|...................*.................... .....|.....................
85 * . v * . . | post_release .
86 * .INIT_STATUS_WATCHER * * (error) . .NOTIFY_LOCK_RELEASED .
87 * . | ^ . .....^.....................
89 * .INIT_INSTANCES *> SHUT_DOWN_STATUS_WATCHER . RELEASE_LEADER_LOCK
91 * . v . .....|.....................
92 * .NOTIFY_LISTENER . .SHUT_DOWN_STATUS_WATCHER .
95 * .NOTIFY_LOCK_ACQUIRED post_acquire . .SHUT_DOWN_INSTANCES .
96 * ....|........................................ . ^ .
98 * <leader> -----------------------------------> .NOTIFY_LISTENER .
99 * (shut_down, release_leader, . pre_release .
100 * notify error) ...........................
104 class LeaderLock
: public librbd::ManagedLock
<ImageCtxT
> {
106 typedef librbd::ManagedLock
<ImageCtxT
> Parent
;
108 LeaderLock(librados::IoCtx
& ioctx
, ContextWQ
*work_queue
,
109 const std::string
& oid
, LeaderWatcher
*watcher
,
110 bool blacklist_on_break_lock
,
111 uint32_t blacklist_expire_seconds
)
112 : Parent(ioctx
, work_queue
, oid
, watcher
, librbd::managed_lock::EXCLUSIVE
,
113 blacklist_on_break_lock
, blacklist_expire_seconds
),
117 bool is_leader() const {
118 Mutex::Locker
locker(Parent::m_lock
);
119 return Parent::is_state_post_acquiring() || Parent::is_state_locked();
122 bool is_releasing_leader() const {
123 Mutex::Locker
locker(Parent::m_lock
);
124 return Parent::is_state_pre_releasing();
128 void post_acquire_lock_handler(int r
, Context
*on_finish
) {
130 // lock is owned at this point
131 Mutex::Locker
locker(Parent::m_lock
);
132 Parent::set_state_post_acquiring();
134 watcher
->handle_post_acquire_leader_lock(r
, on_finish
);
136 void pre_release_lock_handler(bool shutting_down
,
137 Context
*on_finish
) {
138 watcher
->handle_pre_release_leader_lock(on_finish
);
140 void post_release_lock_handler(bool shutting_down
, int r
,
141 Context
*on_finish
) {
142 watcher
->handle_post_release_leader_lock(r
, on_finish
);
145 LeaderWatcher
*watcher
;
148 struct HandlePayloadVisitor
: public boost::static_visitor
<void> {
149 LeaderWatcher
*leader_watcher
;
150 Context
*on_notify_ack
;
152 HandlePayloadVisitor(LeaderWatcher
*leader_watcher
, Context
*on_notify_ack
)
153 : leader_watcher(leader_watcher
), on_notify_ack(on_notify_ack
) {
156 template <typename Payload
>
157 inline void operator()(const Payload
&payload
) const {
158 leader_watcher
->handle_payload(payload
, on_notify_ack
);
162 struct C_GetLocker
: public Context
{
163 LeaderWatcher
*leader_watcher
;
164 librbd::managed_lock::Locker locker
;
166 C_GetLocker(LeaderWatcher
*leader_watcher
)
167 : leader_watcher(leader_watcher
) {
170 void finish(int r
) override
{
171 leader_watcher
->handle_get_locker(r
, locker
);
175 typedef void (LeaderWatcher
<ImageCtxT
>::*TimerCallback
)();
177 struct C_TimerGate
: public Context
{
178 LeaderWatcher
*leader_watcher
;
181 TimerCallback timer_callback
= nullptr;
183 C_TimerGate(LeaderWatcher
*leader_watcher
)
184 : leader_watcher(leader_watcher
) {
187 void finish(int r
) override
{
188 leader_watcher
->m_timer_gate
= nullptr;
189 leader_watcher
->execute_timer_task(leader
, timer_callback
);
193 Threads
<ImageCtxT
> *m_threads
;
194 Listener
*m_listener
;
196 mutable Mutex m_lock
;
197 uint64_t m_notifier_id
;
198 LeaderLock
*m_leader_lock
;
199 Context
*m_on_finish
= nullptr;
200 Context
*m_on_shut_down_finish
= nullptr;
201 int m_acquire_attempts
= 0;
203 MirrorStatusWatcher
<ImageCtxT
> *m_status_watcher
= nullptr;
204 Instances
<ImageCtxT
> *m_instances
= nullptr;
205 librbd::managed_lock::Locker m_locker
;
207 AsyncOpTracker m_timer_op_tracker
;
208 Context
*m_timer_task
= nullptr;
209 C_TimerGate
*m_timer_gate
= nullptr;
211 librbd::watcher::NotifyResponse m_heartbeat_response
;
213 bool is_leader(Mutex
&m_lock
) const;
214 bool is_releasing_leader(Mutex
&m_lock
) const;
216 void cancel_timer_task();
217 void schedule_timer_task(const std::string
&name
,
218 int delay_factor
, bool leader
,
219 TimerCallback callback
, bool shutting_down
);
220 void execute_timer_task(bool leader
, TimerCallback timer_callback
);
222 void create_leader_object();
223 void handle_create_leader_object(int r
);
225 void register_watch();
226 void handle_register_watch(int r
);
228 void shut_down_leader_lock();
229 void handle_shut_down_leader_lock(int r
);
231 void unregister_watch();
232 void handle_unregister_watch(int r
);
234 void wait_for_tasks();
235 void handle_wait_for_tasks();
237 void break_leader_lock();
238 void handle_break_leader_lock(int r
);
240 void schedule_get_locker(bool reset_leader
, uint32_t delay_factor
);
242 void handle_get_locker(int r
, librbd::managed_lock::Locker
& locker
);
244 void schedule_acquire_leader_lock(uint32_t delay_factor
);
245 void acquire_leader_lock();
246 void handle_acquire_leader_lock(int r
);
248 void release_leader_lock();
249 void handle_release_leader_lock(int r
);
251 void init_status_watcher();
252 void handle_init_status_watcher(int r
);
254 void shut_down_status_watcher();
255 void handle_shut_down_status_watcher(int r
);
257 void init_instances();
258 void handle_init_instances(int r
);
260 void shut_down_instances();
261 void handle_shut_down_instances(int r
);
263 void notify_listener();
264 void handle_notify_listener(int r
);
266 void notify_lock_acquired();
267 void handle_notify_lock_acquired(int r
);
269 void notify_lock_released();
270 void handle_notify_lock_released(int r
);
272 void notify_heartbeat();
273 void handle_notify_heartbeat(int r
);
275 void get_instances();
276 void handle_get_instances(int r
);
278 void handle_post_acquire_leader_lock(int r
, Context
*on_finish
);
279 void handle_pre_release_leader_lock(Context
*on_finish
);
280 void handle_post_release_leader_lock(int r
, Context
*on_finish
);
282 void handle_notify(uint64_t notify_id
, uint64_t handle
,
283 uint64_t notifier_id
, bufferlist
&bl
) override
;
285 void handle_heartbeat(Context
*on_ack
);
286 void handle_lock_acquired(Context
*on_ack
);
287 void handle_lock_released(Context
*on_ack
);
289 void handle_payload(const leader_watcher::HeartbeatPayload
&payload
,
290 Context
*on_notify_ack
);
291 void handle_payload(const leader_watcher::LockAcquiredPayload
&payload
,
292 Context
*on_notify_ack
);
293 void handle_payload(const leader_watcher::LockReleasedPayload
&payload
,
294 Context
*on_notify_ack
);
295 void handle_payload(const leader_watcher::UnknownPayload
&payload
,
296 Context
*on_notify_ack
);
299 } // namespace mirror
302 #endif // CEPH_RBD_MIRROR_LEADER_WATCHER_H