1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #ifndef CEPH_RBD_MIRROR_LEADER_WATCHER_H
5 #define CEPH_RBD_MIRROR_LEADER_WATCHER_H
11 #include "common/AsyncOpTracker.h"
12 #include "librbd/ManagedLock.h"
13 #include "librbd/Watcher.h"
14 #include "librbd/managed_lock/Types.h"
15 #include "librbd/watcher/Types.h"
16 #include "Instances.h"
17 #include "tools/rbd_mirror/instances/Types.h"
18 #include "tools/rbd_mirror/leader_watcher/Types.h"
22 namespace asio
{ struct ContextWQ
; }
28 template <typename
> struct Threads
;
30 template <typename ImageCtxT
= librbd::ImageCtx
>
31 class LeaderWatcher
: protected librbd::Watcher
{
32 using librbd::Watcher::unregister_watch
; // Silence overloaded virtual warning
34 static LeaderWatcher
* create(Threads
<ImageCtxT
> *threads
,
35 librados::IoCtx
&io_ctx
,
36 leader_watcher::Listener
*listener
) {
37 return new LeaderWatcher(threads
, io_ctx
, listener
);
40 LeaderWatcher(Threads
<ImageCtxT
> *threads
, librados::IoCtx
&io_ctx
,
41 leader_watcher::Listener
*listener
);
42 ~LeaderWatcher() override
;
47 void init(Context
*on_finish
);
48 void shut_down(Context
*on_finish
);
50 bool is_blocklisted() const;
51 bool is_leader() const;
52 bool is_releasing_leader() const;
53 bool get_leader_instance_id(std::string
*instance_id
) const;
54 void release_leader();
55 void list_instances(std::vector
<std::string
> *instance_ids
);
57 std::string
get_instance_id();
63 * <uninitialized> <------------------------------ WAIT_FOR_TASKS
66 * CREATE_OBJECT * * * * * (error) UNREGISTER_WATCH
69 * REGISTER_WATCH * * * * * SHUT_DOWN_LEADER_LOCK
71 * | (no leader heartbeat and acquire failed) |
72 * | BREAK_LOCK <-------------------------------------\ |
73 * | | (no leader heartbeat) | | (shut down)
74 * | | /----------------------------------------\ | |
75 * | | | (lock_released received) | |
76 * | | | /-------------------------------------\ | |
77 * | | | | (lock_acquired or | | |
78 * | | | | heartbeat received) | | |
79 * | | | | (ENOENT) /-----------\ | | |
80 * | | | | * * * * * * * * * * | | | | |
81 * v v v v v (error) * v | | | |
82 * ACQUIRE_LEADER_LOCK * * * * *> GET_LOCKER ---> <secondary>
84 * ....|...................*.................... .....|.....................
85 * . v * . . | post_release .
86 * .INIT_INSTANCES * * * * * . .NOTIFY_LOCK_RELEASED .
87 * . | . .....^.....................
89 * .NOTIFY_LISTENER . RELEASE_LEADER_LOCK
91 * . v . .....|.....................
92 * .NOTIFY_LOCK_ACQUIRED . . | .
93 * . | post_acquire . .SHUT_DOWN_INSTANCES .
94 * ....|........................................ . ^ .
96 * <leader> -----------------------------------> .NOTIFY_LISTENER .
97 * (shut_down, release_leader, . pre_release .
98 * notify error) ...........................
102 struct InstancesListener
: public instances::Listener
{
103 LeaderWatcher
* leader_watcher
;
105 InstancesListener(LeaderWatcher
* leader_watcher
)
106 : leader_watcher(leader_watcher
) {
109 void handle_added(const InstanceIds
& instance_ids
) override
{
110 leader_watcher
->m_listener
->handle_instances_added(instance_ids
);
113 void handle_removed(const InstanceIds
& instance_ids
) override
{
114 leader_watcher
->m_listener
->handle_instances_removed(instance_ids
);
118 class LeaderLock
: public librbd::ManagedLock
<ImageCtxT
> {
120 typedef librbd::ManagedLock
<ImageCtxT
> Parent
;
122 LeaderLock(librados::IoCtx
& ioctx
, librbd::AsioEngine
& asio_engine
,
123 const std::string
& oid
, LeaderWatcher
*watcher
,
124 bool blocklist_on_break_lock
,
125 uint32_t blocklist_expire_seconds
)
126 : Parent(ioctx
, asio_engine
, oid
, watcher
,
127 librbd::managed_lock::EXCLUSIVE
, blocklist_on_break_lock
,
128 blocklist_expire_seconds
),
132 bool is_leader() const {
133 std::lock_guard locker
{Parent::m_lock
};
134 return Parent::is_state_post_acquiring() || Parent::is_state_locked();
137 bool is_releasing_leader() const {
138 std::lock_guard locker
{Parent::m_lock
};
139 return Parent::is_state_pre_releasing();
143 void post_acquire_lock_handler(int r
, Context
*on_finish
) {
145 // lock is owned at this point
146 std::lock_guard locker
{Parent::m_lock
};
147 Parent::set_state_post_acquiring();
149 watcher
->handle_post_acquire_leader_lock(r
, on_finish
);
151 void pre_release_lock_handler(bool shutting_down
,
152 Context
*on_finish
) {
153 watcher
->handle_pre_release_leader_lock(on_finish
);
155 void post_release_lock_handler(bool shutting_down
, int r
,
156 Context
*on_finish
) {
157 watcher
->handle_post_release_leader_lock(r
, on_finish
);
160 LeaderWatcher
*watcher
;
163 struct HandlePayloadVisitor
: public boost::static_visitor
<void> {
164 LeaderWatcher
*leader_watcher
;
165 Context
*on_notify_ack
;
167 HandlePayloadVisitor(LeaderWatcher
*leader_watcher
, Context
*on_notify_ack
)
168 : leader_watcher(leader_watcher
), on_notify_ack(on_notify_ack
) {
171 template <typename Payload
>
172 inline void operator()(const Payload
&payload
) const {
173 leader_watcher
->handle_payload(payload
, on_notify_ack
);
177 struct C_GetLocker
: public Context
{
178 LeaderWatcher
*leader_watcher
;
179 librbd::managed_lock::Locker locker
;
181 C_GetLocker(LeaderWatcher
*leader_watcher
)
182 : leader_watcher(leader_watcher
) {
185 void finish(int r
) override
{
186 leader_watcher
->handle_get_locker(r
, locker
);
190 typedef void (LeaderWatcher
<ImageCtxT
>::*TimerCallback
)();
192 struct C_TimerGate
: public Context
{
193 LeaderWatcher
*leader_watcher
;
196 TimerCallback timer_callback
= nullptr;
198 C_TimerGate(LeaderWatcher
*leader_watcher
)
199 : leader_watcher(leader_watcher
) {
202 void finish(int r
) override
{
203 leader_watcher
->m_timer_gate
= nullptr;
204 leader_watcher
->execute_timer_task(leader
, timer_callback
);
208 Threads
<ImageCtxT
> *m_threads
;
209 leader_watcher::Listener
*m_listener
;
211 InstancesListener m_instances_listener
;
212 mutable ceph::mutex m_lock
;
213 uint64_t m_notifier_id
;
214 std::string m_instance_id
;
215 LeaderLock
*m_leader_lock
;
216 Context
*m_on_finish
= nullptr;
217 Context
*m_on_shut_down_finish
= nullptr;
218 uint64_t m_acquire_attempts
= 0;
220 Instances
<ImageCtxT
> *m_instances
= nullptr;
221 librbd::managed_lock::Locker m_locker
;
223 bool m_blocklisted
= false;
225 AsyncOpTracker m_timer_op_tracker
;
226 Context
*m_timer_task
= nullptr;
227 C_TimerGate
*m_timer_gate
= nullptr;
229 librbd::watcher::NotifyResponse m_heartbeat_response
;
231 bool is_leader(ceph::mutex
&m_lock
) const;
232 bool is_releasing_leader(ceph::mutex
&m_lock
) const;
234 void cancel_timer_task();
235 void schedule_timer_task(const std::string
&name
,
236 int delay_factor
, bool leader
,
237 TimerCallback callback
, bool shutting_down
);
238 void execute_timer_task(bool leader
, TimerCallback timer_callback
);
240 void create_leader_object();
241 void handle_create_leader_object(int r
);
243 void register_watch();
244 void handle_register_watch(int r
);
246 void shut_down_leader_lock();
247 void handle_shut_down_leader_lock(int r
);
249 void unregister_watch();
250 void handle_unregister_watch(int r
);
252 void wait_for_tasks();
253 void handle_wait_for_tasks();
255 void break_leader_lock();
256 void handle_break_leader_lock(int r
);
258 void schedule_get_locker(bool reset_leader
, uint32_t delay_factor
);
260 void handle_get_locker(int r
, librbd::managed_lock::Locker
& locker
);
262 void schedule_acquire_leader_lock(uint32_t delay_factor
);
263 void acquire_leader_lock();
264 void handle_acquire_leader_lock(int r
);
266 void release_leader_lock();
267 void handle_release_leader_lock(int r
);
269 void init_instances();
270 void handle_init_instances(int r
);
272 void shut_down_instances();
273 void handle_shut_down_instances(int r
);
275 void notify_listener();
276 void handle_notify_listener(int r
);
278 void notify_lock_acquired();
279 void handle_notify_lock_acquired(int r
);
281 void notify_lock_released();
282 void handle_notify_lock_released(int r
);
284 void notify_heartbeat();
285 void handle_notify_heartbeat(int r
);
287 void handle_post_acquire_leader_lock(int r
, Context
*on_finish
);
288 void handle_pre_release_leader_lock(Context
*on_finish
);
289 void handle_post_release_leader_lock(int r
, Context
*on_finish
);
291 void handle_notify(uint64_t notify_id
, uint64_t handle
,
292 uint64_t notifier_id
, bufferlist
&bl
) override
;
294 void handle_rewatch_complete(int r
) override
;
296 void handle_heartbeat(Context
*on_ack
);
297 void handle_lock_acquired(Context
*on_ack
);
298 void handle_lock_released(Context
*on_ack
);
300 void handle_payload(const leader_watcher::HeartbeatPayload
&payload
,
301 Context
*on_notify_ack
);
302 void handle_payload(const leader_watcher::LockAcquiredPayload
&payload
,
303 Context
*on_notify_ack
);
304 void handle_payload(const leader_watcher::LockReleasedPayload
&payload
,
305 Context
*on_notify_ack
);
306 void handle_payload(const leader_watcher::UnknownPayload
&payload
,
307 Context
*on_notify_ack
);
310 } // namespace mirror
313 #endif // CEPH_RBD_MIRROR_LEADER_WATCHER_H