]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #ifndef CEPH_RBD_MIRROR_LEADER_WATCHER_H | |
5 | #define CEPH_RBD_MIRROR_LEADER_WATCHER_H | |
6 | ||
7 | #include <list> | |
8 | #include <memory> | |
9 | #include <string> | |
10 | ||
11 | #include "common/AsyncOpTracker.h" | |
12 | #include "librbd/ManagedLock.h" | |
13 | #include "librbd/Watcher.h" | |
14 | #include "librbd/managed_lock/Types.h" | |
15 | #include "librbd/watcher/Types.h" | |
16 | #include "Instances.h" | |
17 | #include "MirrorStatusWatcher.h" | |
18 | #include "tools/rbd_mirror/leader_watcher/Types.h" | |
19 | ||
20 | namespace librbd { class ImageCtx; } | |
21 | ||
22 | namespace rbd { | |
23 | namespace mirror { | |
24 | ||
25 | template <typename> struct Threads; | |
26 | ||
27 | template <typename ImageCtxT = librbd::ImageCtx> | |
28 | class LeaderWatcher : protected librbd::Watcher { | |
29 | public: | |
30 | struct Listener { | |
31 | virtual ~Listener() { | |
32 | } | |
33 | ||
34 | virtual void post_acquire_handler(Context *on_finish) = 0; | |
35 | virtual void pre_release_handler(Context *on_finish) = 0; | |
36 | }; | |
37 | ||
38 | LeaderWatcher(Threads<ImageCtxT> *threads, librados::IoCtx &io_ctx, | |
39 | Listener *listener); | |
40 | ~LeaderWatcher() override; | |
41 | ||
42 | int init(); | |
43 | void shut_down(); | |
44 | ||
45 | void init(Context *on_finish); | |
46 | void shut_down(Context *on_finish); | |
47 | ||
48 | bool is_leader() const; | |
49 | bool is_releasing_leader() const; | |
50 | bool get_leader_instance_id(std::string *instance_id) const; | |
51 | void release_leader(); | |
52 | void list_instances(std::vector<std::string> *instance_ids); | |
53 | ||
54 | private: | |
55 | /** | |
56 | * @verbatim | |
57 | * | |
58 | * <uninitialized> <------------------------------ WAIT_FOR_TASKS | |
59 | * | (init) ^ ^ | |
60 | * v * | | |
61 | * CREATE_OBJECT * * (error) UNREGISTER_WATCH | |
62 | * | * ^ | |
63 | * v * | | |
64 | * REGISTER_WATCH * * SHUT_DOWN_LEADER_LOCK | |
65 | * | ^ | |
66 | * | (no leader heartbeat and acquire failed) | | |
67 | * | BREAK_LOCK <-------------------------------------\ | | |
68 | * | | (no leader heartbeat) | | (shut down) | |
69 | * | | /----------------------------------------\ | | | |
70 | * | | | (lock_released received) | | | |
71 | * | | | /-------------------------------------\ | | | |
72 | * | | | | (lock_acquired or | | | | |
73 | * | | | | heartbeat received) | | | | |
74 | * | | | | (ENOENT) /-----------\ | | | | |
75 | * | | | | * * * * * * * * * * | | | | | | |
76 | * v v v v v (error) * v | | | | | |
77 | * ACQUIRE_LEADER_LOCK * * * * *> GET_LOCKER ---> <secondary> | |
78 | * | * ^ | |
79 | * ....|...................*.................... .....|..................... | |
80 | * . v * . . | post_release . | |
81 | * .INIT_STATUS_WATCHER * * (error) . .NOTIFY_LOCK_RELEASED . | |
82 | * . | ^ . .....^..................... | |
83 | * . v (error) | . | | |
84 | * .INIT_INSTANCES *> SHUT_DOWN_STATUS_WATCHER . RELEASE_LEADER_LOCK | |
85 | * . | . ^ | |
86 | * . v . .....|..................... | |
87 | * .NOTIFY_LISTENER . .SHUT_DOWN_STATUS_WATCHER . | |
88 | * . | . . ^ . | |
89 | * . v . . | . | |
90 | * .NOTIFY_LOCK_ACQUIRED post_acquire . .SHUT_DOWN_INSTANCES . | |
91 | * ....|........................................ . ^ . | |
92 | * v . | . | |
93 | * <leader> -----------------------------------> .NOTIFY_LISTENER . | |
94 | * (shut_down, release_leader, . pre_release . | |
95 | * notify error) ........................... | |
96 | * @endverbatim | |
97 | */ | |
98 | ||
99 | class LeaderLock : public librbd::ManagedLock<ImageCtxT> { | |
100 | public: | |
101 | typedef librbd::ManagedLock<ImageCtxT> Parent; | |
102 | ||
103 | LeaderLock(librados::IoCtx& ioctx, ContextWQ *work_queue, | |
104 | const std::string& oid, LeaderWatcher *watcher, | |
105 | bool blacklist_on_break_lock, | |
106 | uint32_t blacklist_expire_seconds) | |
107 | : Parent(ioctx, work_queue, oid, watcher, librbd::managed_lock::EXCLUSIVE, | |
108 | blacklist_on_break_lock, blacklist_expire_seconds), | |
109 | watcher(watcher) { | |
110 | } | |
111 | ||
112 | bool is_leader() const { | |
113 | Mutex::Locker locker(Parent::m_lock); | |
114 | return Parent::is_state_post_acquiring() || Parent::is_state_locked(); | |
115 | } | |
116 | ||
117 | bool is_releasing_leader() const { | |
118 | Mutex::Locker locker(Parent::m_lock); | |
119 | return Parent::is_state_pre_releasing(); | |
120 | } | |
121 | ||
122 | protected: | |
123 | void post_acquire_lock_handler(int r, Context *on_finish) { | |
124 | if (r == 0) { | |
125 | // lock is owned at this point | |
126 | Mutex::Locker locker(Parent::m_lock); | |
127 | Parent::set_state_post_acquiring(); | |
128 | } | |
129 | watcher->handle_post_acquire_leader_lock(r, on_finish); | |
130 | } | |
131 | void pre_release_lock_handler(bool shutting_down, | |
132 | Context *on_finish) { | |
133 | watcher->handle_pre_release_leader_lock(on_finish); | |
134 | } | |
135 | void post_release_lock_handler(bool shutting_down, int r, | |
136 | Context *on_finish) { | |
137 | watcher->handle_post_release_leader_lock(r, on_finish); | |
138 | } | |
139 | private: | |
140 | LeaderWatcher *watcher; | |
141 | }; | |
142 | ||
143 | struct HandlePayloadVisitor : public boost::static_visitor<void> { | |
144 | LeaderWatcher *leader_watcher; | |
145 | Context *on_notify_ack; | |
146 | ||
147 | HandlePayloadVisitor(LeaderWatcher *leader_watcher, Context *on_notify_ack) | |
148 | : leader_watcher(leader_watcher), on_notify_ack(on_notify_ack) { | |
149 | } | |
150 | ||
151 | template <typename Payload> | |
152 | inline void operator()(const Payload &payload) const { | |
153 | leader_watcher->handle_payload(payload, on_notify_ack); | |
154 | } | |
155 | }; | |
156 | ||
157 | struct C_GetLocker : public Context { | |
158 | LeaderWatcher *leader_watcher; | |
159 | librbd::managed_lock::Locker locker; | |
160 | ||
161 | C_GetLocker(LeaderWatcher *leader_watcher) | |
162 | : leader_watcher(leader_watcher) { | |
163 | } | |
164 | ||
165 | void finish(int r) override { | |
166 | leader_watcher->handle_get_locker(r, locker); | |
167 | } | |
168 | }; | |
169 | ||
170 | typedef void (LeaderWatcher<ImageCtxT>::*TimerCallback)(); | |
171 | ||
172 | struct C_TimerGate : public Context { | |
173 | LeaderWatcher *leader_watcher; | |
174 | ||
175 | bool leader = false; | |
176 | TimerCallback timer_callback = nullptr; | |
177 | ||
178 | C_TimerGate(LeaderWatcher *leader_watcher) | |
179 | : leader_watcher(leader_watcher) { | |
180 | } | |
181 | ||
182 | void finish(int r) override { | |
183 | leader_watcher->m_timer_gate = nullptr; | |
184 | leader_watcher->execute_timer_task(leader, timer_callback); | |
185 | } | |
186 | }; | |
187 | ||
188 | Threads<ImageCtxT> *m_threads; | |
189 | Listener *m_listener; | |
190 | ||
191 | mutable Mutex m_lock; | |
192 | uint64_t m_notifier_id; | |
193 | LeaderLock *m_leader_lock; | |
194 | Context *m_on_finish = nullptr; | |
195 | Context *m_on_shut_down_finish = nullptr; | |
196 | int m_acquire_attempts = 0; | |
197 | int m_ret_val = 0; | |
198 | MirrorStatusWatcher<ImageCtxT> *m_status_watcher = nullptr; | |
199 | Instances<ImageCtxT> *m_instances = nullptr; | |
200 | librbd::managed_lock::Locker m_locker; | |
201 | ||
202 | AsyncOpTracker m_timer_op_tracker; | |
203 | Context *m_timer_task = nullptr; | |
204 | C_TimerGate *m_timer_gate = nullptr; | |
205 | ||
206 | librbd::watcher::NotifyResponse m_heartbeat_response; | |
207 | ||
208 | bool is_leader(Mutex &m_lock) const; | |
209 | bool is_releasing_leader(Mutex &m_lock) const; | |
210 | ||
211 | void cancel_timer_task(); | |
212 | void schedule_timer_task(const std::string &name, | |
213 | int delay_factor, bool leader, | |
214 | TimerCallback callback, bool shutting_down); | |
215 | void execute_timer_task(bool leader, TimerCallback timer_callback); | |
216 | ||
217 | void create_leader_object(); | |
218 | void handle_create_leader_object(int r); | |
219 | ||
220 | void register_watch(); | |
221 | void handle_register_watch(int r); | |
222 | ||
223 | void shut_down_leader_lock(); | |
224 | void handle_shut_down_leader_lock(int r); | |
225 | ||
226 | void unregister_watch(); | |
227 | void handle_unregister_watch(int r); | |
228 | ||
229 | void wait_for_tasks(); | |
230 | void handle_wait_for_tasks(); | |
231 | ||
232 | void break_leader_lock(); | |
233 | void handle_break_leader_lock(int r); | |
234 | ||
235 | void schedule_get_locker(bool reset_leader, uint32_t delay_factor); | |
236 | void get_locker(); | |
237 | void handle_get_locker(int r, librbd::managed_lock::Locker& locker); | |
238 | ||
239 | void schedule_acquire_leader_lock(uint32_t delay_factor); | |
240 | void acquire_leader_lock(); | |
241 | void handle_acquire_leader_lock(int r); | |
242 | ||
243 | void release_leader_lock(); | |
244 | void handle_release_leader_lock(int r); | |
245 | ||
246 | void init_status_watcher(); | |
247 | void handle_init_status_watcher(int r); | |
248 | ||
249 | void shut_down_status_watcher(); | |
250 | void handle_shut_down_status_watcher(int r); | |
251 | ||
252 | void init_instances(); | |
253 | void handle_init_instances(int r); | |
254 | ||
255 | void shut_down_instances(); | |
256 | void handle_shut_down_instances(int r); | |
257 | ||
258 | void notify_listener(); | |
259 | void handle_notify_listener(int r); | |
260 | ||
261 | void notify_lock_acquired(); | |
262 | void handle_notify_lock_acquired(int r); | |
263 | ||
264 | void notify_lock_released(); | |
265 | void handle_notify_lock_released(int r); | |
266 | ||
267 | void notify_heartbeat(); | |
268 | void handle_notify_heartbeat(int r); | |
269 | ||
270 | void get_instances(); | |
271 | void handle_get_instances(int r); | |
272 | ||
273 | void handle_post_acquire_leader_lock(int r, Context *on_finish); | |
274 | void handle_pre_release_leader_lock(Context *on_finish); | |
275 | void handle_post_release_leader_lock(int r, Context *on_finish); | |
276 | ||
277 | void handle_notify(uint64_t notify_id, uint64_t handle, | |
278 | uint64_t notifier_id, bufferlist &bl) override; | |
279 | ||
280 | void handle_heartbeat(Context *on_ack); | |
281 | void handle_lock_acquired(Context *on_ack); | |
282 | void handle_lock_released(Context *on_ack); | |
283 | ||
284 | void handle_payload(const leader_watcher::HeartbeatPayload &payload, | |
285 | Context *on_notify_ack); | |
286 | void handle_payload(const leader_watcher::LockAcquiredPayload &payload, | |
287 | Context *on_notify_ack); | |
288 | void handle_payload(const leader_watcher::LockReleasedPayload &payload, | |
289 | Context *on_notify_ack); | |
290 | void handle_payload(const leader_watcher::UnknownPayload &payload, | |
291 | Context *on_notify_ack); | |
292 | }; | |
293 | ||
294 | } // namespace mirror | |
295 | } // namespace rbd | |
296 | ||
297 | #endif // CEPH_RBD_MIRROR_LEADER_WATCHER_H |