]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #ifndef CEPH_RBD_MIRROR_LEADER_WATCHER_H | |
5 | #define CEPH_RBD_MIRROR_LEADER_WATCHER_H | |
6 | ||
7 | #include <list> | |
8 | #include <memory> | |
9 | #include <string> | |
10 | ||
11 | #include "common/AsyncOpTracker.h" | |
12 | #include "librbd/ManagedLock.h" | |
13 | #include "librbd/Watcher.h" | |
14 | #include "librbd/managed_lock/Types.h" | |
15 | #include "librbd/watcher/Types.h" | |
16 | #include "Instances.h" | |
17 | #include "MirrorStatusWatcher.h" | |
18 | #include "tools/rbd_mirror/leader_watcher/Types.h" | |
19 | ||
20 | namespace librbd { class ImageCtx; } | |
21 | ||
22 | namespace rbd { | |
23 | namespace mirror { | |
24 | ||
25 | template <typename> struct Threads; | |
26 | ||
27 | template <typename ImageCtxT = librbd::ImageCtx> | |
28 | class LeaderWatcher : protected librbd::Watcher { | |
29 | public: | |
30 | struct Listener { | |
31 | virtual ~Listener() { | |
32 | } | |
33 | ||
34 | virtual void post_acquire_handler(Context *on_finish) = 0; | |
35 | virtual void pre_release_handler(Context *on_finish) = 0; | |
31f18b77 FG |
36 | |
37 | virtual void update_leader_handler( | |
38 | const std::string &leader_instance_id) = 0; | |
7c673cae FG |
39 | }; |
40 | ||
41 | LeaderWatcher(Threads<ImageCtxT> *threads, librados::IoCtx &io_ctx, | |
42 | Listener *listener); | |
43 | ~LeaderWatcher() override; | |
44 | ||
45 | int init(); | |
46 | void shut_down(); | |
47 | ||
48 | void init(Context *on_finish); | |
49 | void shut_down(Context *on_finish); | |
50 | ||
51 | bool is_leader() const; | |
52 | bool is_releasing_leader() const; | |
53 | bool get_leader_instance_id(std::string *instance_id) const; | |
54 | void release_leader(); | |
55 | void list_instances(std::vector<std::string> *instance_ids); | |
56 | ||
31f18b77 FG |
57 | std::string get_instance_id(); |
58 | ||
7c673cae FG |
59 | private: |
60 | /** | |
61 | * @verbatim | |
62 | * | |
63 | * <uninitialized> <------------------------------ WAIT_FOR_TASKS | |
64 | * | (init) ^ ^ | |
65 | * v * | | |
66 | * CREATE_OBJECT * * (error) UNREGISTER_WATCH | |
67 | * | * ^ | |
68 | * v * | | |
69 | * REGISTER_WATCH * * SHUT_DOWN_LEADER_LOCK | |
70 | * | ^ | |
71 | * | (no leader heartbeat and acquire failed) | | |
72 | * | BREAK_LOCK <-------------------------------------\ | | |
73 | * | | (no leader heartbeat) | | (shut down) | |
74 | * | | /----------------------------------------\ | | | |
75 | * | | | (lock_released received) | | | |
76 | * | | | /-------------------------------------\ | | | |
77 | * | | | | (lock_acquired or | | | | |
78 | * | | | | heartbeat received) | | | | |
79 | * | | | | (ENOENT) /-----------\ | | | | |
80 | * | | | | * * * * * * * * * * | | | | | | |
81 | * v v v v v (error) * v | | | | | |
82 | * ACQUIRE_LEADER_LOCK * * * * *> GET_LOCKER ---> <secondary> | |
83 | * | * ^ | |
84 | * ....|...................*.................... .....|..................... | |
85 | * . v * . . | post_release . | |
86 | * .INIT_STATUS_WATCHER * * (error) . .NOTIFY_LOCK_RELEASED . | |
87 | * . | ^ . .....^..................... | |
88 | * . v (error) | . | | |
89 | * .INIT_INSTANCES *> SHUT_DOWN_STATUS_WATCHER . RELEASE_LEADER_LOCK | |
90 | * . | . ^ | |
91 | * . v . .....|..................... | |
92 | * .NOTIFY_LISTENER . .SHUT_DOWN_STATUS_WATCHER . | |
93 | * . | . . ^ . | |
94 | * . v . . | . | |
95 | * .NOTIFY_LOCK_ACQUIRED post_acquire . .SHUT_DOWN_INSTANCES . | |
96 | * ....|........................................ . ^ . | |
97 | * v . | . | |
98 | * <leader> -----------------------------------> .NOTIFY_LISTENER . | |
99 | * (shut_down, release_leader, . pre_release . | |
100 | * notify error) ........................... | |
101 | * @endverbatim | |
102 | */ | |
103 | ||
104 | class LeaderLock : public librbd::ManagedLock<ImageCtxT> { | |
105 | public: | |
106 | typedef librbd::ManagedLock<ImageCtxT> Parent; | |
107 | ||
108 | LeaderLock(librados::IoCtx& ioctx, ContextWQ *work_queue, | |
109 | const std::string& oid, LeaderWatcher *watcher, | |
110 | bool blacklist_on_break_lock, | |
111 | uint32_t blacklist_expire_seconds) | |
112 | : Parent(ioctx, work_queue, oid, watcher, librbd::managed_lock::EXCLUSIVE, | |
113 | blacklist_on_break_lock, blacklist_expire_seconds), | |
114 | watcher(watcher) { | |
115 | } | |
116 | ||
117 | bool is_leader() const { | |
118 | Mutex::Locker locker(Parent::m_lock); | |
119 | return Parent::is_state_post_acquiring() || Parent::is_state_locked(); | |
120 | } | |
121 | ||
122 | bool is_releasing_leader() const { | |
123 | Mutex::Locker locker(Parent::m_lock); | |
124 | return Parent::is_state_pre_releasing(); | |
125 | } | |
126 | ||
127 | protected: | |
128 | void post_acquire_lock_handler(int r, Context *on_finish) { | |
129 | if (r == 0) { | |
130 | // lock is owned at this point | |
131 | Mutex::Locker locker(Parent::m_lock); | |
132 | Parent::set_state_post_acquiring(); | |
133 | } | |
134 | watcher->handle_post_acquire_leader_lock(r, on_finish); | |
135 | } | |
136 | void pre_release_lock_handler(bool shutting_down, | |
137 | Context *on_finish) { | |
138 | watcher->handle_pre_release_leader_lock(on_finish); | |
139 | } | |
140 | void post_release_lock_handler(bool shutting_down, int r, | |
141 | Context *on_finish) { | |
142 | watcher->handle_post_release_leader_lock(r, on_finish); | |
143 | } | |
144 | private: | |
145 | LeaderWatcher *watcher; | |
146 | }; | |
147 | ||
148 | struct HandlePayloadVisitor : public boost::static_visitor<void> { | |
149 | LeaderWatcher *leader_watcher; | |
150 | Context *on_notify_ack; | |
151 | ||
152 | HandlePayloadVisitor(LeaderWatcher *leader_watcher, Context *on_notify_ack) | |
153 | : leader_watcher(leader_watcher), on_notify_ack(on_notify_ack) { | |
154 | } | |
155 | ||
156 | template <typename Payload> | |
157 | inline void operator()(const Payload &payload) const { | |
158 | leader_watcher->handle_payload(payload, on_notify_ack); | |
159 | } | |
160 | }; | |
161 | ||
162 | struct C_GetLocker : public Context { | |
163 | LeaderWatcher *leader_watcher; | |
164 | librbd::managed_lock::Locker locker; | |
165 | ||
166 | C_GetLocker(LeaderWatcher *leader_watcher) | |
167 | : leader_watcher(leader_watcher) { | |
168 | } | |
169 | ||
170 | void finish(int r) override { | |
171 | leader_watcher->handle_get_locker(r, locker); | |
172 | } | |
173 | }; | |
174 | ||
175 | typedef void (LeaderWatcher<ImageCtxT>::*TimerCallback)(); | |
176 | ||
177 | struct C_TimerGate : public Context { | |
178 | LeaderWatcher *leader_watcher; | |
179 | ||
180 | bool leader = false; | |
181 | TimerCallback timer_callback = nullptr; | |
182 | ||
183 | C_TimerGate(LeaderWatcher *leader_watcher) | |
184 | : leader_watcher(leader_watcher) { | |
185 | } | |
186 | ||
187 | void finish(int r) override { | |
188 | leader_watcher->m_timer_gate = nullptr; | |
189 | leader_watcher->execute_timer_task(leader, timer_callback); | |
190 | } | |
191 | }; | |
192 | ||
193 | Threads<ImageCtxT> *m_threads; | |
194 | Listener *m_listener; | |
195 | ||
196 | mutable Mutex m_lock; | |
197 | uint64_t m_notifier_id; | |
198 | LeaderLock *m_leader_lock; | |
199 | Context *m_on_finish = nullptr; | |
200 | Context *m_on_shut_down_finish = nullptr; | |
201 | int m_acquire_attempts = 0; | |
202 | int m_ret_val = 0; | |
203 | MirrorStatusWatcher<ImageCtxT> *m_status_watcher = nullptr; | |
204 | Instances<ImageCtxT> *m_instances = nullptr; | |
205 | librbd::managed_lock::Locker m_locker; | |
206 | ||
207 | AsyncOpTracker m_timer_op_tracker; | |
208 | Context *m_timer_task = nullptr; | |
209 | C_TimerGate *m_timer_gate = nullptr; | |
210 | ||
211 | librbd::watcher::NotifyResponse m_heartbeat_response; | |
212 | ||
213 | bool is_leader(Mutex &m_lock) const; | |
214 | bool is_releasing_leader(Mutex &m_lock) const; | |
215 | ||
216 | void cancel_timer_task(); | |
217 | void schedule_timer_task(const std::string &name, | |
218 | int delay_factor, bool leader, | |
219 | TimerCallback callback, bool shutting_down); | |
220 | void execute_timer_task(bool leader, TimerCallback timer_callback); | |
221 | ||
222 | void create_leader_object(); | |
223 | void handle_create_leader_object(int r); | |
224 | ||
225 | void register_watch(); | |
226 | void handle_register_watch(int r); | |
227 | ||
228 | void shut_down_leader_lock(); | |
229 | void handle_shut_down_leader_lock(int r); | |
230 | ||
231 | void unregister_watch(); | |
232 | void handle_unregister_watch(int r); | |
233 | ||
234 | void wait_for_tasks(); | |
235 | void handle_wait_for_tasks(); | |
236 | ||
237 | void break_leader_lock(); | |
238 | void handle_break_leader_lock(int r); | |
239 | ||
240 | void schedule_get_locker(bool reset_leader, uint32_t delay_factor); | |
241 | void get_locker(); | |
242 | void handle_get_locker(int r, librbd::managed_lock::Locker& locker); | |
243 | ||
244 | void schedule_acquire_leader_lock(uint32_t delay_factor); | |
245 | void acquire_leader_lock(); | |
246 | void handle_acquire_leader_lock(int r); | |
247 | ||
248 | void release_leader_lock(); | |
249 | void handle_release_leader_lock(int r); | |
250 | ||
251 | void init_status_watcher(); | |
252 | void handle_init_status_watcher(int r); | |
253 | ||
254 | void shut_down_status_watcher(); | |
255 | void handle_shut_down_status_watcher(int r); | |
256 | ||
257 | void init_instances(); | |
258 | void handle_init_instances(int r); | |
259 | ||
260 | void shut_down_instances(); | |
261 | void handle_shut_down_instances(int r); | |
262 | ||
263 | void notify_listener(); | |
264 | void handle_notify_listener(int r); | |
265 | ||
266 | void notify_lock_acquired(); | |
267 | void handle_notify_lock_acquired(int r); | |
268 | ||
269 | void notify_lock_released(); | |
270 | void handle_notify_lock_released(int r); | |
271 | ||
272 | void notify_heartbeat(); | |
273 | void handle_notify_heartbeat(int r); | |
274 | ||
275 | void get_instances(); | |
276 | void handle_get_instances(int r); | |
277 | ||
278 | void handle_post_acquire_leader_lock(int r, Context *on_finish); | |
279 | void handle_pre_release_leader_lock(Context *on_finish); | |
280 | void handle_post_release_leader_lock(int r, Context *on_finish); | |
281 | ||
282 | void handle_notify(uint64_t notify_id, uint64_t handle, | |
283 | uint64_t notifier_id, bufferlist &bl) override; | |
284 | ||
285 | void handle_heartbeat(Context *on_ack); | |
286 | void handle_lock_acquired(Context *on_ack); | |
287 | void handle_lock_released(Context *on_ack); | |
288 | ||
289 | void handle_payload(const leader_watcher::HeartbeatPayload &payload, | |
290 | Context *on_notify_ack); | |
291 | void handle_payload(const leader_watcher::LockAcquiredPayload &payload, | |
292 | Context *on_notify_ack); | |
293 | void handle_payload(const leader_watcher::LockReleasedPayload &payload, | |
294 | Context *on_notify_ack); | |
295 | void handle_payload(const leader_watcher::UnknownPayload &payload, | |
296 | Context *on_notify_ack); | |
297 | }; | |
298 | ||
299 | } // namespace mirror | |
300 | } // namespace rbd | |
301 | ||
302 | #endif // CEPH_RBD_MIRROR_LEADER_WATCHER_H |