ceph/src/tools/rbd_mirror/LeaderWatcher.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #ifndef CEPH_RBD_MIRROR_LEADER_WATCHER_H
   5 #define CEPH_RBD_MIRROR_LEADER_WATCHER_H
   6
   7 #include <list>
   8 #include <memory>
   9 #include <string>
  10
  11 #include "common/AsyncOpTracker.h"
  12 #include "librbd/ManagedLock.h"
  13 #include "librbd/Watcher.h"
  14 #include "librbd/managed_lock/Types.h"
  15 #include "librbd/watcher/Types.h"
  16 #include "Instances.h"
  17 #include "MirrorStatusWatcher.h"
  18 #include "tools/rbd_mirror/leader_watcher/Types.h"
  19
  20 namespace librbd { class ImageCtx; }
  21
  22 namespace rbd {
  23 namespace mirror {
  24
  25 template <typename> struct Threads;
  26
  27 template <typename ImageCtxT = librbd::ImageCtx>
  28 class LeaderWatcher : protected librbd::Watcher {
  29 public:
  30   struct Listener {
  31     virtual ~Listener() {
  32     }
  33
  34     virtual void post_acquire_handler(Context *on_finish) = 0;
  35     virtual void pre_release_handler(Context *on_finish) = 0;
  36   };
  37
  38   LeaderWatcher(Threads<ImageCtxT> *threads, librados::IoCtx &io_ctx,
  39                 Listener *listener);
  40   ~LeaderWatcher() override;
  41
  42   int init();
  43   void shut_down();
  44
  45   void init(Context *on_finish);
  46   void shut_down(Context *on_finish);
  47
  48   bool is_leader() const;
  49   bool is_releasing_leader() const;
  50   bool get_leader_instance_id(std::string *instance_id) const;
  51   void release_leader();
  52   void list_instances(std::vector<std::string> *instance_ids);
  53
  54 private:
  55   /**
  56    * @verbatim
  57    *
  58    *  <uninitialized> <------------------------------ WAIT_FOR_TASKS
  59    *     | (init)      ^                                      ^
  60    *     v             *                                      |
  61    *  CREATE_OBJECT  * *  (error)                     UNREGISTER_WATCH
  62    *     |             *                                      ^
  63    *     v             *                                      |
  64    *  REGISTER_WATCH * *                              SHUT_DOWN_LEADER_LOCK
  65    *     |                                                    ^
  66    *     |           (no leader heartbeat and acquire failed) |
  67    *     | BREAK_LOCK <-------------------------------------\ |
  68    *     |    |                 (no leader heartbeat)       | | (shut down)
  69    *     |    |  /----------------------------------------\ | |
  70    *     |    |  |              (lock_released received)    | |
  71    *     |    |  |  /-------------------------------------\ | |
  72    *     |    |  |  |                   (lock_acquired or | | |
  73    *     |    |  |  |                 heartbeat received) | | |
  74    *     |    |  |  |       (ENOENT)        /-----------\ | | |
  75    *     |    |  |  |  * * * * * * * * * *  |           | | | |
  76    *     v    v  v  v  v  (error)        *  v           | | | |
  77    *  ACQUIRE_LEADER_LOCK  * * * * *> GET_LOCKER ---> <secondary>
  78    *     |                   *                           ^
  79    * ....|...................*....................  .....|.....................
  80    * .   v                   *                   .  .    |       post_release .
  81    * .INIT_STATUS_WATCHER  * * (error)           .  .NOTIFY_LOCK_RELEASED     .
  82    * .   |                   ^                   .  .....^.....................
  83    * .   v         (error)   |                   .       |
  84    * .INIT_INSTANCES *> SHUT_DOWN_STATUS_WATCHER .   RELEASE_LEADER_LOCK
  85    * .   |                                       .       ^
  86    * .   v                                       .  .....|.....................
  87    * .NOTIFY_LISTENER                            .  .SHUT_DOWN_STATUS_WATCHER .
  88    * .   |                                       .  .    ^                    .
  89    * .   v                                       .  .    |                    .
  90    * .NOTIFY_LOCK_ACQUIRED          post_acquire .  .SHUT_DOWN_INSTANCES      .
  91    * ....|........................................  .    ^                    .
  92    *     v                                          .    |                    .
  93    *  <leader> -----------------------------------> .NOTIFY_LISTENER          .
  94    *            (shut_down, release_leader,         .             pre_release .
  95    *             notify error)                      ...........................
  96    * @endverbatim
  97    */
  98
  99   class LeaderLock : public librbd::ManagedLock<ImageCtxT> {
 100   public:
 101     typedef librbd::ManagedLock<ImageCtxT> Parent;
 102
 103     LeaderLock(librados::IoCtx& ioctx, ContextWQ *work_queue,
 104                const std::string& oid, LeaderWatcher *watcher,
 105                bool blacklist_on_break_lock,
 106                uint32_t blacklist_expire_seconds)
 107       : Parent(ioctx, work_queue, oid, watcher, librbd::managed_lock::EXCLUSIVE,
 108                blacklist_on_break_lock, blacklist_expire_seconds),
 109         watcher(watcher) {
 110     }
 111
 112     bool is_leader() const {
 113       Mutex::Locker locker(Parent::m_lock);
 114       return Parent::is_state_post_acquiring() || Parent::is_state_locked();
 115     }
 116
 117     bool is_releasing_leader() const {
 118       Mutex::Locker locker(Parent::m_lock);
 119       return Parent::is_state_pre_releasing();
 120     }
 121
 122   protected:
 123     void post_acquire_lock_handler(int r, Context *on_finish) {
 124       if (r == 0) {
 125         // lock is owned at this point
 126         Mutex::Locker locker(Parent::m_lock);
 127         Parent::set_state_post_acquiring();
 128       }
 129       watcher->handle_post_acquire_leader_lock(r, on_finish);
 130     }
 131     void pre_release_lock_handler(bool shutting_down,
 132                                   Context *on_finish) {
 133       watcher->handle_pre_release_leader_lock(on_finish);
 134     }
 135     void post_release_lock_handler(bool shutting_down, int r,
 136                                    Context *on_finish) {
 137       watcher->handle_post_release_leader_lock(r, on_finish);
 138     }
 139   private:
 140     LeaderWatcher *watcher;
 141   };
 142
 143   struct HandlePayloadVisitor : public boost::static_visitor<void> {
 144     LeaderWatcher *leader_watcher;
 145     Context *on_notify_ack;
 146
 147     HandlePayloadVisitor(LeaderWatcher *leader_watcher, Context *on_notify_ack)
 148       : leader_watcher(leader_watcher), on_notify_ack(on_notify_ack) {
 149     }
 150
 151     template <typename Payload>
 152     inline void operator()(const Payload &payload) const {
 153       leader_watcher->handle_payload(payload, on_notify_ack);
 154     }
 155   };
 156
 157   struct C_GetLocker : public Context {
 158     LeaderWatcher *leader_watcher;
 159     librbd::managed_lock::Locker locker;
 160
 161     C_GetLocker(LeaderWatcher *leader_watcher)
 162       : leader_watcher(leader_watcher) {
 163     }
 164
 165     void finish(int r) override {
 166       leader_watcher->handle_get_locker(r, locker);
 167     }
 168   };
 169
 170   typedef void (LeaderWatcher<ImageCtxT>::*TimerCallback)();
 171
 172   struct C_TimerGate : public Context {
 173     LeaderWatcher *leader_watcher;
 174
 175     bool leader = false;
 176     TimerCallback timer_callback = nullptr;
 177
 178     C_TimerGate(LeaderWatcher *leader_watcher)
 179       : leader_watcher(leader_watcher) {
 180     }
 181
 182     void finish(int r) override {
 183       leader_watcher->m_timer_gate = nullptr;
 184       leader_watcher->execute_timer_task(leader, timer_callback);
 185     }
 186   };
 187
 188   Threads<ImageCtxT> *m_threads;
 189   Listener *m_listener;
 190
 191   mutable Mutex m_lock;
 192   uint64_t m_notifier_id;
 193   LeaderLock *m_leader_lock;
 194   Context *m_on_finish = nullptr;
 195   Context *m_on_shut_down_finish = nullptr;
 196   int m_acquire_attempts = 0;
 197   int m_ret_val = 0;
 198   MirrorStatusWatcher<ImageCtxT> *m_status_watcher = nullptr;
 199   Instances<ImageCtxT> *m_instances = nullptr;
 200   librbd::managed_lock::Locker m_locker;
 201
 202   AsyncOpTracker m_timer_op_tracker;
 203   Context *m_timer_task = nullptr;
 204   C_TimerGate *m_timer_gate = nullptr;
 205
 206   librbd::watcher::NotifyResponse m_heartbeat_response;
 207
 208   bool is_leader(Mutex &m_lock) const;
 209   bool is_releasing_leader(Mutex &m_lock) const;
 210
 211   void cancel_timer_task();
 212   void schedule_timer_task(const std::string &name,
 213                            int delay_factor, bool leader,
 214                            TimerCallback callback, bool shutting_down);
 215   void execute_timer_task(bool leader, TimerCallback timer_callback);
 216
 217   void create_leader_object();
 218   void handle_create_leader_object(int r);
 219
 220   void register_watch();
 221   void handle_register_watch(int r);
 222
 223   void shut_down_leader_lock();
 224   void handle_shut_down_leader_lock(int r);
 225
 226   void unregister_watch();
 227   void handle_unregister_watch(int r);
 228
 229   void wait_for_tasks();
 230   void handle_wait_for_tasks();
 231
 232   void break_leader_lock();
 233   void handle_break_leader_lock(int r);
 234
 235   void schedule_get_locker(bool reset_leader, uint32_t delay_factor);
 236   void get_locker();
 237   void handle_get_locker(int r, librbd::managed_lock::Locker& locker);
 238
 239   void schedule_acquire_leader_lock(uint32_t delay_factor);
 240   void acquire_leader_lock();
 241   void handle_acquire_leader_lock(int r);
 242
 243   void release_leader_lock();
 244   void handle_release_leader_lock(int r);
 245
 246   void init_status_watcher();
 247   void handle_init_status_watcher(int r);
 248
 249   void shut_down_status_watcher();
 250   void handle_shut_down_status_watcher(int r);
 251
 252   void init_instances();
 253   void handle_init_instances(int r);
 254
 255   void shut_down_instances();
 256   void handle_shut_down_instances(int r);
 257
 258   void notify_listener();
 259   void handle_notify_listener(int r);
 260
 261   void notify_lock_acquired();
 262   void handle_notify_lock_acquired(int r);
 263
 264   void notify_lock_released();
 265   void handle_notify_lock_released(int r);
 266
 267   void notify_heartbeat();
 268   void handle_notify_heartbeat(int r);
 269
 270   void get_instances();
 271   void handle_get_instances(int r);
 272
 273   void handle_post_acquire_leader_lock(int r, Context *on_finish);
 274   void handle_pre_release_leader_lock(Context *on_finish);
 275   void handle_post_release_leader_lock(int r, Context *on_finish);
 276
 277   void handle_notify(uint64_t notify_id, uint64_t handle,
 278                      uint64_t notifier_id, bufferlist &bl) override;
 279
 280   void handle_heartbeat(Context *on_ack);
 281   void handle_lock_acquired(Context *on_ack);
 282   void handle_lock_released(Context *on_ack);
 283
 284   void handle_payload(const leader_watcher::HeartbeatPayload &payload,
 285                       Context *on_notify_ack);
 286   void handle_payload(const leader_watcher::LockAcquiredPayload &payload,
 287                       Context *on_notify_ack);
 288   void handle_payload(const leader_watcher::LockReleasedPayload &payload,
 289                       Context *on_notify_ack);
 290   void handle_payload(const leader_watcher::UnknownPayload &payload,
 291                       Context *on_notify_ack);
 292 };
 293
 294 } // namespace mirror
 295 } // namespace rbd
 296
 297 #endif // CEPH_RBD_MIRROR_LEADER_WATCHER_H