ceph/src/tools/rbd_mirror/RemotePoolPoller.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "RemotePoolPoller.h"
   5 #include "include/ceph_assert.h"
   6 #include "common/debug.h"
   7 #include "common/errno.h"
   8 #include "common/Timer.h"
   9 #include "cls/rbd/cls_rbd_client.h"
  10 #include "librbd/ImageCtx.h"
  11 #include "librbd/Utils.h"
  12 #include "librbd/asio/ContextWQ.h"
  13 #include "tools/rbd_mirror/Threads.h"
  14 #include "tools/rbd_mirror/Types.h"
  15
  16 #define dout_context g_ceph_context
  17 #define dout_subsys ceph_subsys_rbd_mirror
  18 #undef dout_prefix
  19 #define dout_prefix *_dout << "rbd::mirror::RemotePollPoller: " << this << " " \
  20                            << __func__ << ": "
  21
  22 namespace rbd {
  23 namespace mirror {
  24
  25 static const double POLL_INTERVAL_SECONDS = 30;
  26
  27 using librbd::util::create_rados_callback;
  28
  29 template <typename I>
  30 RemotePoolPoller<I>::~RemotePoolPoller() {
  31   ceph_assert(m_timer_task == nullptr);
  32 }
  33
  34 template <typename I>
  35 void RemotePoolPoller<I>::init(Context* on_finish) {
  36   dout(10) << dendl;
  37
  38   ceph_assert(m_state == STATE_INITIALIZING);
  39   ceph_assert(m_on_finish == nullptr);
  40   m_on_finish = on_finish;
  41
  42   get_mirror_uuid();
  43 }
  44
  45 template <typename I>
  46 void RemotePoolPoller<I>::shut_down(Context* on_finish) {
  47   dout(10) << dendl;
  48
  49   std::unique_lock locker(m_threads->timer_lock);
  50   ceph_assert(m_state == STATE_POLLING);
  51   m_state = STATE_SHUTTING_DOWN;
  52
  53   if (m_timer_task == nullptr) {
  54     // currently executing a poll
  55     ceph_assert(m_on_finish == nullptr);
  56     m_on_finish = on_finish;
  57     return;
  58   }
  59
  60   m_threads->timer->cancel_event(m_timer_task);
  61   m_timer_task = nullptr;
  62   m_threads->work_queue->queue(on_finish, 0);
  63 }
  64
  65 template <typename I>
  66 void RemotePoolPoller<I>::get_mirror_uuid() {
  67   dout(10) << dendl;
  68
  69   librados::ObjectReadOperation op;
  70   librbd::cls_client::mirror_uuid_get_start(&op);
  71
  72   auto aio_comp = create_rados_callback<
  73     RemotePoolPoller<I>, &RemotePoolPoller<I>::handle_get_mirror_uuid>(this);
  74   m_out_bl.clear();
  75   int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
  76   ceph_assert(r == 0);
  77   aio_comp->release();
  78 }
  79
  80 template <typename I>
  81 void RemotePoolPoller<I>::handle_get_mirror_uuid(int r) {
  82   dout(10) << "r=" << r << dendl;
  83   std::string remote_mirror_uuid;
  84   if (r >= 0) {
  85     auto it = m_out_bl.cbegin();
  86     r = librbd::cls_client::mirror_uuid_get_finish(&it, &remote_mirror_uuid);
  87     if (r >= 0 && remote_mirror_uuid.empty()) {
  88       r = -ENOENT;
  89     }
  90   }
  91
  92   if (r < 0) {
  93     if (r == -ENOENT) {
  94       dout(5) << "remote mirror uuid missing" << dendl;
  95     } else {
  96       derr << "failed to retrieve remote mirror uuid: " << cpp_strerror(r)
  97            << dendl;
  98     }
  99
 100     m_remote_pool_meta.mirror_uuid = "";
 101   }
 102
 103   // if we have the mirror uuid, we will poll until shut down
 104   if (m_state == STATE_INITIALIZING) {
 105     if (r < 0) {
 106       schedule_task(r);
 107       return;
 108     }
 109
 110     m_state = STATE_POLLING;
 111   }
 112
 113   dout(10) << "remote_mirror_uuid=" << remote_mirror_uuid << dendl;
 114   if (m_remote_pool_meta.mirror_uuid != remote_mirror_uuid) {
 115     m_remote_pool_meta.mirror_uuid = remote_mirror_uuid;
 116     m_updated = true;
 117   }
 118
 119   mirror_peer_ping();
 120 }
 121
 122 template <typename I>
 123 void RemotePoolPoller<I>::mirror_peer_ping() {
 124   dout(10) << dendl;
 125
 126   librados::ObjectWriteOperation op;
 127   librbd::cls_client::mirror_peer_ping(&op, m_site_name, m_local_mirror_uuid);
 128
 129   auto aio_comp = create_rados_callback<
 130     RemotePoolPoller<I>, &RemotePoolPoller<I>::handle_mirror_peer_ping>(this);
 131   int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
 132   ceph_assert(r == 0);
 133   aio_comp->release();
 134 }
 135
 136 template <typename I>
 137 void RemotePoolPoller<I>::handle_mirror_peer_ping(int r) {
 138   dout(10) << "r=" << r << dendl;
 139
 140   if (r == -EOPNOTSUPP) {
 141     // older OSD that doesn't support snaphot-based mirroring, so no need
 142     // to query remote peers
 143     dout(10) << "remote peer does not support snapshot-based mirroring"
 144              << dendl;
 145     notify_listener();
 146     return;
 147   } else if (r < 0) {
 148     // we can still see if we can perform a peer list and find outselves
 149     derr << "failed to ping remote mirror peer: " << cpp_strerror(r) << dendl;
 150   }
 151
 152   mirror_peer_list();
 153 }
 154
 155 template <typename I>
 156 void RemotePoolPoller<I>::mirror_peer_list() {
 157   dout(10) << dendl;
 158
 159   librados::ObjectReadOperation op;
 160   librbd::cls_client::mirror_peer_list_start(&op);
 161
 162   auto aio_comp = create_rados_callback<
 163     RemotePoolPoller<I>, &RemotePoolPoller<I>::handle_mirror_peer_list>(this);
 164   m_out_bl.clear();
 165   int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
 166   ceph_assert(r == 0);
 167   aio_comp->release();
 168 }
 169
 170 template <typename I>
 171 void RemotePoolPoller<I>::handle_mirror_peer_list(int r) {
 172   dout(10) << "r=" << r << dendl;
 173
 174   std::vector<cls::rbd::MirrorPeer> peers;
 175   if (r == 0) {
 176     auto iter = m_out_bl.cbegin();
 177     r = librbd::cls_client::mirror_peer_list_finish(&iter, &peers);
 178   }
 179
 180   if (r < 0) {
 181     derr << "failed to retrieve mirror peers: " << cpp_strerror(r) << dendl;
 182   }
 183
 184   cls::rbd::MirrorPeer* matched_peer = nullptr;
 185   for (auto& peer : peers) {
 186     if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_TX) {
 187       continue;
 188     }
 189
 190     if (peer.mirror_uuid == m_local_mirror_uuid) {
 191       matched_peer = &peer;
 192       break;
 193     } else if (peer.site_name == m_site_name) {
 194       // keep searching in case we hit an exact match by fsid
 195       matched_peer = &peer;
 196     }
 197   }
 198
 199   // older OSDs don't support peer ping so we might fail to find a match,
 200   // which will prevent snapshot mirroring from functioning
 201   std::string remote_mirror_peer_uuid;
 202   if (matched_peer != nullptr) {
 203     remote_mirror_peer_uuid = matched_peer->uuid;
 204   }
 205
 206   dout(10) << "remote_mirror_peer_uuid=" << remote_mirror_peer_uuid << dendl;
 207   if (m_remote_pool_meta.mirror_peer_uuid != remote_mirror_peer_uuid) {
 208     m_remote_pool_meta.mirror_peer_uuid = remote_mirror_peer_uuid;
 209     m_updated = true;
 210   }
 211
 212   notify_listener();
 213 }
 214
 215 template <typename I>
 216 void RemotePoolPoller<I>::notify_listener() {
 217   bool updated = false;
 218   std::swap(updated, m_updated);
 219   if (updated) {
 220     dout(10) << dendl;
 221     m_listener.handle_updated(m_remote_pool_meta);
 222   }
 223
 224   schedule_task(0);
 225 }
 226
 227 template <typename I>
 228 void RemotePoolPoller<I>::schedule_task(int r) {
 229   std::unique_lock locker{m_threads->timer_lock};
 230
 231   if (m_state == STATE_POLLING) {
 232     dout(10) << dendl;
 233
 234     ceph_assert(m_timer_task == nullptr);
 235     m_timer_task = new LambdaContext([this](int) {
 236       handle_task();
 237     });
 238
 239     m_threads->timer->add_event_after(POLL_INTERVAL_SECONDS, m_timer_task);
 240   }
 241
 242   // finish init or shut down callback
 243   if (m_on_finish != nullptr) {
 244     locker.unlock();
 245     Context* on_finish = nullptr;
 246     std::swap(on_finish, m_on_finish);
 247     on_finish->complete(m_state == STATE_SHUTTING_DOWN ? 0 : r);
 248   }
 249 }
 250
 251 template <typename I>
 252 void RemotePoolPoller<I>::handle_task() {
 253   dout(10) << dendl;
 254
 255   ceph_assert(ceph_mutex_is_locked_by_me(m_threads->timer_lock));
 256   m_timer_task = nullptr;
 257
 258   auto ctx = new LambdaContext([this](int) {
 259     get_mirror_uuid();
 260   });
 261   m_threads->work_queue->queue(ctx);
 262 }
 263
 264 } // namespace mirror
 265 } // namespace rbd
 266
 267 template class rbd::mirror::RemotePoolPoller<librbd::ImageCtx>;