ceph/src/tools/rbd_mirror/image_deleter/TrashWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "tools/rbd_mirror/image_deleter/TrashWatcher.h"
   5 #include "include/rbd_types.h"
   6 #include "cls/rbd/cls_rbd_client.h"
   7 #include "common/debug.h"
   8 #include "common/errno.h"
   9 #include "common/Timer.h"
  10 #include "librbd/ImageCtx.h"
  11 #include "librbd/Utils.h"
  12 #include "tools/rbd_mirror/Threads.h"
  13 #include "tools/rbd_mirror/image_deleter/Types.h"
  14
  15 #define dout_context g_ceph_context
  16 #define dout_subsys ceph_subsys_rbd_mirror
  17 #undef dout_prefix
  18 #define dout_prefix *_dout << "rbd::mirror::image_deleter::TrashWatcher: " \
  19                            << this << " " << __func__ << ": "
  20
  21 using librbd::util::create_context_callback;
  22 using librbd::util::create_rados_callback;
  23
  24 namespace rbd {
  25 namespace mirror {
  26 namespace image_deleter {
  27
  28 namespace {
  29
  30 const size_t MAX_RETURN = 1024;
  31
  32 } // anonymous namespace
  33
  34 template <typename I>
  35 TrashWatcher<I>::TrashWatcher(librados::IoCtx &io_ctx, Threads<I> *threads,
  36                               TrashListener& trash_listener)
  37   : librbd::TrashWatcher<I>(io_ctx, threads->work_queue),
  38     m_io_ctx(io_ctx), m_threads(threads), m_trash_listener(trash_listener),
  39     m_lock(librbd::util::unique_lock_name(
  40       "rbd::mirror::image_deleter::TrashWatcher", this)) {
  41 }
  42
  43 template <typename I>
  44 void TrashWatcher<I>::init(Context *on_finish) {
  45   dout(5) << dendl;
  46
  47   {
  48     Mutex::Locker locker(m_lock);
  49     m_on_init_finish = on_finish;
  50
  51     ceph_assert(!m_trash_list_in_progress);
  52     m_trash_list_in_progress = true;
  53   }
  54
  55   create_trash();
  56 }
  57
  58 template <typename I>
  59 void TrashWatcher<I>::shut_down(Context *on_finish) {
  60   dout(5) << dendl;
  61
  62   {
  63     Mutex::Locker timer_locker(m_threads->timer_lock);
  64     Mutex::Locker locker(m_lock);
  65
  66     ceph_assert(!m_shutting_down);
  67     m_shutting_down = true;
  68     if (m_timer_ctx != nullptr) {
  69       m_threads->timer->cancel_event(m_timer_ctx);
  70       m_timer_ctx = nullptr;
  71     }
  72   }
  73
  74   auto ctx = new FunctionContext([this, on_finish](int r) {
  75       unregister_watcher(on_finish);
  76     });
  77   m_async_op_tracker.wait_for_ops(ctx);
  78 }
  79
  80 template <typename I>
  81 void TrashWatcher<I>::handle_image_added(const std::string &image_id,
  82                                          const cls::rbd::TrashImageSpec& spec) {
  83   dout(10) << "image_id=" << image_id << dendl;
  84
  85   Mutex::Locker locker(m_lock);
  86   add_image(image_id, spec);
  87 }
  88
  89 template <typename I>
  90 void TrashWatcher<I>::handle_image_removed(const std::string &image_id) {
  91   // ignore removals -- the image deleter will ignore -ENOENTs
  92 }
  93
  94 template <typename I>
  95 void TrashWatcher<I>::handle_rewatch_complete(int r) {
  96   dout(5) << "r=" << r << dendl;
  97
  98   if (r == -EBLACKLISTED) {
  99     dout(0) << "detected client is blacklisted" << dendl;
 100     return;
 101   } else if (r == -ENOENT) {
 102     dout(5) << "trash directory deleted" << dendl;
 103   } else if (r < 0) {
 104     derr << "unexpected error re-registering trash directory watch: "
 105          << cpp_strerror(r) << dendl;
 106   }
 107   schedule_trash_list(30);
 108 }
 109
 110 template <typename I>
 111 void TrashWatcher<I>::create_trash() {
 112   dout(20) << dendl;
 113   {
 114     Mutex::Locker locker(m_lock);
 115     ceph_assert(m_trash_list_in_progress);
 116   }
 117
 118   librados::ObjectWriteOperation op;
 119   op.create(false);
 120
 121   m_async_op_tracker.start_op();
 122   auto aio_comp = create_rados_callback<
 123     TrashWatcher<I>, &TrashWatcher<I>::handle_create_trash>(this);
 124   int r = m_io_ctx.aio_operate(RBD_TRASH, aio_comp, &op);
 125   ceph_assert(r == 0);
 126   aio_comp->release();
 127 }
 128
 129 template <typename I>
 130 void TrashWatcher<I>::handle_create_trash(int r) {
 131   dout(20) << "r=" << r << dendl;
 132   {
 133     Mutex::Locker locker(m_lock);
 134     ceph_assert(m_trash_list_in_progress);
 135   }
 136
 137   Context* on_init_finish = nullptr;
 138   if (r == -EBLACKLISTED || r == -ENOENT) {
 139     if (r == -EBLACKLISTED) {
 140       dout(0) << "detected client is blacklisted" << dendl;
 141     } else {
 142       dout(0) << "detected pool no longer exists" << dendl;
 143     }
 144
 145     Mutex::Locker locker(m_lock);
 146     std::swap(on_init_finish, m_on_init_finish);
 147     m_trash_list_in_progress = false;
 148   } else if (r < 0 && r != -EEXIST) {
 149     derr << "failed to create trash object: " << cpp_strerror(r) << dendl;
 150     {
 151       Mutex::Locker locker(m_lock);
 152       m_trash_list_in_progress = false;
 153     }
 154
 155     schedule_trash_list(30);
 156   } else {
 157     register_watcher();
 158   }
 159
 160   m_async_op_tracker.finish_op();
 161   if (on_init_finish != nullptr) {
 162     on_init_finish->complete(r);
 163   }
 164 }
 165
 166 template <typename I>
 167 void TrashWatcher<I>::register_watcher() {
 168   {
 169     Mutex::Locker locker(m_lock);
 170     ceph_assert(m_trash_list_in_progress);
 171   }
 172
 173   // if the watch registration is in-flight, let the watcher
 174   // handle the transition -- only (re-)register if it's not registered
 175   if (!this->is_unregistered()) {
 176     trash_list(true);
 177     return;
 178   }
 179
 180   // first time registering or the watch failed
 181   dout(5) << dendl;
 182   m_async_op_tracker.start_op();
 183
 184   Context *ctx = create_context_callback<
 185     TrashWatcher, &TrashWatcher<I>::handle_register_watcher>(this);
 186   this->register_watch(ctx);
 187 }
 188
 189 template <typename I>
 190 void TrashWatcher<I>::handle_register_watcher(int r) {
 191   dout(5) << "r=" << r << dendl;
 192
 193   {
 194     Mutex::Locker locker(m_lock);
 195     ceph_assert(m_trash_list_in_progress);
 196     if (r < 0) {
 197       m_trash_list_in_progress = false;
 198     }
 199   }
 200
 201   Context *on_init_finish = nullptr;
 202   if (r >= 0) {
 203     trash_list(true);
 204   } else if (r == -EBLACKLISTED) {
 205     dout(0) << "detected client is blacklisted" << dendl;
 206
 207     Mutex::Locker locker(m_lock);
 208     std::swap(on_init_finish, m_on_init_finish);
 209   } else {
 210     derr << "unexpected error registering trash directory watch: "
 211          << cpp_strerror(r) << dendl;
 212     schedule_trash_list(10);
 213   }
 214
 215   m_async_op_tracker.finish_op();
 216   if (on_init_finish != nullptr) {
 217     on_init_finish->complete(r);
 218   }
 219 }
 220
 221 template <typename I>
 222 void TrashWatcher<I>::unregister_watcher(Context* on_finish) {
 223   dout(5) << dendl;
 224
 225   m_async_op_tracker.start_op();
 226   Context *ctx = new FunctionContext([this, on_finish](int r) {
 227       handle_unregister_watcher(r, on_finish);
 228     });
 229   this->unregister_watch(ctx);
 230 }
 231
 232 template <typename I>
 233 void TrashWatcher<I>::handle_unregister_watcher(int r, Context* on_finish) {
 234   dout(5) << "unregister_watcher: r=" << r << dendl;
 235   if (r < 0) {
 236     derr << "error unregistering watcher for trash directory: "
 237          << cpp_strerror(r) << dendl;
 238   }
 239   m_async_op_tracker.finish_op();
 240   on_finish->complete(0);
 241 }
 242
 243 template <typename I>
 244 void TrashWatcher<I>::trash_list(bool initial_request) {
 245   if (initial_request) {
 246     m_async_op_tracker.start_op();
 247     m_last_image_id = "";
 248   }
 249
 250   dout(5) << "last_image_id=" << m_last_image_id << dendl;
 251
 252   {
 253     Mutex::Locker locker(m_lock);
 254     ceph_assert(m_trash_list_in_progress);
 255   }
 256
 257   librados::ObjectReadOperation op;
 258   librbd::cls_client::trash_list_start(&op, m_last_image_id, MAX_RETURN);
 259
 260   librados::AioCompletion *aio_comp = create_rados_callback<
 261     TrashWatcher<I>, &TrashWatcher<I>::handle_trash_list>(this);
 262   m_out_bl.clear();
 263   int r = m_io_ctx.aio_operate(RBD_TRASH, aio_comp, &op, &m_out_bl);
 264   ceph_assert(r == 0);
 265   aio_comp->release();
 266 }
 267
 268 template <typename I>
 269 void TrashWatcher<I>::handle_trash_list(int r) {
 270   dout(5) << "r=" << r << dendl;
 271
 272   std::map<std::string, cls::rbd::TrashImageSpec> images;
 273   if (r >= 0) {
 274     auto bl_it = m_out_bl.cbegin();
 275     r = librbd::cls_client::trash_list_finish(&bl_it, &images);
 276   }
 277
 278   Context *on_init_finish = nullptr;
 279   {
 280     Mutex::Locker locker(m_lock);
 281     ceph_assert(m_trash_list_in_progress);
 282     if (r >= 0) {
 283       for (auto& image : images) {
 284         add_image(image.first, image.second);
 285       }
 286     } else if (r == -ENOENT) {
 287       r = 0;
 288     }
 289
 290     if (r == -EBLACKLISTED) {
 291       dout(0) << "detected client is blacklisted during trash refresh" << dendl;
 292       m_trash_list_in_progress = false;
 293       std::swap(on_init_finish, m_on_init_finish);
 294     } else if (r >= 0 && images.size() < MAX_RETURN) {
 295       m_trash_list_in_progress = false;
 296       std::swap(on_init_finish, m_on_init_finish);
 297     } else if (r < 0) {
 298       m_trash_list_in_progress = false;
 299     }
 300   }
 301
 302   if (r >= 0 && images.size() == MAX_RETURN) {
 303     m_last_image_id = images.rbegin()->first;
 304     trash_list(false);
 305     return;
 306   } else if (r < 0 && r != -EBLACKLISTED) {
 307     derr << "failed to retrieve trash directory: " << cpp_strerror(r) << dendl;
 308     schedule_trash_list(10);
 309   }
 310
 311   m_async_op_tracker.finish_op();
 312   if (on_init_finish != nullptr) {
 313     on_init_finish->complete(r);
 314   }
 315 }
 316
 317 template <typename I>
 318 void TrashWatcher<I>::schedule_trash_list(double interval) {
 319   Mutex::Locker timer_locker(m_threads->timer_lock);
 320   Mutex::Locker locker(m_lock);
 321   if (m_shutting_down || m_trash_list_in_progress || m_timer_ctx != nullptr) {
 322     if (m_trash_list_in_progress && !m_deferred_trash_list) {
 323       dout(5) << "deferring refresh until in-flight refresh completes" << dendl;
 324       m_deferred_trash_list = true;
 325     }
 326     return;
 327   }
 328
 329   dout(5) << dendl;
 330   m_timer_ctx = m_threads->timer->add_event_after(
 331     interval,
 332     new FunctionContext([this](int r) {
 333         process_trash_list();
 334       }));
 335 }
 336
 337 template <typename I>
 338 void TrashWatcher<I>::process_trash_list() {
 339   dout(5) << dendl;
 340
 341   ceph_assert(m_threads->timer_lock.is_locked());
 342   ceph_assert(m_timer_ctx != nullptr);
 343   m_timer_ctx = nullptr;
 344
 345   {
 346     Mutex::Locker locker(m_lock);
 347     ceph_assert(!m_trash_list_in_progress);
 348     m_trash_list_in_progress = true;
 349   }
 350
 351   // execute outside of the timer's lock
 352   m_async_op_tracker.start_op();
 353   Context *ctx = new FunctionContext([this](int r) {
 354       create_trash();
 355       m_async_op_tracker.finish_op();
 356     });
 357   m_threads->work_queue->queue(ctx, 0);
 358 }
 359
 360 template <typename I>
 361 void TrashWatcher<I>::add_image(const std::string& image_id,
 362                                 const cls::rbd::TrashImageSpec& spec) {
 363   if (spec.source != cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING) {
 364     return;
 365   }
 366
 367   ceph_assert(m_lock.is_locked());
 368   auto& deferment_end_time = spec.deferment_end_time;
 369   dout(10) << "image_id=" << image_id << ", "
 370            << "deferment_end_time=" << deferment_end_time << dendl;
 371
 372   m_async_op_tracker.start_op();
 373   auto ctx = new FunctionContext([this, image_id, deferment_end_time](int r) {
 374       m_trash_listener.handle_trash_image(image_id, deferment_end_time);
 375       m_async_op_tracker.finish_op();
 376     });
 377   m_threads->work_queue->queue(ctx, 0);
 378 }
 379
 380 } // namespace image_deleter;
 381 } // namespace mirror
 382 } // namespace rbd
 383
 384 template class rbd::mirror::image_deleter::TrashWatcher<librbd::ImageCtx>;