ceph/src/tools/rbd_mirror/ImageDeleter.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 SUSE LINUX GmbH
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #include "include/rados/librados.hpp"
  16 #include "common/Formatter.h"
  17 #include "common/admin_socket.h"
  18 #include "common/debug.h"
  19 #include "common/errno.h"
  20 #include "common/Timer.h"
  21 #include "global/global_context.h"
  22 #include "librbd/internal.h"
  23 #include "librbd/ImageCtx.h"
  24 #include "librbd/ImageState.h"
  25 #include "librbd/Operations.h"
  26 #include "librbd/asio/ContextWQ.h"
  27 #include "cls/rbd/cls_rbd_client.h"
  28 #include "cls/rbd/cls_rbd_types.h"
  29 #include "librbd/Utils.h"
  30 #include "ImageDeleter.h"
  31 #include "tools/rbd_mirror/Threads.h"
  32 #include "tools/rbd_mirror/Throttler.h"
  33 #include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h"
  34 #include "tools/rbd_mirror/image_deleter/TrashRemoveRequest.h"
  35 #include "tools/rbd_mirror/image_deleter/TrashWatcher.h"
  36 #include <map>
  37 #include <sstream>
  38
  39 #define dout_context g_ceph_context
  40 #define dout_subsys ceph_subsys_rbd_mirror
  41
  42 using std::string;
  43 using std::stringstream;
  44 using std::vector;
  45 using std::pair;
  46 using std::make_pair;
  47
  48 using librados::IoCtx;
  49 using namespace librbd;
  50
  51 namespace rbd {
  52 namespace mirror {
  53
  54 using librbd::util::create_async_context_callback;
  55
  56 namespace {
  57
  58 class ImageDeleterAdminSocketCommand {
  59 public:
  60   virtual ~ImageDeleterAdminSocketCommand() {}
  61   virtual int call(Formatter *f) = 0;
  62 };
  63
  64 template <typename I>
  65 class StatusCommand : public ImageDeleterAdminSocketCommand {
  66 public:
  67   explicit StatusCommand(ImageDeleter<I> *image_del) : image_del(image_del) {}
  68
  69   int call(Formatter *f) override {
  70     image_del->print_status(f);
  71     return 0;
  72   }
  73
  74 private:
  75   ImageDeleter<I> *image_del;
  76 };
  77
  78 } // anonymous namespace
  79
  80 template <typename I>
  81 class ImageDeleterAdminSocketHook : public AdminSocketHook {
  82 public:
  83   ImageDeleterAdminSocketHook(CephContext *cct, const std::string& pool_name,
  84                               ImageDeleter<I> *image_del) :
  85     admin_socket(cct->get_admin_socket()) {
  86
  87     std::string command;
  88     int r;
  89
  90     command = "rbd mirror deletion status " + pool_name;
  91     r = admin_socket->register_command(command, this,
  92                                        "get status for image deleter");
  93     if (r == 0) {
  94       commands[command] = new StatusCommand<I>(image_del);
  95     }
  96
  97   }
  98
  99   ~ImageDeleterAdminSocketHook() override {
 100     (void)admin_socket->unregister_commands(this);
 101     for (Commands::const_iterator i = commands.begin(); i != commands.end();
 102          ++i) {
 103       delete i->second;
 104     }
 105   }
 106
 107   int call(std::string_view command, const cmdmap_t& cmdmap,
 108            Formatter *f,
 109            std::ostream& errss,
 110            bufferlist& out) override {
 111     Commands::const_iterator i = commands.find(command);
 112     ceph_assert(i != commands.end());
 113     return i->second->call(f);
 114   }
 115
 116 private:
 117   typedef std::map<std::string, ImageDeleterAdminSocketCommand*,
 118                    std::less<>> Commands;
 119   AdminSocket *admin_socket;
 120   Commands commands;
 121 };
 122
 123 template <typename I>
 124 ImageDeleter<I>::ImageDeleter(
 125     librados::IoCtx& local_io_ctx, Threads<librbd::ImageCtx>* threads,
 126     Throttler<librbd::ImageCtx>* image_deletion_throttler,
 127     ServiceDaemon<librbd::ImageCtx>* service_daemon)
 128   : m_local_io_ctx(local_io_ctx), m_threads(threads),
 129     m_image_deletion_throttler(image_deletion_throttler),
 130     m_service_daemon(service_daemon), m_trash_listener(this),
 131     m_lock(ceph::make_mutex(
 132       librbd::util::unique_lock_name("rbd::mirror::ImageDeleter::m_lock",
 133                                      this))) {
 134 }
 135
 136 #undef dout_prefix
 137 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << " " \
 138                            << __func__ << ": "
 139
 140 template <typename I>
 141 void ImageDeleter<I>::trash_move(librados::IoCtx& local_io_ctx,
 142                                  const std::string& global_image_id,
 143                                  bool resync,
 144                                  librbd::asio::ContextWQ* work_queue,
 145                                  Context* on_finish) {
 146   dout(10) << "global_image_id=" << global_image_id << ", "
 147            << "resync=" << resync << dendl;
 148
 149   auto req = rbd::mirror::image_deleter::TrashMoveRequest<>::create(
 150     local_io_ctx, global_image_id, resync, work_queue, on_finish);
 151   req->send();
 152 }
 153
 154 #undef dout_prefix
 155 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
 156                            << __func__ << ": "
 157
 158 template <typename I>
 159 void ImageDeleter<I>::init(Context* on_finish) {
 160   dout(10) << dendl;
 161
 162   m_asok_hook = new ImageDeleterAdminSocketHook<I>(
 163     g_ceph_context, m_local_io_ctx.get_pool_name(), this);
 164
 165   m_trash_watcher = image_deleter::TrashWatcher<I>::create(m_local_io_ctx,
 166                                                            m_threads,
 167                                                            m_trash_listener);
 168   m_trash_watcher->init(on_finish);
 169 }
 170
 171 template <typename I>
 172 void ImageDeleter<I>::shut_down(Context* on_finish) {
 173   dout(10) << dendl;
 174
 175   delete m_asok_hook;
 176   m_asok_hook = nullptr;
 177
 178   m_image_deletion_throttler->drain(m_local_io_ctx.get_namespace(),
 179                                     -ESTALE);
 180
 181   shut_down_trash_watcher(on_finish);
 182 }
 183
 184 template <typename I>
 185 void ImageDeleter<I>::shut_down_trash_watcher(Context* on_finish) {
 186   dout(10) << dendl;
 187   ceph_assert(m_trash_watcher);
 188   auto ctx = new LambdaContext([this, on_finish](int r) {
 189       delete m_trash_watcher;
 190       m_trash_watcher = nullptr;
 191
 192       wait_for_ops(on_finish);
 193     });
 194   m_trash_watcher->shut_down(ctx);
 195 }
 196
 197 template <typename I>
 198 void ImageDeleter<I>::wait_for_ops(Context* on_finish) {
 199   {
 200     std::scoped_lock locker{m_threads->timer_lock, m_lock};
 201     m_running = false;
 202     cancel_retry_timer();
 203   }
 204
 205   auto ctx = new LambdaContext([this, on_finish](int) {
 206       cancel_all_deletions(on_finish);
 207     });
 208   m_async_op_tracker.wait_for_ops(ctx);
 209 }
 210
 211 template <typename I>
 212 void ImageDeleter<I>::cancel_all_deletions(Context* on_finish) {
 213   m_image_deletion_throttler->drain(m_local_io_ctx.get_namespace(),
 214                                     -ECANCELED);
 215   {
 216     std::lock_guard locker{m_lock};
 217     // wake up any external state machines waiting on deletions
 218     ceph_assert(m_in_flight_delete_queue.empty());
 219     for (auto& queue : {&m_delete_queue, &m_retry_delete_queue}) {
 220       for (auto& info : *queue) {
 221         notify_on_delete(info->image_id, -ECANCELED);
 222       }
 223       queue->clear();
 224     }
 225   }
 226   on_finish->complete(0);
 227 }
 228
 229 template <typename I>
 230 void ImageDeleter<I>::wait_for_deletion(const std::string& image_id,
 231                                         bool scheduled_only,
 232                                         Context* on_finish) {
 233   dout(5) << "image_id=" << image_id << dendl;
 234
 235   on_finish = new LambdaContext([this, on_finish](int r) {
 236       m_threads->work_queue->queue(on_finish, r);
 237     });
 238
 239   std::lock_guard locker{m_lock};
 240   auto del_info = find_delete_info(image_id);
 241   if (!del_info && scheduled_only) {
 242     // image not scheduled for deletion
 243     on_finish->complete(0);
 244     return;
 245   }
 246
 247   notify_on_delete(image_id, -ESTALE);
 248   m_on_delete_contexts[image_id] = on_finish;
 249 }
 250
 251 template <typename I>
 252 void ImageDeleter<I>::complete_active_delete(DeleteInfoRef* delete_info,
 253                                              int r) {
 254   dout(20) << "info=" << *delete_info << ", r=" << r << dendl;
 255   std::lock_guard locker{m_lock};
 256   notify_on_delete((*delete_info)->image_id, r);
 257   delete_info->reset();
 258 }
 259
 260 template <typename I>
 261 void ImageDeleter<I>::enqueue_failed_delete(DeleteInfoRef* delete_info,
 262                                             int error_code,
 263                                             double retry_delay) {
 264   dout(20) << "info=" << *delete_info << ", r=" << error_code << dendl;
 265   if (error_code == -EBLOCKLISTED) {
 266     std::lock_guard locker{m_lock};
 267     derr << "blocklisted while deleting local image" << dendl;
 268     complete_active_delete(delete_info, error_code);
 269     return;
 270   }
 271
 272   std::scoped_lock locker{m_threads->timer_lock, m_lock};
 273   auto& delete_info_ref = *delete_info;
 274   notify_on_delete(delete_info_ref->image_id, error_code);
 275   delete_info_ref->error_code = error_code;
 276   ++delete_info_ref->retries;
 277   delete_info_ref->retry_time = (clock_t::now() +
 278                                  ceph::make_timespan(retry_delay));
 279   m_retry_delete_queue.push_back(delete_info_ref);
 280
 281   schedule_retry_timer();
 282 }
 283
 284 template <typename I>
 285 typename ImageDeleter<I>::DeleteInfoRef
 286 ImageDeleter<I>::find_delete_info(const std::string &image_id) {
 287   ceph_assert(ceph_mutex_is_locked(m_lock));
 288   DeleteQueue delete_queues[] = {m_in_flight_delete_queue,
 289                                  m_retry_delete_queue,
 290                                  m_delete_queue};
 291
 292   DeleteInfo delete_info{image_id};
 293   for (auto& queue : delete_queues) {
 294     auto it = std::find_if(queue.begin(), queue.end(),
 295                            [&delete_info](const DeleteInfoRef& ref) {
 296                              return delete_info == *ref;
 297                            });
 298     if (it != queue.end()) {
 299       return *it;
 300     }
 301   }
 302   return {};
 303 }
 304
 305 template <typename I>
 306 void ImageDeleter<I>::print_status(Formatter *f) {
 307   dout(20) << dendl;
 308
 309   f->open_object_section("image_deleter_status");
 310   f->open_array_section("delete_images_queue");
 311
 312   std::lock_guard l{m_lock};
 313   for (const auto& image : m_delete_queue) {
 314     image->print_status(f);
 315   }
 316
 317   f->close_section();
 318   f->open_array_section("failed_deletes_queue");
 319   for (const auto& image : m_retry_delete_queue) {
 320     image->print_status(f, true);
 321   }
 322
 323   f->close_section();
 324   f->close_section();
 325 }
 326
 327 template <typename I>
 328 vector<string> ImageDeleter<I>::get_delete_queue_items() {
 329   vector<string> items;
 330
 331   std::lock_guard l{m_lock};
 332   for (const auto& del_info : m_delete_queue) {
 333     items.push_back(del_info->image_id);
 334   }
 335
 336   return items;
 337 }
 338
 339 template <typename I>
 340 vector<pair<string, int> > ImageDeleter<I>::get_failed_queue_items() {
 341   vector<pair<string, int> > items;
 342
 343   std::lock_guard l{m_lock};
 344   for (const auto& del_info : m_retry_delete_queue) {
 345     items.push_back(make_pair(del_info->image_id,
 346                               del_info->error_code));
 347   }
 348
 349   return items;
 350 }
 351
 352 template <typename I>
 353 void ImageDeleter<I>::remove_images() {
 354   dout(10) << dendl;
 355
 356   std::lock_guard locker{m_lock};
 357   while (m_running && !m_delete_queue.empty()) {
 358
 359     DeleteInfoRef delete_info = m_delete_queue.front();
 360     m_delete_queue.pop_front();
 361
 362     ceph_assert(delete_info);
 363
 364     auto on_start = create_async_context_callback(
 365         m_threads->work_queue, new LambdaContext(
 366             [this, delete_info](int r) {
 367               if (r < 0) {
 368                 notify_on_delete(delete_info->image_id, r);
 369                 return;
 370               }
 371               remove_image(delete_info);
 372             }));
 373
 374     m_image_deletion_throttler->start_op(m_local_io_ctx.get_namespace(),
 375                                          delete_info->image_id, on_start);
 376   }
 377 }
 378
 379 template <typename I>
 380 void ImageDeleter<I>::remove_image(DeleteInfoRef delete_info) {
 381   dout(10) << "info=" << *delete_info << dendl;
 382
 383   std::lock_guard locker{m_lock};
 384
 385   m_in_flight_delete_queue.push_back(delete_info);
 386   m_async_op_tracker.start_op();
 387
 388   auto ctx = new LambdaContext([this, delete_info](int r) {
 389       handle_remove_image(delete_info, r);
 390       m_async_op_tracker.finish_op();
 391     });
 392
 393   auto req = image_deleter::TrashRemoveRequest<I>::create(
 394     m_local_io_ctx, delete_info->image_id, &delete_info->error_result,
 395     m_threads->work_queue, ctx);
 396   req->send();
 397 }
 398
 399 template <typename I>
 400 void ImageDeleter<I>::handle_remove_image(DeleteInfoRef delete_info,
 401                                           int r) {
 402   dout(10) << "info=" << *delete_info << ", r=" << r << dendl;
 403
 404   m_image_deletion_throttler->finish_op(m_local_io_ctx.get_namespace(),
 405                                         delete_info->image_id);
 406   {
 407     std::lock_guard locker{m_lock};
 408     ceph_assert(ceph_mutex_is_locked(m_lock));
 409     auto it = std::find(m_in_flight_delete_queue.begin(),
 410                         m_in_flight_delete_queue.end(), delete_info);
 411     ceph_assert(it != m_in_flight_delete_queue.end());
 412     m_in_flight_delete_queue.erase(it);
 413   }
 414
 415   if (r < 0) {
 416     if (delete_info->error_result == image_deleter::ERROR_RESULT_COMPLETE) {
 417       complete_active_delete(&delete_info, r);
 418     } else if (delete_info->error_result ==
 419                  image_deleter::ERROR_RESULT_RETRY_IMMEDIATELY) {
 420       enqueue_failed_delete(&delete_info, r, m_busy_interval);
 421     } else {
 422       auto cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
 423       double failed_interval = cct->_conf.get_val<double>(
 424         "rbd_mirror_delete_retry_interval");
 425       enqueue_failed_delete(&delete_info, r, failed_interval);
 426     }
 427   } else {
 428     complete_active_delete(&delete_info, 0);
 429   }
 430
 431   // process the next queued image to delete
 432   remove_images();
 433 }
 434
 435 template <typename I>
 436 void ImageDeleter<I>::schedule_retry_timer() {
 437   ceph_assert(ceph_mutex_is_locked(m_threads->timer_lock));
 438   ceph_assert(ceph_mutex_is_locked(m_lock));
 439   if (!m_running || m_timer_ctx != nullptr || m_retry_delete_queue.empty()) {
 440     return;
 441   }
 442
 443   dout(10) << dendl;
 444   auto &delete_info = m_retry_delete_queue.front();
 445   m_timer_ctx = new LambdaContext([this](int r) {
 446       handle_retry_timer();
 447     });
 448   m_threads->timer->add_event_at(delete_info->retry_time, m_timer_ctx);
 449 }
 450
 451 template <typename I>
 452 void ImageDeleter<I>::cancel_retry_timer() {
 453   dout(10) << dendl;
 454   ceph_assert(ceph_mutex_is_locked(m_threads->timer_lock));
 455   if (m_timer_ctx != nullptr) {
 456     bool canceled = m_threads->timer->cancel_event(m_timer_ctx);
 457     m_timer_ctx = nullptr;
 458     ceph_assert(canceled);
 459   }
 460 }
 461
 462 template <typename I>
 463 void ImageDeleter<I>::handle_retry_timer() {
 464   dout(10) << dendl;
 465   ceph_assert(ceph_mutex_is_locked(m_threads->timer_lock));
 466   std::lock_guard locker{m_lock};
 467
 468   ceph_assert(m_timer_ctx != nullptr);
 469   m_timer_ctx = nullptr;
 470
 471   ceph_assert(m_running);
 472   ceph_assert(!m_retry_delete_queue.empty());
 473
 474   // move all ready-to-ready items back to main queue
 475   auto now = clock_t::now();
 476   while (!m_retry_delete_queue.empty()) {
 477     auto &delete_info = m_retry_delete_queue.front();
 478     if (delete_info->retry_time > now) {
 479       break;
 480     }
 481
 482     m_delete_queue.push_back(delete_info);
 483     m_retry_delete_queue.pop_front();
 484   }
 485
 486   // schedule wake up for any future retries
 487   schedule_retry_timer();
 488
 489   // start (concurrent) removal of images
 490   m_async_op_tracker.start_op();
 491   auto ctx = new LambdaContext([this](int r) {
 492       remove_images();
 493       m_async_op_tracker.finish_op();
 494     });
 495   m_threads->work_queue->queue(ctx, 0);
 496 }
 497
 498 template <typename I>
 499 void ImageDeleter<I>::handle_trash_image(const std::string& image_id,
 500   const ImageDeleter<I>::clock_t::time_point& deferment_end_time) {
 501   std::scoped_lock locker{m_threads->timer_lock, m_lock};
 502
 503   auto del_info = find_delete_info(image_id);
 504   if (del_info != nullptr) {
 505     dout(20) << "image " << image_id << " "
 506              << "was already scheduled for deletion" << dendl;
 507     return;
 508   }
 509
 510   dout(10) << "image_id=" << image_id << ", "
 511            << "deferment_end_time=" << utime_t{deferment_end_time} << dendl;
 512
 513   del_info.reset(new DeleteInfo(image_id));
 514   del_info->retry_time = deferment_end_time;
 515   m_retry_delete_queue.push_back(del_info);
 516
 517   schedule_retry_timer();
 518 }
 519
 520 template <typename I>
 521 void ImageDeleter<I>::notify_on_delete(const std::string& image_id,
 522                                        int r) {
 523   dout(10) << "image_id=" << image_id << ", r=" << r << dendl;
 524   auto it = m_on_delete_contexts.find(image_id);
 525   if (it == m_on_delete_contexts.end()) {
 526     return;
 527   }
 528
 529   it->second->complete(r);
 530   m_on_delete_contexts.erase(it);
 531 }
 532
 533 template <typename I>
 534 void ImageDeleter<I>::DeleteInfo::print_status(Formatter *f,
 535                                                bool print_failure_info) {
 536   f->open_object_section("delete_info");
 537   f->dump_string("image_id", image_id);
 538   if (print_failure_info) {
 539     f->dump_string("error_code", cpp_strerror(error_code));
 540     f->dump_int("retries", retries);
 541   }
 542   f->close_section();
 543 }
 544
 545 } // namespace mirror
 546 } // namespace rbd
 547
 548 template class rbd::mirror::ImageDeleter<librbd::ImageCtx>;