ceph/src/tools/rbd_mirror/ImageDeleter.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 SUSE LINUX GmbH
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #include "include/rados/librados.hpp"
  16 #include "common/Formatter.h"
  17 #include "common/admin_socket.h"
  18 #include "common/debug.h"
  19 #include "common/errno.h"
  20 #include "common/Timer.h"
  21 #include "global/global_context.h"
  22 #include "librbd/internal.h"
  23 #include "librbd/ImageCtx.h"
  24 #include "librbd/ImageState.h"
  25 #include "librbd/Operations.h"
  26 #include "librbd/asio/ContextWQ.h"
  27 #include "cls/rbd/cls_rbd_client.h"
  28 #include "cls/rbd/cls_rbd_types.h"
  29 #include "librbd/Utils.h"
  30 #include "ImageDeleter.h"
  31 #include "tools/rbd_mirror/Threads.h"
  32 #include "tools/rbd_mirror/Throttler.h"
  33 #include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h"
  34 #include "tools/rbd_mirror/image_deleter/TrashRemoveRequest.h"
  35 #include "tools/rbd_mirror/image_deleter/TrashWatcher.h"
  36 #include <map>
  37 #include <sstream>
  38
  39 #define dout_context g_ceph_context
  40 #define dout_subsys ceph_subsys_rbd_mirror
  41
  42 using std::string;
  43 using std::stringstream;
  44 using std::vector;
  45 using std::pair;
  46 using std::make_pair;
  47
  48 using librados::IoCtx;
  49 using namespace librbd;
  50
  51 namespace rbd {
  52 namespace mirror {
  53
  54 using librbd::util::create_async_context_callback;
  55
  56 namespace {
  57
  58 class ImageDeleterAdminSocketCommand {
  59 public:
  60   virtual ~ImageDeleterAdminSocketCommand() {}
  61   virtual int call(Formatter *f) = 0;
  62 };
  63
  64 template <typename I>
  65 class StatusCommand : public ImageDeleterAdminSocketCommand {
  66 public:
  67   explicit StatusCommand(ImageDeleter<I> *image_del) : image_del(image_del) {}
  68
  69   int call(Formatter *f) override {
  70     image_del->print_status(f);
  71     return 0;
  72   }
  73
  74 private:
  75   ImageDeleter<I> *image_del;
  76 };
  77
  78 } // anonymous namespace
  79
  80 template <typename I>
  81 class ImageDeleterAdminSocketHook : public AdminSocketHook {
  82 public:
  83   ImageDeleterAdminSocketHook(CephContext *cct, const std::string& pool_name,
  84                               ImageDeleter<I> *image_del) :
  85     admin_socket(cct->get_admin_socket()) {
  86
  87     std::string command;
  88     int r;
  89
  90     command = "rbd mirror deletion status " + pool_name;
  91     r = admin_socket->register_command(command, this,
  92                                        "get status for image deleter");
  93     if (r == 0) {
  94       commands[command] = new StatusCommand<I>(image_del);
  95     }
  96
  97   }
  98
  99   ~ImageDeleterAdminSocketHook() override {
 100     (void)admin_socket->unregister_commands(this);
 101     for (Commands::const_iterator i = commands.begin(); i != commands.end();
 102          ++i) {
 103       delete i->second;
 104     }
 105   }
 106
 107   int call(std::string_view command, const cmdmap_t& cmdmap,
 108            const bufferlist&,
 109            Formatter *f,
 110            std::ostream& errss,
 111            bufferlist& out) override {
 112     Commands::const_iterator i = commands.find(command);
 113     ceph_assert(i != commands.end());
 114     return i->second->call(f);
 115   }
 116
 117 private:
 118   typedef std::map<std::string, ImageDeleterAdminSocketCommand*,
 119                    std::less<>> Commands;
 120   AdminSocket *admin_socket;
 121   Commands commands;
 122 };
 123
 124 template <typename I>
 125 ImageDeleter<I>::ImageDeleter(
 126     librados::IoCtx& local_io_ctx, Threads<librbd::ImageCtx>* threads,
 127     Throttler<librbd::ImageCtx>* image_deletion_throttler,
 128     ServiceDaemon<librbd::ImageCtx>* service_daemon)
 129   : m_local_io_ctx(local_io_ctx), m_threads(threads),
 130     m_image_deletion_throttler(image_deletion_throttler),
 131     m_service_daemon(service_daemon), m_trash_listener(this),
 132     m_lock(ceph::make_mutex(
 133       librbd::util::unique_lock_name("rbd::mirror::ImageDeleter::m_lock",
 134                                      this))) {
 135 }
 136
 137 #undef dout_prefix
 138 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << " " \
 139                            << __func__ << ": "
 140
 141 template <typename I>
 142 void ImageDeleter<I>::trash_move(librados::IoCtx& local_io_ctx,
 143                                  const std::string& global_image_id,
 144                                  bool resync,
 145                                  librbd::asio::ContextWQ* work_queue,
 146                                  Context* on_finish) {
 147   dout(10) << "global_image_id=" << global_image_id << ", "
 148            << "resync=" << resync << dendl;
 149
 150   auto req = rbd::mirror::image_deleter::TrashMoveRequest<>::create(
 151     local_io_ctx, global_image_id, resync, work_queue, on_finish);
 152   req->send();
 153 }
 154
 155 #undef dout_prefix
 156 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
 157                            << __func__ << ": "
 158
 159 template <typename I>
 160 void ImageDeleter<I>::init(Context* on_finish) {
 161   dout(10) << dendl;
 162
 163   m_asok_hook = new ImageDeleterAdminSocketHook<I>(
 164     g_ceph_context, m_local_io_ctx.get_pool_name(), this);
 165
 166   m_trash_watcher = image_deleter::TrashWatcher<I>::create(m_local_io_ctx,
 167                                                            m_threads,
 168                                                            m_trash_listener);
 169   m_trash_watcher->init(on_finish);
 170 }
 171
 172 template <typename I>
 173 void ImageDeleter<I>::shut_down(Context* on_finish) {
 174   dout(10) << dendl;
 175
 176   delete m_asok_hook;
 177   m_asok_hook = nullptr;
 178
 179   m_image_deletion_throttler->drain(m_local_io_ctx.get_namespace(),
 180                                     -ESTALE);
 181
 182   shut_down_trash_watcher(on_finish);
 183 }
 184
 185 template <typename I>
 186 void ImageDeleter<I>::shut_down_trash_watcher(Context* on_finish) {
 187   dout(10) << dendl;
 188   ceph_assert(m_trash_watcher);
 189   auto ctx = new LambdaContext([this, on_finish](int r) {
 190       delete m_trash_watcher;
 191       m_trash_watcher = nullptr;
 192
 193       wait_for_ops(on_finish);
 194     });
 195   m_trash_watcher->shut_down(ctx);
 196 }
 197
 198 template <typename I>
 199 void ImageDeleter<I>::wait_for_ops(Context* on_finish) {
 200   {
 201     std::scoped_lock locker{m_threads->timer_lock, m_lock};
 202     m_running = false;
 203     cancel_retry_timer();
 204   }
 205
 206   auto ctx = new LambdaContext([this, on_finish](int) {
 207       cancel_all_deletions(on_finish);
 208     });
 209   m_async_op_tracker.wait_for_ops(ctx);
 210 }
 211
 212 template <typename I>
 213 void ImageDeleter<I>::cancel_all_deletions(Context* on_finish) {
 214   m_image_deletion_throttler->drain(m_local_io_ctx.get_namespace(),
 215                                     -ECANCELED);
 216   {
 217     std::lock_guard locker{m_lock};
 218     // wake up any external state machines waiting on deletions
 219     ceph_assert(m_in_flight_delete_queue.empty());
 220     for (auto& queue : {&m_delete_queue, &m_retry_delete_queue}) {
 221       for (auto& info : *queue) {
 222         notify_on_delete(info->image_id, -ECANCELED);
 223       }
 224       queue->clear();
 225     }
 226   }
 227   on_finish->complete(0);
 228 }
 229
 230 template <typename I>
 231 void ImageDeleter<I>::wait_for_deletion(const std::string& image_id,
 232                                         bool scheduled_only,
 233                                         Context* on_finish) {
 234   dout(5) << "image_id=" << image_id << dendl;
 235
 236   on_finish = new LambdaContext([this, on_finish](int r) {
 237       m_threads->work_queue->queue(on_finish, r);
 238     });
 239
 240   std::lock_guard locker{m_lock};
 241   auto del_info = find_delete_info(image_id);
 242   if (!del_info && scheduled_only) {
 243     // image not scheduled for deletion
 244     on_finish->complete(0);
 245     return;
 246   }
 247
 248   notify_on_delete(image_id, -ESTALE);
 249   m_on_delete_contexts[image_id] = on_finish;
 250 }
 251
 252 template <typename I>
 253 void ImageDeleter<I>::complete_active_delete(DeleteInfoRef* delete_info,
 254                                              int r) {
 255   dout(20) << "info=" << *delete_info << ", r=" << r << dendl;
 256   std::lock_guard locker{m_lock};
 257   notify_on_delete((*delete_info)->image_id, r);
 258   delete_info->reset();
 259 }
 260
 261 template <typename I>
 262 void ImageDeleter<I>::enqueue_failed_delete(DeleteInfoRef* delete_info,
 263                                             int error_code,
 264                                             double retry_delay) {
 265   dout(20) << "info=" << *delete_info << ", r=" << error_code << dendl;
 266   if (error_code == -EBLOCKLISTED) {
 267     std::lock_guard locker{m_lock};
 268     derr << "blocklisted while deleting local image" << dendl;
 269     complete_active_delete(delete_info, error_code);
 270     return;
 271   }
 272
 273   std::scoped_lock locker{m_threads->timer_lock, m_lock};
 274   auto& delete_info_ref = *delete_info;
 275   notify_on_delete(delete_info_ref->image_id, error_code);
 276   delete_info_ref->error_code = error_code;
 277   ++delete_info_ref->retries;
 278   delete_info_ref->retry_time = (clock_t::now() +
 279                                  ceph::make_timespan(retry_delay));
 280   m_retry_delete_queue.push_back(delete_info_ref);
 281
 282   schedule_retry_timer();
 283 }
 284
 285 template <typename I>
 286 typename ImageDeleter<I>::DeleteInfoRef
 287 ImageDeleter<I>::find_delete_info(const std::string &image_id) {
 288   ceph_assert(ceph_mutex_is_locked(m_lock));
 289   DeleteQueue delete_queues[] = {m_in_flight_delete_queue,
 290                                  m_retry_delete_queue,
 291                                  m_delete_queue};
 292
 293   DeleteInfo delete_info{image_id};
 294   for (auto& queue : delete_queues) {
 295     auto it = std::find_if(queue.begin(), queue.end(),
 296                            [&delete_info](const DeleteInfoRef& ref) {
 297                              return delete_info == *ref;
 298                            });
 299     if (it != queue.end()) {
 300       return *it;
 301     }
 302   }
 303   return {};
 304 }
 305
 306 template <typename I>
 307 void ImageDeleter<I>::print_status(Formatter *f) {
 308   dout(20) << dendl;
 309
 310   f->open_object_section("image_deleter_status");
 311   f->open_array_section("delete_images_queue");
 312
 313   std::lock_guard l{m_lock};
 314   for (const auto& image : m_delete_queue) {
 315     image->print_status(f);
 316   }
 317
 318   f->close_section();
 319   f->open_array_section("failed_deletes_queue");
 320   for (const auto& image : m_retry_delete_queue) {
 321     image->print_status(f, true);
 322   }
 323
 324   f->close_section();
 325   f->close_section();
 326 }
 327
 328 template <typename I>
 329 vector<string> ImageDeleter<I>::get_delete_queue_items() {
 330   vector<string> items;
 331
 332   std::lock_guard l{m_lock};
 333   for (const auto& del_info : m_delete_queue) {
 334     items.push_back(del_info->image_id);
 335   }
 336
 337   return items;
 338 }
 339
 340 template <typename I>
 341 vector<pair<string, int> > ImageDeleter<I>::get_failed_queue_items() {
 342   vector<pair<string, int> > items;
 343
 344   std::lock_guard l{m_lock};
 345   for (const auto& del_info : m_retry_delete_queue) {
 346     items.push_back(make_pair(del_info->image_id,
 347                               del_info->error_code));
 348   }
 349
 350   return items;
 351 }
 352
 353 template <typename I>
 354 void ImageDeleter<I>::remove_images() {
 355   dout(10) << dendl;
 356
 357   std::lock_guard locker{m_lock};
 358   while (m_running && !m_delete_queue.empty()) {
 359
 360     DeleteInfoRef delete_info = m_delete_queue.front();
 361     m_delete_queue.pop_front();
 362
 363     ceph_assert(delete_info);
 364
 365     auto on_start = create_async_context_callback(
 366         m_threads->work_queue, new LambdaContext(
 367             [this, delete_info](int r) {
 368               if (r < 0) {
 369                 notify_on_delete(delete_info->image_id, r);
 370                 return;
 371               }
 372               remove_image(delete_info);
 373             }));
 374
 375     m_image_deletion_throttler->start_op(m_local_io_ctx.get_namespace(),
 376                                          delete_info->image_id, on_start);
 377   }
 378 }
 379
 380 template <typename I>
 381 void ImageDeleter<I>::remove_image(DeleteInfoRef delete_info) {
 382   dout(10) << "info=" << *delete_info << dendl;
 383
 384   std::lock_guard locker{m_lock};
 385
 386   m_in_flight_delete_queue.push_back(delete_info);
 387   m_async_op_tracker.start_op();
 388
 389   auto ctx = new LambdaContext([this, delete_info](int r) {
 390       handle_remove_image(delete_info, r);
 391       m_async_op_tracker.finish_op();
 392     });
 393
 394   auto req = image_deleter::TrashRemoveRequest<I>::create(
 395     m_local_io_ctx, delete_info->image_id, &delete_info->error_result,
 396     m_threads->work_queue, ctx);
 397   req->send();
 398 }
 399
 400 template <typename I>
 401 void ImageDeleter<I>::handle_remove_image(DeleteInfoRef delete_info,
 402                                           int r) {
 403   dout(10) << "info=" << *delete_info << ", r=" << r << dendl;
 404
 405   m_image_deletion_throttler->finish_op(m_local_io_ctx.get_namespace(),
 406                                         delete_info->image_id);
 407   {
 408     std::lock_guard locker{m_lock};
 409     ceph_assert(ceph_mutex_is_locked(m_lock));
 410     auto it = std::find(m_in_flight_delete_queue.begin(),
 411                         m_in_flight_delete_queue.end(), delete_info);
 412     ceph_assert(it != m_in_flight_delete_queue.end());
 413     m_in_flight_delete_queue.erase(it);
 414   }
 415
 416   if (r < 0) {
 417     if (delete_info->error_result == image_deleter::ERROR_RESULT_COMPLETE) {
 418       complete_active_delete(&delete_info, r);
 419     } else if (delete_info->error_result ==
 420                  image_deleter::ERROR_RESULT_RETRY_IMMEDIATELY) {
 421       enqueue_failed_delete(&delete_info, r, m_busy_interval);
 422     } else {
 423       auto cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
 424       double failed_interval = cct->_conf.get_val<double>(
 425         "rbd_mirror_delete_retry_interval");
 426       enqueue_failed_delete(&delete_info, r, failed_interval);
 427     }
 428   } else {
 429     complete_active_delete(&delete_info, 0);
 430   }
 431
 432   // process the next queued image to delete
 433   remove_images();
 434 }
 435
 436 template <typename I>
 437 void ImageDeleter<I>::schedule_retry_timer() {
 438   ceph_assert(ceph_mutex_is_locked(m_threads->timer_lock));
 439   ceph_assert(ceph_mutex_is_locked(m_lock));
 440   if (!m_running || m_timer_ctx != nullptr || m_retry_delete_queue.empty()) {
 441     return;
 442   }
 443
 444   dout(10) << dendl;
 445   auto &delete_info = m_retry_delete_queue.front();
 446   m_timer_ctx = new LambdaContext([this](int r) {
 447       handle_retry_timer();
 448     });
 449   m_threads->timer->add_event_at(delete_info->retry_time, m_timer_ctx);
 450 }
 451
 452 template <typename I>
 453 void ImageDeleter<I>::cancel_retry_timer() {
 454   dout(10) << dendl;
 455   ceph_assert(ceph_mutex_is_locked(m_threads->timer_lock));
 456   if (m_timer_ctx != nullptr) {
 457     bool canceled = m_threads->timer->cancel_event(m_timer_ctx);
 458     m_timer_ctx = nullptr;
 459     ceph_assert(canceled);
 460   }
 461 }
 462
 463 template <typename I>
 464 void ImageDeleter<I>::handle_retry_timer() {
 465   dout(10) << dendl;
 466   ceph_assert(ceph_mutex_is_locked(m_threads->timer_lock));
 467   std::lock_guard locker{m_lock};
 468
 469   ceph_assert(m_timer_ctx != nullptr);
 470   m_timer_ctx = nullptr;
 471
 472   ceph_assert(m_running);
 473   ceph_assert(!m_retry_delete_queue.empty());
 474
 475   // move all ready-to-ready items back to main queue
 476   auto now = clock_t::now();
 477   while (!m_retry_delete_queue.empty()) {
 478     auto &delete_info = m_retry_delete_queue.front();
 479     if (delete_info->retry_time > now) {
 480       break;
 481     }
 482
 483     m_delete_queue.push_back(delete_info);
 484     m_retry_delete_queue.pop_front();
 485   }
 486
 487   // schedule wake up for any future retries
 488   schedule_retry_timer();
 489
 490   // start (concurrent) removal of images
 491   m_async_op_tracker.start_op();
 492   auto ctx = new LambdaContext([this](int r) {
 493       remove_images();
 494       m_async_op_tracker.finish_op();
 495     });
 496   m_threads->work_queue->queue(ctx, 0);
 497 }
 498
 499 template <typename I>
 500 void ImageDeleter<I>::handle_trash_image(const std::string& image_id,
 501   const ImageDeleter<I>::clock_t::time_point& deferment_end_time) {
 502   std::scoped_lock locker{m_threads->timer_lock, m_lock};
 503
 504   auto del_info = find_delete_info(image_id);
 505   if (del_info != nullptr) {
 506     dout(20) << "image " << image_id << " "
 507              << "was already scheduled for deletion" << dendl;
 508     return;
 509   }
 510
 511   dout(10) << "image_id=" << image_id << ", "
 512            << "deferment_end_time=" << utime_t{deferment_end_time} << dendl;
 513
 514   del_info.reset(new DeleteInfo(image_id));
 515   del_info->retry_time = deferment_end_time;
 516   m_retry_delete_queue.push_back(del_info);
 517
 518   schedule_retry_timer();
 519 }
 520
 521 template <typename I>
 522 void ImageDeleter<I>::notify_on_delete(const std::string& image_id,
 523                                        int r) {
 524   dout(10) << "image_id=" << image_id << ", r=" << r << dendl;
 525   auto it = m_on_delete_contexts.find(image_id);
 526   if (it == m_on_delete_contexts.end()) {
 527     return;
 528   }
 529
 530   it->second->complete(r);
 531   m_on_delete_contexts.erase(it);
 532 }
 533
 534 template <typename I>
 535 void ImageDeleter<I>::DeleteInfo::print_status(Formatter *f,
 536                                                bool print_failure_info) {
 537   f->open_object_section("delete_info");
 538   f->dump_string("image_id", image_id);
 539   if (print_failure_info) {
 540     f->dump_string("error_code", cpp_strerror(error_code));
 541     f->dump_int("retries", retries);
 542   }
 543   f->close_section();
 544 }
 545
 546 } // namespace mirror
 547 } // namespace rbd
 548
 549 template class rbd::mirror::ImageDeleter<librbd::ImageCtx>;