ceph/src/tools/rbd_mirror/ImageDeleter.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 SUSE LINUX GmbH
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #include <boost/bind.hpp>
  16 #include <map>
  17 #include <set>
  18 #include <sstream>
  19
  20 #include "include/rados/librados.hpp"
  21 #include "common/Formatter.h"
  22 #include "common/admin_socket.h"
  23 #include "common/debug.h"
  24 #include "common/errno.h"
  25 #include "common/WorkQueue.h"
  26 #include "global/global_context.h"
  27 #include "librbd/internal.h"
  28 #include "librbd/ImageCtx.h"
  29 #include "librbd/ImageState.h"
  30 #include "librbd/Journal.h"
  31 #include "librbd/Operations.h"
  32 #include "librbd/journal/Policy.h"
  33 #include "cls/rbd/cls_rbd_client.h"
  34 #include "cls/rbd/cls_rbd_types.h"
  35 #include "librbd/Utils.h"
  36 #include "ImageDeleter.h"
  37
  38 #define dout_context g_ceph_context
  39 #define dout_subsys ceph_subsys_rbd_mirror
  40 #undef dout_prefix
  41 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
  42                            << __func__ << ": "
  43
  44 using std::string;
  45 using std::map;
  46 using std::stringstream;
  47 using std::vector;
  48 using std::pair;
  49 using std::make_pair;
  50
  51 using librados::IoCtx;
  52 using namespace librbd;
  53
  54 namespace rbd {
  55 namespace mirror {
  56
  57 namespace {
  58
  59 class ImageDeleterAdminSocketCommand {
  60 public:
  61   virtual ~ImageDeleterAdminSocketCommand() {}
  62   virtual bool call(Formatter *f, stringstream *ss) = 0;
  63 };
  64
  65 template <typename I>
  66 class StatusCommand : public ImageDeleterAdminSocketCommand {
  67 public:
  68   explicit StatusCommand(ImageDeleter<I> *image_del) : image_del(image_del) {}
  69
  70   bool call(Formatter *f, stringstream *ss) override {
  71     image_del->print_status(f, ss);
  72     return true;
  73   }
  74
  75 private:
  76   ImageDeleter<I> *image_del;
  77 };
  78
  79 struct DeleteJournalPolicy : public librbd::journal::Policy {
  80   bool append_disabled() const override {
  81     return true;
  82   }
  83   bool journal_disabled() const override {
  84     return false;
  85   }
  86
  87   void allocate_tag_on_lock(Context *on_finish) override {
  88     on_finish->complete(0);
  89   }
  90 };
  91
  92 } // anonymous namespace
  93
  94 template <typename I>
  95 class ImageDeleterAdminSocketHook : public AdminSocketHook {
  96 public:
  97   ImageDeleterAdminSocketHook(CephContext *cct, ImageDeleter<I> *image_del) :
  98     admin_socket(cct->get_admin_socket()) {
  99
 100     std::string command;
 101     int r;
 102
 103     command = "rbd mirror deletion status";
 104     r = admin_socket->register_command(command, command, this,
 105                                        "get status for image deleter");
 106     if (r == 0) {
 107       commands[command] = new StatusCommand<I>(image_del);
 108     }
 109
 110   }
 111
 112   ~ImageDeleterAdminSocketHook() override {
 113     for (Commands::const_iterator i = commands.begin(); i != commands.end();
 114          ++i) {
 115       (void)admin_socket->unregister_command(i->first);
 116       delete i->second;
 117     }
 118   }
 119
 120   bool call(std::string command, cmdmap_t& cmdmap, std::string format,
 121             bufferlist& out) override {
 122     Commands::const_iterator i = commands.find(command);
 123     assert(i != commands.end());
 124     Formatter *f = Formatter::create(format);
 125     stringstream ss;
 126     bool r = i->second->call(f, &ss);
 127     delete f;
 128     out.append(ss);
 129     return r;
 130   }
 131
 132 private:
 133   typedef std::map<std::string, ImageDeleterAdminSocketCommand*> Commands;
 134   AdminSocket *admin_socket;
 135   Commands commands;
 136 };
 137
 138 template <typename I>
 139 ImageDeleter<I>::ImageDeleter(ContextWQ *work_queue, SafeTimer *timer,
 140                               Mutex *timer_lock,
 141                               ServiceDaemon<librbd::ImageCtx>* service_daemon)
 142   : m_work_queue(work_queue),
 143     m_service_daemon(service_daemon),
 144     m_delete_lock("rbd::mirror::ImageDeleter::Delete"),
 145     m_image_deleter_thread(this),
 146     m_failed_timer(timer),
 147     m_failed_timer_lock(timer_lock),
 148     m_asok_hook(new ImageDeleterAdminSocketHook<I>(g_ceph_context, this))
 149 {
 150   set_failed_timer_interval(g_ceph_context->_conf->rbd_mirror_delete_retry_interval);
 151   m_image_deleter_thread.create("image_deleter");
 152 }
 153
 154 template <typename I>
 155 ImageDeleter<I>::~ImageDeleter() {
 156   dout(20) << "enter" << dendl;
 157
 158   m_running = false;
 159   {
 160     Mutex::Locker l (m_delete_lock);
 161     m_delete_queue_cond.Signal();
 162   }
 163   if (m_image_deleter_thread.is_started()) {
 164     m_image_deleter_thread.join();
 165   }
 166
 167   delete m_asok_hook;
 168   dout(20) << "return" << dendl;
 169 }
 170
 171 template <typename I>
 172 void ImageDeleter<I>::run() {
 173   dout(20) << "enter" << dendl;
 174   while(m_running) {
 175     m_delete_lock.Lock();
 176     while (m_delete_queue.empty()) {
 177       dout(20) << "waiting for delete requests" << dendl;
 178       m_delete_queue_cond.Wait(m_delete_lock);
 179
 180       if (!m_running) {
 181         m_delete_lock.Unlock();
 182         dout(20) << "return" << dendl;
 183         return;
 184       }
 185     }
 186
 187     m_active_delete = std::move(m_delete_queue.back());
 188     m_delete_queue.pop_back();
 189     m_delete_lock.Unlock();
 190
 191     bool move_to_next = process_image_delete();
 192     if (!move_to_next) {
 193       if (!m_running) {
 194        dout(20) << "return" << dendl;
 195        return;
 196       }
 197
 198       Mutex::Locker l(m_delete_lock);
 199       if (m_delete_queue.size() == 1) {
 200         m_delete_queue_cond.Wait(m_delete_lock);
 201       }
 202     }
 203   }
 204 }
 205
 206 template <typename I>
 207 void ImageDeleter<I>::schedule_image_delete(RadosRef local_rados,
 208                                             int64_t local_pool_id,
 209                                             const std::string& global_image_id,
 210                                             bool ignore_orphaned) {
 211   dout(20) << "enter" << dendl;
 212
 213   Mutex::Locker locker(m_delete_lock);
 214
 215   auto del_info = find_delete_info(local_pool_id, global_image_id);
 216   if (del_info != nullptr) {
 217     dout(20) << "image " << global_image_id << " "
 218              << "was already scheduled for deletion" << dendl;
 219     if (ignore_orphaned) {
 220       (*del_info)->ignore_orphaned = true;
 221     }
 222     return;
 223   }
 224
 225   m_delete_queue.push_front(
 226     unique_ptr<DeleteInfo>(new DeleteInfo(local_rados, local_pool_id,
 227                                           global_image_id, ignore_orphaned)));
 228   m_delete_queue_cond.Signal();
 229 }
 230
 231 template <typename I>
 232 void ImageDeleter<I>::wait_for_scheduled_deletion(int64_t local_pool_id,
 233                                                   const std::string &global_image_id,
 234                                                   Context *ctx,
 235                                                   bool notify_on_failed_retry) {
 236
 237   ctx = new FunctionContext([this, ctx](int r) {
 238       m_work_queue->queue(ctx, r);
 239     });
 240
 241   Mutex::Locker locker(m_delete_lock);
 242   auto del_info = find_delete_info(local_pool_id, global_image_id);
 243   if (!del_info) {
 244     // image not scheduled for deletion
 245     ctx->complete(0);
 246     return;
 247   }
 248
 249   dout(20) << "local_pool_id=" << local_pool_id << ", "
 250            << "global_image_id=" << global_image_id << dendl;
 251
 252   if ((*del_info)->on_delete != nullptr) {
 253     (*del_info)->on_delete->complete(-ESTALE);
 254   }
 255   (*del_info)->on_delete = ctx;
 256   (*del_info)->notify_on_failed_retry = notify_on_failed_retry;
 257 }
 258
 259 template <typename I>
 260 void ImageDeleter<I>::cancel_waiter(int64_t local_pool_id,
 261                                     const std::string &global_image_id) {
 262   Mutex::Locker locker(m_delete_lock);
 263   auto del_info = find_delete_info(local_pool_id, global_image_id);
 264   if (!del_info) {
 265     return;
 266   }
 267
 268   if ((*del_info)->on_delete != nullptr) {
 269     (*del_info)->on_delete->complete(-ECANCELED);
 270     (*del_info)->on_delete = nullptr;
 271   }
 272 }
 273
 274 template <typename I>
 275 bool ImageDeleter<I>::process_image_delete() {
 276   stringstream ss;
 277   m_active_delete->to_string(ss);
 278   std::string del_info_str = ss.str();
 279   dout(10) << "start processing delete request: " << del_info_str << dendl;
 280   int r;
 281   cls::rbd::MirrorImage mirror_image;
 282
 283   // remote image was disabled, now we need to delete local image
 284   IoCtx ioctx;
 285   r = m_active_delete->local_rados->ioctx_create2(
 286     m_active_delete->local_pool_id, ioctx);
 287   if (r < 0) {
 288     derr << "error accessing local pool " << m_active_delete->local_pool_id
 289          << ": " << cpp_strerror(r) << dendl;
 290     enqueue_failed_delete(r);
 291     return true;
 292   }
 293
 294   dout(20) << "connected to local pool: " << ioctx.get_pool_name() << dendl;
 295
 296   auto &global_image_id = m_active_delete->global_image_id;
 297   std::string local_image_id;
 298   r = librbd::cls_client::mirror_image_get_image_id(
 299     &ioctx, global_image_id, &local_image_id);
 300   if (r == -ENOENT) {
 301     dout(10) << "image " << global_image_id << " is not mirrored" << dendl;
 302     complete_active_delete(r);
 303     return true;
 304   } else if (r < 0) {
 305     derr << "error retrieving local id for image " << global_image_id
 306          << ": " << cpp_strerror(r) << dendl;
 307     enqueue_failed_delete(r);
 308     return true;
 309   }
 310
 311   std::string mirror_uuid;
 312   C_SaferCond tag_owner_ctx;
 313   Journal<>::get_tag_owner(ioctx, local_image_id, &mirror_uuid, m_work_queue,
 314                            &tag_owner_ctx);
 315   r = tag_owner_ctx.wait();
 316   if (r < 0 && r != -ENOENT) {
 317     derr << "error retrieving image primary info for image " << global_image_id
 318          << ": " << cpp_strerror(r) << dendl;
 319     enqueue_failed_delete(r);
 320     return true;
 321   } else if (r != -ENOENT) {
 322     if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) {
 323       dout(10) << "image " << global_image_id << " is local primary" << dendl;
 324       complete_active_delete(-EISPRM);
 325       return true;
 326     } else if (mirror_uuid == Journal<>::ORPHAN_MIRROR_UUID &&
 327                !m_active_delete->ignore_orphaned) {
 328       dout(10) << "image " << global_image_id << " is orphaned" << dendl;
 329       complete_active_delete(-EISPRM);
 330       return true;
 331     }
 332   }
 333
 334   dout(20) << "local image is not the primary" << dendl;
 335   bool has_snapshots;
 336   r = image_has_snapshots_and_children(&ioctx, local_image_id, &has_snapshots);
 337   if (r < 0) {
 338     enqueue_failed_delete(r);
 339     return true;
 340   }
 341
 342   mirror_image.global_image_id = global_image_id;
 343   mirror_image.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING;
 344   r = cls_client::mirror_image_set(&ioctx, local_image_id, mirror_image);
 345   if (r == -ENOENT) {
 346     dout(10) << "local image is not mirrored, aborting deletion..." << dendl;
 347     complete_active_delete(r);
 348     return true;
 349   } else if (r == -EEXIST || r == -EINVAL) {
 350     derr << "cannot disable mirroring for image " << global_image_id
 351          << ": global_image_id has changed/reused: "
 352          << cpp_strerror(r) << dendl;
 353     complete_active_delete(r);
 354     return true;
 355   } else if (r < 0) {
 356     derr << "cannot disable mirroring for image " << global_image_id
 357          << ": " << cpp_strerror(r) << dendl;
 358     enqueue_failed_delete(r);
 359     return true;
 360   }
 361
 362   dout(20) << "set local image mirroring to disable" << dendl;
 363
 364   if (has_snapshots) {
 365     dout(20) << "local image has snapshots" << dendl;
 366
 367     ImageCtx *imgctx = new ImageCtx("", local_image_id, nullptr, ioctx, false);
 368     r = imgctx->state->open(false);
 369     if (r < 0) {
 370       derr << "error opening image " << global_image_id << " ("
 371            << local_image_id << "): " << cpp_strerror(r) << dendl;
 372       enqueue_failed_delete(r);
 373       return true;
 374     }
 375
 376     {
 377       RWLock::WLocker snap_locker(imgctx->snap_lock);
 378       imgctx->set_journal_policy(new DeleteJournalPolicy());
 379     }
 380
 381     std::vector<librbd::snap_info_t> snaps;
 382     r = librbd::snap_list(imgctx, snaps);
 383     if (r < 0) {
 384       derr << "error listing snapshot of image " << imgctx->name
 385            << cpp_strerror(r) << dendl;
 386       imgctx->state->close();
 387       enqueue_failed_delete(r);
 388       return true;
 389     }
 390
 391     for (const auto& snap : snaps) {
 392       dout(20) << "processing deletion of snapshot " << imgctx->name << "@"
 393                << snap.name << dendl;
 394
 395       bool is_protected;
 396       r = librbd::snap_is_protected(imgctx, snap.name.c_str(), &is_protected);
 397       if (r < 0) {
 398         derr << "error checking snapshot protection of snapshot "
 399              << imgctx->name << "@" << snap.name << ": " << cpp_strerror(r)
 400              << dendl;
 401         imgctx->state->close();
 402         enqueue_failed_delete(r);
 403         return true;
 404       }
 405       if (is_protected) {
 406         dout(20) << "snapshot " << imgctx->name << "@" << snap.name
 407                  << " is protected, issuing unprotect command" << dendl;
 408
 409         r = imgctx->operations->snap_unprotect(
 410           cls::rbd::UserSnapshotNamespace(), snap.name.c_str());
 411         if (r == -EBUSY) {
 412           // there are still clones of snapshots of this image, therefore send
 413           // the delete request to the end of the queue
 414           dout(10) << "local image id " << local_image_id << " has "
 415                    << "snapshots with cloned children, postponing deletion..."
 416                    << dendl;
 417           imgctx->state->close();
 418           Mutex::Locker l(m_delete_lock);
 419           m_active_delete->notify(r);
 420           m_delete_queue.push_front(std::move(m_active_delete));
 421           return false;
 422         } else if (r < 0) {
 423           derr << "error unprotecting snapshot " << imgctx->name << "@"
 424                << snap.name << ": " << cpp_strerror(r) << dendl;
 425           imgctx->state->close();
 426           enqueue_failed_delete(r);
 427           return true;
 428         }
 429       }
 430
 431       r = imgctx->operations->snap_remove(cls::rbd::UserSnapshotNamespace(),
 432                                           snap.name.c_str());
 433       if (r < 0) {
 434         derr << "error removing snapshot " << imgctx->name << "@"
 435              << snap.name << ": " << cpp_strerror(r) << dendl;
 436         imgctx->state->close();
 437         enqueue_failed_delete(r);
 438         return true;
 439       }
 440
 441       dout(10) << "snapshot " << imgctx->name << "@" << snap.name
 442                << " was deleted" << dendl;
 443     }
 444
 445     imgctx->state->close();
 446   }
 447
 448   librbd::NoOpProgressContext ctx;
 449   r = librbd::remove(ioctx, "", local_image_id, ctx, true);
 450   if (r < 0 && r != -ENOENT) {
 451     derr << "error removing image " << global_image_id << " "
 452          << "(" << local_image_id << ") from local pool: "
 453          << cpp_strerror(r) << dendl;
 454     enqueue_failed_delete(r);
 455     return true;
 456   }
 457
 458   // image was already deleted from rbd_directory, now we will make sure
 459   // that will be also removed from rbd_mirroring
 460   if (r == -ENOENT) {
 461     dout(20) << "local image does not exist, removing image from rbd_mirroring"
 462              << dendl;
 463   }
 464
 465   r = cls_client::mirror_image_remove(&ioctx, local_image_id);
 466   if (r < 0 && r != -ENOENT) {
 467     derr << "error removing image from mirroring directory: "
 468          << cpp_strerror(r) << dendl;
 469     enqueue_failed_delete(r);
 470     return true;
 471   }
 472
 473   dout(10) << "Successfully deleted image "
 474            << global_image_id << " " << "(" << local_image_id << ")" << dendl;
 475
 476   complete_active_delete(0);
 477   return true;
 478 }
 479
 480 template <typename I>
 481 int ImageDeleter<I>::image_has_snapshots_and_children(IoCtx *ioctx,
 482                                                       string& image_id,
 483                                                       bool *has_snapshots) {
 484   string header_oid = librbd::util::header_name(image_id);
 485   ::SnapContext snapc;
 486   int r = cls_client::get_snapcontext(ioctx, header_oid, &snapc);
 487   if (r < 0 && r != -ENOENT) {
 488     derr << "error retrieving snapshot context for image id " << image_id
 489          << ": " << cpp_strerror(r) << dendl;
 490     return r;
 491   }
 492
 493   *has_snapshots = !snapc.snaps.empty();
 494
 495   return 0;
 496 }
 497
 498 template <typename I>
 499 void ImageDeleter<I>::complete_active_delete(int r) {
 500   dout(20) << dendl;
 501
 502   Mutex::Locker delete_locker(m_delete_lock);
 503   m_active_delete->notify(r);
 504   m_active_delete.reset();
 505 }
 506
 507 template <typename I>
 508 void ImageDeleter<I>::enqueue_failed_delete(int error_code) {
 509   dout(20) << "enter" << dendl;
 510
 511   if (error_code == -EBLACKLISTED) {
 512     derr << "blacklisted while deleting local image" << dendl;
 513     complete_active_delete(error_code);
 514     return;
 515   }
 516
 517   m_delete_lock.Lock();
 518   if (m_active_delete->notify_on_failed_retry) {
 519     m_active_delete->notify(error_code);
 520   }
 521   m_active_delete->error_code = error_code;
 522   bool was_empty = m_failed_queue.empty();
 523   m_failed_queue.push_front(std::move(m_active_delete));
 524   m_delete_lock.Unlock();
 525   if (was_empty) {
 526     FunctionContext *ctx = new FunctionContext(
 527       boost::bind(&ImageDeleter<I>::retry_failed_deletions, this));
 528     Mutex::Locker l(*m_failed_timer_lock);
 529     m_failed_timer->add_event_after(m_failed_interval, ctx);
 530   }
 531 }
 532
 533 template <typename I>
 534 void ImageDeleter<I>::retry_failed_deletions() {
 535   dout(20) << "enter" << dendl;
 536
 537   Mutex::Locker l(m_delete_lock);
 538
 539   bool empty = m_failed_queue.empty();
 540   while (!m_failed_queue.empty()) {
 541     m_delete_queue.push_back(std::move(m_failed_queue.back()));
 542     m_delete_queue.back()->retries++;
 543     m_failed_queue.pop_back();
 544   }
 545   if (!empty) {
 546     m_delete_queue_cond.Signal();
 547   }
 548 }
 549
 550 template <typename I>
 551 unique_ptr<typename ImageDeleter<I>::DeleteInfo> const*
 552 ImageDeleter<I>::find_delete_info(int64_t local_pool_id,
 553                                   const std::string &global_image_id) {
 554   assert(m_delete_lock.is_locked());
 555
 556   if (m_active_delete && m_active_delete->match(local_pool_id,
 557                                                 global_image_id)) {
 558     return &m_active_delete;
 559   }
 560
 561   for (const auto& del_info : m_delete_queue) {
 562     if (del_info->match(local_pool_id, global_image_id)) {
 563       return &del_info;
 564     }
 565   }
 566
 567   for (const auto& del_info : m_failed_queue) {
 568     if (del_info->match(local_pool_id, global_image_id)) {
 569       return &del_info;
 570     }
 571   }
 572
 573   return nullptr;
 574 }
 575
 576 template <typename I>
 577 void ImageDeleter<I>::print_status(Formatter *f, stringstream *ss) {
 578   dout(20) << "enter" << dendl;
 579
 580   if (f) {
 581     f->open_object_section("image_deleter_status");
 582     f->open_array_section("delete_images_queue");
 583   }
 584
 585   Mutex::Locker l(m_delete_lock);
 586   for (const auto& image : m_delete_queue) {
 587     image->print_status(f, ss);
 588   }
 589
 590   if (f) {
 591     f->close_section();
 592     f->open_array_section("failed_deletes_queue");
 593   }
 594
 595   for (const auto& image : m_failed_queue) {
 596     image->print_status(f, ss, true);
 597   }
 598
 599   if (f) {
 600     f->close_section();
 601     f->close_section();
 602     f->flush(*ss);
 603   }
 604 }
 605
 606 template <typename I>
 607 void ImageDeleter<I>::DeleteInfo::notify(int r) {
 608   if (on_delete) {
 609     dout(20) << "executing image deletion handler r=" << r << dendl;
 610
 611     Context *ctx = on_delete;
 612     on_delete = nullptr;
 613     ctx->complete(r);
 614   }
 615 }
 616
 617 template <typename I>
 618 void ImageDeleter<I>::DeleteInfo::to_string(stringstream& ss) {
 619   ss << "[" << "local_pool_id=" << local_pool_id << ", ";
 620   ss << "global_image_id=" << global_image_id << "]";
 621 }
 622
 623 template <typename I>
 624 void ImageDeleter<I>::DeleteInfo::print_status(Formatter *f, stringstream *ss,
 625                                                bool print_failure_info) {
 626   if (f) {
 627     f->open_object_section("delete_info");
 628     f->dump_int("local_pool_id", local_pool_id);
 629     f->dump_string("global_image_id", global_image_id);
 630     if (print_failure_info) {
 631       f->dump_string("error_code", cpp_strerror(error_code));
 632       f->dump_int("retries", retries);
 633     }
 634     f->close_section();
 635     f->flush(*ss);
 636   } else {
 637     this->to_string(*ss);
 638   }
 639 }
 640
 641 template <typename I>
 642 vector<string> ImageDeleter<I>::get_delete_queue_items() {
 643   vector<string> items;
 644
 645   Mutex::Locker l(m_delete_lock);
 646   for (const auto& del_info : m_delete_queue) {
 647     items.push_back(del_info->global_image_id);
 648   }
 649
 650   return items;
 651 }
 652
 653 template <typename I>
 654 vector<pair<string, int> > ImageDeleter<I>::get_failed_queue_items() {
 655   vector<pair<string, int> > items;
 656
 657   Mutex::Locker l(m_delete_lock);
 658   for (const auto& del_info : m_failed_queue) {
 659     items.push_back(make_pair(del_info->global_image_id,
 660                               del_info->error_code));
 661   }
 662
 663   return items;
 664 }
 665
 666 template <typename I>
 667 void ImageDeleter<I>::set_failed_timer_interval(double interval) {
 668   this->m_failed_interval = interval;
 669 }
 670
 671 } // namespace mirror
 672 } // namespace rbd
 673
 674 template class rbd::mirror::ImageDeleter<librbd::ImageCtx>;