[ceph.git] / ceph / src / tools / rbd_mirror / ImageDeleter.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2016 SUSE LINUX GmbH
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include <boost/bind.hpp>
#include <map>
#include <set>
#include <sstream>

#include "include/rados/librados.hpp"
#include "common/Formatter.h"
#include "common/admin_socket.h"
#include "common/debug.h"
#include "common/errno.h"
#include "common/WorkQueue.h"
#include "global/global_context.h"
#include "librbd/internal.h"
#include "librbd/ImageCtx.h"
#include "librbd/ImageState.h"
#include "librbd/Journal.h"
#include "librbd/Operations.h"
#include "librbd/journal/Policy.h"
#include "cls/rbd/cls_rbd_client.h"
#include "cls/rbd/cls_rbd_types.h"
#include "librbd/Utils.h"
#include "ImageDeleter.h"

#define dout_context g_ceph_context
#define dout_subsys ceph_subsys_rbd_mirror
#undef dout_prefix
#define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
                           << __func__ << ": "

using std::string;
using std::map;
using std::stringstream;
using std::vector;
using std::pair;
using std::make_pair;

using librados::IoCtx;
using namespace librbd;

namespace rbd {
namespace mirror {

namespace {

class ImageDeleterAdminSocketCommand {
public:
  virtual ~ImageDeleterAdminSocketCommand() {}
  virtual bool call(Formatter *f, stringstream *ss) = 0;
};

template <typename I>
class StatusCommand : public ImageDeleterAdminSocketCommand {
public:
  explicit StatusCommand(ImageDeleter<I> *image_del) : image_del(image_del) {}

  bool call(Formatter *f, stringstream *ss) override {
    image_del->print_status(f, ss);
    return true;
  }

private:
  ImageDeleter<I> *image_del;
};

struct DeleteJournalPolicy : public librbd::journal::Policy {
  bool append_disabled() const override {
    return true;
  }
  bool journal_disabled() const override {
    return false;
  }

  void allocate_tag_on_lock(Context *on_finish) override {
    on_finish->complete(0);
  }
};

} // anonymous namespace

template <typename I>
class ImageDeleterAdminSocketHook : public AdminSocketHook {
public:
  ImageDeleterAdminSocketHook(CephContext *cct, ImageDeleter<I> *image_del) :
    admin_socket(cct->get_admin_socket()) {

    std::string command;
    int r;

    command = "rbd mirror deletion status";
    r = admin_socket->register_command(command, command, this,
				       "get status for image deleter");
    if (r == 0) {
      commands[command] = new StatusCommand<I>(image_del);
    }

  }

  ~ImageDeleterAdminSocketHook() override {
    for (Commands::const_iterator i = commands.begin(); i != commands.end();
	 ++i) {
      (void)admin_socket->unregister_command(i->first);
      delete i->second;
    }
  }

  bool call(std::string command, cmdmap_t& cmdmap, std::string format,
	    bufferlist& out) override {
    Commands::const_iterator i = commands.find(command);
    assert(i != commands.end());
    Formatter *f = Formatter::create(format);
    stringstream ss;
    bool r = i->second->call(f, &ss);
    delete f;
    out.append(ss);
    return r;
  }

private:
  typedef std::map<std::string, ImageDeleterAdminSocketCommand*> Commands;
  AdminSocket *admin_socket;
  Commands commands;
};

template <typename I>
ImageDeleter<I>::ImageDeleter(ContextWQ *work_queue, SafeTimer *timer,
                              Mutex *timer_lock,
                              ServiceDaemon<librbd::ImageCtx>* service_daemon)
  : m_work_queue(work_queue),
    m_service_daemon(service_daemon),
    m_delete_lock("rbd::mirror::ImageDeleter::Delete"),
    m_image_deleter_thread(this),
    m_failed_timer(timer),
    m_failed_timer_lock(timer_lock),
    m_asok_hook(new ImageDeleterAdminSocketHook<I>(g_ceph_context, this))
{
  set_failed_timer_interval(g_ceph_context->_conf->get_val<double>(
    "rbd_mirror_delete_retry_interval"));
  m_image_deleter_thread.create("image_deleter");
}

template <typename I>
ImageDeleter<I>::~ImageDeleter() {
  dout(20) << "enter" << dendl;

  m_running = false;
  {
    Mutex::Locker l (m_delete_lock);
    m_delete_queue_cond.Signal();
  }
  if (m_image_deleter_thread.is_started()) {
    m_image_deleter_thread.join();
  }

  delete m_asok_hook;
  dout(20) << "return" << dendl;
}

template <typename I>
void ImageDeleter<I>::run() {
  dout(20) << "enter" << dendl;
  while(m_running) {
    m_delete_lock.Lock();
    while (m_delete_queue.empty()) {
      dout(20) << "waiting for delete requests" << dendl;
      m_delete_queue_cond.Wait(m_delete_lock);

      if (!m_running) {
        m_delete_lock.Unlock();
        dout(20) << "return" << dendl;
        return;
      }
    }

    m_active_delete = std::move(m_delete_queue.back());
    m_delete_queue.pop_back();
    m_delete_lock.Unlock();

    bool move_to_next = process_image_delete();
    if (!move_to_next) {
      if (!m_running) {
       dout(20) << "return" << dendl;
       return;
      }

      Mutex::Locker l(m_delete_lock);
      if (m_delete_queue.size() == 1) {
        m_delete_queue_cond.Wait(m_delete_lock);
      }
    }
  }
}

template <typename I>
void ImageDeleter<I>::schedule_image_delete(RadosRef local_rados,
                                            int64_t local_pool_id,
                                            const std::string& global_image_id,
                                            bool ignore_orphaned) {
  dout(20) << "enter" << dendl;

  Mutex::Locker locker(m_delete_lock);

  auto del_info = find_delete_info(local_pool_id, global_image_id);
  if (del_info != nullptr) {
    dout(20) << "image " << global_image_id << " "
             << "was already scheduled for deletion" << dendl;
    if (ignore_orphaned) {
      (*del_info)->ignore_orphaned = true;
    }
    return;
  }

  m_delete_queue.push_front(
    unique_ptr<DeleteInfo>(new DeleteInfo(local_rados, local_pool_id,
                                          global_image_id, ignore_orphaned)));
  m_delete_queue_cond.Signal();
}

template <typename I>
void ImageDeleter<I>::wait_for_scheduled_deletion(int64_t local_pool_id,
                                                  const std::string &global_image_id,
                                                  Context *ctx,
                                                  bool notify_on_failed_retry) {

  ctx = new FunctionContext([this, ctx](int r) {
      m_work_queue->queue(ctx, r);
    });

  Mutex::Locker locker(m_delete_lock);
  auto del_info = find_delete_info(local_pool_id, global_image_id);
  if (!del_info) {
    // image not scheduled for deletion
    ctx->complete(0);
    return;
  }

  dout(20) << "local_pool_id=" << local_pool_id << ", "
           << "global_image_id=" << global_image_id << dendl;

  if ((*del_info)->on_delete != nullptr) {
    (*del_info)->on_delete->complete(-ESTALE);
  }
  (*del_info)->on_delete = ctx;
  (*del_info)->notify_on_failed_retry = notify_on_failed_retry;
}

template <typename I>
void ImageDeleter<I>::cancel_waiter(int64_t local_pool_id,
                                    const std::string &global_image_id) {
  Mutex::Locker locker(m_delete_lock);
  auto del_info = find_delete_info(local_pool_id, global_image_id);
  if (!del_info) {
    return;
  }

  if ((*del_info)->on_delete != nullptr) {
    (*del_info)->on_delete->complete(-ECANCELED);
    (*del_info)->on_delete = nullptr;
  }
}

template <typename I>
bool ImageDeleter<I>::process_image_delete() {
  stringstream ss;
  m_active_delete->to_string(ss);
  std::string del_info_str = ss.str();
  dout(10) << "start processing delete request: " << del_info_str << dendl;
  int r;
  cls::rbd::MirrorImage mirror_image;

  // remote image was disabled, now we need to delete local image
  IoCtx ioctx;
  r = m_active_delete->local_rados->ioctx_create2(
    m_active_delete->local_pool_id, ioctx);
  if (r < 0) {
    derr << "error accessing local pool " << m_active_delete->local_pool_id
         << ": " << cpp_strerror(r) << dendl;
    enqueue_failed_delete(r);
    return true;
  }

  dout(20) << "connected to local pool: " << ioctx.get_pool_name() << dendl;

  auto &global_image_id = m_active_delete->global_image_id;
  std::string local_image_id;
  r = librbd::cls_client::mirror_image_get_image_id(
    &ioctx, global_image_id, &local_image_id);
  if (r == -ENOENT) {
    dout(10) << "image " << global_image_id << " is not mirrored" << dendl;
    complete_active_delete(r);
    return true;
  } else if (r < 0) {
    derr << "error retrieving local id for image " << global_image_id
         << ": " << cpp_strerror(r) << dendl;
    enqueue_failed_delete(r);
    return true;
  }

  std::string mirror_uuid;
  C_SaferCond tag_owner_ctx;
  Journal<>::get_tag_owner(ioctx, local_image_id, &mirror_uuid, m_work_queue,
                           &tag_owner_ctx);
  r = tag_owner_ctx.wait();
  if (r < 0 && r != -ENOENT) {
    derr << "error retrieving image primary info for image " << global_image_id
         << ": " << cpp_strerror(r) << dendl;
    enqueue_failed_delete(r);
    return true;
  } else if (r != -ENOENT) {
    if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) {
      dout(10) << "image " << global_image_id << " is local primary" << dendl;
      complete_active_delete(-EISPRM);
      return true;
    } else if (mirror_uuid == Journal<>::ORPHAN_MIRROR_UUID &&
               !m_active_delete->ignore_orphaned) {
      dout(10) << "image " << global_image_id << " is orphaned" << dendl;
      complete_active_delete(-EISPRM);
      return true;
    }
  }

  dout(20) << "local image is not the primary" << dendl;
  bool has_snapshots;
  r = image_has_snapshots_and_children(&ioctx, local_image_id, &has_snapshots);
  if (r < 0) {
    enqueue_failed_delete(r);
    return true;
  }

  mirror_image.global_image_id = global_image_id;
  mirror_image.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING;
  r = cls_client::mirror_image_set(&ioctx, local_image_id, mirror_image);
  if (r == -ENOENT) {
    dout(10) << "local image is not mirrored, aborting deletion..." << dendl;
    complete_active_delete(r);
    return true;
  } else if (r == -EEXIST || r == -EINVAL) {
    derr << "cannot disable mirroring for image " << global_image_id
         << ": global_image_id has changed/reused: "
         << cpp_strerror(r) << dendl;
    complete_active_delete(r);
    return true;
  } else if (r < 0) {
    derr << "cannot disable mirroring for image " << global_image_id
         << ": " << cpp_strerror(r) << dendl;
    enqueue_failed_delete(r);
    return true;
  }

  dout(20) << "set local image mirroring to disable" << dendl;

  if (has_snapshots) {
    dout(20) << "local image has snapshots" << dendl;

    ImageCtx *imgctx = new ImageCtx("", local_image_id, nullptr, ioctx, false);
    r = imgctx->state->open(false);
    if (r < 0) {
      derr << "error opening image " << global_image_id << " ("
           << local_image_id << "): " << cpp_strerror(r) << dendl;
      enqueue_failed_delete(r);
      return true;
    }

    {
      RWLock::WLocker snap_locker(imgctx->snap_lock);
      imgctx->set_journal_policy(new DeleteJournalPolicy());
    }

    std::vector<librbd::snap_info_t> snaps;
    r = librbd::snap_list(imgctx, snaps);
    if (r < 0) {
      derr << "error listing snapshot of image " << imgctx->name
           << cpp_strerror(r) << dendl;
      imgctx->state->close();
      enqueue_failed_delete(r);
      return true;
    }

    for (const auto& snap : snaps) {
      dout(20) << "processing deletion of snapshot " << imgctx->name << "@"
               << snap.name << dendl;

      bool is_protected;
      r = librbd::snap_is_protected(imgctx, snap.name.c_str(), &is_protected);
      if (r < 0) {
        derr << "error checking snapshot protection of snapshot "
             << imgctx->name << "@" << snap.name << ": " << cpp_strerror(r)
             << dendl;
        imgctx->state->close();
        enqueue_failed_delete(r);
        return true;
      }
      if (is_protected) {
        dout(20) << "snapshot " << imgctx->name << "@" << snap.name
                 << " is protected, issuing unprotect command" << dendl;

        r = imgctx->operations->snap_unprotect(
          cls::rbd::UserSnapshotNamespace(), snap.name.c_str());
        if (r == -EBUSY) {
          // there are still clones of snapshots of this image, therefore send
          // the delete request to the end of the queue
          dout(10) << "local image id " << local_image_id << " has "
                   << "snapshots with cloned children, postponing deletion..."
                   << dendl;
          imgctx->state->close();
          Mutex::Locker l(m_delete_lock);
          m_active_delete->notify(r);
          m_delete_queue.push_front(std::move(m_active_delete));
          return false;
        } else if (r < 0) {
          derr << "error unprotecting snapshot " << imgctx->name << "@"
               << snap.name << ": " << cpp_strerror(r) << dendl;
          imgctx->state->close();
          enqueue_failed_delete(r);
          return true;
        }
      }

      r = imgctx->operations->snap_remove(cls::rbd::UserSnapshotNamespace(),
					  snap.name.c_str());
      if (r < 0) {
        derr << "error removing snapshot " << imgctx->name << "@"
             << snap.name << ": " << cpp_strerror(r) << dendl;
        imgctx->state->close();
        enqueue_failed_delete(r);
        return true;
      }

      dout(10) << "snapshot " << imgctx->name << "@" << snap.name
               << " was deleted" << dendl;
    }

    imgctx->state->close();
  }

  librbd::NoOpProgressContext ctx;
  r = librbd::remove(ioctx, "", local_image_id, ctx, true);
  if (r < 0 && r != -ENOENT) {
    derr << "error removing image " << global_image_id << " "
         << "(" << local_image_id << ") from local pool: "
         << cpp_strerror(r) << dendl;
    enqueue_failed_delete(r);
    return true;
  }

  // image was already deleted from rbd_directory, now we will make sure
  // that will be also removed from rbd_mirroring
  if (r == -ENOENT) {
    dout(20) << "local image does not exist, removing image from rbd_mirroring"
             << dendl;
  }

  r = cls_client::mirror_image_remove(&ioctx, local_image_id);
  if (r < 0 && r != -ENOENT) {
    derr << "error removing image from mirroring directory: "
         << cpp_strerror(r) << dendl;
    enqueue_failed_delete(r);
    return true;
  }

  dout(10) << "Successfully deleted image "
           << global_image_id << " " << "(" << local_image_id << ")" << dendl;

  complete_active_delete(0);
  return true;
}

template <typename I>
int ImageDeleter<I>::image_has_snapshots_and_children(IoCtx *ioctx,
                                                      string& image_id,
                                                      bool *has_snapshots) {
  string header_oid = librbd::util::header_name(image_id);
  ::SnapContext snapc;
  int r = cls_client::get_snapcontext(ioctx, header_oid, &snapc);
  if (r < 0 && r != -ENOENT) {
    derr << "error retrieving snapshot context for image id " << image_id
         << ": " << cpp_strerror(r) << dendl;
    return r;
  }

  *has_snapshots = !snapc.snaps.empty();

  return 0;
}

template <typename I>
void ImageDeleter<I>::complete_active_delete(int r) {
  dout(20) << dendl;

  Mutex::Locker delete_locker(m_delete_lock);
  m_active_delete->notify(r);
  m_active_delete.reset();
}

template <typename I>
void ImageDeleter<I>::enqueue_failed_delete(int error_code) {
  dout(20) << "enter" << dendl;

  if (error_code == -EBLACKLISTED) {
    derr << "blacklisted while deleting local image" << dendl;
    complete_active_delete(error_code);
    return;
  }

  m_delete_lock.Lock();
  if (m_active_delete->notify_on_failed_retry) {
    m_active_delete->notify(error_code);
  }
  m_active_delete->error_code = error_code;
  bool was_empty = m_failed_queue.empty();
  m_failed_queue.push_front(std::move(m_active_delete));
  m_delete_lock.Unlock();
  if (was_empty) {
    FunctionContext *ctx = new FunctionContext(
      boost::bind(&ImageDeleter<I>::retry_failed_deletions, this));
    Mutex::Locker l(*m_failed_timer_lock);
    m_failed_timer->add_event_after(m_failed_interval, ctx);
  }
}

template <typename I>
void ImageDeleter<I>::retry_failed_deletions() {
  dout(20) << "enter" << dendl;

  Mutex::Locker l(m_delete_lock);

  bool empty = m_failed_queue.empty();
  while (!m_failed_queue.empty()) {
    m_delete_queue.push_back(std::move(m_failed_queue.back()));
    m_delete_queue.back()->retries++;
    m_failed_queue.pop_back();
  }
  if (!empty) {
    m_delete_queue_cond.Signal();
  }
}

template <typename I>
unique_ptr<typename ImageDeleter<I>::DeleteInfo> const*
ImageDeleter<I>::find_delete_info(int64_t local_pool_id,
                                  const std::string &global_image_id) {
  assert(m_delete_lock.is_locked());

  if (m_active_delete && m_active_delete->match(local_pool_id,
                                                global_image_id)) {
    return &m_active_delete;
  }

  for (const auto& del_info : m_delete_queue) {
    if (del_info->match(local_pool_id, global_image_id)) {
      return &del_info;
    }
  }

  for (const auto& del_info : m_failed_queue) {
    if (del_info->match(local_pool_id, global_image_id)) {
      return &del_info;
    }
  }

  return nullptr;
}

template <typename I>
void ImageDeleter<I>::print_status(Formatter *f, stringstream *ss) {
  dout(20) << "enter" << dendl;

  if (f) {
    f->open_object_section("image_deleter_status");
    f->open_array_section("delete_images_queue");
  }

  Mutex::Locker l(m_delete_lock);
  for (const auto& image : m_delete_queue) {
    image->print_status(f, ss);
  }

  if (f) {
    f->close_section();
    f->open_array_section("failed_deletes_queue");
  }

  for (const auto& image : m_failed_queue) {
    image->print_status(f, ss, true);
  }

  if (f) {
    f->close_section();
    f->close_section();
    f->flush(*ss);
  }
}

template <typename I>
void ImageDeleter<I>::DeleteInfo::notify(int r) {
  if (on_delete) {
    dout(20) << "executing image deletion handler r=" << r << dendl;

    Context *ctx = on_delete;
    on_delete = nullptr;
    ctx->complete(r);
  }
}

template <typename I>
void ImageDeleter<I>::DeleteInfo::to_string(stringstream& ss) {
  ss << "[" << "local_pool_id=" << local_pool_id << ", ";
  ss << "global_image_id=" << global_image_id << "]";
}

template <typename I>
void ImageDeleter<I>::DeleteInfo::print_status(Formatter *f, stringstream *ss,
                                               bool print_failure_info) {
  if (f) {
    f->open_object_section("delete_info");
    f->dump_int("local_pool_id", local_pool_id);
    f->dump_string("global_image_id", global_image_id);
    if (print_failure_info) {
      f->dump_string("error_code", cpp_strerror(error_code));
      f->dump_int("retries", retries);
    }
    f->close_section();
    f->flush(*ss);
  } else {
    this->to_string(*ss);
  }
}

template <typename I>
vector<string> ImageDeleter<I>::get_delete_queue_items() {
  vector<string> items;

  Mutex::Locker l(m_delete_lock);
  for (const auto& del_info : m_delete_queue) {
    items.push_back(del_info->global_image_id);
  }

  return items;
}

template <typename I>
vector<pair<string, int> > ImageDeleter<I>::get_failed_queue_items() {
  vector<pair<string, int> > items;

  Mutex::Locker l(m_delete_lock);
  for (const auto& del_info : m_failed_queue) {
    items.push_back(make_pair(del_info->global_image_id,
                              del_info->error_code));
  }

  return items;
}

template <typename I>
void ImageDeleter<I>::set_failed_timer_interval(double interval) {
  this->m_failed_interval = interval;
}

} // namespace mirror
} // namespace rbd

template class rbd::mirror::ImageDeleter<librbd::ImageCtx>;
Commit	Line	Data
7c673cae FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3	/*
	4	* Ceph - scalable distributed file system
	5	*
	6	* Copyright (C) 2016 SUSE LINUX GmbH
	7	*
	8	* This is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License version 2.1, as published by the Free Software
	11	* Foundation. See file COPYING.
	12	*
	13	*/
c07f9fc5	14
7c673cae FG	15	#include <boost/bind.hpp>
	16	#include <map>
	17	#include <set>
	18	#include <sstream>
	19
	20	#include "include/rados/librados.hpp"
	21	#include "common/Formatter.h"
	22	#include "common/admin_socket.h"
	23	#include "common/debug.h"
	24	#include "common/errno.h"
	25	#include "common/WorkQueue.h"
	26	#include "global/global_context.h"
	27	#include "librbd/internal.h"
	28	#include "librbd/ImageCtx.h"
	29	#include "librbd/ImageState.h"
	30	#include "librbd/Journal.h"
	31	#include "librbd/Operations.h"
	32	#include "librbd/journal/Policy.h"
	33	#include "cls/rbd/cls_rbd_client.h"
	34	#include "cls/rbd/cls_rbd_types.h"
	35	#include "librbd/Utils.h"
	36	#include "ImageDeleter.h"
	37
	38	#define dout_context g_ceph_context
	39	#define dout_subsys ceph_subsys_rbd_mirror
	40	#undef dout_prefix
	41	#define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
	42	<< __func__ << ": "
	43
	44	using std::string;
	45	using std::map;
	46	using std::stringstream;
	47	using std::vector;
	48	using std::pair;
	49	using std::make_pair;
	50
	51	using librados::IoCtx;
	52	using namespace librbd;
	53
	54	namespace rbd {
	55	namespace mirror {
	56
	57	namespace {
	58
	59	class ImageDeleterAdminSocketCommand {
	60	public:
	61	virtual ~ImageDeleterAdminSocketCommand() {}
	62	virtual bool call(Formatter f, stringstream ss) = 0;
	63	};
	64
c07f9fc5	65	template <typename I>
7c673cae FG	66	class StatusCommand : public ImageDeleterAdminSocketCommand {
7c673cae FG	67	public:
c07f9fc5	68	explicit StatusCommand(ImageDeleter<I> *image_del) : image_del(image_del) {}
7c673cae FG	69
	70	bool call(Formatter f, stringstream ss) override {
	71	image_del->print_status(f, ss);
	72	return true;
	73	}
	74
	75	private:
c07f9fc5	76	ImageDeleter<I> *image_del;
7c673cae FG	77	};
	78
	79	struct DeleteJournalPolicy : public librbd::journal::Policy {
	80	bool append_disabled() const override {
	81	return true;
	82	}
	83	bool journal_disabled() const override {
	84	return false;
	85	}
	86
	87	void allocate_tag_on_lock(Context *on_finish) override {
	88	on_finish->complete(0);
	89	}
	90	};
	91
	92	} // anonymous namespace
	93
c07f9fc5	94	template <typename I>
7c673cae FG	95	class ImageDeleterAdminSocketHook : public AdminSocketHook {
7c673cae FG	96	public:
c07f9fc5	97	ImageDeleterAdminSocketHook(CephContext cct, ImageDeleter<I> image_del) :
7c673cae FG	98	admin_socket(cct->get_admin_socket()) {
	99
	100	std::string command;
	101	int r;
	102
	103	command = "rbd mirror deletion status";
	104	r = admin_socket->register_command(command, command, this,
	105	"get status for image deleter");
	106	if (r == 0) {
c07f9fc5	107	commands[command] = new StatusCommand<I>(image_del);
7c673cae FG	108	}
	109
	110	}
	111
	112	~ImageDeleterAdminSocketHook() override {
	113	for (Commands::const_iterator i = commands.begin(); i != commands.end();
	114	++i) {
	115	(void)admin_socket->unregister_command(i->first);
	116	delete i->second;
	117	}
	118	}
	119
	120	bool call(std::string command, cmdmap_t& cmdmap, std::string format,
	121	bufferlist& out) override {
	122	Commands::const_iterator i = commands.find(command);
	123	assert(i != commands.end());
	124	Formatter *f = Formatter::create(format);
	125	stringstream ss;
	126	bool r = i->second->call(f, &ss);
	127	delete f;
	128	out.append(ss);
	129	return r;
	130	}
	131
	132	private:
	133	typedef std::map<std::string, ImageDeleterAdminSocketCommand*> Commands;
	134	AdminSocket *admin_socket;
	135	Commands commands;
	136	};
	137
c07f9fc5 FG	138	template <typename I>
	139	ImageDeleter<I>::ImageDeleter(ContextWQ work_queue, SafeTimer timer,
	140	Mutex *timer_lock,
	141	ServiceDaemon<librbd::ImageCtx>* service_daemon)
	142	: m_work_queue(work_queue),
	143	m_service_daemon(service_daemon),
7c673cae FG	144	m_delete_lock("rbd::mirror::ImageDeleter::Delete"),
	145	m_image_deleter_thread(this),
	146	m_failed_timer(timer),
	147	m_failed_timer_lock(timer_lock),
c07f9fc5	148	m_asok_hook(new ImageDeleterAdminSocketHook<I>(g_ceph_context, this))
7c673cae	149	{
181888fb FG	150	set_failed_timer_interval(g_ceph_context->_conf->get_val<double>(
181888fb FG	151	"rbd_mirror_delete_retry_interval"));
7c673cae FG	152	m_image_deleter_thread.create("image_deleter");
	153	}
	154
c07f9fc5 FG	155	template <typename I>
c07f9fc5 FG	156	ImageDeleter<I>::~ImageDeleter() {
7c673cae FG	157	dout(20) << "enter" << dendl;
	158
	159	m_running = false;
	160	{
	161	Mutex::Locker l (m_delete_lock);
	162	m_delete_queue_cond.Signal();
	163	}
	164	if (m_image_deleter_thread.is_started()) {
	165	m_image_deleter_thread.join();
	166	}
	167
	168	delete m_asok_hook;
	169	dout(20) << "return" << dendl;
	170	}
	171
c07f9fc5 FG	172	template <typename I>
c07f9fc5 FG	173	void ImageDeleter<I>::run() {
7c673cae FG	174	dout(20) << "enter" << dendl;
	175	while(m_running) {
	176	m_delete_lock.Lock();
	177	while (m_delete_queue.empty()) {
	178	dout(20) << "waiting for delete requests" << dendl;
	179	m_delete_queue_cond.Wait(m_delete_lock);
	180
	181	if (!m_running) {
	182	m_delete_lock.Unlock();
	183	dout(20) << "return" << dendl;
	184	return;
	185	}
	186	}
	187
	188	m_active_delete = std::move(m_delete_queue.back());
	189	m_delete_queue.pop_back();
	190	m_delete_lock.Unlock();
	191
	192	bool move_to_next = process_image_delete();
	193	if (!move_to_next) {
	194	if (!m_running) {
	195	dout(20) << "return" << dendl;
	196	return;
	197	}
	198
	199	Mutex::Locker l(m_delete_lock);
	200	if (m_delete_queue.size() == 1) {
	201	m_delete_queue_cond.Wait(m_delete_lock);
	202	}
	203	}
	204	}
	205	}
	206
c07f9fc5 FG	207	template <typename I>
	208	void ImageDeleter<I>::schedule_image_delete(RadosRef local_rados,
	209	int64_t local_pool_id,
	210	const std::string& global_image_id,
	211	bool ignore_orphaned) {
7c673cae FG	212	dout(20) << "enter" << dendl;
	213
	214	Mutex::Locker locker(m_delete_lock);
	215
	216	auto del_info = find_delete_info(local_pool_id, global_image_id);
	217	if (del_info != nullptr) {
	218	dout(20) << "image " << global_image_id << " "
	219	<< "was already scheduled for deletion" << dendl;
c07f9fc5 FG	220	if (ignore_orphaned) {
	221	(*del_info)->ignore_orphaned = true;
	222	}
7c673cae FG	223	return;
	224	}
	225
	226	m_delete_queue.push_front(
	227	unique_ptr<DeleteInfo>(new DeleteInfo(local_rados, local_pool_id,
c07f9fc5	228	global_image_id, ignore_orphaned)));
7c673cae FG	229	m_delete_queue_cond.Signal();
	230	}
	231
c07f9fc5 FG	232	template <typename I>
	233	void ImageDeleter<I>::wait_for_scheduled_deletion(int64_t local_pool_id,
	234	const std::string &global_image_id,
	235	Context *ctx,
	236	bool notify_on_failed_retry) {
7c673cae FG	237
	238	ctx = new FunctionContext([this, ctx](int r) {
	239	m_work_queue->queue(ctx, r);
	240	});
	241
	242	Mutex::Locker locker(m_delete_lock);
	243	auto del_info = find_delete_info(local_pool_id, global_image_id);
	244	if (!del_info) {
	245	// image not scheduled for deletion
	246	ctx->complete(0);
	247	return;
	248	}
	249
	250	dout(20) << "local_pool_id=" << local_pool_id << ", "
	251	<< "global_image_id=" << global_image_id << dendl;
	252
	253	if ((*del_info)->on_delete != nullptr) {
	254	(*del_info)->on_delete->complete(-ESTALE);
	255	}
	256	(*del_info)->on_delete = ctx;
	257	(*del_info)->notify_on_failed_retry = notify_on_failed_retry;
	258	}
	259
c07f9fc5 FG	260	template <typename I>
	261	void ImageDeleter<I>::cancel_waiter(int64_t local_pool_id,
	262	const std::string &global_image_id) {
7c673cae FG	263	Mutex::Locker locker(m_delete_lock);
	264	auto del_info = find_delete_info(local_pool_id, global_image_id);
	265	if (!del_info) {
	266	return;
	267	}
	268
	269	if ((*del_info)->on_delete != nullptr) {
	270	(*del_info)->on_delete->complete(-ECANCELED);
	271	(*del_info)->on_delete = nullptr;
	272	}
	273	}
	274
c07f9fc5 FG	275	template <typename I>
c07f9fc5 FG	276	bool ImageDeleter<I>::process_image_delete() {
7c673cae FG	277	stringstream ss;
	278	m_active_delete->to_string(ss);
	279	std::string del_info_str = ss.str();
	280	dout(10) << "start processing delete request: " << del_info_str << dendl;
	281	int r;
	282	cls::rbd::MirrorImage mirror_image;
	283
	284	// remote image was disabled, now we need to delete local image
	285	IoCtx ioctx;
	286	r = m_active_delete->local_rados->ioctx_create2(
	287	m_active_delete->local_pool_id, ioctx);
	288	if (r < 0) {
	289	derr << "error accessing local pool " << m_active_delete->local_pool_id
	290	<< ": " << cpp_strerror(r) << dendl;
	291	enqueue_failed_delete(r);
	292	return true;
	293	}
	294
	295	dout(20) << "connected to local pool: " << ioctx.get_pool_name() << dendl;
	296
	297	auto &global_image_id = m_active_delete->global_image_id;
	298	std::string local_image_id;
	299	r = librbd::cls_client::mirror_image_get_image_id(
	300	&ioctx, global_image_id, &local_image_id);
	301	if (r == -ENOENT) {
	302	dout(10) << "image " << global_image_id << " is not mirrored" << dendl;
	303	complete_active_delete(r);
	304	return true;
	305	} else if (r < 0) {
	306	derr << "error retrieving local id for image " << global_image_id
	307	<< ": " << cpp_strerror(r) << dendl;
	308	enqueue_failed_delete(r);
	309	return true;
	310	}
	311
c07f9fc5	312	std::string mirror_uuid;
7c673cae	313	C_SaferCond tag_owner_ctx;
c07f9fc5 FG	314	Journal<>::get_tag_owner(ioctx, local_image_id, &mirror_uuid, m_work_queue,
c07f9fc5 FG	315	&tag_owner_ctx);
7c673cae FG	316	r = tag_owner_ctx.wait();
	317	if (r < 0 && r != -ENOENT) {
	318	derr << "error retrieving image primary info for image " << global_image_id
	319	<< ": " << cpp_strerror(r) << dendl;
	320	enqueue_failed_delete(r);
	321	return true;
c07f9fc5 FG	322	} else if (r != -ENOENT) {
	323	if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) {
	324	dout(10) << "image " << global_image_id << " is local primary" << dendl;
	325	complete_active_delete(-EISPRM);
	326	return true;
	327	} else if (mirror_uuid == Journal<>::ORPHAN_MIRROR_UUID &&
	328	!m_active_delete->ignore_orphaned) {
	329	dout(10) << "image " << global_image_id << " is orphaned" << dendl;
	330	complete_active_delete(-EISPRM);
	331	return true;
	332	}
7c673cae FG	333	}
	334
	335	dout(20) << "local image is not the primary" << dendl;
7c673cae FG	336	bool has_snapshots;
	337	r = image_has_snapshots_and_children(&ioctx, local_image_id, &has_snapshots);
	338	if (r < 0) {
	339	enqueue_failed_delete(r);
	340	return true;
	341	}
	342
	343	mirror_image.global_image_id = global_image_id;
	344	mirror_image.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING;
	345	r = cls_client::mirror_image_set(&ioctx, local_image_id, mirror_image);
	346	if (r == -ENOENT) {
	347	dout(10) << "local image is not mirrored, aborting deletion..." << dendl;
	348	complete_active_delete(r);
	349	return true;
	350	} else if (r == -EEXIST \|\| r == -EINVAL) {
	351	derr << "cannot disable mirroring for image " << global_image_id
	352	<< ": global_image_id has changed/reused: "
	353	<< cpp_strerror(r) << dendl;
	354	complete_active_delete(r);
	355	return true;
	356	} else if (r < 0) {
	357	derr << "cannot disable mirroring for image " << global_image_id
	358	<< ": " << cpp_strerror(r) << dendl;
	359	enqueue_failed_delete(r);
	360	return true;
	361	}
	362
	363	dout(20) << "set local image mirroring to disable" << dendl;
	364
	365	if (has_snapshots) {
	366	dout(20) << "local image has snapshots" << dendl;
	367
	368	ImageCtx *imgctx = new ImageCtx("", local_image_id, nullptr, ioctx, false);
	369	r = imgctx->state->open(false);
	370	if (r < 0) {
	371	derr << "error opening image " << global_image_id << " ("
	372	<< local_image_id << "): " << cpp_strerror(r) << dendl;
	373	enqueue_failed_delete(r);
	374	return true;
	375	}
	376
	377	{
	378	RWLock::WLocker snap_locker(imgctx->snap_lock);
	379	imgctx->set_journal_policy(new DeleteJournalPolicy());
	380	}
	381
	382	std::vector<librbd::snap_info_t> snaps;
	383	r = librbd::snap_list(imgctx, snaps);
	384	if (r < 0) {
	385	derr << "error listing snapshot of image " << imgctx->name
	386	<< cpp_strerror(r) << dendl;
	387	imgctx->state->close();
	388	enqueue_failed_delete(r);
	389	return true;
	390	}
	391
	392	for (const auto& snap : snaps) {
	393	dout(20) << "processing deletion of snapshot " << imgctx->name << "@"
	394	<< snap.name << dendl;
	395
	396	bool is_protected;
	397	r = librbd::snap_is_protected(imgctx, snap.name.c_str(), &is_protected);
	398	if (r < 0) {
	399	derr << "error checking snapshot protection of snapshot "
400	<< imgctx->name << "@" << snap.name << ": " << cpp_strerror(r)
401	<< dendl;
402	imgctx->state->close();
403	enqueue_failed_delete(r);
404	return true;
405	}
406	if (is_protected) {
407	dout(20) << "snapshot " << imgctx->name << "@" << snap.name
408	<< " is protected, issuing unprotect command" << dendl;
409
410	r = imgctx->operations->snap_unprotect(
411	cls::rbd::UserSnapshotNamespace(), snap.name.c_str());
412	if (r == -EBUSY) {
413	// there are still clones of snapshots of this image, therefore send
414	// the delete request to the end of the queue
415	dout(10) << "local image id " << local_image_id << " has "
416	<< "snapshots with cloned children, postponing deletion..."
417	<< dendl;
418	imgctx->state->close();
419	Mutex::Locker l(m_delete_lock);
420	m_active_delete->notify(r);
421	m_delete_queue.push_front(std::move(m_active_delete));
422	return false;
423	} else if (r < 0) {
424	derr << "error unprotecting snapshot " << imgctx->name << "@"
425	<< snap.name << ": " << cpp_strerror(r) << dendl;
426	imgctx->state->close();
427	enqueue_failed_delete(r);
428	return true;
429	}
430	}
431
432	r = imgctx->operations->snap_remove(cls::rbd::UserSnapshotNamespace(),
433	snap.name.c_str());
434	if (r < 0) {
435	derr << "error removing snapshot " << imgctx->name << "@"
436	<< snap.name << ": " << cpp_strerror(r) << dendl;
437	imgctx->state->close();
438	enqueue_failed_delete(r);
439	return true;
440	}
441
442	dout(10) << "snapshot " << imgctx->name << "@" << snap.name
443	<< " was deleted" << dendl;
444	}
445
446	imgctx->state->close();
447	}
448
449	librbd::NoOpProgressContext ctx;
450	r = librbd::remove(ioctx, "", local_image_id, ctx, true);
451	if (r < 0 && r != -ENOENT) {
452	derr << "error removing image " << global_image_id << " "
453	<< "(" << local_image_id << ") from local pool: "
454	<< cpp_strerror(r) << dendl;
455	enqueue_failed_delete(r);
456	return true;
457	}
458
459	// image was already deleted from rbd_directory, now we will make sure
460	// that will be also removed from rbd_mirroring
461	if (r == -ENOENT) {
462	dout(20) << "local image does not exist, removing image from rbd_mirroring"
463	<< dendl;
464	}
465
466	r = cls_client::mirror_image_remove(&ioctx, local_image_id);
467	if (r < 0 && r != -ENOENT) {
468	derr << "error removing image from mirroring directory: "
469	<< cpp_strerror(r) << dendl;
470	enqueue_failed_delete(r);
471	return true;
472	}
473
474	dout(10) << "Successfully deleted image "
475	<< global_image_id << " " << "(" << local_image_id << ")" << dendl;
476
477	complete_active_delete(0);
478	return true;
479	}
480
c07f9fc5 FG	481	template <typename I>
	482	int ImageDeleter<I>::image_has_snapshots_and_children(IoCtx *ioctx,
	483	string& image_id,
	484	bool *has_snapshots) {
7c673cae FG	485	string header_oid = librbd::util::header_name(image_id);
	486	::SnapContext snapc;
	487	int r = cls_client::get_snapcontext(ioctx, header_oid, &snapc);
	488	if (r < 0 && r != -ENOENT) {
	489	derr << "error retrieving snapshot context for image id " << image_id
	490	<< ": " << cpp_strerror(r) << dendl;
	491	return r;
	492	}
	493
	494	*has_snapshots = !snapc.snaps.empty();
	495
	496	return 0;
	497	}
	498
c07f9fc5 FG	499	template <typename I>
c07f9fc5 FG	500	void ImageDeleter<I>::complete_active_delete(int r) {
7c673cae FG	501	dout(20) << dendl;
7c673cae FG	502
31f18b77 FG	503	Mutex::Locker delete_locker(m_delete_lock);
	504	m_active_delete->notify(r);
	505	m_active_delete.reset();
7c673cae FG	506	}
7c673cae FG	507
c07f9fc5 FG	508	template <typename I>
c07f9fc5 FG	509	void ImageDeleter<I>::enqueue_failed_delete(int error_code) {
7c673cae FG	510	dout(20) << "enter" << dendl;
	511
	512	if (error_code == -EBLACKLISTED) {
	513	derr << "blacklisted while deleting local image" << dendl;
	514	complete_active_delete(error_code);
	515	return;
	516	}
	517
	518	m_delete_lock.Lock();
	519	if (m_active_delete->notify_on_failed_retry) {
	520	m_active_delete->notify(error_code);
	521	}
	522	m_active_delete->error_code = error_code;
	523	bool was_empty = m_failed_queue.empty();
	524	m_failed_queue.push_front(std::move(m_active_delete));
	525	m_delete_lock.Unlock();
	526	if (was_empty) {
	527	FunctionContext *ctx = new FunctionContext(
c07f9fc5	528	boost::bind(&ImageDeleter<I>::retry_failed_deletions, this));
7c673cae FG	529	Mutex::Locker l(*m_failed_timer_lock);
	530	m_failed_timer->add_event_after(m_failed_interval, ctx);
	531	}
	532	}
	533
c07f9fc5 FG	534	template <typename I>
c07f9fc5 FG	535	void ImageDeleter<I>::retry_failed_deletions() {
7c673cae FG	536	dout(20) << "enter" << dendl;
	537
	538	Mutex::Locker l(m_delete_lock);
	539
	540	bool empty = m_failed_queue.empty();
	541	while (!m_failed_queue.empty()) {
	542	m_delete_queue.push_back(std::move(m_failed_queue.back()));
	543	m_delete_queue.back()->retries++;
	544	m_failed_queue.pop_back();
	545	}
	546	if (!empty) {
	547	m_delete_queue_cond.Signal();
	548	}
	549	}
	550
c07f9fc5 FG	551	template <typename I>
	552	unique_ptr<typename ImageDeleter<I>::DeleteInfo> const*
	553	ImageDeleter<I>::find_delete_info(int64_t local_pool_id,
	554	const std::string &global_image_id) {
7c673cae FG	555	assert(m_delete_lock.is_locked());
	556
	557	if (m_active_delete && m_active_delete->match(local_pool_id,
	558	global_image_id)) {
	559	return &m_active_delete;
	560	}
	561
	562	for (const auto& del_info : m_delete_queue) {
	563	if (del_info->match(local_pool_id, global_image_id)) {
	564	return &del_info;
	565	}
	566	}
	567
	568	for (const auto& del_info : m_failed_queue) {
	569	if (del_info->match(local_pool_id, global_image_id)) {
	570	return &del_info;
	571	}
	572	}
	573
	574	return nullptr;
	575	}
	576
c07f9fc5 FG	577	template <typename I>
c07f9fc5 FG	578	void ImageDeleter<I>::print_status(Formatter f, stringstream ss) {
7c673cae FG	579	dout(20) << "enter" << dendl;
	580
	581	if (f) {
	582	f->open_object_section("image_deleter_status");
	583	f->open_array_section("delete_images_queue");
	584	}
	585
	586	Mutex::Locker l(m_delete_lock);
	587	for (const auto& image : m_delete_queue) {
	588	image->print_status(f, ss);
	589	}
	590
	591	if (f) {
	592	f->close_section();
	593	f->open_array_section("failed_deletes_queue");
	594	}
	595
	596	for (const auto& image : m_failed_queue) {
	597	image->print_status(f, ss, true);
	598	}
	599
	600	if (f) {
	601	f->close_section();
	602	f->close_section();
	603	f->flush(*ss);
	604	}
	605	}
	606
c07f9fc5 FG	607	template <typename I>
c07f9fc5 FG	608	void ImageDeleter<I>::DeleteInfo::notify(int r) {
7c673cae FG	609	if (on_delete) {
	610	dout(20) << "executing image deletion handler r=" << r << dendl;
	611
	612	Context *ctx = on_delete;
	613	on_delete = nullptr;
	614	ctx->complete(r);
	615	}
	616	}
	617
c07f9fc5 FG	618	template <typename I>
c07f9fc5 FG	619	void ImageDeleter<I>::DeleteInfo::to_string(stringstream& ss) {
7c673cae FG	620	ss << "[" << "local_pool_id=" << local_pool_id << ", ";
	621	ss << "global_image_id=" << global_image_id << "]";
	622	}
	623
c07f9fc5 FG	624	template <typename I>
	625	void ImageDeleter<I>::DeleteInfo::print_status(Formatter f, stringstream ss,
	626	bool print_failure_info) {
7c673cae FG	627	if (f) {
	628	f->open_object_section("delete_info");
	629	f->dump_int("local_pool_id", local_pool_id);
	630	f->dump_string("global_image_id", global_image_id);
	631	if (print_failure_info) {
	632	f->dump_string("error_code", cpp_strerror(error_code));
	633	f->dump_int("retries", retries);
	634	}
	635	f->close_section();
	636	f->flush(*ss);
	637	} else {
	638	this->to_string(*ss);
	639	}
	640	}
	641
c07f9fc5 FG	642	template <typename I>
c07f9fc5 FG	643	vector<string> ImageDeleter<I>::get_delete_queue_items() {
7c673cae FG	644	vector<string> items;
	645
	646	Mutex::Locker l(m_delete_lock);
	647	for (const auto& del_info : m_delete_queue) {
	648	items.push_back(del_info->global_image_id);
	649	}
	650
	651	return items;
	652	}
	653
c07f9fc5 FG	654	template <typename I>
c07f9fc5 FG	655	vector<pair<string, int> > ImageDeleter<I>::get_failed_queue_items() {
7c673cae FG	656	vector<pair<string, int> > items;
	657
	658	Mutex::Locker l(m_delete_lock);
	659	for (const auto& del_info : m_failed_queue) {
	660	items.push_back(make_pair(del_info->global_image_id,
	661	del_info->error_code));
	662	}
	663
	664	return items;
	665	}
	666
c07f9fc5 FG	667	template <typename I>
c07f9fc5 FG	668	void ImageDeleter<I>::set_failed_timer_interval(double interval) {
7c673cae FG	669	this->m_failed_interval = interval;
	670	}
	671
	672	} // namespace mirror
	673	} // namespace rbd
c07f9fc5 FG	674
c07f9fc5 FG	675	template class rbd::mirror::ImageDeleter<librbd::ImageCtx>;