1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 SUSE LINUX GmbH
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
14 #include <boost/bind.hpp>
19 #include "include/rados/librados.hpp"
20 #include "common/Formatter.h"
21 #include "common/admin_socket.h"
22 #include "common/debug.h"
23 #include "common/errno.h"
24 #include "common/WorkQueue.h"
25 #include "global/global_context.h"
26 #include "librbd/internal.h"
27 #include "librbd/ImageCtx.h"
28 #include "librbd/ImageState.h"
29 #include "librbd/Journal.h"
30 #include "librbd/Operations.h"
31 #include "librbd/journal/Policy.h"
32 #include "cls/rbd/cls_rbd_client.h"
33 #include "cls/rbd/cls_rbd_types.h"
34 #include "librbd/Utils.h"
35 #include "ImageDeleter.h"
37 #define dout_context g_ceph_context
38 #define dout_subsys ceph_subsys_rbd_mirror
40 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
45 using std::stringstream
;
50 using librados::IoCtx
;
51 using namespace librbd
;
58 class ImageDeleterAdminSocketCommand
{
60 virtual ~ImageDeleterAdminSocketCommand() {}
61 virtual bool call(Formatter
*f
, stringstream
*ss
) = 0;
64 class StatusCommand
: public ImageDeleterAdminSocketCommand
{
66 explicit StatusCommand(ImageDeleter
*image_del
) : image_del(image_del
) {}
68 bool call(Formatter
*f
, stringstream
*ss
) override
{
69 image_del
->print_status(f
, ss
);
74 ImageDeleter
*image_del
;
77 struct DeleteJournalPolicy
: public librbd::journal::Policy
{
78 bool append_disabled() const override
{
81 bool journal_disabled() const override
{
85 void allocate_tag_on_lock(Context
*on_finish
) override
{
86 on_finish
->complete(0);
90 } // anonymous namespace
92 class ImageDeleterAdminSocketHook
: public AdminSocketHook
{
94 ImageDeleterAdminSocketHook(CephContext
*cct
, ImageDeleter
*image_del
) :
95 admin_socket(cct
->get_admin_socket()) {
100 command
= "rbd mirror deletion status";
101 r
= admin_socket
->register_command(command
, command
, this,
102 "get status for image deleter");
104 commands
[command
] = new StatusCommand(image_del
);
109 ~ImageDeleterAdminSocketHook() override
{
110 for (Commands::const_iterator i
= commands
.begin(); i
!= commands
.end();
112 (void)admin_socket
->unregister_command(i
->first
);
117 bool call(std::string command
, cmdmap_t
& cmdmap
, std::string format
,
118 bufferlist
& out
) override
{
119 Commands::const_iterator i
= commands
.find(command
);
120 assert(i
!= commands
.end());
121 Formatter
*f
= Formatter::create(format
);
123 bool r
= i
->second
->call(f
, &ss
);
130 typedef std::map
<std::string
, ImageDeleterAdminSocketCommand
*> Commands
;
131 AdminSocket
*admin_socket
;
135 ImageDeleter::ImageDeleter(ContextWQ
*work_queue
, SafeTimer
*timer
,
138 m_work_queue(work_queue
),
139 m_delete_lock("rbd::mirror::ImageDeleter::Delete"),
140 m_image_deleter_thread(this),
141 m_failed_timer(timer
),
142 m_failed_timer_lock(timer_lock
),
143 m_asok_hook(new ImageDeleterAdminSocketHook(g_ceph_context
, this))
145 set_failed_timer_interval(g_ceph_context
->_conf
->rbd_mirror_delete_retry_interval
);
146 m_image_deleter_thread
.create("image_deleter");
149 ImageDeleter::~ImageDeleter() {
150 dout(20) << "enter" << dendl
;
154 Mutex::Locker
l (m_delete_lock
);
155 m_delete_queue_cond
.Signal();
157 if (m_image_deleter_thread
.is_started()) {
158 m_image_deleter_thread
.join();
162 dout(20) << "return" << dendl
;
165 void ImageDeleter::run() {
166 dout(20) << "enter" << dendl
;
168 m_delete_lock
.Lock();
169 while (m_delete_queue
.empty()) {
170 dout(20) << "waiting for delete requests" << dendl
;
171 m_delete_queue_cond
.Wait(m_delete_lock
);
174 m_delete_lock
.Unlock();
175 dout(20) << "return" << dendl
;
180 m_active_delete
= std::move(m_delete_queue
.back());
181 m_delete_queue
.pop_back();
182 m_delete_lock
.Unlock();
184 bool move_to_next
= process_image_delete();
187 dout(20) << "return" << dendl
;
191 Mutex::Locker
l(m_delete_lock
);
192 if (m_delete_queue
.size() == 1) {
193 m_delete_queue_cond
.Wait(m_delete_lock
);
199 void ImageDeleter::schedule_image_delete(RadosRef local_rados
,
200 int64_t local_pool_id
,
201 const std::string
& global_image_id
) {
202 dout(20) << "enter" << dendl
;
204 Mutex::Locker
locker(m_delete_lock
);
206 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
207 if (del_info
!= nullptr) {
208 dout(20) << "image " << global_image_id
<< " "
209 << "was already scheduled for deletion" << dendl
;
213 m_delete_queue
.push_front(
214 unique_ptr
<DeleteInfo
>(new DeleteInfo(local_rados
, local_pool_id
,
216 m_delete_queue_cond
.Signal();
219 void ImageDeleter::wait_for_scheduled_deletion(int64_t local_pool_id
,
220 const std::string
&global_image_id
,
222 bool notify_on_failed_retry
) {
224 ctx
= new FunctionContext([this, ctx
](int r
) {
225 m_work_queue
->queue(ctx
, r
);
228 Mutex::Locker
locker(m_delete_lock
);
229 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
231 // image not scheduled for deletion
236 dout(20) << "local_pool_id=" << local_pool_id
<< ", "
237 << "global_image_id=" << global_image_id
<< dendl
;
239 if ((*del_info
)->on_delete
!= nullptr) {
240 (*del_info
)->on_delete
->complete(-ESTALE
);
242 (*del_info
)->on_delete
= ctx
;
243 (*del_info
)->notify_on_failed_retry
= notify_on_failed_retry
;
246 void ImageDeleter::cancel_waiter(int64_t local_pool_id
,
247 const std::string
&global_image_id
) {
248 Mutex::Locker
locker(m_delete_lock
);
249 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
254 if ((*del_info
)->on_delete
!= nullptr) {
255 (*del_info
)->on_delete
->complete(-ECANCELED
);
256 (*del_info
)->on_delete
= nullptr;
260 bool ImageDeleter::process_image_delete() {
263 m_active_delete
->to_string(ss
);
264 std::string del_info_str
= ss
.str();
265 dout(10) << "start processing delete request: " << del_info_str
<< dendl
;
267 cls::rbd::MirrorImage mirror_image
;
269 // remote image was disabled, now we need to delete local image
271 r
= m_active_delete
->local_rados
->ioctx_create2(
272 m_active_delete
->local_pool_id
, ioctx
);
274 derr
<< "error accessing local pool " << m_active_delete
->local_pool_id
275 << ": " << cpp_strerror(r
) << dendl
;
276 enqueue_failed_delete(r
);
280 dout(20) << "connected to local pool: " << ioctx
.get_pool_name() << dendl
;
282 auto &global_image_id
= m_active_delete
->global_image_id
;
283 std::string local_image_id
;
284 r
= librbd::cls_client::mirror_image_get_image_id(
285 &ioctx
, global_image_id
, &local_image_id
);
287 dout(10) << "image " << global_image_id
<< " is not mirrored" << dendl
;
288 complete_active_delete(r
);
291 derr
<< "error retrieving local id for image " << global_image_id
292 << ": " << cpp_strerror(r
) << dendl
;
293 enqueue_failed_delete(r
);
297 bool is_primary
= false;
298 C_SaferCond tag_owner_ctx
;
299 Journal
<>::is_tag_owner(ioctx
, local_image_id
, &is_primary
,
300 m_work_queue
, &tag_owner_ctx
);
301 r
= tag_owner_ctx
.wait();
302 if (r
< 0 && r
!= -ENOENT
) {
303 derr
<< "error retrieving image primary info for image " << global_image_id
304 << ": " << cpp_strerror(r
) << dendl
;
305 enqueue_failed_delete(r
);
309 dout(10) << "image " << global_image_id
<< " is local primary" << dendl
;
310 complete_active_delete(-EISPRM
);
314 dout(20) << "local image is not the primary" << dendl
;
317 r
= image_has_snapshots_and_children(&ioctx
, local_image_id
, &has_snapshots
);
319 enqueue_failed_delete(r
);
323 mirror_image
.global_image_id
= global_image_id
;
324 mirror_image
.state
= cls::rbd::MIRROR_IMAGE_STATE_DISABLING
;
325 r
= cls_client::mirror_image_set(&ioctx
, local_image_id
, mirror_image
);
327 dout(10) << "local image is not mirrored, aborting deletion..." << dendl
;
328 complete_active_delete(r
);
330 } else if (r
== -EEXIST
|| r
== -EINVAL
) {
331 derr
<< "cannot disable mirroring for image " << global_image_id
332 << ": global_image_id has changed/reused: "
333 << cpp_strerror(r
) << dendl
;
334 complete_active_delete(r
);
337 derr
<< "cannot disable mirroring for image " << global_image_id
338 << ": " << cpp_strerror(r
) << dendl
;
339 enqueue_failed_delete(r
);
343 dout(20) << "set local image mirroring to disable" << dendl
;
346 dout(20) << "local image has snapshots" << dendl
;
348 ImageCtx
*imgctx
= new ImageCtx("", local_image_id
, nullptr, ioctx
, false);
349 r
= imgctx
->state
->open(false);
351 derr
<< "error opening image " << global_image_id
<< " ("
352 << local_image_id
<< "): " << cpp_strerror(r
) << dendl
;
353 enqueue_failed_delete(r
);
358 RWLock::WLocker
snap_locker(imgctx
->snap_lock
);
359 imgctx
->set_journal_policy(new DeleteJournalPolicy());
362 std::vector
<librbd::snap_info_t
> snaps
;
363 r
= librbd::snap_list(imgctx
, snaps
);
365 derr
<< "error listing snapshot of image " << imgctx
->name
366 << cpp_strerror(r
) << dendl
;
367 imgctx
->state
->close();
368 enqueue_failed_delete(r
);
372 for (const auto& snap
: snaps
) {
373 dout(20) << "processing deletion of snapshot " << imgctx
->name
<< "@"
374 << snap
.name
<< dendl
;
377 r
= librbd::snap_is_protected(imgctx
, snap
.name
.c_str(), &is_protected
);
379 derr
<< "error checking snapshot protection of snapshot "
380 << imgctx
->name
<< "@" << snap
.name
<< ": " << cpp_strerror(r
)
382 imgctx
->state
->close();
383 enqueue_failed_delete(r
);
387 dout(20) << "snapshot " << imgctx
->name
<< "@" << snap
.name
388 << " is protected, issuing unprotect command" << dendl
;
390 r
= imgctx
->operations
->snap_unprotect(
391 cls::rbd::UserSnapshotNamespace(), snap
.name
.c_str());
393 // there are still clones of snapshots of this image, therefore send
394 // the delete request to the end of the queue
395 dout(10) << "local image id " << local_image_id
<< " has "
396 << "snapshots with cloned children, postponing deletion..."
398 imgctx
->state
->close();
399 Mutex::Locker
l(m_delete_lock
);
400 m_active_delete
->notify(r
);
401 m_delete_queue
.push_front(std::move(m_active_delete
));
404 derr
<< "error unprotecting snapshot " << imgctx
->name
<< "@"
405 << snap
.name
<< ": " << cpp_strerror(r
) << dendl
;
406 imgctx
->state
->close();
407 enqueue_failed_delete(r
);
412 r
= imgctx
->operations
->snap_remove(cls::rbd::UserSnapshotNamespace(),
415 derr
<< "error removing snapshot " << imgctx
->name
<< "@"
416 << snap
.name
<< ": " << cpp_strerror(r
) << dendl
;
417 imgctx
->state
->close();
418 enqueue_failed_delete(r
);
422 dout(10) << "snapshot " << imgctx
->name
<< "@" << snap
.name
423 << " was deleted" << dendl
;
426 imgctx
->state
->close();
429 librbd::NoOpProgressContext ctx
;
430 r
= librbd::remove(ioctx
, "", local_image_id
, ctx
, true);
431 if (r
< 0 && r
!= -ENOENT
) {
432 derr
<< "error removing image " << global_image_id
<< " "
433 << "(" << local_image_id
<< ") from local pool: "
434 << cpp_strerror(r
) << dendl
;
435 enqueue_failed_delete(r
);
439 // image was already deleted from rbd_directory, now we will make sure
440 // that will be also removed from rbd_mirroring
442 dout(20) << "local image does not exist, removing image from rbd_mirroring"
446 r
= cls_client::mirror_image_remove(&ioctx
, local_image_id
);
447 if (r
< 0 && r
!= -ENOENT
) {
448 derr
<< "error removing image from mirroring directory: "
449 << cpp_strerror(r
) << dendl
;
450 enqueue_failed_delete(r
);
454 dout(10) << "Successfully deleted image "
455 << global_image_id
<< " " << "(" << local_image_id
<< ")" << dendl
;
457 complete_active_delete(0);
461 int ImageDeleter::image_has_snapshots_and_children(IoCtx
*ioctx
,
463 bool *has_snapshots
) {
465 string header_oid
= librbd::util::header_name(image_id
);
467 int r
= cls_client::get_snapcontext(ioctx
, header_oid
, &snapc
);
468 if (r
< 0 && r
!= -ENOENT
) {
469 derr
<< "error retrieving snapshot context for image id " << image_id
470 << ": " << cpp_strerror(r
) << dendl
;
474 *has_snapshots
= !snapc
.snaps
.empty();
479 void ImageDeleter::complete_active_delete(int r
) {
482 m_delete_lock
.Lock();
483 DeleteInfo
*del_info
= m_active_delete
.release();
484 assert(del_info
!= nullptr);
485 m_delete_lock
.Unlock();
489 void ImageDeleter::enqueue_failed_delete(int error_code
) {
490 dout(20) << "enter" << dendl
;
492 if (error_code
== -EBLACKLISTED
) {
493 derr
<< "blacklisted while deleting local image" << dendl
;
494 complete_active_delete(error_code
);
498 m_delete_lock
.Lock();
499 if (m_active_delete
->notify_on_failed_retry
) {
500 m_active_delete
->notify(error_code
);
502 m_active_delete
->error_code
= error_code
;
503 bool was_empty
= m_failed_queue
.empty();
504 m_failed_queue
.push_front(std::move(m_active_delete
));
505 m_delete_lock
.Unlock();
507 FunctionContext
*ctx
= new FunctionContext(
508 boost::bind(&ImageDeleter::retry_failed_deletions
, this));
509 Mutex::Locker
l(*m_failed_timer_lock
);
510 m_failed_timer
->add_event_after(m_failed_interval
, ctx
);
514 void ImageDeleter::retry_failed_deletions() {
515 dout(20) << "enter" << dendl
;
517 Mutex::Locker
l(m_delete_lock
);
519 bool empty
= m_failed_queue
.empty();
520 while (!m_failed_queue
.empty()) {
521 m_delete_queue
.push_back(std::move(m_failed_queue
.back()));
522 m_delete_queue
.back()->retries
++;
523 m_failed_queue
.pop_back();
526 m_delete_queue_cond
.Signal();
530 unique_ptr
<ImageDeleter::DeleteInfo
> const* ImageDeleter::find_delete_info(
531 int64_t local_pool_id
, const std::string
&global_image_id
) {
532 assert(m_delete_lock
.is_locked());
534 if (m_active_delete
&& m_active_delete
->match(local_pool_id
,
536 return &m_active_delete
;
539 for (const auto& del_info
: m_delete_queue
) {
540 if (del_info
->match(local_pool_id
, global_image_id
)) {
545 for (const auto& del_info
: m_failed_queue
) {
546 if (del_info
->match(local_pool_id
, global_image_id
)) {
554 void ImageDeleter::print_status(Formatter
*f
, stringstream
*ss
) {
555 dout(20) << "enter" << dendl
;
558 f
->open_object_section("image_deleter_status");
559 f
->open_array_section("delete_images_queue");
562 Mutex::Locker
l(m_delete_lock
);
563 for (const auto& image
: m_delete_queue
) {
564 image
->print_status(f
, ss
);
569 f
->open_array_section("failed_deletes_queue");
572 for (const auto& image
: m_failed_queue
) {
573 image
->print_status(f
, ss
, true);
583 void ImageDeleter::DeleteInfo::notify(int r
) {
585 dout(20) << "executing image deletion handler r=" << r
<< dendl
;
587 Context
*ctx
= on_delete
;
593 void ImageDeleter::DeleteInfo::to_string(stringstream
& ss
) {
594 ss
<< "[" << "local_pool_id=" << local_pool_id
<< ", ";
595 ss
<< "global_image_id=" << global_image_id
<< "]";
598 void ImageDeleter::DeleteInfo::print_status(Formatter
*f
, stringstream
*ss
,
599 bool print_failure_info
) {
601 f
->open_object_section("delete_info");
602 f
->dump_int("local_pool_id", local_pool_id
);
603 f
->dump_string("global_image_id", global_image_id
);
604 if (print_failure_info
) {
605 f
->dump_string("error_code", cpp_strerror(error_code
));
606 f
->dump_int("retries", retries
);
611 this->to_string(*ss
);
615 vector
<string
> ImageDeleter::get_delete_queue_items() {
616 vector
<string
> items
;
618 Mutex::Locker
l(m_delete_lock
);
619 for (const auto& del_info
: m_delete_queue
) {
620 items
.push_back(del_info
->global_image_id
);
626 vector
<pair
<string
, int> > ImageDeleter::get_failed_queue_items() {
627 vector
<pair
<string
, int> > items
;
629 Mutex::Locker
l(m_delete_lock
);
630 for (const auto& del_info
: m_failed_queue
) {
631 items
.push_back(make_pair(del_info
->global_image_id
,
632 del_info
->error_code
));
638 void ImageDeleter::set_failed_timer_interval(double interval
) {
639 this->m_failed_interval
= interval
;
642 } // namespace mirror