1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 SUSE LINUX GmbH
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include <boost/bind.hpp>
20 #include "include/rados/librados.hpp"
21 #include "common/Formatter.h"
22 #include "common/admin_socket.h"
23 #include "common/debug.h"
24 #include "common/errno.h"
25 #include "common/WorkQueue.h"
26 #include "global/global_context.h"
27 #include "librbd/internal.h"
28 #include "librbd/ImageCtx.h"
29 #include "librbd/ImageState.h"
30 #include "librbd/Journal.h"
31 #include "librbd/Operations.h"
32 #include "librbd/journal/Policy.h"
33 #include "cls/rbd/cls_rbd_client.h"
34 #include "cls/rbd/cls_rbd_types.h"
35 #include "librbd/Utils.h"
36 #include "ImageDeleter.h"
38 #define dout_context g_ceph_context
39 #define dout_subsys ceph_subsys_rbd_mirror
41 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
46 using std::stringstream
;
51 using librados::IoCtx
;
52 using namespace librbd
;
59 class ImageDeleterAdminSocketCommand
{
61 virtual ~ImageDeleterAdminSocketCommand() {}
62 virtual bool call(Formatter
*f
, stringstream
*ss
) = 0;
66 class StatusCommand
: public ImageDeleterAdminSocketCommand
{
68 explicit StatusCommand(ImageDeleter
<I
> *image_del
) : image_del(image_del
) {}
70 bool call(Formatter
*f
, stringstream
*ss
) override
{
71 image_del
->print_status(f
, ss
);
76 ImageDeleter
<I
> *image_del
;
79 struct DeleteJournalPolicy
: public librbd::journal::Policy
{
80 bool append_disabled() const override
{
83 bool journal_disabled() const override
{
87 void allocate_tag_on_lock(Context
*on_finish
) override
{
88 on_finish
->complete(0);
92 } // anonymous namespace
95 class ImageDeleterAdminSocketHook
: public AdminSocketHook
{
97 ImageDeleterAdminSocketHook(CephContext
*cct
, ImageDeleter
<I
> *image_del
) :
98 admin_socket(cct
->get_admin_socket()) {
103 command
= "rbd mirror deletion status";
104 r
= admin_socket
->register_command(command
, command
, this,
105 "get status for image deleter");
107 commands
[command
] = new StatusCommand
<I
>(image_del
);
112 ~ImageDeleterAdminSocketHook() override
{
113 for (Commands::const_iterator i
= commands
.begin(); i
!= commands
.end();
115 (void)admin_socket
->unregister_command(i
->first
);
120 bool call(std::string command
, cmdmap_t
& cmdmap
, std::string format
,
121 bufferlist
& out
) override
{
122 Commands::const_iterator i
= commands
.find(command
);
123 assert(i
!= commands
.end());
124 Formatter
*f
= Formatter::create(format
);
126 bool r
= i
->second
->call(f
, &ss
);
133 typedef std::map
<std::string
, ImageDeleterAdminSocketCommand
*> Commands
;
134 AdminSocket
*admin_socket
;
138 template <typename I
>
139 ImageDeleter
<I
>::ImageDeleter(ContextWQ
*work_queue
, SafeTimer
*timer
,
141 ServiceDaemon
<librbd::ImageCtx
>* service_daemon
)
142 : m_work_queue(work_queue
),
143 m_service_daemon(service_daemon
),
144 m_delete_lock("rbd::mirror::ImageDeleter::Delete"),
145 m_image_deleter_thread(this),
146 m_failed_timer(timer
),
147 m_failed_timer_lock(timer_lock
),
148 m_asok_hook(new ImageDeleterAdminSocketHook
<I
>(g_ceph_context
, this))
150 set_failed_timer_interval(g_ceph_context
->_conf
->rbd_mirror_delete_retry_interval
);
151 m_image_deleter_thread
.create("image_deleter");
154 template <typename I
>
155 ImageDeleter
<I
>::~ImageDeleter() {
156 dout(20) << "enter" << dendl
;
160 Mutex::Locker
l (m_delete_lock
);
161 m_delete_queue_cond
.Signal();
163 if (m_image_deleter_thread
.is_started()) {
164 m_image_deleter_thread
.join();
168 dout(20) << "return" << dendl
;
171 template <typename I
>
172 void ImageDeleter
<I
>::run() {
173 dout(20) << "enter" << dendl
;
175 m_delete_lock
.Lock();
176 while (m_delete_queue
.empty()) {
177 dout(20) << "waiting for delete requests" << dendl
;
178 m_delete_queue_cond
.Wait(m_delete_lock
);
181 m_delete_lock
.Unlock();
182 dout(20) << "return" << dendl
;
187 m_active_delete
= std::move(m_delete_queue
.back());
188 m_delete_queue
.pop_back();
189 m_delete_lock
.Unlock();
191 bool move_to_next
= process_image_delete();
194 dout(20) << "return" << dendl
;
198 Mutex::Locker
l(m_delete_lock
);
199 if (m_delete_queue
.size() == 1) {
200 m_delete_queue_cond
.Wait(m_delete_lock
);
206 template <typename I
>
207 void ImageDeleter
<I
>::schedule_image_delete(RadosRef local_rados
,
208 int64_t local_pool_id
,
209 const std::string
& global_image_id
,
210 bool ignore_orphaned
) {
211 dout(20) << "enter" << dendl
;
213 Mutex::Locker
locker(m_delete_lock
);
215 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
216 if (del_info
!= nullptr) {
217 dout(20) << "image " << global_image_id
<< " "
218 << "was already scheduled for deletion" << dendl
;
219 if (ignore_orphaned
) {
220 (*del_info
)->ignore_orphaned
= true;
225 m_delete_queue
.push_front(
226 unique_ptr
<DeleteInfo
>(new DeleteInfo(local_rados
, local_pool_id
,
227 global_image_id
, ignore_orphaned
)));
228 m_delete_queue_cond
.Signal();
231 template <typename I
>
232 void ImageDeleter
<I
>::wait_for_scheduled_deletion(int64_t local_pool_id
,
233 const std::string
&global_image_id
,
235 bool notify_on_failed_retry
) {
237 ctx
= new FunctionContext([this, ctx
](int r
) {
238 m_work_queue
->queue(ctx
, r
);
241 Mutex::Locker
locker(m_delete_lock
);
242 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
244 // image not scheduled for deletion
249 dout(20) << "local_pool_id=" << local_pool_id
<< ", "
250 << "global_image_id=" << global_image_id
<< dendl
;
252 if ((*del_info
)->on_delete
!= nullptr) {
253 (*del_info
)->on_delete
->complete(-ESTALE
);
255 (*del_info
)->on_delete
= ctx
;
256 (*del_info
)->notify_on_failed_retry
= notify_on_failed_retry
;
259 template <typename I
>
260 void ImageDeleter
<I
>::cancel_waiter(int64_t local_pool_id
,
261 const std::string
&global_image_id
) {
262 Mutex::Locker
locker(m_delete_lock
);
263 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
268 if ((*del_info
)->on_delete
!= nullptr) {
269 (*del_info
)->on_delete
->complete(-ECANCELED
);
270 (*del_info
)->on_delete
= nullptr;
274 template <typename I
>
275 bool ImageDeleter
<I
>::process_image_delete() {
277 m_active_delete
->to_string(ss
);
278 std::string del_info_str
= ss
.str();
279 dout(10) << "start processing delete request: " << del_info_str
<< dendl
;
281 cls::rbd::MirrorImage mirror_image
;
283 // remote image was disabled, now we need to delete local image
285 r
= m_active_delete
->local_rados
->ioctx_create2(
286 m_active_delete
->local_pool_id
, ioctx
);
288 derr
<< "error accessing local pool " << m_active_delete
->local_pool_id
289 << ": " << cpp_strerror(r
) << dendl
;
290 enqueue_failed_delete(r
);
294 dout(20) << "connected to local pool: " << ioctx
.get_pool_name() << dendl
;
296 auto &global_image_id
= m_active_delete
->global_image_id
;
297 std::string local_image_id
;
298 r
= librbd::cls_client::mirror_image_get_image_id(
299 &ioctx
, global_image_id
, &local_image_id
);
301 dout(10) << "image " << global_image_id
<< " is not mirrored" << dendl
;
302 complete_active_delete(r
);
305 derr
<< "error retrieving local id for image " << global_image_id
306 << ": " << cpp_strerror(r
) << dendl
;
307 enqueue_failed_delete(r
);
311 std::string mirror_uuid
;
312 C_SaferCond tag_owner_ctx
;
313 Journal
<>::get_tag_owner(ioctx
, local_image_id
, &mirror_uuid
, m_work_queue
,
315 r
= tag_owner_ctx
.wait();
316 if (r
< 0 && r
!= -ENOENT
) {
317 derr
<< "error retrieving image primary info for image " << global_image_id
318 << ": " << cpp_strerror(r
) << dendl
;
319 enqueue_failed_delete(r
);
321 } else if (r
!= -ENOENT
) {
322 if (mirror_uuid
== Journal
<>::LOCAL_MIRROR_UUID
) {
323 dout(10) << "image " << global_image_id
<< " is local primary" << dendl
;
324 complete_active_delete(-EISPRM
);
326 } else if (mirror_uuid
== Journal
<>::ORPHAN_MIRROR_UUID
&&
327 !m_active_delete
->ignore_orphaned
) {
328 dout(10) << "image " << global_image_id
<< " is orphaned" << dendl
;
329 complete_active_delete(-EISPRM
);
334 dout(20) << "local image is not the primary" << dendl
;
336 r
= image_has_snapshots_and_children(&ioctx
, local_image_id
, &has_snapshots
);
338 enqueue_failed_delete(r
);
342 mirror_image
.global_image_id
= global_image_id
;
343 mirror_image
.state
= cls::rbd::MIRROR_IMAGE_STATE_DISABLING
;
344 r
= cls_client::mirror_image_set(&ioctx
, local_image_id
, mirror_image
);
346 dout(10) << "local image is not mirrored, aborting deletion..." << dendl
;
347 complete_active_delete(r
);
349 } else if (r
== -EEXIST
|| r
== -EINVAL
) {
350 derr
<< "cannot disable mirroring for image " << global_image_id
351 << ": global_image_id has changed/reused: "
352 << cpp_strerror(r
) << dendl
;
353 complete_active_delete(r
);
356 derr
<< "cannot disable mirroring for image " << global_image_id
357 << ": " << cpp_strerror(r
) << dendl
;
358 enqueue_failed_delete(r
);
362 dout(20) << "set local image mirroring to disable" << dendl
;
365 dout(20) << "local image has snapshots" << dendl
;
367 ImageCtx
*imgctx
= new ImageCtx("", local_image_id
, nullptr, ioctx
, false);
368 r
= imgctx
->state
->open(false);
370 derr
<< "error opening image " << global_image_id
<< " ("
371 << local_image_id
<< "): " << cpp_strerror(r
) << dendl
;
372 enqueue_failed_delete(r
);
377 RWLock::WLocker
snap_locker(imgctx
->snap_lock
);
378 imgctx
->set_journal_policy(new DeleteJournalPolicy());
381 std::vector
<librbd::snap_info_t
> snaps
;
382 r
= librbd::snap_list(imgctx
, snaps
);
384 derr
<< "error listing snapshot of image " << imgctx
->name
385 << cpp_strerror(r
) << dendl
;
386 imgctx
->state
->close();
387 enqueue_failed_delete(r
);
391 for (const auto& snap
: snaps
) {
392 dout(20) << "processing deletion of snapshot " << imgctx
->name
<< "@"
393 << snap
.name
<< dendl
;
396 r
= librbd::snap_is_protected(imgctx
, snap
.name
.c_str(), &is_protected
);
398 derr
<< "error checking snapshot protection of snapshot "
399 << imgctx
->name
<< "@" << snap
.name
<< ": " << cpp_strerror(r
)
401 imgctx
->state
->close();
402 enqueue_failed_delete(r
);
406 dout(20) << "snapshot " << imgctx
->name
<< "@" << snap
.name
407 << " is protected, issuing unprotect command" << dendl
;
409 r
= imgctx
->operations
->snap_unprotect(
410 cls::rbd::UserSnapshotNamespace(), snap
.name
.c_str());
412 // there are still clones of snapshots of this image, therefore send
413 // the delete request to the end of the queue
414 dout(10) << "local image id " << local_image_id
<< " has "
415 << "snapshots with cloned children, postponing deletion..."
417 imgctx
->state
->close();
418 Mutex::Locker
l(m_delete_lock
);
419 m_active_delete
->notify(r
);
420 m_delete_queue
.push_front(std::move(m_active_delete
));
423 derr
<< "error unprotecting snapshot " << imgctx
->name
<< "@"
424 << snap
.name
<< ": " << cpp_strerror(r
) << dendl
;
425 imgctx
->state
->close();
426 enqueue_failed_delete(r
);
431 r
= imgctx
->operations
->snap_remove(cls::rbd::UserSnapshotNamespace(),
434 derr
<< "error removing snapshot " << imgctx
->name
<< "@"
435 << snap
.name
<< ": " << cpp_strerror(r
) << dendl
;
436 imgctx
->state
->close();
437 enqueue_failed_delete(r
);
441 dout(10) << "snapshot " << imgctx
->name
<< "@" << snap
.name
442 << " was deleted" << dendl
;
445 imgctx
->state
->close();
448 librbd::NoOpProgressContext ctx
;
449 r
= librbd::remove(ioctx
, "", local_image_id
, ctx
, true);
450 if (r
< 0 && r
!= -ENOENT
) {
451 derr
<< "error removing image " << global_image_id
<< " "
452 << "(" << local_image_id
<< ") from local pool: "
453 << cpp_strerror(r
) << dendl
;
454 enqueue_failed_delete(r
);
458 // image was already deleted from rbd_directory, now we will make sure
459 // that will be also removed from rbd_mirroring
461 dout(20) << "local image does not exist, removing image from rbd_mirroring"
465 r
= cls_client::mirror_image_remove(&ioctx
, local_image_id
);
466 if (r
< 0 && r
!= -ENOENT
) {
467 derr
<< "error removing image from mirroring directory: "
468 << cpp_strerror(r
) << dendl
;
469 enqueue_failed_delete(r
);
473 dout(10) << "Successfully deleted image "
474 << global_image_id
<< " " << "(" << local_image_id
<< ")" << dendl
;
476 complete_active_delete(0);
480 template <typename I
>
481 int ImageDeleter
<I
>::image_has_snapshots_and_children(IoCtx
*ioctx
,
483 bool *has_snapshots
) {
484 string header_oid
= librbd::util::header_name(image_id
);
486 int r
= cls_client::get_snapcontext(ioctx
, header_oid
, &snapc
);
487 if (r
< 0 && r
!= -ENOENT
) {
488 derr
<< "error retrieving snapshot context for image id " << image_id
489 << ": " << cpp_strerror(r
) << dendl
;
493 *has_snapshots
= !snapc
.snaps
.empty();
498 template <typename I
>
499 void ImageDeleter
<I
>::complete_active_delete(int r
) {
502 Mutex::Locker
delete_locker(m_delete_lock
);
503 m_active_delete
->notify(r
);
504 m_active_delete
.reset();
507 template <typename I
>
508 void ImageDeleter
<I
>::enqueue_failed_delete(int error_code
) {
509 dout(20) << "enter" << dendl
;
511 if (error_code
== -EBLACKLISTED
) {
512 derr
<< "blacklisted while deleting local image" << dendl
;
513 complete_active_delete(error_code
);
517 m_delete_lock
.Lock();
518 if (m_active_delete
->notify_on_failed_retry
) {
519 m_active_delete
->notify(error_code
);
521 m_active_delete
->error_code
= error_code
;
522 bool was_empty
= m_failed_queue
.empty();
523 m_failed_queue
.push_front(std::move(m_active_delete
));
524 m_delete_lock
.Unlock();
526 FunctionContext
*ctx
= new FunctionContext(
527 boost::bind(&ImageDeleter
<I
>::retry_failed_deletions
, this));
528 Mutex::Locker
l(*m_failed_timer_lock
);
529 m_failed_timer
->add_event_after(m_failed_interval
, ctx
);
533 template <typename I
>
534 void ImageDeleter
<I
>::retry_failed_deletions() {
535 dout(20) << "enter" << dendl
;
537 Mutex::Locker
l(m_delete_lock
);
539 bool empty
= m_failed_queue
.empty();
540 while (!m_failed_queue
.empty()) {
541 m_delete_queue
.push_back(std::move(m_failed_queue
.back()));
542 m_delete_queue
.back()->retries
++;
543 m_failed_queue
.pop_back();
546 m_delete_queue_cond
.Signal();
550 template <typename I
>
551 unique_ptr
<typename ImageDeleter
<I
>::DeleteInfo
> const*
552 ImageDeleter
<I
>::find_delete_info(int64_t local_pool_id
,
553 const std::string
&global_image_id
) {
554 assert(m_delete_lock
.is_locked());
556 if (m_active_delete
&& m_active_delete
->match(local_pool_id
,
558 return &m_active_delete
;
561 for (const auto& del_info
: m_delete_queue
) {
562 if (del_info
->match(local_pool_id
, global_image_id
)) {
567 for (const auto& del_info
: m_failed_queue
) {
568 if (del_info
->match(local_pool_id
, global_image_id
)) {
576 template <typename I
>
577 void ImageDeleter
<I
>::print_status(Formatter
*f
, stringstream
*ss
) {
578 dout(20) << "enter" << dendl
;
581 f
->open_object_section("image_deleter_status");
582 f
->open_array_section("delete_images_queue");
585 Mutex::Locker
l(m_delete_lock
);
586 for (const auto& image
: m_delete_queue
) {
587 image
->print_status(f
, ss
);
592 f
->open_array_section("failed_deletes_queue");
595 for (const auto& image
: m_failed_queue
) {
596 image
->print_status(f
, ss
, true);
606 template <typename I
>
607 void ImageDeleter
<I
>::DeleteInfo::notify(int r
) {
609 dout(20) << "executing image deletion handler r=" << r
<< dendl
;
611 Context
*ctx
= on_delete
;
617 template <typename I
>
618 void ImageDeleter
<I
>::DeleteInfo::to_string(stringstream
& ss
) {
619 ss
<< "[" << "local_pool_id=" << local_pool_id
<< ", ";
620 ss
<< "global_image_id=" << global_image_id
<< "]";
623 template <typename I
>
624 void ImageDeleter
<I
>::DeleteInfo::print_status(Formatter
*f
, stringstream
*ss
,
625 bool print_failure_info
) {
627 f
->open_object_section("delete_info");
628 f
->dump_int("local_pool_id", local_pool_id
);
629 f
->dump_string("global_image_id", global_image_id
);
630 if (print_failure_info
) {
631 f
->dump_string("error_code", cpp_strerror(error_code
));
632 f
->dump_int("retries", retries
);
637 this->to_string(*ss
);
641 template <typename I
>
642 vector
<string
> ImageDeleter
<I
>::get_delete_queue_items() {
643 vector
<string
> items
;
645 Mutex::Locker
l(m_delete_lock
);
646 for (const auto& del_info
: m_delete_queue
) {
647 items
.push_back(del_info
->global_image_id
);
653 template <typename I
>
654 vector
<pair
<string
, int> > ImageDeleter
<I
>::get_failed_queue_items() {
655 vector
<pair
<string
, int> > items
;
657 Mutex::Locker
l(m_delete_lock
);
658 for (const auto& del_info
: m_failed_queue
) {
659 items
.push_back(make_pair(del_info
->global_image_id
,
660 del_info
->error_code
));
666 template <typename I
>
667 void ImageDeleter
<I
>::set_failed_timer_interval(double interval
) {
668 this->m_failed_interval
= interval
;
671 } // namespace mirror
674 template class rbd::mirror::ImageDeleter
<librbd::ImageCtx
>;