1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 SUSE LINUX GmbH
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include <boost/bind.hpp>
20 #include "include/rados/librados.hpp"
21 #include "common/Formatter.h"
22 #include "common/admin_socket.h"
23 #include "common/debug.h"
24 #include "common/errno.h"
25 #include "common/WorkQueue.h"
26 #include "global/global_context.h"
27 #include "librbd/internal.h"
28 #include "librbd/ImageCtx.h"
29 #include "librbd/ImageState.h"
30 #include "librbd/Journal.h"
31 #include "librbd/Operations.h"
32 #include "librbd/journal/Policy.h"
33 #include "cls/rbd/cls_rbd_client.h"
34 #include "cls/rbd/cls_rbd_types.h"
35 #include "librbd/Utils.h"
36 #include "ImageDeleter.h"
38 #define dout_context g_ceph_context
39 #define dout_subsys ceph_subsys_rbd_mirror
41 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
46 using std::stringstream
;
51 using librados::IoCtx
;
52 using namespace librbd
;
59 class ImageDeleterAdminSocketCommand
{
61 virtual ~ImageDeleterAdminSocketCommand() {}
62 virtual bool call(Formatter
*f
, stringstream
*ss
) = 0;
66 class StatusCommand
: public ImageDeleterAdminSocketCommand
{
68 explicit StatusCommand(ImageDeleter
<I
> *image_del
) : image_del(image_del
) {}
70 bool call(Formatter
*f
, stringstream
*ss
) override
{
71 image_del
->print_status(f
, ss
);
76 ImageDeleter
<I
> *image_del
;
79 struct DeleteJournalPolicy
: public librbd::journal::Policy
{
80 bool append_disabled() const override
{
83 bool journal_disabled() const override
{
87 void allocate_tag_on_lock(Context
*on_finish
) override
{
88 on_finish
->complete(0);
92 } // anonymous namespace
95 class ImageDeleterAdminSocketHook
: public AdminSocketHook
{
97 ImageDeleterAdminSocketHook(CephContext
*cct
, ImageDeleter
<I
> *image_del
) :
98 admin_socket(cct
->get_admin_socket()) {
103 command
= "rbd mirror deletion status";
104 r
= admin_socket
->register_command(command
, command
, this,
105 "get status for image deleter");
107 commands
[command
] = new StatusCommand
<I
>(image_del
);
112 ~ImageDeleterAdminSocketHook() override
{
113 for (Commands::const_iterator i
= commands
.begin(); i
!= commands
.end();
115 (void)admin_socket
->unregister_command(i
->first
);
120 bool call(std::string command
, cmdmap_t
& cmdmap
, std::string format
,
121 bufferlist
& out
) override
{
122 Commands::const_iterator i
= commands
.find(command
);
123 assert(i
!= commands
.end());
124 Formatter
*f
= Formatter::create(format
);
126 bool r
= i
->second
->call(f
, &ss
);
133 typedef std::map
<std::string
, ImageDeleterAdminSocketCommand
*> Commands
;
134 AdminSocket
*admin_socket
;
138 template <typename I
>
139 ImageDeleter
<I
>::ImageDeleter(ContextWQ
*work_queue
, SafeTimer
*timer
,
141 ServiceDaemon
<librbd::ImageCtx
>* service_daemon
)
142 : m_work_queue(work_queue
),
143 m_service_daemon(service_daemon
),
144 m_delete_lock("rbd::mirror::ImageDeleter::Delete"),
145 m_image_deleter_thread(this),
146 m_failed_timer(timer
),
147 m_failed_timer_lock(timer_lock
),
148 m_asok_hook(new ImageDeleterAdminSocketHook
<I
>(g_ceph_context
, this))
150 set_failed_timer_interval(g_ceph_context
->_conf
->get_val
<double>(
151 "rbd_mirror_delete_retry_interval"));
152 m_image_deleter_thread
.create("image_deleter");
155 template <typename I
>
156 ImageDeleter
<I
>::~ImageDeleter() {
157 dout(20) << "enter" << dendl
;
161 Mutex::Locker
l (m_delete_lock
);
162 m_delete_queue_cond
.Signal();
164 if (m_image_deleter_thread
.is_started()) {
165 m_image_deleter_thread
.join();
169 dout(20) << "return" << dendl
;
172 template <typename I
>
173 void ImageDeleter
<I
>::run() {
174 dout(20) << "enter" << dendl
;
176 m_delete_lock
.Lock();
177 while (m_delete_queue
.empty()) {
178 dout(20) << "waiting for delete requests" << dendl
;
179 m_delete_queue_cond
.Wait(m_delete_lock
);
182 m_delete_lock
.Unlock();
183 dout(20) << "return" << dendl
;
188 m_active_delete
= std::move(m_delete_queue
.back());
189 m_delete_queue
.pop_back();
190 m_delete_lock
.Unlock();
192 bool move_to_next
= process_image_delete();
195 dout(20) << "return" << dendl
;
199 Mutex::Locker
l(m_delete_lock
);
200 if (m_delete_queue
.size() == 1) {
201 m_delete_queue_cond
.Wait(m_delete_lock
);
207 template <typename I
>
208 void ImageDeleter
<I
>::schedule_image_delete(RadosRef local_rados
,
209 int64_t local_pool_id
,
210 const std::string
& global_image_id
,
211 bool ignore_orphaned
) {
212 dout(20) << "enter" << dendl
;
214 Mutex::Locker
locker(m_delete_lock
);
216 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
217 if (del_info
!= nullptr) {
218 dout(20) << "image " << global_image_id
<< " "
219 << "was already scheduled for deletion" << dendl
;
220 if (ignore_orphaned
) {
221 (*del_info
)->ignore_orphaned
= true;
226 m_delete_queue
.push_front(
227 unique_ptr
<DeleteInfo
>(new DeleteInfo(local_rados
, local_pool_id
,
228 global_image_id
, ignore_orphaned
)));
229 m_delete_queue_cond
.Signal();
232 template <typename I
>
233 void ImageDeleter
<I
>::wait_for_scheduled_deletion(int64_t local_pool_id
,
234 const std::string
&global_image_id
,
236 bool notify_on_failed_retry
) {
238 ctx
= new FunctionContext([this, ctx
](int r
) {
239 m_work_queue
->queue(ctx
, r
);
242 Mutex::Locker
locker(m_delete_lock
);
243 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
245 // image not scheduled for deletion
250 dout(20) << "local_pool_id=" << local_pool_id
<< ", "
251 << "global_image_id=" << global_image_id
<< dendl
;
253 if ((*del_info
)->on_delete
!= nullptr) {
254 (*del_info
)->on_delete
->complete(-ESTALE
);
256 (*del_info
)->on_delete
= ctx
;
257 (*del_info
)->notify_on_failed_retry
= notify_on_failed_retry
;
260 template <typename I
>
261 void ImageDeleter
<I
>::cancel_waiter(int64_t local_pool_id
,
262 const std::string
&global_image_id
) {
263 Mutex::Locker
locker(m_delete_lock
);
264 auto del_info
= find_delete_info(local_pool_id
, global_image_id
);
269 if ((*del_info
)->on_delete
!= nullptr) {
270 (*del_info
)->on_delete
->complete(-ECANCELED
);
271 (*del_info
)->on_delete
= nullptr;
275 template <typename I
>
276 bool ImageDeleter
<I
>::process_image_delete() {
278 m_active_delete
->to_string(ss
);
279 std::string del_info_str
= ss
.str();
280 dout(10) << "start processing delete request: " << del_info_str
<< dendl
;
282 cls::rbd::MirrorImage mirror_image
;
284 // remote image was disabled, now we need to delete local image
286 r
= m_active_delete
->local_rados
->ioctx_create2(
287 m_active_delete
->local_pool_id
, ioctx
);
289 derr
<< "error accessing local pool " << m_active_delete
->local_pool_id
290 << ": " << cpp_strerror(r
) << dendl
;
291 enqueue_failed_delete(r
);
295 dout(20) << "connected to local pool: " << ioctx
.get_pool_name() << dendl
;
297 auto &global_image_id
= m_active_delete
->global_image_id
;
298 std::string local_image_id
;
299 r
= librbd::cls_client::mirror_image_get_image_id(
300 &ioctx
, global_image_id
, &local_image_id
);
302 dout(10) << "image " << global_image_id
<< " is not mirrored" << dendl
;
303 complete_active_delete(r
);
306 derr
<< "error retrieving local id for image " << global_image_id
307 << ": " << cpp_strerror(r
) << dendl
;
308 enqueue_failed_delete(r
);
312 std::string mirror_uuid
;
313 C_SaferCond tag_owner_ctx
;
314 Journal
<>::get_tag_owner(ioctx
, local_image_id
, &mirror_uuid
, m_work_queue
,
316 r
= tag_owner_ctx
.wait();
317 if (r
< 0 && r
!= -ENOENT
) {
318 derr
<< "error retrieving image primary info for image " << global_image_id
319 << ": " << cpp_strerror(r
) << dendl
;
320 enqueue_failed_delete(r
);
322 } else if (r
!= -ENOENT
) {
323 if (mirror_uuid
== Journal
<>::LOCAL_MIRROR_UUID
) {
324 dout(10) << "image " << global_image_id
<< " is local primary" << dendl
;
325 complete_active_delete(-EISPRM
);
327 } else if (mirror_uuid
== Journal
<>::ORPHAN_MIRROR_UUID
&&
328 !m_active_delete
->ignore_orphaned
) {
329 dout(10) << "image " << global_image_id
<< " is orphaned" << dendl
;
330 complete_active_delete(-EISPRM
);
335 dout(20) << "local image is not the primary" << dendl
;
337 r
= image_has_snapshots_and_children(&ioctx
, local_image_id
, &has_snapshots
);
339 enqueue_failed_delete(r
);
343 mirror_image
.global_image_id
= global_image_id
;
344 mirror_image
.state
= cls::rbd::MIRROR_IMAGE_STATE_DISABLING
;
345 r
= cls_client::mirror_image_set(&ioctx
, local_image_id
, mirror_image
);
347 dout(10) << "local image is not mirrored, aborting deletion..." << dendl
;
348 complete_active_delete(r
);
350 } else if (r
== -EEXIST
|| r
== -EINVAL
) {
351 derr
<< "cannot disable mirroring for image " << global_image_id
352 << ": global_image_id has changed/reused: "
353 << cpp_strerror(r
) << dendl
;
354 complete_active_delete(r
);
357 derr
<< "cannot disable mirroring for image " << global_image_id
358 << ": " << cpp_strerror(r
) << dendl
;
359 enqueue_failed_delete(r
);
363 dout(20) << "set local image mirroring to disable" << dendl
;
366 dout(20) << "local image has snapshots" << dendl
;
368 ImageCtx
*imgctx
= new ImageCtx("", local_image_id
, nullptr, ioctx
, false);
369 r
= imgctx
->state
->open(false);
371 derr
<< "error opening image " << global_image_id
<< " ("
372 << local_image_id
<< "): " << cpp_strerror(r
) << dendl
;
373 enqueue_failed_delete(r
);
378 RWLock::WLocker
snap_locker(imgctx
->snap_lock
);
379 imgctx
->set_journal_policy(new DeleteJournalPolicy());
382 std::vector
<librbd::snap_info_t
> snaps
;
383 r
= librbd::snap_list(imgctx
, snaps
);
385 derr
<< "error listing snapshot of image " << imgctx
->name
386 << cpp_strerror(r
) << dendl
;
387 imgctx
->state
->close();
388 enqueue_failed_delete(r
);
392 for (const auto& snap
: snaps
) {
393 dout(20) << "processing deletion of snapshot " << imgctx
->name
<< "@"
394 << snap
.name
<< dendl
;
397 r
= librbd::snap_is_protected(imgctx
, snap
.name
.c_str(), &is_protected
);
399 derr
<< "error checking snapshot protection of snapshot "
400 << imgctx
->name
<< "@" << snap
.name
<< ": " << cpp_strerror(r
)
402 imgctx
->state
->close();
403 enqueue_failed_delete(r
);
407 dout(20) << "snapshot " << imgctx
->name
<< "@" << snap
.name
408 << " is protected, issuing unprotect command" << dendl
;
410 r
= imgctx
->operations
->snap_unprotect(
411 cls::rbd::UserSnapshotNamespace(), snap
.name
.c_str());
413 // there are still clones of snapshots of this image, therefore send
414 // the delete request to the end of the queue
415 dout(10) << "local image id " << local_image_id
<< " has "
416 << "snapshots with cloned children, postponing deletion..."
418 imgctx
->state
->close();
419 Mutex::Locker
l(m_delete_lock
);
420 m_active_delete
->notify(r
);
421 m_delete_queue
.push_front(std::move(m_active_delete
));
424 derr
<< "error unprotecting snapshot " << imgctx
->name
<< "@"
425 << snap
.name
<< ": " << cpp_strerror(r
) << dendl
;
426 imgctx
->state
->close();
427 enqueue_failed_delete(r
);
432 r
= imgctx
->operations
->snap_remove(cls::rbd::UserSnapshotNamespace(),
435 derr
<< "error removing snapshot " << imgctx
->name
<< "@"
436 << snap
.name
<< ": " << cpp_strerror(r
) << dendl
;
437 imgctx
->state
->close();
438 enqueue_failed_delete(r
);
442 dout(10) << "snapshot " << imgctx
->name
<< "@" << snap
.name
443 << " was deleted" << dendl
;
446 imgctx
->state
->close();
449 librbd::NoOpProgressContext ctx
;
450 r
= librbd::remove(ioctx
, "", local_image_id
, ctx
, true);
451 if (r
< 0 && r
!= -ENOENT
) {
452 derr
<< "error removing image " << global_image_id
<< " "
453 << "(" << local_image_id
<< ") from local pool: "
454 << cpp_strerror(r
) << dendl
;
455 enqueue_failed_delete(r
);
459 // image was already deleted from rbd_directory, now we will make sure
460 // that will be also removed from rbd_mirroring
462 dout(20) << "local image does not exist, removing image from rbd_mirroring"
466 r
= cls_client::mirror_image_remove(&ioctx
, local_image_id
);
467 if (r
< 0 && r
!= -ENOENT
) {
468 derr
<< "error removing image from mirroring directory: "
469 << cpp_strerror(r
) << dendl
;
470 enqueue_failed_delete(r
);
474 dout(10) << "Successfully deleted image "
475 << global_image_id
<< " " << "(" << local_image_id
<< ")" << dendl
;
477 complete_active_delete(0);
481 template <typename I
>
482 int ImageDeleter
<I
>::image_has_snapshots_and_children(IoCtx
*ioctx
,
484 bool *has_snapshots
) {
485 string header_oid
= librbd::util::header_name(image_id
);
487 int r
= cls_client::get_snapcontext(ioctx
, header_oid
, &snapc
);
488 if (r
< 0 && r
!= -ENOENT
) {
489 derr
<< "error retrieving snapshot context for image id " << image_id
490 << ": " << cpp_strerror(r
) << dendl
;
494 *has_snapshots
= !snapc
.snaps
.empty();
499 template <typename I
>
500 void ImageDeleter
<I
>::complete_active_delete(int r
) {
503 Mutex::Locker
delete_locker(m_delete_lock
);
504 m_active_delete
->notify(r
);
505 m_active_delete
.reset();
508 template <typename I
>
509 void ImageDeleter
<I
>::enqueue_failed_delete(int error_code
) {
510 dout(20) << "enter" << dendl
;
512 if (error_code
== -EBLACKLISTED
) {
513 derr
<< "blacklisted while deleting local image" << dendl
;
514 complete_active_delete(error_code
);
518 m_delete_lock
.Lock();
519 if (m_active_delete
->notify_on_failed_retry
) {
520 m_active_delete
->notify(error_code
);
522 m_active_delete
->error_code
= error_code
;
523 bool was_empty
= m_failed_queue
.empty();
524 m_failed_queue
.push_front(std::move(m_active_delete
));
525 m_delete_lock
.Unlock();
527 FunctionContext
*ctx
= new FunctionContext(
528 boost::bind(&ImageDeleter
<I
>::retry_failed_deletions
, this));
529 Mutex::Locker
l(*m_failed_timer_lock
);
530 m_failed_timer
->add_event_after(m_failed_interval
, ctx
);
534 template <typename I
>
535 void ImageDeleter
<I
>::retry_failed_deletions() {
536 dout(20) << "enter" << dendl
;
538 Mutex::Locker
l(m_delete_lock
);
540 bool empty
= m_failed_queue
.empty();
541 while (!m_failed_queue
.empty()) {
542 m_delete_queue
.push_back(std::move(m_failed_queue
.back()));
543 m_delete_queue
.back()->retries
++;
544 m_failed_queue
.pop_back();
547 m_delete_queue_cond
.Signal();
551 template <typename I
>
552 unique_ptr
<typename ImageDeleter
<I
>::DeleteInfo
> const*
553 ImageDeleter
<I
>::find_delete_info(int64_t local_pool_id
,
554 const std::string
&global_image_id
) {
555 assert(m_delete_lock
.is_locked());
557 if (m_active_delete
&& m_active_delete
->match(local_pool_id
,
559 return &m_active_delete
;
562 for (const auto& del_info
: m_delete_queue
) {
563 if (del_info
->match(local_pool_id
, global_image_id
)) {
568 for (const auto& del_info
: m_failed_queue
) {
569 if (del_info
->match(local_pool_id
, global_image_id
)) {
577 template <typename I
>
578 void ImageDeleter
<I
>::print_status(Formatter
*f
, stringstream
*ss
) {
579 dout(20) << "enter" << dendl
;
582 f
->open_object_section("image_deleter_status");
583 f
->open_array_section("delete_images_queue");
586 Mutex::Locker
l(m_delete_lock
);
587 for (const auto& image
: m_delete_queue
) {
588 image
->print_status(f
, ss
);
593 f
->open_array_section("failed_deletes_queue");
596 for (const auto& image
: m_failed_queue
) {
597 image
->print_status(f
, ss
, true);
607 template <typename I
>
608 void ImageDeleter
<I
>::DeleteInfo::notify(int r
) {
610 dout(20) << "executing image deletion handler r=" << r
<< dendl
;
612 Context
*ctx
= on_delete
;
618 template <typename I
>
619 void ImageDeleter
<I
>::DeleteInfo::to_string(stringstream
& ss
) {
620 ss
<< "[" << "local_pool_id=" << local_pool_id
<< ", ";
621 ss
<< "global_image_id=" << global_image_id
<< "]";
624 template <typename I
>
625 void ImageDeleter
<I
>::DeleteInfo::print_status(Formatter
*f
, stringstream
*ss
,
626 bool print_failure_info
) {
628 f
->open_object_section("delete_info");
629 f
->dump_int("local_pool_id", local_pool_id
);
630 f
->dump_string("global_image_id", global_image_id
);
631 if (print_failure_info
) {
632 f
->dump_string("error_code", cpp_strerror(error_code
));
633 f
->dump_int("retries", retries
);
638 this->to_string(*ss
);
642 template <typename I
>
643 vector
<string
> ImageDeleter
<I
>::get_delete_queue_items() {
644 vector
<string
> items
;
646 Mutex::Locker
l(m_delete_lock
);
647 for (const auto& del_info
: m_delete_queue
) {
648 items
.push_back(del_info
->global_image_id
);
654 template <typename I
>
655 vector
<pair
<string
, int> > ImageDeleter
<I
>::get_failed_queue_items() {
656 vector
<pair
<string
, int> > items
;
658 Mutex::Locker
l(m_delete_lock
);
659 for (const auto& del_info
: m_failed_queue
) {
660 items
.push_back(make_pair(del_info
->global_image_id
,
661 del_info
->error_code
));
667 template <typename I
>
668 void ImageDeleter
<I
>::set_failed_timer_interval(double interval
) {
669 this->m_failed_interval
= interval
;
672 } // namespace mirror
675 template class rbd::mirror::ImageDeleter
<librbd::ImageCtx
>;