1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "common/Cond.h"
5 #include "common/errno.h"
7 #include "librbd/ExclusiveLock.h"
8 #include "librbd/api/Group.h"
9 #include "librbd/ImageCtx.h"
10 #include "librbd/ImageState.h"
11 #include "librbd/ImageWatcher.h"
12 #include "librbd/Operations.h"
13 #include "librbd/Utils.h"
14 #include "librbd/internal.h"
15 #include "librbd/io/AioCompletion.h"
17 #define dout_subsys ceph_subsys_rbd
19 #define dout_prefix *_dout << "librbd::api::Group: " << __func__ << ": "
26 // list binds to list() here, so std::list is explicitly used below
28 using ceph::bufferlist
;
29 using librados::snap_t
;
30 using librados::IoCtx
;
31 using librados::Rados
;
40 snap_t
get_group_snap_id(I
* ictx
,
41 const cls::rbd::SnapshotNamespace
& in_snap_namespace
) {
42 ceph_assert(ceph_mutex_is_locked(ictx
->image_lock
));
43 auto it
= ictx
->snap_ids
.lower_bound({cls::rbd::GroupSnapshotNamespace
{},
45 for (; it
!= ictx
->snap_ids
.end(); ++it
) {
46 if (it
->first
.first
== in_snap_namespace
) {
48 } else if (boost::get
<cls::rbd::GroupSnapshotNamespace
>(&it
->first
.first
) ==
56 string
generate_uuid(librados::IoCtx
& io_ctx
)
59 uint64_t bid
= rados
.get_instance_id();
61 uint32_t extra
= rand() % 0xFFFFFFFF;
63 bid_ss
<< std::hex
<< bid
<< std::hex
<< extra
;
67 int group_snap_list(librados::IoCtx
& group_ioctx
, const char *group_name
,
68 std::vector
<cls::rbd::GroupSnapshot
> *cls_snaps
)
70 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
73 vector
<string
> ind_snap_names
;
75 int r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
,
76 group_name
, &group_id
);
78 lderr(cct
) << "error reading group id object: "
83 string group_header_oid
= util::group_header_name(group_id
);
85 const int max_read
= 1024;
86 cls::rbd::GroupSnapshot snap_last
;
89 vector
<cls::rbd::GroupSnapshot
> snaps_page
;
91 r
= cls_client::group_snap_list(&group_ioctx
, group_header_oid
,
92 snap_last
, max_read
, &snaps_page
);
95 lderr(cct
) << "error reading snap list from group: "
96 << cpp_strerror(-r
) << dendl
;
99 cls_snaps
->insert(cls_snaps
->end(), snaps_page
.begin(), snaps_page
.end());
100 if (snaps_page
.size() < max_read
) {
103 snap_last
= *snaps_page
.rbegin();
109 std::string
calc_ind_image_snap_name(uint64_t pool_id
,
110 const std::string
&group_id
,
111 const std::string
&snap_id
)
113 std::stringstream ind_snap_name_stream
;
114 ind_snap_name_stream
<< ".group." << std::hex
<< pool_id
<< "_"
115 << group_id
<< "_" << snap_id
;
116 return ind_snap_name_stream
.str();
119 int group_image_list(librados::IoCtx
& group_ioctx
, const char *group_name
,
120 std::vector
<cls::rbd::GroupImageStatus
> *image_ids
)
122 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
126 int r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
,
127 group_name
, &group_id
);
129 lderr(cct
) << "error reading group id object: "
134 string group_header_oid
= util::group_header_name(group_id
);
136 ldout(cct
, 20) << "listing images in group name "
137 << group_name
<< " group id " << group_header_oid
<< dendl
;
140 const int max_read
= 1024;
141 cls::rbd::GroupImageSpec start_last
;
143 std::vector
<cls::rbd::GroupImageStatus
> image_ids_page
;
145 r
= cls_client::group_image_list(&group_ioctx
, group_header_oid
,
146 start_last
, max_read
, &image_ids_page
);
149 lderr(cct
) << "error reading image list from group: "
150 << cpp_strerror(-r
) << dendl
;
153 image_ids
->insert(image_ids
->end(),
154 image_ids_page
.begin(), image_ids_page
.end());
156 if (image_ids_page
.size() > 0)
157 start_last
= image_ids_page
.rbegin()->spec
;
159 r
= image_ids_page
.size();
160 } while (r
== max_read
);
165 int group_image_remove(librados::IoCtx
& group_ioctx
, string group_id
,
166 librados::IoCtx
& image_ioctx
, string image_id
)
168 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
170 string group_header_oid
= util::group_header_name(group_id
);
172 string image_header_oid
= util::header_name(image_id
);
174 ldout(cct
, 20) << "removing image " << image_id
175 << " image id " << image_header_oid
<< dendl
;
177 cls::rbd::GroupSpec
group_spec(group_id
, group_ioctx
.get_id());
179 cls::rbd::GroupImageStatus
incomplete_st(image_id
, image_ioctx
.get_id(),
180 cls::rbd::GROUP_IMAGE_LINK_STATE_INCOMPLETE
);
182 cls::rbd::GroupImageSpec
spec(image_id
, image_ioctx
.get_id());
184 int r
= cls_client::group_image_set(&group_ioctx
, group_header_oid
,
188 lderr(cct
) << "couldn't put image into removing state: "
189 << cpp_strerror(-r
) << dendl
;
193 r
= cls_client::image_group_remove(&image_ioctx
, image_header_oid
,
195 if ((r
< 0) && (r
!= -ENOENT
)) {
196 lderr(cct
) << "couldn't remove group reference from image"
197 << cpp_strerror(-r
) << dendl
;
200 ImageWatcher
<>::notify_header_update(image_ioctx
, image_header_oid
);
203 r
= cls_client::group_image_remove(&group_ioctx
, group_header_oid
, spec
);
205 lderr(cct
) << "couldn't remove image from group"
206 << cpp_strerror(-r
) << dendl
;
213 int group_snap_remove_by_record(librados::IoCtx
& group_ioctx
,
214 const cls::rbd::GroupSnapshot
& group_snap
,
215 const std::string
& group_id
,
216 const std::string
& group_header_oid
) {
218 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
219 std::vector
<C_SaferCond
*> on_finishes
;
222 std::vector
<librbd::ImageCtx
*> ictxs
;
224 cls::rbd::GroupSnapshotNamespace ne
{group_ioctx
.get_id(), group_id
,
227 ldout(cct
, 20) << "Removing snapshots" << dendl
;
228 int snap_count
= group_snap
.snaps
.size();
230 for (int i
= 0; i
< snap_count
; ++i
) {
231 librbd::IoCtx image_io_ctx
;
232 r
= util::create_ioctx(group_ioctx
, "image", group_snap
.snaps
[i
].pool
, {},
238 librbd::ImageCtx
* image_ctx
= new ImageCtx("", group_snap
.snaps
[i
].image_id
,
239 nullptr, image_io_ctx
, false);
241 C_SaferCond
* on_finish
= new C_SaferCond
;
243 image_ctx
->state
->open(0, on_finish
);
245 ictxs
.push_back(image_ctx
);
246 on_finishes
.push_back(on_finish
);
250 for (int i
= 0; i
< snap_count
; ++i
) {
251 r
= on_finishes
[i
]->wait();
252 delete on_finishes
[i
];
262 ldout(cct
, 20) << "Opened participating images. " <<
263 "Deleting snapshots themselves." << dendl
;
265 for (int i
= 0; i
< snap_count
; ++i
) {
266 ImageCtx
*ictx
= ictxs
[i
];
267 on_finishes
[i
] = new C_SaferCond
;
269 std::string snap_name
;
270 ictx
->image_lock
.lock_shared();
271 snap_t snap_id
= get_group_snap_id(ictx
, ne
);
272 r
= ictx
->get_snap_name(snap_id
, &snap_name
);
273 ictx
->image_lock
.unlock_shared();
276 ldout(cct
, 20) << "removing individual snapshot from image " << ictx
->name
278 ictx
->operations
->snap_remove(ne
, snap_name
, on_finishes
[i
]);
280 // We are ok to ignore missing image snapshots. The snapshot could have
281 // been inconsistent in the first place.
282 on_finishes
[i
]->complete(0);
286 for (int i
= 0; i
< snap_count
; ++i
) {
287 r
= on_finishes
[i
]->wait();
288 delete on_finishes
[i
];
289 if (r
< 0 && r
!= -ENOENT
) {
290 // if previous attempts to remove this snapshot failed then the image's
291 // snapshot may not exist
292 lderr(cct
) << "Failed deleting image snapshot. Ret code: " << r
<< dendl
;
301 ldout(cct
, 20) << "Removed images snapshots removing snapshot record."
304 r
= cls_client::group_snap_remove(&group_ioctx
, group_header_oid
,
312 for (int i
= 0; i
< snap_count
; ++i
) {
313 if (ictxs
[i
] != nullptr) {
314 ictxs
[i
]->state
->close();
320 int group_snap_rollback_by_record(librados::IoCtx
& group_ioctx
,
321 const cls::rbd::GroupSnapshot
& group_snap
,
322 const std::string
& group_id
,
323 const std::string
& group_header_oid
,
324 ProgressContext
& pctx
) {
325 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
326 std::vector
<C_SaferCond
*> on_finishes
;
329 std::vector
<librbd::ImageCtx
*> ictxs
;
331 cls::rbd::GroupSnapshotNamespace ne
{group_ioctx
.get_id(), group_id
,
334 ldout(cct
, 20) << "Rolling back snapshots" << dendl
;
335 int snap_count
= group_snap
.snaps
.size();
337 for (int i
= 0; i
< snap_count
; ++i
) {
338 librados::IoCtx image_io_ctx
;
339 r
= util::create_ioctx(group_ioctx
, "image", group_snap
.snaps
[i
].pool
, {},
345 librbd::ImageCtx
* image_ctx
= new ImageCtx("", group_snap
.snaps
[i
].image_id
,
346 nullptr, image_io_ctx
, false);
348 C_SaferCond
* on_finish
= new C_SaferCond
;
350 image_ctx
->state
->open(0, on_finish
);
352 ictxs
.push_back(image_ctx
);
353 on_finishes
.push_back(on_finish
);
357 for (int i
= 0; i
< snap_count
; ++i
) {
358 r
= on_finishes
[i
]->wait();
359 delete on_finishes
[i
];
369 ldout(cct
, 20) << "Requesting exclusive locks for images" << dendl
;
370 for (auto ictx
: ictxs
) {
371 std::shared_lock owner_lock
{ictx
->owner_lock
};
372 if (ictx
->exclusive_lock
!= nullptr) {
373 ictx
->exclusive_lock
->block_requests(-EBUSY
);
376 for (int i
= 0; i
< snap_count
; ++i
) {
377 ImageCtx
*ictx
= ictxs
[i
];
378 std::shared_lock owner_lock
{ictx
->owner_lock
};
380 on_finishes
[i
] = new C_SaferCond
;
381 if (ictx
->exclusive_lock
!= nullptr) {
382 ictx
->exclusive_lock
->acquire_lock(on_finishes
[i
]);
387 for (int i
= 0; i
< snap_count
; ++i
) {
389 ImageCtx
*ictx
= ictxs
[i
];
390 if (ictx
->exclusive_lock
!= nullptr) {
391 r
= on_finishes
[i
]->wait();
393 delete on_finishes
[i
];
402 for (int i
= 0; i
< snap_count
; ++i
) {
403 ImageCtx
*ictx
= ictxs
[i
];
404 on_finishes
[i
] = new C_SaferCond
;
406 std::shared_lock owner_locker
{ictx
->owner_lock
};
407 std::string snap_name
;
408 ictx
->image_lock
.lock_shared();
409 snap_t snap_id
= get_group_snap_id(ictx
, ne
);
410 r
= ictx
->get_snap_name(snap_id
, &snap_name
);
411 ictx
->image_lock
.unlock_shared();
414 ldout(cct
, 20) << "rolling back to individual snapshot for image " << ictx
->name
416 ictx
->operations
->execute_snap_rollback(ne
, snap_name
, pctx
, on_finishes
[i
]);
418 on_finishes
[i
]->complete(r
);
422 for (int i
= 0; i
< snap_count
; ++i
) {
423 r
= on_finishes
[i
]->wait();
424 delete on_finishes
[i
];
425 if (r
< 0 && r
!= -ENOENT
) {
426 lderr(cct
) << "Failed rolling back group to snapshot. Ret code: " << r
<< dendl
;
432 for (int i
= 0; i
< snap_count
; ++i
) {
433 if (ictxs
[i
] != nullptr) {
434 ictxs
[i
]->state
->close();
440 template <typename I
>
441 void notify_unquiesce(std::vector
<I
*> &ictxs
,
442 const std::vector
<uint64_t> &requests
) {
443 if (requests
.empty()) {
447 ceph_assert(requests
.size() == ictxs
.size());
448 int image_count
= ictxs
.size();
449 std::vector
<C_SaferCond
> on_finishes(image_count
);
451 for (int i
= 0; i
< image_count
; ++i
) {
452 ImageCtx
*ictx
= ictxs
[i
];
454 ictx
->image_watcher
->notify_unquiesce(requests
[i
], &on_finishes
[i
]);
457 for (int i
= 0; i
< image_count
; ++i
) {
458 on_finishes
[i
].wait();
462 template <typename I
>
463 int notify_quiesce(std::vector
<I
*> &ictxs
, ProgressContext
&prog_ctx
,
464 std::vector
<uint64_t> *requests
) {
465 int image_count
= ictxs
.size();
466 std::vector
<C_SaferCond
> on_finishes(image_count
);
468 requests
->resize(image_count
);
469 for (int i
= 0; i
< image_count
; ++i
) {
470 auto ictx
= ictxs
[i
];
472 ictx
->image_watcher
->notify_quiesce(&(*requests
)[i
], prog_ctx
,
477 for (int i
= 0; i
< image_count
; ++i
) {
478 int r
= on_finishes
[i
].wait();
485 notify_unquiesce(ictxs
, *requests
);
491 } // anonymous namespace
493 template <typename I
>
494 int Group
<I
>::image_remove_by_id(librados::IoCtx
& group_ioctx
,
495 const char *group_name
,
496 librados::IoCtx
& image_ioctx
,
497 const char *image_id
)
499 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
500 ldout(cct
, 20) << "io_ctx=" << &group_ioctx
501 << " group name " << group_name
<< " image "
502 << &image_ioctx
<< " id " << image_id
<< dendl
;
506 int r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
, group_name
,
509 lderr(cct
) << "error reading group id object: "
515 ldout(cct
, 20) << "removing image from group name " << group_name
516 << " group id " << group_id
<< dendl
;
518 return group_image_remove(group_ioctx
, group_id
, image_ioctx
, string(image_id
));
521 template <typename I
>
522 int Group
<I
>::create(librados::IoCtx
& io_ctx
, const char *group_name
)
524 CephContext
*cct
= (CephContext
*)io_ctx
.cct();
526 string id
= generate_uuid(io_ctx
);
528 ldout(cct
, 2) << "adding group to directory..." << dendl
;
530 int r
= cls_client::group_dir_add(&io_ctx
, RBD_GROUP_DIRECTORY
, group_name
,
533 lderr(cct
) << "error adding group to directory: "
538 string header_oid
= util::group_header_name(id
);
540 r
= io_ctx
.create(header_oid
, true);
542 lderr(cct
) << "error creating group header: " << cpp_strerror(r
) << dendl
;
543 goto err_remove_from_dir
;
549 int remove_r
= cls_client::group_dir_remove(&io_ctx
, RBD_GROUP_DIRECTORY
,
552 lderr(cct
) << "error cleaning up group from rbd_directory "
553 << "object after creation failed: " << cpp_strerror(remove_r
)
560 template <typename I
>
561 int Group
<I
>::remove(librados::IoCtx
& io_ctx
, const char *group_name
)
563 CephContext
*cct((CephContext
*)io_ctx
.cct());
564 ldout(cct
, 20) << "group_remove " << &io_ctx
<< " " << group_name
<< dendl
;
566 std::string group_id
;
567 int r
= cls_client::dir_get_id(&io_ctx
, RBD_GROUP_DIRECTORY
,
568 std::string(group_name
), &group_id
);
569 if (r
< 0 && r
!= -ENOENT
) {
570 lderr(cct
) << "error getting id of group" << dendl
;
573 string group_header_oid
= util::group_header_name(group_id
);
575 std::vector
<cls::rbd::GroupSnapshot
> snaps
;
576 r
= group_snap_list(io_ctx
, group_name
, &snaps
);
577 if (r
< 0 && r
!= -ENOENT
) {
578 lderr(cct
) << "error listing group snapshots" << dendl
;
582 for (auto &snap
: snaps
) {
583 r
= group_snap_remove_by_record(io_ctx
, snap
, group_id
, group_header_oid
);
589 std::vector
<cls::rbd::GroupImageStatus
> images
;
590 r
= group_image_list(io_ctx
, group_name
, &images
);
591 if (r
< 0 && r
!= -ENOENT
) {
592 lderr(cct
) << "error listing group images" << dendl
;
596 for (auto image
: images
) {
598 r
= util::create_ioctx(io_ctx
, "image", image
.spec
.pool_id
, {},
604 r
= group_image_remove(io_ctx
, group_id
, image_ioctx
, image
.spec
.image_id
);
605 if (r
< 0 && r
!= -ENOENT
) {
606 lderr(cct
) << "error removing image from a group" << dendl
;
611 string header_oid
= util::group_header_name(group_id
);
613 r
= io_ctx
.remove(header_oid
);
614 if (r
< 0 && r
!= -ENOENT
) {
615 lderr(cct
) << "error removing header: " << cpp_strerror(-r
) << dendl
;
619 r
= cls_client::group_dir_remove(&io_ctx
, RBD_GROUP_DIRECTORY
,
620 group_name
, group_id
);
621 if (r
< 0 && r
!= -ENOENT
) {
622 lderr(cct
) << "error removing group from directory" << dendl
;
629 template <typename I
>
630 int Group
<I
>::list(IoCtx
& io_ctx
, vector
<string
> *names
)
632 CephContext
*cct
= (CephContext
*)io_ctx
.cct();
633 ldout(cct
, 20) << "io_ctx=" << &io_ctx
<< dendl
;
636 string last_read
= "";
639 map
<string
, string
> groups
;
640 r
= cls_client::group_dir_list(&io_ctx
, RBD_GROUP_DIRECTORY
, last_read
,
643 return 0; // Ignore missing rbd group directory. It means we don't have any groups yet.
647 lderr(cct
) << "error listing group in directory: "
648 << cpp_strerror(r
) << dendl
;
654 for (pair
<string
, string
> group
: groups
) {
655 names
->push_back(group
.first
);
657 if (!groups
.empty()) {
658 last_read
= groups
.rbegin()->first
;
661 } while (r
== max_read
);
666 template <typename I
>
667 int Group
<I
>::image_add(librados::IoCtx
& group_ioctx
, const char *group_name
,
668 librados::IoCtx
& image_ioctx
, const char *image_name
)
670 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
671 ldout(cct
, 20) << "io_ctx=" << &group_ioctx
672 << " group name " << group_name
<< " image "
673 << &image_ioctx
<< " name " << image_name
<< dendl
;
675 if (group_ioctx
.get_namespace() != image_ioctx
.get_namespace()) {
676 lderr(cct
) << "group and image cannot be in different namespaces" << dendl
;
682 int r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
, group_name
,
685 lderr(cct
) << "error reading group id object: "
690 string group_header_oid
= util::group_header_name(group_id
);
693 ldout(cct
, 20) << "adding image to group name " << group_name
694 << " group id " << group_header_oid
<< dendl
;
698 r
= cls_client::dir_get_id(&image_ioctx
, RBD_DIRECTORY
, image_name
,
701 lderr(cct
) << "error reading image id object: "
702 << cpp_strerror(-r
) << dendl
;
706 string image_header_oid
= util::header_name(image_id
);
708 ldout(cct
, 20) << "adding image " << image_name
709 << " image id " << image_header_oid
<< dendl
;
711 cls::rbd::GroupImageStatus
incomplete_st(
712 image_id
, image_ioctx
.get_id(),
713 cls::rbd::GROUP_IMAGE_LINK_STATE_INCOMPLETE
);
714 cls::rbd::GroupImageStatus
attached_st(
715 image_id
, image_ioctx
.get_id(), cls::rbd::GROUP_IMAGE_LINK_STATE_ATTACHED
);
717 r
= cls_client::group_image_set(&group_ioctx
, group_header_oid
,
720 cls::rbd::GroupSpec
group_spec(group_id
, group_ioctx
.get_id());
723 lderr(cct
) << "error adding image reference to group: "
724 << cpp_strerror(-r
) << dendl
;
728 r
= cls_client::image_group_add(&image_ioctx
, image_header_oid
, group_spec
);
730 lderr(cct
) << "error adding group reference to image: "
731 << cpp_strerror(-r
) << dendl
;
732 cls::rbd::GroupImageSpec
spec(image_id
, image_ioctx
.get_id());
733 cls_client::group_image_remove(&group_ioctx
, group_header_oid
, spec
);
734 // Ignore errors in the clean up procedure.
737 ImageWatcher
<>::notify_header_update(image_ioctx
, image_header_oid
);
739 r
= cls_client::group_image_set(&group_ioctx
, group_header_oid
,
745 template <typename I
>
746 int Group
<I
>::image_remove(librados::IoCtx
& group_ioctx
, const char *group_name
,
747 librados::IoCtx
& image_ioctx
, const char *image_name
)
749 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
750 ldout(cct
, 20) << "io_ctx=" << &group_ioctx
751 << " group name " << group_name
<< " image "
752 << &image_ioctx
<< " name " << image_name
<< dendl
;
754 if (group_ioctx
.get_namespace() != image_ioctx
.get_namespace()) {
755 lderr(cct
) << "group and image cannot be in different namespaces" << dendl
;
761 int r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
, group_name
,
764 lderr(cct
) << "error reading group id object: "
770 ldout(cct
, 20) << "removing image from group name " << group_name
771 << " group id " << group_id
<< dendl
;
774 r
= cls_client::dir_get_id(&image_ioctx
, RBD_DIRECTORY
, image_name
,
777 lderr(cct
) << "error reading image id object: "
778 << cpp_strerror(-r
) << dendl
;
782 r
= group_image_remove(group_ioctx
, group_id
, image_ioctx
, image_id
);
787 template <typename I
>
788 int Group
<I
>::image_list(librados::IoCtx
& group_ioctx
,
789 const char *group_name
,
790 std::vector
<group_image_info_t
>* images
)
792 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
793 ldout(cct
, 20) << "io_ctx=" << &group_ioctx
794 << " group name " << group_name
<< dendl
;
796 std::vector
<cls::rbd::GroupImageStatus
> image_ids
;
798 group_image_list(group_ioctx
, group_name
, &image_ids
);
800 for (auto image_id
: image_ids
) {
802 int r
= util::create_ioctx(group_ioctx
, "image", image_id
.spec
.pool_id
, {},
808 std::string image_name
;
809 r
= cls_client::dir_get_name(&ioctx
, RBD_DIRECTORY
,
810 image_id
.spec
.image_id
, &image_name
);
819 static_cast<group_image_state_t
>(image_id
.state
)});
825 template <typename I
>
826 int Group
<I
>::rename(librados::IoCtx
& io_ctx
, const char *src_name
,
827 const char *dest_name
)
829 CephContext
*cct((CephContext
*)io_ctx
.cct());
830 ldout(cct
, 20) << "group_rename " << &io_ctx
<< " " << src_name
831 << " -> " << dest_name
<< dendl
;
833 std::string group_id
;
834 int r
= cls_client::dir_get_id(&io_ctx
, RBD_GROUP_DIRECTORY
,
835 std::string(src_name
), &group_id
);
838 lderr(cct
) << "error getting id of group" << dendl
;
842 r
= cls_client::group_dir_rename(&io_ctx
, RBD_GROUP_DIRECTORY
,
843 src_name
, dest_name
, group_id
);
844 if (r
< 0 && r
!= -ENOENT
) {
845 lderr(cct
) << "error renaming group from directory" << dendl
;
853 template <typename I
>
854 int Group
<I
>::image_get_group(I
*ictx
, group_info_t
*group_info
)
856 int r
= ictx
->state
->refresh_if_required();
860 if (RBD_GROUP_INVALID_POOL
!= ictx
->group_spec
.pool_id
) {
862 r
= util::create_ioctx(ictx
->md_ctx
, "group", ictx
->group_spec
.pool_id
, {},
868 std::string group_name
;
869 r
= cls_client::dir_get_name(&ioctx
, RBD_GROUP_DIRECTORY
,
870 ictx
->group_spec
.group_id
, &group_name
);
873 group_info
->pool
= ioctx
.get_id();
874 group_info
->name
= group_name
;
876 group_info
->pool
= RBD_GROUP_INVALID_POOL
;
877 group_info
->name
= "";
883 template <typename I
>
884 int Group
<I
>::snap_create(librados::IoCtx
& group_ioctx
,
885 const char *group_name
, const char *snap_name
,
887 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
890 cls::rbd::GroupSnapshot group_snap
;
891 vector
<cls::rbd::ImageSnapshotSpec
> image_snaps
;
892 std::string ind_snap_name
;
894 std::vector
<librbd::ImageCtx
*> ictxs
;
895 std::vector
<C_SaferCond
*> on_finishes
;
896 std::vector
<uint64_t> quiesce_requests
;
897 NoOpProgressContext prog_ctx
;
898 uint64_t internal_flags
= 0;
900 int r
= util::snap_create_flags_api_to_internal(cct
, flags
, &internal_flags
);
904 internal_flags
&= ~(SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE
|
905 SNAP_CREATE_FLAG_IGNORE_NOTIFY_QUIESCE_ERROR
);
907 r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
, group_name
,
910 lderr(cct
) << "error reading group id object: "
916 std::vector
<cls::rbd::GroupImageStatus
> images
;
917 r
= group_image_list(group_ioctx
, group_name
, &images
);
921 int image_count
= images
.size();
923 ldout(cct
, 20) << "Found " << image_count
<< " images in group" << dendl
;
925 image_snaps
= vector
<cls::rbd::ImageSnapshotSpec
>(image_count
,
926 cls::rbd::ImageSnapshotSpec());
928 for (int i
= 0; i
< image_count
; ++i
) {
929 image_snaps
[i
].pool
= images
[i
].spec
.pool_id
;
930 image_snaps
[i
].image_id
= images
[i
].spec
.image_id
;
933 string group_header_oid
= util::group_header_name(group_id
);
935 group_snap
.id
= generate_uuid(group_ioctx
);
936 group_snap
.name
= string(snap_name
);
937 group_snap
.state
= cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE
;
938 group_snap
.snaps
= image_snaps
;
940 cls::rbd::GroupSnapshotNamespace ne
{group_ioctx
.get_id(), group_id
,
943 r
= cls_client::group_snap_set(&group_ioctx
, group_header_oid
, group_snap
);
945 lderr(cct
) << "snapshot with this name already exists: "
955 for (auto image
: images
) {
956 librbd::IoCtx image_io_ctx
;
957 r
= util::create_ioctx(group_ioctx
, "image", image
.spec
.pool_id
, {},
964 ldout(cct
, 20) << "Opening image with id " << image
.spec
.image_id
<< dendl
;
966 librbd::ImageCtx
* image_ctx
= new ImageCtx("", image
.spec
.image_id
.c_str(),
967 nullptr, image_io_ctx
, false);
969 C_SaferCond
* on_finish
= new C_SaferCond
;
971 image_ctx
->state
->open(0, on_finish
);
973 ictxs
.push_back(image_ctx
);
974 on_finishes
.push_back(on_finish
);
976 ldout(cct
, 20) << "Issued open request waiting for the completion" << dendl
;
978 for (int i
= 0; i
< image_count
; ++i
) {
980 ldout(cct
, 20) << "Waiting for completion on on_finish: " <<
981 on_finishes
[i
] << dendl
;
983 r
= on_finishes
[i
]->wait();
984 delete on_finishes
[i
];
994 if ((flags
& RBD_SNAP_CREATE_SKIP_QUIESCE
) == 0) {
995 ldout(cct
, 20) << "Sending quiesce notification" << dendl
;
996 ret_code
= notify_quiesce(ictxs
, prog_ctx
, &quiesce_requests
);
997 if (ret_code
!= 0 && (flags
& RBD_SNAP_CREATE_IGNORE_QUIESCE_ERROR
) == 0) {
1002 ldout(cct
, 20) << "Requesting exclusive locks for images" << dendl
;
1004 for (auto ictx
: ictxs
) {
1005 std::shared_lock owner_lock
{ictx
->owner_lock
};
1006 if (ictx
->exclusive_lock
!= nullptr) {
1007 ictx
->exclusive_lock
->block_requests(-EBUSY
);
1010 for (int i
= 0; i
< image_count
; ++i
) {
1011 ImageCtx
*ictx
= ictxs
[i
];
1012 std::shared_lock owner_lock
{ictx
->owner_lock
};
1014 on_finishes
[i
] = new C_SaferCond
;
1015 if (ictx
->exclusive_lock
!= nullptr) {
1016 ictx
->exclusive_lock
->acquire_lock(on_finishes
[i
]);
1021 for (int i
= 0; i
< image_count
; ++i
) {
1023 ImageCtx
*ictx
= ictxs
[i
];
1024 if (ictx
->exclusive_lock
!= nullptr) {
1025 r
= on_finishes
[i
]->wait();
1027 delete on_finishes
[i
];
1032 if (ret_code
!= 0) {
1033 notify_unquiesce(ictxs
, quiesce_requests
);
1037 ind_snap_name
= calc_ind_image_snap_name(group_ioctx
.get_id(), group_id
,
1040 for (int i
= 0; i
< image_count
; ++i
) {
1041 ImageCtx
*ictx
= ictxs
[i
];
1043 C_SaferCond
* on_finish
= new C_SaferCond
;
1045 std::shared_lock owner_locker
{ictx
->owner_lock
};
1046 ictx
->operations
->execute_snap_create(
1047 ne
, ind_snap_name
.c_str(), on_finish
, 0,
1048 SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE
, prog_ctx
);
1050 on_finishes
[i
] = on_finish
;
1054 for (int i
= 0; i
< image_count
; ++i
) {
1055 r
= on_finishes
[i
]->wait();
1056 delete on_finishes
[i
];
1060 ImageCtx
*ictx
= ictxs
[i
];
1061 ictx
->image_lock
.lock_shared();
1062 snap_t snap_id
= get_group_snap_id(ictx
, ne
);
1063 ictx
->image_lock
.unlock_shared();
1064 if (snap_id
== CEPH_NOSNAP
) {
1065 ldout(cct
, 20) << "Couldn't find created snapshot with namespace: "
1069 image_snaps
[i
].snap_id
= snapid_t(snap_id
);
1070 image_snaps
[i
].pool
= ictx
->md_ctx
.get_id();
1071 image_snaps
[i
].image_id
= ictx
->id
;
1075 if (ret_code
!= 0) {
1076 goto remove_image_snaps
;
1079 group_snap
.snaps
= image_snaps
;
1080 group_snap
.state
= cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE
;
1082 r
= cls_client::group_snap_set(&group_ioctx
, group_header_oid
, group_snap
);
1085 goto remove_image_snaps
;
1088 ldout(cct
, 20) << "Sending unquiesce notification" << dendl
;
1089 notify_unquiesce(ictxs
, quiesce_requests
);
1094 notify_unquiesce(ictxs
, quiesce_requests
);
1096 for (int i
= 0; i
< image_count
; ++i
) {
1097 ImageCtx
*ictx
= ictxs
[i
];
1098 ldout(cct
, 20) << "Removing individual snapshot with name: " <<
1099 ind_snap_name
<< dendl
;
1101 on_finishes
[i
] = new C_SaferCond
;
1102 std::string snap_name
;
1103 ictx
->image_lock
.lock_shared();
1104 snap_t snap_id
= get_group_snap_id(ictx
, ne
);
1105 r
= ictx
->get_snap_name(snap_id
, &snap_name
);
1106 ictx
->image_lock
.unlock_shared();
1108 ictx
->operations
->snap_remove(ne
, snap_name
.c_str(), on_finishes
[i
]);
1110 // Ignore missing image snapshots. The whole snapshot could have been
1112 on_finishes
[i
]->complete(0);
1116 for (int i
= 0, n
= on_finishes
.size(); i
< n
; ++i
) {
1117 r
= on_finishes
[i
]->wait();
1118 delete on_finishes
[i
];
1119 if (r
< 0 && r
!= -ENOENT
) { // if previous attempts to remove this snapshot failed then the image's snapshot may not exist
1120 lderr(cct
) << "Failed cleaning up image snapshot. Ret code: " << r
<< dendl
;
1121 // just report error, but don't abort the process
1126 r
= cls_client::group_snap_remove(&group_ioctx
, group_header_oid
,
1129 lderr(cct
) << "error while cleaning up group snapshot" << dendl
;
1130 // we ignore return value in clean up
1134 for (int i
= 0, n
= ictxs
.size(); i
< n
; ++i
) {
1135 if (ictxs
[i
] != nullptr) {
1136 ictxs
[i
]->state
->close();
1142 template <typename I
>
1143 int Group
<I
>::snap_remove(librados::IoCtx
& group_ioctx
, const char *group_name
,
1144 const char *snap_name
)
1146 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
1149 int r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
,
1150 group_name
, &group_id
);
1152 lderr(cct
) << "error reading group id object: "
1158 std::vector
<cls::rbd::GroupSnapshot
> snaps
;
1159 r
= group_snap_list(group_ioctx
, group_name
, &snaps
);
1164 cls::rbd::GroupSnapshot
*group_snap
= nullptr;
1165 for (auto &snap
: snaps
) {
1166 if (snap
.name
== string(snap_name
)) {
1171 if (group_snap
== nullptr) {
1175 string group_header_oid
= util::group_header_name(group_id
);
1176 r
= group_snap_remove_by_record(group_ioctx
, *group_snap
, group_id
,
1181 template <typename I
>
1182 int Group
<I
>::snap_rename(librados::IoCtx
& group_ioctx
, const char *group_name
,
1183 const char *old_snap_name
,
1184 const char *new_snap_name
) {
1185 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
1186 if (0 == strcmp(old_snap_name
, new_snap_name
))
1189 std::string group_id
;
1190 int r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
,
1191 group_name
, &group_id
);
1195 lderr(cct
) << "error reading group id object: " << cpp_strerror(r
) << dendl
;
1199 std::vector
<cls::rbd::GroupSnapshot
> group_snaps
;
1200 r
= group_snap_list(group_ioctx
, group_name
, &group_snaps
);
1205 cls::rbd::GroupSnapshot group_snap
;
1206 for (auto &snap
: group_snaps
) {
1207 if (snap
.name
== old_snap_name
) {
1213 if (group_snap
.id
.empty()) {
1217 std::string group_header_oid
= util::group_header_name(group_id
);
1218 group_snap
.name
= new_snap_name
;
1219 r
= cls_client::group_snap_set(&group_ioctx
, group_header_oid
, group_snap
);
1227 template <typename I
>
1228 int Group
<I
>::snap_list(librados::IoCtx
& group_ioctx
, const char *group_name
,
1229 std::vector
<group_snap_info_t
> *snaps
)
1231 std::vector
<cls::rbd::GroupSnapshot
> cls_snaps
;
1233 int r
= group_snap_list(group_ioctx
, group_name
, &cls_snaps
);
1238 for (auto snap
: cls_snaps
) {
1242 static_cast<group_snap_state_t
>(snap
.state
)});
1248 template <typename I
>
1249 int Group
<I
>::snap_rollback(librados::IoCtx
& group_ioctx
,
1250 const char *group_name
, const char *snap_name
,
1251 ProgressContext
& pctx
)
1253 CephContext
*cct
= (CephContext
*)group_ioctx
.cct();
1256 int r
= cls_client::dir_get_id(&group_ioctx
, RBD_GROUP_DIRECTORY
,
1257 group_name
, &group_id
);
1259 lderr(cct
) << "error reading group id object: "
1260 << cpp_strerror(r
) << dendl
;
1264 std::vector
<cls::rbd::GroupSnapshot
> snaps
;
1265 r
= group_snap_list(group_ioctx
, group_name
, &snaps
);
1270 cls::rbd::GroupSnapshot
*group_snap
= nullptr;
1271 for (auto &snap
: snaps
) {
1272 if (snap
.name
== string(snap_name
)) {
1277 if (group_snap
== nullptr) {
1281 string group_header_oid
= util::group_header_name(group_id
);
1282 r
= group_snap_rollback_by_record(group_ioctx
, *group_snap
, group_id
,
1283 group_header_oid
, pctx
);
1288 } // namespace librbd
1290 template class librbd::api::Group
<librbd::ImageCtx
>;