1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "librbd/api/Mirror.h"
5 #include "include/rados/librados.hpp"
6 #include "include/stringify.h"
7 #include "common/ceph_json.h"
8 #include "common/dout.h"
9 #include "common/errno.h"
10 #include "cls/rbd/cls_rbd_client.h"
11 #include "librbd/ImageCtx.h"
12 #include "librbd/ImageState.h"
13 #include "librbd/Journal.h"
14 #include "librbd/MirroringWatcher.h"
15 #include "librbd/Operations.h"
16 #include "librbd/Utils.h"
17 #include "librbd/api/Image.h"
18 #include "librbd/api/Namespace.h"
19 #include "librbd/mirror/DemoteRequest.h"
20 #include "librbd/mirror/DisableRequest.h"
21 #include "librbd/mirror/EnableRequest.h"
22 #include "librbd/mirror/GetInfoRequest.h"
23 #include "librbd/mirror/GetStatusRequest.h"
24 #include "librbd/mirror/GetUuidRequest.h"
25 #include "librbd/mirror/PromoteRequest.h"
26 #include "librbd/mirror/Types.h"
27 #include "librbd/MirroringWatcher.h"
28 #include "librbd/mirror/snapshot/CreatePrimaryRequest.h"
29 #include "librbd/mirror/snapshot/ImageMeta.h"
30 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
31 #include "librbd/mirror/snapshot/Utils.h"
32 #include <boost/algorithm/string/trim.hpp>
33 #include <boost/algorithm/string/replace.hpp>
34 #include <boost/scope_exit.hpp>
35 #include "json_spirit/json_spirit.h"
39 #define dout_subsys ceph_subsys_rbd
41 #define dout_prefix *_dout << "librbd::api::Mirror: " << __func__ << ": "
48 int get_config_key(librados::Rados
& rados
, const std::string
& key
,
52 "\"prefix\": \"config-key get\", "
53 "\"key\": \"" + key
+ "\""
59 int r
= rados
.mon_command(cmd
, in_bl
, &out_bl
, nullptr);
62 } else if (r
< 0 && r
!= -ENOENT
) {
66 *value
= out_bl
.to_str();
70 int set_config_key(librados::Rados
& rados
, const std::string
& key
,
71 const std::string
& value
) {
75 "\"prefix\": \"config-key rm\", "
76 "\"key\": \"" + key
+ "\""
80 "\"prefix\": \"config-key set\", "
81 "\"key\": \"" + key
+ "\", "
82 "\"val\": \"" + value
+ "\""
88 int r
= rados
.mon_command(cmd
, in_bl
, &out_bl
, nullptr);
98 std::string
get_peer_config_key_name(int64_t pool_id
,
99 const std::string
& peer_uuid
) {
100 return RBD_MIRROR_PEER_CONFIG_KEY_PREFIX
+ stringify(pool_id
) + "/" +
104 int remove_peer_config_key(librados::IoCtx
& io_ctx
,
105 const std::string
& peer_uuid
) {
106 int64_t pool_id
= io_ctx
.get_id();
107 auto key
= get_peer_config_key_name(pool_id
, peer_uuid
);
109 librados::Rados
rados(io_ctx
);
110 int r
= set_config_key(rados
, key
, "");
111 if (r
< 0 && r
!= -ENOENT
&& r
!= -EPERM
) {
117 int create_bootstrap_user(CephContext
* cct
, librados::Rados
& rados
,
118 std::string
* peer_client_id
, std::string
* cephx_key
) {
119 ldout(cct
, 20) << dendl
;
121 // retrieve peer CephX user from config-key
122 int r
= get_config_key(rados
, RBD_MIRROR_PEER_CLIENT_ID_CONFIG_KEY
,
125 ldout(cct
, 5) << "insufficient permissions to get peer-client-id "
126 << "config-key" << dendl
;
128 } else if (r
< 0 && r
!= -ENOENT
) {
129 lderr(cct
) << "failed to retrieve peer client id key: "
130 << cpp_strerror(r
) << dendl
;
132 } else if (r
== -ENOENT
|| peer_client_id
->empty()) {
133 ldout(cct
, 20) << "creating new peer-client-id config-key" << dendl
;
135 *peer_client_id
= "rbd-mirror-peer";
136 r
= set_config_key(rados
, RBD_MIRROR_PEER_CLIENT_ID_CONFIG_KEY
,
139 ldout(cct
, 5) << "insufficient permissions to update peer-client-id "
140 << "config-key" << dendl
;
143 lderr(cct
) << "failed to update peer client id key: "
144 << cpp_strerror(r
) << dendl
;
148 ldout(cct
, 20) << "peer_client_id=" << *peer_client_id
<< dendl
;
150 // create peer client user
153 R
"( "prefix
": "auth get
-or-create
",)" \
154 R
"( "entity
": "client
.)" + *peer_client_id + R"(",)" \
156 R
"( "mon
", "profile rbd
-mirror
-peer
",)" \
157 R
"( "osd
", "profile rbd
"],)" \
158 R
"( "format
": "json
")" \
164 r
= rados
.mon_command(cmd
, in_bl
, &out_bl
, nullptr);
166 ldout(cct
, 5) << "caps mismatch for existing user" << dendl
;
168 } else if (r
== -EACCES
) {
169 ldout(cct
, 5) << "insufficient permissions to create user" << dendl
;
172 lderr(cct
) << "failed to create or update RBD mirroring bootstrap user: "
173 << cpp_strerror(r
) << dendl
;
177 // extract key from response
178 bool json_valid
= false;
179 json_spirit::mValue json_root
;
180 if(json_spirit::read(out_bl
.to_str(), json_root
)) {
182 auto& json_obj
= json_root
.get_array()[0].get_obj();
183 *cephx_key
= json_obj
["key"].get_str();
185 } catch (std::runtime_error
&) {
190 lderr(cct
) << "invalid auth keyring JSON received" << dendl
;
197 int create_bootstrap_peer(CephContext
* cct
, librados::IoCtx
& io_ctx
,
198 mirror_peer_direction_t direction
,
199 const std::string
& site_name
, const std::string
& fsid
,
200 const std::string
& client_id
, const std::string
& key
,
201 const std::string
& mon_host
,
202 const std::string
& cluster1
,
203 const std::string
& cluster2
) {
204 ldout(cct
, 20) << dendl
;
206 std::string peer_uuid
;
207 std::vector
<mirror_peer_site_t
> peers
;
208 int r
= Mirror
<>::peer_site_list(io_ctx
, &peers
);
209 if (r
< 0 && r
!= -ENOENT
) {
210 lderr(cct
) << "failed to list mirror peers: " << cpp_strerror(r
) << dendl
;
215 r
= Mirror
<>::peer_site_add(io_ctx
, &peer_uuid
, direction
, site_name
,
216 "client." + client_id
);
218 lderr(cct
) << "failed to add " << cluster1
<< " peer to "
219 << cluster2
<< " " << "cluster: " << cpp_strerror(r
) << dendl
;
222 } else if (peers
[0].site_name
!= site_name
&&
223 peers
[0].site_name
!= fsid
) {
224 // only support a single peer
225 lderr(cct
) << "multiple peers are not currently supported" << dendl
;
228 peer_uuid
= peers
[0].uuid
;
230 if (peers
[0].site_name
!= site_name
) {
231 r
= Mirror
<>::peer_site_set_name(io_ctx
, peer_uuid
, site_name
);
233 // non-fatal attempt to update site name
234 lderr(cct
) << "failed to update peer site name" << dendl
;
239 Mirror
<>::Attributes attributes
{
240 {"mon_host", mon_host
},
242 r
= Mirror
<>::peer_site_set_attributes(io_ctx
, peer_uuid
, attributes
);
244 lderr(cct
) << "failed to update " << cluster1
<< " cluster connection "
245 << "attributes in " << cluster2
<< " cluster: "
246 << cpp_strerror(r
) << dendl
;
253 int list_mirror_images(librados::IoCtx
& io_ctx
,
254 std::set
<std::string
>& mirror_image_ids
) {
255 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
257 std::string last_read
= "";
261 std::map
<std::string
, std::string
> mirror_images
;
262 r
= cls_client::mirror_image_list(&io_ctx
, last_read
, max_read
,
264 if (r
< 0 && r
!= -ENOENT
) {
265 lderr(cct
) << "error listing mirrored image directory: "
266 << cpp_strerror(r
) << dendl
;
269 for (auto it
= mirror_images
.begin(); it
!= mirror_images
.end(); ++it
) {
270 mirror_image_ids
.insert(it
->first
);
272 if (!mirror_images
.empty()) {
273 last_read
= mirror_images
.rbegin()->first
;
275 r
= mirror_images
.size();
276 } while (r
== max_read
);
281 struct C_ImageGetInfo
: public Context
{
282 mirror_image_info_t
*mirror_image_info
;
283 mirror_image_mode_t
*mirror_image_mode
;
286 cls::rbd::MirrorImage mirror_image
;
287 mirror::PromotionState promotion_state
= mirror::PROMOTION_STATE_PRIMARY
;
288 std::string primary_mirror_uuid
;
290 C_ImageGetInfo(mirror_image_info_t
*mirror_image_info
,
291 mirror_image_mode_t
*mirror_image_mode
, Context
*on_finish
)
292 : mirror_image_info(mirror_image_info
),
293 mirror_image_mode(mirror_image_mode
), on_finish(on_finish
) {
296 void finish(int r
) override
{
297 if (r
< 0 && r
!= -ENOENT
) {
298 on_finish
->complete(r
);
302 if (mirror_image_info
!= nullptr) {
303 mirror_image_info
->global_id
= mirror_image
.global_image_id
;
304 mirror_image_info
->state
= static_cast<rbd_mirror_image_state_t
>(
306 mirror_image_info
->primary
= (
307 promotion_state
== mirror::PROMOTION_STATE_PRIMARY
);
310 if (mirror_image_mode
!= nullptr) {
312 static_cast<rbd_mirror_image_mode_t
>(mirror_image
.mode
);
315 on_finish
->complete(0);
319 struct C_ImageGetGlobalStatus
: public C_ImageGetInfo
{
320 std::string image_name
;
321 mirror_image_global_status_t
*mirror_image_global_status
;
323 cls::rbd::MirrorImageStatus mirror_image_status_internal
;
325 C_ImageGetGlobalStatus(
326 const std::string
&image_name
,
327 mirror_image_global_status_t
*mirror_image_global_status
,
329 : C_ImageGetInfo(&mirror_image_global_status
->info
, nullptr, on_finish
),
330 image_name(image_name
),
331 mirror_image_global_status(mirror_image_global_status
) {
334 void finish(int r
) override
{
335 if (r
< 0 && r
!= -ENOENT
) {
336 on_finish
->complete(r
);
340 mirror_image_global_status
->name
= image_name
;
341 mirror_image_global_status
->site_statuses
.clear();
342 mirror_image_global_status
->site_statuses
.reserve(
343 mirror_image_status_internal
.mirror_image_site_statuses
.size());
344 for (auto& site_status
:
345 mirror_image_status_internal
.mirror_image_site_statuses
) {
346 mirror_image_global_status
->site_statuses
.push_back({
347 site_status
.mirror_uuid
,
348 static_cast<mirror_image_status_state_t
>(site_status
.state
),
349 site_status
.description
, site_status
.last_update
.sec(),
352 C_ImageGetInfo::finish(0);
356 } // anonymous namespace
358 template <typename I
>
359 int Mirror
<I
>::image_enable(I
*ictx
, mirror_image_mode_t mode
,
360 bool relax_same_pool_parent_check
) {
361 CephContext
*cct
= ictx
->cct
;
362 ldout(cct
, 20) << "ictx=" << ictx
<< " mode=" << mode
363 << " relax_same_pool_parent_check="
364 << relax_same_pool_parent_check
<< dendl
;
366 int r
= ictx
->state
->refresh_if_required();
371 cls::rbd::MirrorMode mirror_mode
;
372 r
= cls_client::mirror_mode_get(&ictx
->md_ctx
, &mirror_mode
);
374 lderr(cct
) << "cannot enable mirroring: failed to retrieve mirror mode: "
375 << cpp_strerror(r
) << dendl
;
379 if (mirror_mode
!= cls::rbd::MIRROR_MODE_IMAGE
) {
380 lderr(cct
) << "cannot enable mirroring in the current pool mirroring mode"
385 // is mirroring not enabled for the parent?
387 std::shared_lock image_locker
{ictx
->image_lock
};
388 ImageCtx
*parent
= ictx
->parent
;
390 if (parent
->md_ctx
.get_id() != ictx
->md_ctx
.get_id() ||
391 !relax_same_pool_parent_check
) {
392 cls::rbd::MirrorImage mirror_image_internal
;
393 r
= cls_client::mirror_image_get(&(parent
->md_ctx
), parent
->id
,
394 &mirror_image_internal
);
396 lderr(cct
) << "mirroring is not enabled for the parent" << dendl
;
403 if (mode
== RBD_MIRROR_IMAGE_MODE_JOURNAL
&&
404 !ictx
->test_features(RBD_FEATURE_JOURNALING
)) {
405 uint64_t features
= RBD_FEATURE_JOURNALING
;
406 if (!ictx
->test_features(RBD_FEATURE_EXCLUSIVE_LOCK
)) {
407 features
|= RBD_FEATURE_EXCLUSIVE_LOCK
;
409 r
= ictx
->operations
->update_features(features
, true);
411 lderr(cct
) << "cannot enable journaling: " << cpp_strerror(r
) << dendl
;
417 auto req
= mirror::EnableRequest
<ImageCtx
>::create(
418 ictx
, static_cast<cls::rbd::MirrorImageMode
>(mode
), "", false, &ctx
);
423 lderr(cct
) << "cannot enable mirroring: " << cpp_strerror(r
) << dendl
;
430 template <typename I
>
431 int Mirror
<I
>::image_disable(I
*ictx
, bool force
) {
432 CephContext
*cct
= ictx
->cct
;
433 ldout(cct
, 20) << "ictx=" << ictx
<< dendl
;
435 int r
= ictx
->state
->refresh_if_required();
440 cls::rbd::MirrorMode mirror_mode
;
441 r
= cls_client::mirror_mode_get(&ictx
->md_ctx
, &mirror_mode
);
443 lderr(cct
) << "cannot disable mirroring: failed to retrieve pool "
444 "mirroring mode: " << cpp_strerror(r
) << dendl
;
448 if (mirror_mode
!= cls::rbd::MIRROR_MODE_IMAGE
) {
449 lderr(cct
) << "cannot disable mirroring in the current pool mirroring "
454 // is mirroring enabled for the image?
455 cls::rbd::MirrorImage mirror_image_internal
;
456 r
= cls_client::mirror_image_get(&ictx
->md_ctx
, ictx
->id
,
457 &mirror_image_internal
);
459 // mirroring is not enabled for this image
460 ldout(cct
, 20) << "ignoring disable command: mirroring is not enabled for "
461 << "this image" << dendl
;
463 } else if (r
== -EOPNOTSUPP
) {
464 ldout(cct
, 5) << "mirroring not supported by OSD" << dendl
;
467 lderr(cct
) << "failed to retrieve mirror image metadata: "
468 << cpp_strerror(r
) << dendl
;
472 mirror_image_internal
.state
= cls::rbd::MIRROR_IMAGE_STATE_DISABLING
;
473 r
= cls_client::mirror_image_set(&ictx
->md_ctx
, ictx
->id
,
474 mirror_image_internal
);
476 lderr(cct
) << "cannot disable mirroring: " << cpp_strerror(r
) << dendl
;
480 bool rollback
= false;
481 BOOST_SCOPE_EXIT_ALL(ictx
, &mirror_image_internal
, &rollback
) {
483 // restore the mask bit for treating the non-primary feature as read-only
484 ictx
->image_lock
.lock();
485 ictx
->read_only_mask
|= IMAGE_READ_ONLY_FLAG_NON_PRIMARY
;
486 ictx
->image_lock
.unlock();
488 ictx
->state
->handle_update_notification();
490 // attempt to restore the image state
491 CephContext
*cct
= ictx
->cct
;
492 mirror_image_internal
.state
= cls::rbd::MIRROR_IMAGE_STATE_ENABLED
;
493 int r
= cls_client::mirror_image_set(&ictx
->md_ctx
, ictx
->id
,
494 mirror_image_internal
);
496 lderr(cct
) << "failed to re-enable image mirroring: "
497 << cpp_strerror(r
) << dendl
;
502 std::unique_lock image_locker
{ictx
->image_lock
};
503 map
<librados::snap_t
, SnapInfo
> snap_info
= ictx
->snap_info
;
504 for (auto &info
: snap_info
) {
505 cls::rbd::ParentImageSpec parent_spec
{ictx
->md_ctx
.get_id(),
506 ictx
->md_ctx
.get_namespace(),
507 ictx
->id
, info
.first
};
508 std::vector
<librbd::linked_image_spec_t
> child_images
;
509 r
= Image
<I
>::list_children(ictx
, parent_spec
, &child_images
);
515 if (child_images
.empty()) {
519 librados::IoCtx child_io_ctx
;
520 int64_t child_pool_id
= -1;
521 for (auto &child_image
: child_images
){
522 std::string pool
= child_image
.pool_name
;
523 if (child_pool_id
== -1 ||
524 child_pool_id
!= child_image
.pool_id
||
525 child_io_ctx
.get_namespace() != child_image
.pool_namespace
) {
526 r
= util::create_ioctx(ictx
->md_ctx
, "child image",
528 child_image
.pool_namespace
,
535 child_pool_id
= child_image
.pool_id
;
538 cls::rbd::MirrorImage child_mirror_image_internal
;
539 r
= cls_client::mirror_image_get(&child_io_ctx
, child_image
.image_id
,
540 &child_mirror_image_internal
);
543 lderr(cct
) << "mirroring is enabled on one or more children "
549 image_locker
.unlock();
551 if (mirror_image_internal
.mode
== cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT
) {
552 // don't let the non-primary feature bit prevent image updates
553 ictx
->image_lock
.lock();
554 ictx
->read_only_mask
&= ~IMAGE_READ_ONLY_FLAG_NON_PRIMARY
;
555 ictx
->image_lock
.unlock();
557 r
= ictx
->state
->refresh();
563 // remove any snapshot-based mirroring image-meta from image
564 std::string mirror_uuid
;
565 r
= uuid_get(ictx
->md_ctx
, &mirror_uuid
);
571 r
= ictx
->operations
->metadata_remove(
572 mirror::snapshot::util::get_image_meta_key(mirror_uuid
));
573 if (r
< 0 && r
!= -ENOENT
) {
574 lderr(cct
) << "cannot remove snapshot image-meta key: " << cpp_strerror(r
)
582 auto req
= mirror::DisableRequest
<ImageCtx
>::create(ictx
, force
, true,
588 lderr(cct
) << "cannot disable mirroring: " << cpp_strerror(r
) << dendl
;
593 if (mirror_image_internal
.mode
== cls::rbd::MIRROR_IMAGE_MODE_JOURNAL
) {
594 r
= ictx
->operations
->update_features(RBD_FEATURE_JOURNALING
, false);
596 lderr(cct
) << "cannot disable journaling: " << cpp_strerror(r
) << dendl
;
604 template <typename I
>
605 int Mirror
<I
>::image_promote(I
*ictx
, bool force
) {
606 CephContext
*cct
= ictx
->cct
;
609 Mirror
<I
>::image_promote(ictx
, force
, &ctx
);
612 lderr(cct
) << "failed to promote image" << dendl
;
619 template <typename I
>
620 void Mirror
<I
>::image_promote(I
*ictx
, bool force
, Context
*on_finish
) {
621 CephContext
*cct
= ictx
->cct
;
622 ldout(cct
, 20) << "ictx=" << ictx
<< ", "
623 << "force=" << force
<< dendl
;
625 // don't let the non-primary feature bit prevent image updates
626 ictx
->image_lock
.lock();
627 ictx
->read_only_mask
&= ~IMAGE_READ_ONLY_FLAG_NON_PRIMARY
;
628 ictx
->image_lock
.unlock();
630 auto on_promote
= new LambdaContext([ictx
, on_finish
](int r
) {
631 ictx
->image_lock
.lock();
632 ictx
->read_only_mask
|= IMAGE_READ_ONLY_FLAG_NON_PRIMARY
;
633 ictx
->image_lock
.unlock();
635 ictx
->state
->handle_update_notification();
636 on_finish
->complete(r
);
639 auto on_refresh
= new LambdaContext([ictx
, force
, on_promote
](int r
) {
641 lderr(ictx
->cct
) << "refresh failed: " << cpp_strerror(r
) << dendl
;
642 on_promote
->complete(r
);
646 auto req
= mirror::PromoteRequest
<>::create(*ictx
, force
, on_promote
);
649 ictx
->state
->refresh(on_refresh
);
652 template <typename I
>
653 int Mirror
<I
>::image_demote(I
*ictx
) {
654 CephContext
*cct
= ictx
->cct
;
657 Mirror
<I
>::image_demote(ictx
, &ctx
);
660 lderr(cct
) << "failed to demote image" << dendl
;
667 template <typename I
>
668 void Mirror
<I
>::image_demote(I
*ictx
, Context
*on_finish
) {
669 CephContext
*cct
= ictx
->cct
;
670 ldout(cct
, 20) << "ictx=" << ictx
<< dendl
;
672 auto on_cleanup
= new LambdaContext([ictx
, on_finish
](int r
) {
673 ictx
->image_lock
.lock();
674 ictx
->read_only_mask
|= IMAGE_READ_ONLY_FLAG_NON_PRIMARY
;
675 ictx
->image_lock
.unlock();
677 ictx
->state
->handle_update_notification();
679 on_finish
->complete(r
);
681 auto on_refresh
= new LambdaContext([ictx
, on_cleanup
](int r
) {
683 lderr(ictx
->cct
) << "refresh failed: " << cpp_strerror(r
) << dendl
;
684 on_cleanup
->complete(r
);
688 auto req
= mirror::DemoteRequest
<>::create(*ictx
, on_cleanup
);
692 // ensure we can create a snapshot after setting the non-primary
694 ictx
->image_lock
.lock();
695 ictx
->read_only_mask
&= ~IMAGE_READ_ONLY_FLAG_NON_PRIMARY
;
696 ictx
->image_lock
.unlock();
698 ictx
->state
->refresh(on_refresh
);
701 template <typename I
>
702 int Mirror
<I
>::image_resync(I
*ictx
) {
703 CephContext
*cct
= ictx
->cct
;
704 ldout(cct
, 20) << "ictx=" << ictx
<< dendl
;
706 int r
= ictx
->state
->refresh_if_required();
711 cls::rbd::MirrorImage mirror_image
;
712 mirror::PromotionState promotion_state
;
713 std::string primary_mirror_uuid
;
714 C_SaferCond get_info_ctx
;
715 auto req
= mirror::GetInfoRequest
<I
>::create(*ictx
, &mirror_image
,
717 &primary_mirror_uuid
,
721 r
= get_info_ctx
.wait();
726 if (promotion_state
== mirror::PROMOTION_STATE_PRIMARY
) {
727 lderr(cct
) << "image is primary, cannot resync to itself" << dendl
;
731 if (mirror_image
.mode
== cls::rbd::MIRROR_IMAGE_MODE_JOURNAL
) {
732 // flag the journal indicating that we want to rebuild the local image
733 r
= Journal
<I
>::request_resync(ictx
);
735 lderr(cct
) << "failed to request resync: " << cpp_strerror(r
) << dendl
;
738 } else if (mirror_image
.mode
== cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT
) {
739 std::string mirror_uuid
;
740 r
= uuid_get(ictx
->md_ctx
, &mirror_uuid
);
745 mirror::snapshot::ImageMeta
image_meta(ictx
, mirror_uuid
);
747 C_SaferCond load_meta_ctx
;
748 image_meta
.load(&load_meta_ctx
);
749 r
= load_meta_ctx
.wait();
750 if (r
< 0 && r
!= -ENOENT
) {
751 lderr(cct
) << "failed to load mirror image-meta: " << cpp_strerror(r
)
756 image_meta
.resync_requested
= true;
758 C_SaferCond save_meta_ctx
;
759 image_meta
.save(&save_meta_ctx
);
760 r
= save_meta_ctx
.wait();
762 lderr(cct
) << "failed to request resync: " << cpp_strerror(r
) << dendl
;
766 lderr(cct
) << "unknown mirror mode" << dendl
;
773 template <typename I
>
774 void Mirror
<I
>::image_get_info(I
*ictx
, mirror_image_info_t
*mirror_image_info
,
775 Context
*on_finish
) {
776 CephContext
*cct
= ictx
->cct
;
777 ldout(cct
, 20) << "ictx=" << ictx
<< dendl
;
779 auto on_refresh
= new LambdaContext(
780 [ictx
, mirror_image_info
, on_finish
](int r
) {
782 lderr(ictx
->cct
) << "refresh failed: " << cpp_strerror(r
) << dendl
;
783 on_finish
->complete(r
);
787 auto ctx
= new C_ImageGetInfo(mirror_image_info
, nullptr, on_finish
);
788 auto req
= mirror::GetInfoRequest
<I
>::create(*ictx
, &ctx
->mirror_image
,
789 &ctx
->promotion_state
,
790 &ctx
->primary_mirror_uuid
,
795 if (ictx
->state
->is_refresh_required()) {
796 ictx
->state
->refresh(on_refresh
);
798 on_refresh
->complete(0);
802 template <typename I
>
803 int Mirror
<I
>::image_get_info(I
*ictx
, mirror_image_info_t
*mirror_image_info
) {
805 image_get_info(ictx
, mirror_image_info
, &ctx
);
814 template <typename I
>
815 void Mirror
<I
>::image_get_info(librados::IoCtx
& io_ctx
,
816 ContextWQ
*op_work_queue
,
817 const std::string
&image_id
,
818 mirror_image_info_t
*mirror_image_info
,
819 Context
*on_finish
) {
820 auto cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
821 ldout(cct
, 20) << "pool_id=" << io_ctx
.get_id() << ", image_id=" << image_id
824 auto ctx
= new C_ImageGetInfo(mirror_image_info
, nullptr, on_finish
);
825 auto req
= mirror::GetInfoRequest
<I
>::create(io_ctx
, op_work_queue
, image_id
,
827 &ctx
->promotion_state
,
828 &ctx
->primary_mirror_uuid
, ctx
);
832 template <typename I
>
833 int Mirror
<I
>::image_get_info(librados::IoCtx
& io_ctx
,
834 ContextWQ
*op_work_queue
,
835 const std::string
&image_id
,
836 mirror_image_info_t
*mirror_image_info
) {
838 image_get_info(io_ctx
, op_work_queue
, image_id
, mirror_image_info
, &ctx
);
847 template <typename I
>
848 void Mirror
<I
>::image_get_mode(I
*ictx
, mirror_image_mode_t
*mode
,
849 Context
*on_finish
) {
850 CephContext
*cct
= ictx
->cct
;
851 ldout(cct
, 20) << "ictx=" << ictx
<< dendl
;
853 auto ctx
= new C_ImageGetInfo(nullptr, mode
, on_finish
);
854 auto req
= mirror::GetInfoRequest
<I
>::create(*ictx
, &ctx
->mirror_image
,
855 &ctx
->promotion_state
,
856 &ctx
->primary_mirror_uuid
, ctx
);
860 template <typename I
>
861 int Mirror
<I
>::image_get_mode(I
*ictx
, mirror_image_mode_t
*mode
) {
863 image_get_mode(ictx
, mode
, &ctx
);
872 template <typename I
>
873 void Mirror
<I
>::image_get_global_status(I
*ictx
,
874 mirror_image_global_status_t
*status
,
875 Context
*on_finish
) {
876 CephContext
*cct
= ictx
->cct
;
877 ldout(cct
, 20) << "ictx=" << ictx
<< dendl
;
879 auto ctx
= new C_ImageGetGlobalStatus(ictx
->name
, status
, on_finish
);
880 auto req
= mirror::GetStatusRequest
<I
>::create(
881 *ictx
, &ctx
->mirror_image_status_internal
, &ctx
->mirror_image
,
882 &ctx
->promotion_state
, ctx
);
886 template <typename I
>
887 int Mirror
<I
>::image_get_global_status(I
*ictx
,
888 mirror_image_global_status_t
*status
) {
890 image_get_global_status(ictx
, status
, &ctx
);
899 template <typename I
>
900 int Mirror
<I
>::image_get_instance_id(I
*ictx
, std::string
*instance_id
) {
901 CephContext
*cct
= ictx
->cct
;
902 ldout(cct
, 20) << "ictx=" << ictx
<< dendl
;
904 cls::rbd::MirrorImage mirror_image
;
905 int r
= cls_client::mirror_image_get(&ictx
->md_ctx
, ictx
->id
, &mirror_image
);
906 if (r
< 0 && r
!= -ENOENT
) {
907 lderr(cct
) << "failed to retrieve mirroring state: " << cpp_strerror(r
)
910 } else if (mirror_image
.state
!= cls::rbd::MIRROR_IMAGE_STATE_ENABLED
) {
911 lderr(cct
) << "mirroring is not currently enabled" << dendl
;
915 entity_inst_t instance
;
916 r
= cls_client::mirror_image_instance_get(&ictx
->md_ctx
,
917 mirror_image
.global_image_id
,
920 if (r
!= -ENOENT
&& r
!= -ESTALE
) {
921 lderr(cct
) << "failed to get mirror image instance: " << cpp_strerror(r
)
927 *instance_id
= stringify(instance
.name
.num());
931 template <typename I
>
932 int Mirror
<I
>::site_name_get(librados::Rados
& rados
, std::string
* name
) {
933 CephContext
*cct
= reinterpret_cast<CephContext
*>(rados
.cct());
934 ldout(cct
, 20) << dendl
;
936 int r
= get_config_key(rados
, RBD_MIRROR_SITE_NAME_CONFIG_KEY
, name
);
937 if (r
== -EOPNOTSUPP
) {
939 } else if (r
== -ENOENT
|| name
->empty()) {
940 // default to the cluster fsid
941 r
= rados
.cluster_fsid(name
);
943 lderr(cct
) << "failed to retrieve cluster fsid: " << cpp_strerror(r
)
948 lderr(cct
) << "failed to retrieve site name: " << cpp_strerror(r
)
956 template <typename I
>
957 int Mirror
<I
>::site_name_set(librados::Rados
& rados
, const std::string
& name
) {
958 CephContext
*cct
= reinterpret_cast<CephContext
*>(rados
.cct());
960 std::string site_name
{name
};
961 boost::algorithm::trim(site_name
);
962 ldout(cct
, 20) << "site_name=" << site_name
<< dendl
;
964 int r
= set_config_key(rados
, RBD_MIRROR_SITE_NAME_CONFIG_KEY
, name
);
965 if (r
== -EOPNOTSUPP
) {
967 } else if (r
< 0 && r
!= -ENOENT
) {
968 lderr(cct
) << "failed to update site name: " << cpp_strerror(r
)
976 template <typename I
>
977 int Mirror
<I
>::mode_get(librados::IoCtx
& io_ctx
,
978 rbd_mirror_mode_t
*mirror_mode
) {
979 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
980 ldout(cct
, 20) << dendl
;
982 cls::rbd::MirrorMode mirror_mode_internal
;
983 int r
= cls_client::mirror_mode_get(&io_ctx
, &mirror_mode_internal
);
985 lderr(cct
) << "failed to retrieve mirror mode: " << cpp_strerror(r
)
990 switch (mirror_mode_internal
) {
991 case cls::rbd::MIRROR_MODE_DISABLED
:
992 case cls::rbd::MIRROR_MODE_IMAGE
:
993 case cls::rbd::MIRROR_MODE_POOL
:
994 *mirror_mode
= static_cast<rbd_mirror_mode_t
>(mirror_mode_internal
);
997 lderr(cct
) << "unknown mirror mode ("
998 << static_cast<uint32_t>(mirror_mode_internal
) << ")"
1005 template <typename I
>
1006 int Mirror
<I
>::mode_set(librados::IoCtx
& io_ctx
,
1007 rbd_mirror_mode_t mirror_mode
) {
1008 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1009 ldout(cct
, 20) << dendl
;
1011 cls::rbd::MirrorMode next_mirror_mode
;
1012 switch (mirror_mode
) {
1013 case RBD_MIRROR_MODE_DISABLED
:
1014 case RBD_MIRROR_MODE_IMAGE
:
1015 case RBD_MIRROR_MODE_POOL
:
1016 next_mirror_mode
= static_cast<cls::rbd::MirrorMode
>(mirror_mode
);
1019 lderr(cct
) << "unknown mirror mode ("
1020 << static_cast<uint32_t>(mirror_mode
) << ")" << dendl
;
1025 if (next_mirror_mode
== cls::rbd::MIRROR_MODE_DISABLED
) {
1026 // fail early if pool still has peers registered and attempting to disable
1027 std::vector
<cls::rbd::MirrorPeer
> mirror_peers
;
1028 r
= cls_client::mirror_peer_list(&io_ctx
, &mirror_peers
);
1029 if (r
< 0 && r
!= -ENOENT
) {
1030 lderr(cct
) << "failed to list peers: " << cpp_strerror(r
) << dendl
;
1032 } else if (!mirror_peers
.empty()) {
1033 lderr(cct
) << "mirror peers still registered" << dendl
;
1038 cls::rbd::MirrorMode current_mirror_mode
;
1039 r
= cls_client::mirror_mode_get(&io_ctx
, ¤t_mirror_mode
);
1041 lderr(cct
) << "failed to retrieve mirror mode: " << cpp_strerror(r
)
1046 if (current_mirror_mode
== next_mirror_mode
) {
1048 } else if (current_mirror_mode
== cls::rbd::MIRROR_MODE_DISABLED
) {
1050 uuid_gen
.generate_random();
1051 r
= cls_client::mirror_uuid_set(&io_ctx
, uuid_gen
.to_string());
1053 lderr(cct
) << "failed to allocate mirroring uuid: " << cpp_strerror(r
)
1059 if (current_mirror_mode
!= cls::rbd::MIRROR_MODE_IMAGE
) {
1060 r
= cls_client::mirror_mode_set(&io_ctx
, cls::rbd::MIRROR_MODE_IMAGE
);
1062 lderr(cct
) << "failed to set mirror mode to image: "
1063 << cpp_strerror(r
) << dendl
;
1067 r
= MirroringWatcher
<>::notify_mode_updated(io_ctx
,
1068 cls::rbd::MIRROR_MODE_IMAGE
);
1070 lderr(cct
) << "failed to send update notification: " << cpp_strerror(r
)
1075 if (next_mirror_mode
== cls::rbd::MIRROR_MODE_IMAGE
) {
1079 if (next_mirror_mode
== cls::rbd::MIRROR_MODE_POOL
) {
1080 map
<string
, string
> images
;
1081 r
= Image
<I
>::list_images_v2(io_ctx
, &images
);
1083 lderr(cct
) << "failed listing images: " << cpp_strerror(r
) << dendl
;
1087 for (const auto& img_pair
: images
) {
1089 uint64_t incompatible_features
;
1090 r
= cls_client::get_features(&io_ctx
, util::header_name(img_pair
.second
),
1091 true, &features
, &incompatible_features
);
1093 lderr(cct
) << "error getting features for image " << img_pair
.first
1094 << ": " << cpp_strerror(r
) << dendl
;
1098 // Enable only journal based mirroring
1100 if ((features
& RBD_FEATURE_JOURNALING
) != 0) {
1101 I
*img_ctx
= I::create("", img_pair
.second
, nullptr, io_ctx
, false);
1102 r
= img_ctx
->state
->open(0);
1104 lderr(cct
) << "error opening image "<< img_pair
.first
<< ": "
1105 << cpp_strerror(r
) << dendl
;
1109 r
= image_enable(img_ctx
, RBD_MIRROR_IMAGE_MODE_JOURNAL
, true);
1110 int close_r
= img_ctx
->state
->close();
1112 lderr(cct
) << "error enabling mirroring for image "
1113 << img_pair
.first
<< ": " << cpp_strerror(r
) << dendl
;
1115 } else if (close_r
< 0) {
1116 lderr(cct
) << "failed to close image " << img_pair
.first
<< ": "
1117 << cpp_strerror(close_r
) << dendl
;
1122 } else if (next_mirror_mode
== cls::rbd::MIRROR_MODE_DISABLED
) {
1124 bool retry_busy
= false;
1125 bool pending_busy
= false;
1127 std::set
<std::string
> image_ids
;
1128 r
= list_mirror_images(io_ctx
, image_ids
);
1130 lderr(cct
) << "failed listing images: " << cpp_strerror(r
) << dendl
;
1134 for (const auto& img_id
: image_ids
) {
1135 if (current_mirror_mode
== cls::rbd::MIRROR_MODE_IMAGE
) {
1136 cls::rbd::MirrorImage mirror_image
;
1137 r
= cls_client::mirror_image_get(&io_ctx
, img_id
, &mirror_image
);
1138 if (r
< 0 && r
!= -ENOENT
) {
1139 lderr(cct
) << "failed to retrieve mirroring state for image id "
1140 << img_id
<< ": " << cpp_strerror(r
) << dendl
;
1143 if (mirror_image
.state
== cls::rbd::MIRROR_IMAGE_STATE_ENABLED
) {
1144 lderr(cct
) << "failed to disable mirror mode: there are still "
1145 << "images with mirroring enabled" << dendl
;
1149 I
*img_ctx
= I::create("", img_id
, nullptr, io_ctx
, false);
1150 r
= img_ctx
->state
->open(0);
1152 lderr(cct
) << "error opening image id "<< img_id
<< ": "
1153 << cpp_strerror(r
) << dendl
;
1157 r
= image_disable(img_ctx
, false);
1158 int close_r
= img_ctx
->state
->close();
1160 pending_busy
= true;
1162 lderr(cct
) << "error disabling mirroring for image id " << img_id
1163 << cpp_strerror(r
) << dendl
;
1165 } else if (close_r
< 0) {
1166 lderr(cct
) << "failed to close image id " << img_id
<< ": "
1167 << cpp_strerror(close_r
) << dendl
;
1169 } else if (pending_busy
) {
1170 // at least one mirrored image was successfully disabled, so we can
1171 // retry any failures caused by busy parent/child relationships
1177 if (!retry_busy
&& pending_busy
) {
1178 lderr(cct
) << "error disabling mirroring for one or more images"
1181 } else if (!retry_busy
) {
1187 r
= cls_client::mirror_mode_set(&io_ctx
, next_mirror_mode
);
1189 lderr(cct
) << "failed to set mirror mode: " << cpp_strerror(r
) << dendl
;
1193 r
= MirroringWatcher
<>::notify_mode_updated(io_ctx
, next_mirror_mode
);
1195 lderr(cct
) << "failed to send update notification: " << cpp_strerror(r
)
1201 template <typename I
>
1202 int Mirror
<I
>::uuid_get(librados::IoCtx
& io_ctx
, std::string
* mirror_uuid
) {
1203 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1204 ldout(cct
, 20) << dendl
;
1207 uuid_get(io_ctx
, mirror_uuid
, &ctx
);
1211 lderr(cct
) << "failed to retrieve mirroring uuid: " << cpp_strerror(r
)
1220 template <typename I
>
1221 void Mirror
<I
>::uuid_get(librados::IoCtx
& io_ctx
, std::string
* mirror_uuid
,
1222 Context
* on_finish
) {
1223 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1224 ldout(cct
, 20) << dendl
;
1226 auto req
= mirror::GetUuidRequest
<I
>::create(io_ctx
, mirror_uuid
, on_finish
);
1230 template <typename I
>
1231 int Mirror
<I
>::peer_bootstrap_create(librados::IoCtx
& io_ctx
,
1232 std::string
* token
) {
1233 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1234 ldout(cct
, 20) << dendl
;
1236 auto mirror_mode
= cls::rbd::MIRROR_MODE_DISABLED
;
1237 int r
= cls_client::mirror_mode_get(&io_ctx
, &mirror_mode
);
1238 if (r
< 0 && r
!= -ENOENT
) {
1239 lderr(cct
) << "failed to retrieve mirroring mode: " << cpp_strerror(r
)
1242 } else if (mirror_mode
== cls::rbd::MIRROR_MODE_DISABLED
) {
1246 // retrieve the cluster fsid
1248 librados::Rados
rados(io_ctx
);
1249 r
= rados
.cluster_fsid(&fsid
);
1251 lderr(cct
) << "failed to retrieve cluster fsid: " << cpp_strerror(r
)
1256 std::string peer_client_id
;
1257 std::string cephx_key
;
1258 r
= create_bootstrap_user(cct
, rados
, &peer_client_id
, &cephx_key
);
1263 std::string mon_host
= cct
->_conf
.get_val
<std::string
>("mon_host");
1264 ldout(cct
, 20) << "mon_host=" << mon_host
<< dendl
;
1266 // format the token response
1267 bufferlist token_bl
;
1270 R
"("fsid
":")" + fsid + R"(",)" + \
1271 R
"("client_id
":")" + peer_client_id + R"(",)" + \
1272 R
"("key
":")" + cephx_key + R"(",)" + \
1273 R
"("mon_host
":")" + \
1274 boost::replace_all_copy(mon_host, "\"", "\\\"") + R"(")" + \
1276 ldout(cct
, 20) << "token=" << token_bl
.to_str() << dendl
;
1278 bufferlist base64_bl
;
1279 token_bl
.encode_base64(base64_bl
);
1280 *token
= base64_bl
.to_str();
1285 template <typename I
>
1286 int Mirror
<I
>::peer_bootstrap_import(librados::IoCtx
& io_ctx
,
1287 rbd_mirror_peer_direction_t direction
,
1288 const std::string
& token
) {
1289 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1290 ldout(cct
, 20) << dendl
;
1292 if (direction
!= RBD_MIRROR_PEER_DIRECTION_RX
&&
1293 direction
!= RBD_MIRROR_PEER_DIRECTION_RX_TX
) {
1294 lderr(cct
) << "invalid mirror peer direction" << dendl
;
1298 bufferlist token_bl
;
1300 bufferlist base64_bl
;
1301 base64_bl
.append(token
);
1302 token_bl
.decode_base64(base64_bl
);
1303 } catch (buffer::error
& err
) {
1304 lderr(cct
) << "failed to decode base64" << dendl
;
1308 ldout(cct
, 20) << "token=" << token_bl
.to_str() << dendl
;
1310 bool json_valid
= false;
1311 std::string expected_remote_fsid
;
1312 std::string remote_client_id
;
1313 std::string remote_key
;
1314 std::string remote_mon_host
;
1316 json_spirit::mValue json_root
;
1317 if(json_spirit::read(token_bl
.to_str(), json_root
)) {
1319 auto& json_obj
= json_root
.get_obj();
1320 expected_remote_fsid
= json_obj
["fsid"].get_str();
1321 remote_client_id
= json_obj
["client_id"].get_str();
1322 remote_key
= json_obj
["key"].get_str();
1323 remote_mon_host
= json_obj
["mon_host"].get_str();
1325 } catch (std::runtime_error
&) {
1330 lderr(cct
) << "invalid bootstrap token JSON received" << dendl
;
1334 // sanity check import process
1335 std::string local_fsid
;
1336 librados::Rados
rados(io_ctx
);
1337 int r
= rados
.cluster_fsid(&local_fsid
);
1339 lderr(cct
) << "failed to retrieve cluster fsid: " << cpp_strerror(r
)
1344 std::string local_site_name
;
1345 r
= site_name_get(rados
, &local_site_name
);
1347 lderr(cct
) << "failed to retrieve cluster site name: " << cpp_strerror(r
)
1352 // attempt to connect to remote cluster
1353 librados::Rados remote_rados
;
1354 remote_rados
.init(remote_client_id
.c_str());
1356 auto remote_cct
= reinterpret_cast<CephContext
*>(remote_rados
.cct());
1357 remote_cct
->_conf
.set_val("mon_host", remote_mon_host
);
1358 remote_cct
->_conf
.set_val("key", remote_key
);
1360 r
= remote_rados
.connect();
1362 lderr(cct
) << "failed to connect to peer cluster: " << cpp_strerror(r
)
1367 std::string remote_fsid
;
1368 r
= remote_rados
.cluster_fsid(&remote_fsid
);
1370 lderr(cct
) << "failed to retrieve remote cluster fsid: "
1371 << cpp_strerror(r
) << dendl
;
1373 } else if (local_fsid
== remote_fsid
) {
1374 lderr(cct
) << "cannot import token for local cluster" << dendl
;
1376 } else if (expected_remote_fsid
!= remote_fsid
) {
1377 lderr(cct
) << "unexpected remote cluster fsid" << dendl
;
1381 std::string remote_site_name
;
1382 r
= site_name_get(remote_rados
, &remote_site_name
);
1384 lderr(cct
) << "failed to retrieve remote cluster site name: "
1385 << cpp_strerror(r
) << dendl
;
1387 } else if (local_site_name
== remote_site_name
) {
1388 lderr(cct
) << "cannot import token for duplicate site name" << dendl
;
1392 librados::IoCtx remote_io_ctx
;
1393 r
= remote_rados
.ioctx_create(io_ctx
.get_pool_name().c_str(), remote_io_ctx
);
1395 ldout(cct
, 10) << "remote pool does not exist" << dendl
;
1398 lderr(cct
) << "failed to open remote pool '" << io_ctx
.get_pool_name()
1399 << "': " << cpp_strerror(r
) << dendl
;
1403 auto remote_mirror_mode
= cls::rbd::MIRROR_MODE_DISABLED
;
1404 r
= cls_client::mirror_mode_get(&remote_io_ctx
, &remote_mirror_mode
);
1405 if (r
< 0 && r
!= -ENOENT
) {
1406 lderr(cct
) << "failed to retrieve remote mirroring mode: "
1407 << cpp_strerror(r
) << dendl
;
1409 } else if (remote_mirror_mode
== cls::rbd::MIRROR_MODE_DISABLED
) {
1413 auto local_mirror_mode
= cls::rbd::MIRROR_MODE_DISABLED
;
1414 r
= cls_client::mirror_mode_get(&io_ctx
, &local_mirror_mode
);
1415 if (r
< 0 && r
!= -ENOENT
) {
1416 lderr(cct
) << "failed to retrieve local mirroring mode: " << cpp_strerror(r
)
1419 } else if (local_mirror_mode
== cls::rbd::MIRROR_MODE_DISABLED
) {
1420 // copy mirror mode from remote peer
1421 r
= mode_set(io_ctx
, static_cast<rbd_mirror_mode_t
>(remote_mirror_mode
));
1427 if (direction
== RBD_MIRROR_PEER_DIRECTION_RX_TX
) {
1428 // create a local mirror peer user and export it to the remote cluster
1429 std::string local_client_id
;
1430 std::string local_key
;
1431 r
= create_bootstrap_user(cct
, rados
, &local_client_id
, &local_key
);
1436 std::string local_mon_host
= cct
->_conf
.get_val
<std::string
>("mon_host");
1438 // create local cluster peer in remote cluster
1439 r
= create_bootstrap_peer(cct
, remote_io_ctx
,
1440 RBD_MIRROR_PEER_DIRECTION_RX_TX
, local_site_name
,
1441 local_fsid
, local_client_id
, local_key
,
1442 local_mon_host
, "local", "remote");
1448 // create remote cluster peer in local cluster
1449 r
= create_bootstrap_peer(cct
, io_ctx
, direction
, remote_site_name
,
1450 remote_fsid
, remote_client_id
, remote_key
,
1451 remote_mon_host
, "remote", "local");
1459 template <typename I
>
1460 int Mirror
<I
>::peer_site_add(librados::IoCtx
& io_ctx
, std::string
*uuid
,
1461 mirror_peer_direction_t direction
,
1462 const std::string
&site_name
,
1463 const std::string
&client_name
) {
1464 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1465 ldout(cct
, 20) << "name=" << site_name
<< ", "
1466 << "client=" << client_name
<< dendl
;
1468 if (cct
->_conf
->cluster
== site_name
) {
1469 lderr(cct
) << "cannot add self as remote peer" << dendl
;
1473 if (direction
== RBD_MIRROR_PEER_DIRECTION_TX
) {
1480 uuid_gen
.generate_random();
1482 *uuid
= uuid_gen
.to_string();
1483 r
= cls_client::mirror_peer_add(
1484 &io_ctx
, {*uuid
, static_cast<cls::rbd::MirrorPeerDirection
>(direction
),
1485 site_name
, client_name
, ""});
1487 ldout(cct
, 5) << "duplicate UUID detected, retrying" << dendl
;
1489 lderr(cct
) << "failed to add mirror peer '" << site_name
<< "': "
1490 << cpp_strerror(r
) << dendl
;
1493 } while (r
== -ESTALE
);
1497 template <typename I
>
1498 int Mirror
<I
>::peer_site_remove(librados::IoCtx
& io_ctx
,
1499 const std::string
&uuid
) {
1500 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1501 ldout(cct
, 20) << "uuid=" << uuid
<< dendl
;
1503 int r
= remove_peer_config_key(io_ctx
, uuid
);
1505 lderr(cct
) << "failed to remove peer attributes '" << uuid
<< "': "
1506 << cpp_strerror(r
) << dendl
;
1510 r
= cls_client::mirror_peer_remove(&io_ctx
, uuid
);
1511 if (r
< 0 && r
!= -ENOENT
) {
1512 lderr(cct
) << "failed to remove peer '" << uuid
<< "': "
1513 << cpp_strerror(r
) << dendl
;
1517 vector
<string
> names
;
1518 r
= Namespace
<I
>::list(io_ctx
, &names
);
1523 names
.push_back("");
1525 librados::IoCtx ns_io_ctx
;
1526 ns_io_ctx
.dup(io_ctx
);
1528 for (auto &name
: names
) {
1529 ns_io_ctx
.set_namespace(name
);
1531 std::set
<std::string
> image_ids
;
1532 r
= list_mirror_images(ns_io_ctx
, image_ids
);
1534 lderr(cct
) << "failed listing images in "
1535 << (name
.empty() ? "default" : name
) << " namespace : "
1536 << cpp_strerror(r
) << dendl
;
1540 for (const auto& image_id
: image_ids
) {
1541 cls::rbd::MirrorImage mirror_image
;
1542 r
= cls_client::mirror_image_get(&ns_io_ctx
, image_id
, &mirror_image
);
1547 lderr(cct
) << "error getting mirror info for image " << image_id
1548 << ": " << cpp_strerror(r
) << dendl
;
1551 if (mirror_image
.mode
!= cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT
) {
1555 // Snapshot based mirroring. Unlink the peer from mirroring snapshots.
1558 I
*img_ctx
= I::create("", image_id
, nullptr, ns_io_ctx
, false);
1559 img_ctx
->read_only_mask
&= ~IMAGE_READ_ONLY_FLAG_NON_PRIMARY
;
1561 r
= img_ctx
->state
->open(0);
1566 lderr(cct
) << "error opening image " << image_id
<< ": "
1567 << cpp_strerror(r
) << dendl
;
1571 std::list
<uint64_t> snap_ids
;
1573 std::shared_lock image_locker
{img_ctx
->image_lock
};
1574 for (auto &it
: img_ctx
->snap_info
) {
1575 auto info
= boost::get
<cls::rbd::MirrorSnapshotNamespace
>(
1576 &it
.second
.snap_namespace
);
1577 if (info
&& info
->mirror_peer_uuids
.count(uuid
)) {
1578 snap_ids
.push_back(it
.first
);
1582 for (auto snap_id
: snap_ids
) {
1584 auto req
= mirror::snapshot::UnlinkPeerRequest
<I
>::create(
1585 img_ctx
, snap_id
, uuid
, &cond
);
1596 int close_r
= img_ctx
->state
->close();
1598 lderr(cct
) << "error unlinking peer for image " << image_id
<< ": "
1599 << cpp_strerror(r
) << dendl
;
1601 } else if (close_r
< 0) {
1602 lderr(cct
) << "failed to close image " << image_id
<< ": "
1603 << cpp_strerror(close_r
) << dendl
;
1612 template <typename I
>
1613 int Mirror
<I
>::peer_site_list(librados::IoCtx
& io_ctx
,
1614 std::vector
<mirror_peer_site_t
> *peers
) {
1615 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1616 ldout(cct
, 20) << dendl
;
1618 std::vector
<cls::rbd::MirrorPeer
> mirror_peers
;
1619 int r
= cls_client::mirror_peer_list(&io_ctx
, &mirror_peers
);
1620 if (r
< 0 && r
!= -ENOENT
) {
1621 lderr(cct
) << "failed to list peers: " << cpp_strerror(r
) << dendl
;
1626 peers
->reserve(mirror_peers
.size());
1627 for (auto &mirror_peer
: mirror_peers
) {
1628 mirror_peer_site_t peer
;
1629 peer
.uuid
= mirror_peer
.uuid
;
1630 peer
.direction
= static_cast<mirror_peer_direction_t
>(
1631 mirror_peer
.mirror_peer_direction
);
1632 peer
.site_name
= mirror_peer
.site_name
;
1633 peer
.mirror_uuid
= mirror_peer
.mirror_uuid
;
1634 peer
.client_name
= mirror_peer
.client_name
;
1635 peer
.last_seen
= mirror_peer
.last_seen
.sec();
1636 peers
->push_back(peer
);
1641 template <typename I
>
1642 int Mirror
<I
>::peer_site_set_client(librados::IoCtx
& io_ctx
,
1643 const std::string
&uuid
,
1644 const std::string
&client_name
) {
1645 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1646 ldout(cct
, 20) << "uuid=" << uuid
<< ", "
1647 << "client=" << client_name
<< dendl
;
1649 int r
= cls_client::mirror_peer_set_client(&io_ctx
, uuid
, client_name
);
1651 lderr(cct
) << "failed to update client '" << uuid
<< "': "
1652 << cpp_strerror(r
) << dendl
;
1658 template <typename I
>
1659 int Mirror
<I
>::peer_site_set_name(librados::IoCtx
& io_ctx
,
1660 const std::string
&uuid
,
1661 const std::string
&site_name
) {
1662 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1663 ldout(cct
, 20) << "uuid=" << uuid
<< ", "
1664 << "name=" << site_name
<< dendl
;
1666 if (cct
->_conf
->cluster
== site_name
) {
1667 lderr(cct
) << "cannot set self as remote peer" << dendl
;
1671 int r
= cls_client::mirror_peer_set_cluster(&io_ctx
, uuid
, site_name
);
1673 lderr(cct
) << "failed to update site '" << uuid
<< "': "
1674 << cpp_strerror(r
) << dendl
;
1680 template <typename I
>
1681 int Mirror
<I
>::peer_site_set_direction(librados::IoCtx
& io_ctx
,
1682 const std::string
&uuid
,
1683 mirror_peer_direction_t direction
) {
1684 cls::rbd::MirrorPeerDirection mirror_peer_direction
= static_cast<
1685 cls::rbd::MirrorPeerDirection
>(direction
);
1687 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1688 ldout(cct
, 20) << "uuid=" << uuid
<< ", "
1689 << "direction=" << mirror_peer_direction
<< dendl
;
1691 int r
= cls_client::mirror_peer_set_direction(&io_ctx
, uuid
,
1692 mirror_peer_direction
);
1694 lderr(cct
) << "failed to update direction '" << uuid
<< "': "
1695 << cpp_strerror(r
) << dendl
;
1701 template <typename I
>
1702 int Mirror
<I
>::peer_site_get_attributes(librados::IoCtx
& io_ctx
,
1703 const std::string
&uuid
,
1704 Attributes
* attributes
) {
1705 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1706 ldout(cct
, 20) << "uuid=" << uuid
<< dendl
;
1708 attributes
->clear();
1710 librados::Rados
rados(io_ctx
);
1712 int r
= get_config_key(rados
, get_peer_config_key_name(io_ctx
.get_id(), uuid
),
1714 if (r
== -ENOENT
|| value
.empty()) {
1717 lderr(cct
) << "failed to retrieve peer attributes: " << cpp_strerror(r
)
1722 bool json_valid
= false;
1723 json_spirit::mValue json_root
;
1724 if(json_spirit::read(value
, json_root
)) {
1726 auto& json_obj
= json_root
.get_obj();
1727 for (auto& pairs
: json_obj
) {
1728 (*attributes
)[pairs
.first
] = pairs
.second
.get_str();
1731 } catch (std::runtime_error
&) {
1736 lderr(cct
) << "invalid peer attributes JSON received" << dendl
;
1742 template <typename I
>
1743 int Mirror
<I
>::peer_site_set_attributes(librados::IoCtx
& io_ctx
,
1744 const std::string
&uuid
,
1745 const Attributes
& attributes
) {
1746 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1747 ldout(cct
, 20) << "uuid=" << uuid
<< ", "
1748 << "attributes=" << attributes
<< dendl
;
1750 std::vector
<mirror_peer_site_t
> mirror_peers
;
1751 int r
= peer_site_list(io_ctx
, &mirror_peers
);
1756 if (std::find_if(mirror_peers
.begin(), mirror_peers
.end(),
1757 [&uuid
](const librbd::mirror_peer_site_t
& peer
) {
1758 return uuid
== peer
.uuid
;
1759 }) == mirror_peers
.end()) {
1760 ldout(cct
, 5) << "mirror peer uuid " << uuid
<< " does not exist" << dendl
;
1764 std::stringstream ss
;
1766 for (auto& pair
: attributes
) {
1767 ss
<< "\\\"" << pair
.first
<< "\\\": "
1768 << "\\\"" << pair
.second
<< "\\\"";
1769 if (&pair
!= &(*attributes
.rbegin())) {
1775 librados::Rados
rados(io_ctx
);
1776 r
= set_config_key(rados
, get_peer_config_key_name(io_ctx
.get_id(), uuid
),
1778 if (r
< 0 && r
!= -ENOENT
) {
1779 lderr(cct
) << "failed to update peer attributes: " << cpp_strerror(r
)
1787 template <typename I
>
1788 int Mirror
<I
>::image_global_status_list(
1789 librados::IoCtx
& io_ctx
, const std::string
&start_id
, size_t max
,
1790 IdToMirrorImageGlobalStatus
*images
) {
1791 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1794 map
<string
, string
> id_to_name
;
1796 map
<string
, string
> name_to_id
;
1797 r
= Image
<I
>::list_images_v2(io_ctx
, &name_to_id
);
1801 for (auto it
: name_to_id
) {
1802 id_to_name
[it
.second
] = it
.first
;
1806 map
<std::string
, cls::rbd::MirrorImage
> images_
;
1807 map
<std::string
, cls::rbd::MirrorImageStatus
> statuses_
;
1809 r
= librbd::cls_client::mirror_image_status_list(&io_ctx
, start_id
, max
,
1810 &images_
, &statuses_
);
1811 if (r
< 0 && r
!= -ENOENT
) {
1812 lderr(cct
) << "failed to list mirror image statuses: "
1813 << cpp_strerror(r
) << dendl
;
1817 const std::string
STATUS_NOT_FOUND("status not found");
1818 for (auto it
= images_
.begin(); it
!= images_
.end(); ++it
) {
1819 auto &image_id
= it
->first
;
1820 auto &info
= it
->second
;
1821 if (info
.state
== cls::rbd::MIRROR_IMAGE_STATE_DISABLED
) {
1825 auto &image_name
= id_to_name
[image_id
];
1826 if (image_name
.empty()) {
1827 lderr(cct
) << "failed to find image name for image " << image_id
<< ", "
1828 << "using image id as name" << dendl
;
1829 image_name
= image_id
;
1832 mirror_image_global_status_t
& global_status
= (*images
)[image_id
];
1833 global_status
.name
= image_name
;
1834 global_status
.info
= mirror_image_info_t
{
1835 info
.global_image_id
,
1836 static_cast<mirror_image_state_t
>(info
.state
),
1837 false}; // XXX: To set "primary" right would require an additional call.
1839 auto s_it
= statuses_
.find(image_id
);
1840 if (s_it
!= statuses_
.end()) {
1841 auto& status
= s_it
->second
;
1843 global_status
.site_statuses
.reserve(
1844 status
.mirror_image_site_statuses
.size());
1845 for (auto& site_status
: status
.mirror_image_site_statuses
) {
1846 global_status
.site_statuses
.push_back(mirror_image_site_status_t
{
1847 site_status
.mirror_uuid
,
1848 static_cast<mirror_image_status_state_t
>(site_status
.state
),
1849 site_status
.state
== cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN
?
1850 STATUS_NOT_FOUND
: site_status
.description
,
1851 site_status
.last_update
.sec(), site_status
.up
});
1854 // older OSD that only returns local status
1855 global_status
.site_statuses
.push_back(mirror_image_site_status_t
{
1856 cls::rbd::MirrorImageSiteStatus::LOCAL_MIRROR_UUID
,
1857 MIRROR_IMAGE_STATUS_STATE_UNKNOWN
, STATUS_NOT_FOUND
, 0, false});
1864 template <typename I
>
1865 int Mirror
<I
>::image_status_summary(librados::IoCtx
& io_ctx
,
1866 MirrorImageStatusStates
*states
) {
1867 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1869 std::vector
<cls::rbd::MirrorPeer
> mirror_peers
;
1870 int r
= cls_client::mirror_peer_list(&io_ctx
, &mirror_peers
);
1871 if (r
< 0 && r
!= -ENOENT
) {
1872 lderr(cct
) << "failed to list mirror peers: " << cpp_strerror(r
) << dendl
;
1876 std::map
<cls::rbd::MirrorImageStatusState
, int> states_
;
1877 r
= cls_client::mirror_image_status_get_summary(&io_ctx
, mirror_peers
,
1879 if (r
< 0 && r
!= -ENOENT
) {
1880 lderr(cct
) << "failed to get mirror status summary: "
1881 << cpp_strerror(r
) << dendl
;
1884 for (auto &s
: states_
) {
1885 (*states
)[static_cast<mirror_image_status_state_t
>(s
.first
)] = s
.second
;
1890 template <typename I
>
1891 int Mirror
<I
>::image_instance_id_list(
1892 librados::IoCtx
& io_ctx
, const std::string
&start_image_id
, size_t max
,
1893 std::map
<std::string
, std::string
> *instance_ids
) {
1894 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1895 std::map
<std::string
, entity_inst_t
> instances
;
1897 int r
= librbd::cls_client::mirror_image_instance_list(
1898 &io_ctx
, start_image_id
, max
, &instances
);
1899 if (r
< 0 && r
!= -ENOENT
) {
1900 lderr(cct
) << "failed to list mirror image instances: " << cpp_strerror(r
)
1905 for (auto it
: instances
) {
1906 (*instance_ids
)[it
.first
] = stringify(it
.second
.name
.num());
1912 template <typename I
>
1913 int Mirror
<I
>::image_info_list(
1914 librados::IoCtx
& io_ctx
, mirror_image_mode_t
*mode_filter
,
1915 const std::string
&start_id
, size_t max
,
1916 std::map
<std::string
, std::pair
<mirror_image_mode_t
,
1917 mirror_image_info_t
>> *entries
) {
1918 CephContext
*cct
= reinterpret_cast<CephContext
*>(io_ctx
.cct());
1919 ldout(cct
, 20) << "pool=" << io_ctx
.get_pool_name() << ", mode_filter="
1920 << (mode_filter
? stringify(*mode_filter
) : "null")
1921 << ", start_id=" << start_id
<< ", max=" << max
<< dendl
;
1923 std::string last_read
= start_id
;
1926 while (entries
->size() < max
) {
1927 map
<std::string
, cls::rbd::MirrorImage
> images
;
1928 map
<std::string
, cls::rbd::MirrorImageStatus
> statuses
;
1930 int r
= librbd::cls_client::mirror_image_status_list(&io_ctx
, last_read
,
1933 if (r
< 0 && r
!= -ENOENT
) {
1934 lderr(cct
) << "failed to list mirror image statuses: "
1935 << cpp_strerror(r
) << dendl
;
1939 if (images
.empty()) {
1943 ThreadPool
*thread_pool
;
1944 ContextWQ
*op_work_queue
;
1945 ImageCtx::get_thread_pool_instance(cct
, &thread_pool
, &op_work_queue
);
1947 for (auto &it
: images
) {
1948 auto &image_id
= it
.first
;
1949 auto &image
= it
.second
;
1950 auto mode
= static_cast<mirror_image_mode_t
>(image
.mode
);
1952 if ((mode_filter
&& mode
!= *mode_filter
) ||
1953 image
.state
!= cls::rbd::MIRROR_IMAGE_STATE_ENABLED
) {
1957 // need to call get_info for every image to retrieve promotion state
1959 mirror_image_info_t info
;
1960 r
= image_get_info(io_ctx
, op_work_queue
, image_id
, &info
);
1962 (*entries
)[image_id
] = std::make_pair(mode
, info
);
1966 last_read
= images
.rbegin()->first
;
1972 template <typename I
>
1973 int Mirror
<I
>::image_snapshot_create(I
*ictx
, uint64_t *snap_id
) {
1974 CephContext
*cct
= ictx
->cct
;
1975 ldout(cct
, 20) << "ictx=" << ictx
<< dendl
;
1977 int r
= ictx
->state
->refresh_if_required();
1982 cls::rbd::MirrorImage mirror_image
;
1983 r
= cls_client::mirror_image_get(&ictx
->md_ctx
, ictx
->id
,
1988 lderr(cct
) << "failed to retrieve mirror image" << dendl
;
1992 if (mirror_image
.mode
!= cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT
||
1993 mirror_image
.state
!= cls::rbd::MIRROR_IMAGE_STATE_ENABLED
) {
1994 lderr(cct
) << "snapshot based mirroring is not enabled" << dendl
;
1998 C_SaferCond on_finish
;
1999 auto req
= mirror::snapshot::CreatePrimaryRequest
<I
>::create(
2000 ictx
, mirror_image
.global_image_id
, CEPH_NOSNAP
, 0U, snap_id
, &on_finish
);
2002 return on_finish
.wait();
2006 } // namespace librbd
2008 template class librbd::api::Mirror
<librbd::ImageCtx
>;