1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "include/compat.h"
5 #include "BootstrapRequest.h"
6 #include "CloseImageRequest.h"
7 #include "CreateImageRequest.h"
8 #include "IsPrimaryRequest.h"
9 #include "OpenImageRequest.h"
10 #include "OpenLocalImageRequest.h"
11 #include "common/debug.h"
12 #include "common/dout.h"
13 #include "common/errno.h"
14 #include "common/WorkQueue.h"
15 #include "cls/rbd/cls_rbd_client.h"
16 #include "journal/Journaler.h"
17 #include "librbd/ImageCtx.h"
18 #include "librbd/ImageState.h"
19 #include "librbd/internal.h"
20 #include "librbd/Journal.h"
21 #include "librbd/Utils.h"
22 #include "librbd/journal/Types.h"
23 #include "tools/rbd_mirror/ProgressContext.h"
24 #include "tools/rbd_mirror/ImageSyncThrottler.h"
26 #define dout_context g_ceph_context
27 #define dout_subsys ceph_subsys_rbd_mirror
29 #define dout_prefix *_dout << "rbd::mirror::image_replayer::BootstrapRequest: " \
30 << this << " " << __func__
34 namespace image_replayer
{
36 using librbd::util::create_context_callback
;
37 using librbd::util::create_rados_callback
;
38 using librbd::util::unique_lock_name
;
41 BootstrapRequest
<I
>::BootstrapRequest(
42 librados::IoCtx
&local_io_ctx
,
43 librados::IoCtx
&remote_io_ctx
,
44 std::shared_ptr
<ImageSyncThrottler
<I
>> image_sync_throttler
,
46 const std::string
&local_image_id
,
47 const std::string
&remote_image_id
,
48 const std::string
&global_image_id
,
49 ContextWQ
*work_queue
, SafeTimer
*timer
,
51 const std::string
&local_mirror_uuid
,
52 const std::string
&remote_mirror_uuid
,
54 MirrorPeerClientMeta
*client_meta
,
57 rbd::mirror::ProgressContext
*progress_ctx
)
58 : BaseRequest("rbd::mirror::image_replayer::BootstrapRequest",
59 reinterpret_cast<CephContext
*>(local_io_ctx
.cct()), on_finish
),
60 m_local_io_ctx(local_io_ctx
), m_remote_io_ctx(remote_io_ctx
),
61 m_image_sync_throttler(image_sync_throttler
),
62 m_local_image_ctx(local_image_ctx
), m_local_image_id(local_image_id
),
63 m_remote_image_id(remote_image_id
), m_global_image_id(global_image_id
),
64 m_work_queue(work_queue
), m_timer(timer
), m_timer_lock(timer_lock
),
65 m_local_mirror_uuid(local_mirror_uuid
),
66 m_remote_mirror_uuid(remote_mirror_uuid
), m_journaler(journaler
),
67 m_client_meta(client_meta
), m_progress_ctx(progress_ctx
),
68 m_do_resync(do_resync
),
69 m_lock(unique_lock_name("BootstrapRequest::m_lock", this)) {
73 BootstrapRequest
<I
>::~BootstrapRequest() {
74 assert(m_remote_image_ctx
== nullptr);
78 void BootstrapRequest
<I
>::send() {
81 get_remote_tag_class();
85 void BootstrapRequest
<I
>::cancel() {
88 Mutex::Locker
locker(m_lock
);
91 m_image_sync_throttler
->cancel_sync(m_local_io_ctx
, m_local_image_id
);
95 void BootstrapRequest
<I
>::get_remote_tag_class() {
98 update_progress("GET_REMOTE_TAG_CLASS");
100 Context
*ctx
= create_context_callback
<
101 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_get_remote_tag_class
>(
103 m_journaler
->get_client(librbd::Journal
<>::IMAGE_CLIENT_ID
, &m_client
, ctx
);
106 template <typename I
>
107 void BootstrapRequest
<I
>::handle_get_remote_tag_class(int r
) {
108 dout(20) << ": r=" << r
<< dendl
;
111 derr
<< ": failed to retrieve remote client: " << cpp_strerror(r
) << dendl
;
116 librbd::journal::ClientData client_data
;
117 bufferlist::iterator it
= m_client
.data
.begin();
119 ::decode(client_data
, it
);
120 } catch (const buffer::error
&err
) {
121 derr
<< ": failed to decode remote client meta data: " << err
.what()
127 librbd::journal::ImageClientMeta
*client_meta
=
128 boost::get
<librbd::journal::ImageClientMeta
>(&client_data
.client_meta
);
129 if (client_meta
== nullptr) {
130 derr
<< ": unknown remote client registration" << dendl
;
135 m_remote_tag_class
= client_meta
->tag_class
;
136 dout(10) << ": remote tag class=" << m_remote_tag_class
<< dendl
;
141 template <typename I
>
142 void BootstrapRequest
<I
>::get_client() {
145 update_progress("GET_CLIENT");
147 Context
*ctx
= create_context_callback
<
148 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_get_client
>(
150 m_journaler
->get_client(m_local_mirror_uuid
, &m_client
, ctx
);
153 template <typename I
>
154 void BootstrapRequest
<I
>::handle_get_client(int r
) {
155 dout(20) << ": r=" << r
<< dendl
;
158 dout(10) << ": client not registered" << dendl
;
160 derr
<< ": failed to retrieve client: " << cpp_strerror(r
) << dendl
;
163 } else if (decode_client_meta()) {
164 // skip registration if it already exists
172 template <typename I
>
173 void BootstrapRequest
<I
>::register_client() {
176 update_progress("REGISTER_CLIENT");
178 // record an place-holder record
179 librbd::journal::ClientData client_data
{
180 librbd::journal::MirrorPeerClientMeta
{m_local_image_id
}};
181 bufferlist client_data_bl
;
182 ::encode(client_data
, client_data_bl
);
184 Context
*ctx
= create_context_callback
<
185 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_register_client
>(
187 m_journaler
->register_client(client_data_bl
, ctx
);
190 template <typename I
>
191 void BootstrapRequest
<I
>::handle_register_client(int r
) {
192 dout(20) << ": r=" << r
<< dendl
;
195 derr
<< ": failed to register with remote journal: " << cpp_strerror(r
)
201 *m_client_meta
= librbd::journal::MirrorPeerClientMeta(m_local_image_id
);
205 template <typename I
>
206 void BootstrapRequest
<I
>::open_remote_image() {
209 update_progress("OPEN_REMOTE_IMAGE");
211 Context
*ctx
= create_context_callback
<
212 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_open_remote_image
>(
214 OpenImageRequest
<I
> *request
= OpenImageRequest
<I
>::create(
215 m_remote_io_ctx
, &m_remote_image_ctx
, m_remote_image_id
, false,
220 template <typename I
>
221 void BootstrapRequest
<I
>::handle_open_remote_image(int r
) {
222 dout(20) << ": r=" << r
<< dendl
;
225 derr
<< ": failed to open remote image: " << cpp_strerror(r
) << dendl
;
226 assert(m_remote_image_ctx
== nullptr);
234 template <typename I
>
235 void BootstrapRequest
<I
>::is_primary() {
238 update_progress("OPEN_REMOTE_IMAGE");
240 Context
*ctx
= create_context_callback
<
241 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_is_primary
>(
243 IsPrimaryRequest
<I
> *request
= IsPrimaryRequest
<I
>::create(m_remote_image_ctx
,
248 template <typename I
>
249 void BootstrapRequest
<I
>::handle_is_primary(int r
) {
250 dout(20) << ": r=" << r
<< dendl
;
253 derr
<< ": error querying remote image primary status: " << cpp_strerror(r
)
256 close_remote_image();
261 dout(5) << ": remote image is not primary -- skipping image replay"
263 m_ret_val
= -EREMOTEIO
;
264 update_client_state();
268 if (m_local_image_id
.empty()) {
269 create_local_image();
276 template <typename I
>
277 void BootstrapRequest
<I
>::update_client_state() {
278 if (m_client_meta
->state
== librbd::journal::MIRROR_PEER_STATE_REPLAYING
) {
279 // state already set for replaying upon failover
280 close_remote_image();
285 update_progress("UPDATE_CLIENT_STATE");
287 librbd::journal::MirrorPeerClientMeta
client_meta(*m_client_meta
);
288 client_meta
.state
= librbd::journal::MIRROR_PEER_STATE_REPLAYING
;
290 librbd::journal::ClientData
client_data(client_meta
);
292 ::encode(client_data
, data_bl
);
294 Context
*ctx
= create_context_callback
<
295 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_update_client_state
>(
297 m_journaler
->update_client(data_bl
, ctx
);
300 template <typename I
>
301 void BootstrapRequest
<I
>::handle_update_client_state(int r
) {
302 dout(20) << ": r=" << r
<< dendl
;
304 derr
<< ": failed to update client: " << cpp_strerror(r
) << dendl
;
306 m_client_meta
->state
= librbd::journal::MIRROR_PEER_STATE_REPLAYING
;;
309 close_remote_image();
312 template <typename I
>
313 void BootstrapRequest
<I
>::open_local_image() {
316 update_progress("OPEN_LOCAL_IMAGE");
318 Context
*ctx
= create_context_callback
<
319 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_open_local_image
>(
321 OpenLocalImageRequest
<I
> *request
= OpenLocalImageRequest
<I
>::create(
322 m_local_io_ctx
, m_local_image_ctx
, m_local_image_id
, m_work_queue
,
327 template <typename I
>
328 void BootstrapRequest
<I
>::handle_open_local_image(int r
) {
329 dout(20) << ": r=" << r
<< dendl
;
332 assert(*m_local_image_ctx
== nullptr);
333 dout(10) << ": local image missing" << dendl
;
334 create_local_image();
336 } else if (r
== -EREMOTEIO
) {
337 assert(*m_local_image_ctx
== nullptr);
338 dout(10) << "local image is primary -- skipping image replay" << dendl
;
340 close_remote_image();
343 assert(*m_local_image_ctx
== nullptr);
344 derr
<< ": failed to open local image: " << cpp_strerror(r
) << dendl
;
346 close_remote_image();
350 I
*local_image_ctx
= (*m_local_image_ctx
);
352 RWLock::RLocker
snap_locker(local_image_ctx
->snap_lock
);
353 if (local_image_ctx
->journal
== nullptr) {
354 derr
<< ": local image does not support journaling" << dendl
;
360 r
= (*m_local_image_ctx
)->journal
->is_resync_requested(m_do_resync
);
362 derr
<< ": failed to check if a resync was requested" << dendl
;
370 close_remote_image();
374 if (m_client
.state
== cls::journal::CLIENT_STATE_DISCONNECTED
) {
375 dout(10) << ": client flagged disconnected -- skipping bootstrap" << dendl
;
376 // The caller is expected to detect disconnect initializing remote journal.
378 close_remote_image();
382 update_client_image();
385 template <typename I
>
386 void BootstrapRequest
<I
>::create_local_image() {
389 m_local_image_id
= "";
390 update_progress("CREATE_LOCAL_IMAGE");
392 m_remote_image_ctx
->snap_lock
.get_read();
393 std::string image_name
= m_remote_image_ctx
->name
;
394 m_remote_image_ctx
->snap_lock
.put_read();
396 Context
*ctx
= create_context_callback
<
397 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_create_local_image
>(
399 CreateImageRequest
<I
> *request
= CreateImageRequest
<I
>::create(
400 m_local_io_ctx
, m_work_queue
, m_global_image_id
, m_remote_mirror_uuid
,
401 image_name
, m_remote_image_ctx
, &m_local_image_id
, ctx
);
405 template <typename I
>
406 void BootstrapRequest
<I
>::handle_create_local_image(int r
) {
407 dout(20) << ": r=" << r
<< dendl
;
410 derr
<< ": failed to create local image: " << cpp_strerror(r
) << dendl
;
412 close_remote_image();
419 template <typename I
>
420 void BootstrapRequest
<I
>::update_client_image() {
423 update_progress("UPDATE_CLIENT_IMAGE");
425 if (m_client_meta
->image_id
== (*m_local_image_ctx
)->id
) {
426 // already registered local image with remote journal
430 m_local_image_id
= (*m_local_image_ctx
)->id
;
434 librbd::journal::MirrorPeerClientMeta client_meta
{m_local_image_id
};
435 client_meta
.state
= librbd::journal::MIRROR_PEER_STATE_SYNCING
;
437 librbd::journal::ClientData
client_data(client_meta
);
439 ::encode(client_data
, data_bl
);
441 Context
*ctx
= create_context_callback
<
442 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_update_client_image
>(
444 m_journaler
->update_client(data_bl
, ctx
);
447 template <typename I
>
448 void BootstrapRequest
<I
>::handle_update_client_image(int r
) {
449 dout(20) << ": r=" << r
<< dendl
;
452 derr
<< ": failed to update client: " << cpp_strerror(r
) << dendl
;
459 dout(10) << ": request canceled" << dendl
;
460 m_ret_val
= -ECANCELED
;
465 *m_client_meta
= {m_local_image_id
};
466 m_client_meta
->state
= librbd::journal::MIRROR_PEER_STATE_SYNCING
;
470 template <typename I
>
471 void BootstrapRequest
<I
>::get_remote_tags() {
474 update_progress("GET_REMOTE_TAGS");
476 if (m_client_meta
->state
== librbd::journal::MIRROR_PEER_STATE_SYNCING
) {
477 // optimization -- no need to compare remote tags if we just created
478 // the image locally or sync was interrupted
485 Context
*ctx
= create_context_callback
<
486 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_get_remote_tags
>(this);
487 m_journaler
->get_tags(m_remote_tag_class
, &m_remote_tags
, ctx
);
490 template <typename I
>
491 void BootstrapRequest
<I
>::handle_get_remote_tags(int r
) {
492 dout(20) << ": r=" << r
<< dendl
;
495 derr
<< ": failed to retrieve remote tags: " << cpp_strerror(r
) << dendl
;
502 dout(10) << ": request canceled" << dendl
;
503 m_ret_val
= -ECANCELED
;
508 // At this point, the local image was existing, non-primary, and replaying;
509 // and the remote image is primary. Attempt to link the local image's most
510 // recent tag to the remote image's tag chain.
511 uint64_t local_tag_tid
;
512 librbd::journal::TagData local_tag_data
;
513 I
*local_image_ctx
= (*m_local_image_ctx
);
515 RWLock::RLocker
snap_locker(local_image_ctx
->snap_lock
);
516 if (local_image_ctx
->journal
== nullptr) {
517 derr
<< ": local image does not support journaling" << dendl
;
523 local_tag_tid
= local_image_ctx
->journal
->get_tag_tid();
524 local_tag_data
= local_image_ctx
->journal
->get_tag_data();
525 dout(20) << ": local tag " << local_tag_tid
<< ": "
526 << local_tag_data
<< dendl
;
529 bool remote_tag_data_valid
= false;
530 librbd::journal::TagData remote_tag_data
;
531 boost::optional
<uint64_t> remote_orphan_tag_tid
=
532 boost::make_optional
<uint64_t>(false, 0U);
533 bool reconnect_orphan
= false;
535 // decode the remote tags
536 for (auto &remote_tag
: m_remote_tags
) {
537 if (local_tag_data
.predecessor
.commit_valid
&&
538 local_tag_data
.predecessor
.mirror_uuid
== m_remote_mirror_uuid
&&
539 local_tag_data
.predecessor
.tag_tid
> remote_tag
.tid
) {
540 dout(20) << ": skipping processed predecessor remote tag "
541 << remote_tag
.tid
<< dendl
;
546 bufferlist::iterator it
= remote_tag
.data
.begin();
547 ::decode(remote_tag_data
, it
);
548 remote_tag_data_valid
= true;
549 } catch (const buffer::error
&err
) {
550 derr
<< ": failed to decode remote tag " << remote_tag
.tid
<< ": "
551 << err
.what() << dendl
;
552 m_ret_val
= -EBADMSG
;
557 dout(10) << ": decoded remote tag " << remote_tag
.tid
<< ": "
558 << remote_tag_data
<< dendl
;
560 if (!local_tag_data
.predecessor
.commit_valid
) {
561 // newly synced local image (no predecessor) replays from the first tag
562 if (remote_tag_data
.mirror_uuid
!= librbd::Journal
<>::LOCAL_MIRROR_UUID
) {
563 dout(20) << ": skipping non-primary remote tag" << dendl
;
567 dout(20) << ": using initial primary remote tag" << dendl
;
571 if (local_tag_data
.mirror_uuid
== librbd::Journal
<>::ORPHAN_MIRROR_UUID
) {
572 // demotion last available local epoch
574 if (remote_tag_data
.mirror_uuid
== local_tag_data
.mirror_uuid
&&
575 remote_tag_data
.predecessor
.commit_valid
&&
576 remote_tag_data
.predecessor
.tag_tid
==
577 local_tag_data
.predecessor
.tag_tid
) {
578 // demotion matches remote epoch
580 if (remote_tag_data
.predecessor
.mirror_uuid
== m_local_mirror_uuid
&&
581 local_tag_data
.predecessor
.mirror_uuid
==
582 librbd::Journal
<>::LOCAL_MIRROR_UUID
) {
583 // local demoted and remote has matching event
584 dout(20) << ": found matching local demotion tag" << dendl
;
585 remote_orphan_tag_tid
= remote_tag
.tid
;
589 if (local_tag_data
.predecessor
.mirror_uuid
== m_remote_mirror_uuid
&&
590 remote_tag_data
.predecessor
.mirror_uuid
==
591 librbd::Journal
<>::LOCAL_MIRROR_UUID
) {
592 // remote demoted and local has matching event
593 dout(20) << ": found matching remote demotion tag" << dendl
;
594 remote_orphan_tag_tid
= remote_tag
.tid
;
599 if (remote_tag_data
.mirror_uuid
== librbd::Journal
<>::LOCAL_MIRROR_UUID
&&
600 remote_tag_data
.predecessor
.mirror_uuid
== librbd::Journal
<>::ORPHAN_MIRROR_UUID
&&
601 remote_tag_data
.predecessor
.commit_valid
&& remote_orphan_tag_tid
&&
602 remote_tag_data
.predecessor
.tag_tid
== *remote_orphan_tag_tid
) {
603 // remote promotion tag chained to remote/local demotion tag
604 dout(20) << ": found chained remote promotion tag" << dendl
;
605 reconnect_orphan
= true;
609 // promotion must follow demotion
610 remote_orphan_tag_tid
= boost::none
;
614 if (remote_tag_data_valid
&&
615 local_tag_data
.mirror_uuid
== m_remote_mirror_uuid
) {
616 dout(20) << ": local image is in clean replay state" << dendl
;
617 } else if (reconnect_orphan
) {
618 dout(20) << ": remote image was demoted/promoted" << dendl
;
620 derr
<< ": split-brain detected -- skipping image replay" << dendl
;
629 template <typename I
>
630 void BootstrapRequest
<I
>::image_sync() {
631 if (m_client_meta
->state
== librbd::journal::MIRROR_PEER_STATE_REPLAYING
) {
632 // clean replay state -- no image sync required
633 close_remote_image();
638 update_progress("IMAGE_SYNC");
640 Context
*ctx
= create_context_callback
<
641 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_image_sync
>(
645 Mutex::Locker
locker(m_lock
);
647 m_image_sync_throttler
->start_sync(*m_local_image_ctx
,
648 m_remote_image_ctx
, m_timer
,
650 m_local_mirror_uuid
, m_journaler
,
651 m_client_meta
, m_work_queue
, ctx
,
657 dout(10) << ": request canceled" << dendl
;
658 m_ret_val
= -ECANCELED
;
659 close_remote_image();
662 template <typename I
>
663 void BootstrapRequest
<I
>::handle_image_sync(int r
) {
664 dout(20) << ": r=" << r
<< dendl
;
667 dout(10) << ": request canceled" << dendl
;
668 m_ret_val
= -ECANCELED
;
672 derr
<< ": failed to sync remote image: " << cpp_strerror(r
) << dendl
;
676 close_remote_image();
679 template <typename I
>
680 void BootstrapRequest
<I
>::close_local_image() {
683 update_progress("CLOSE_LOCAL_IMAGE");
685 Context
*ctx
= create_context_callback
<
686 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_close_local_image
>(
688 CloseImageRequest
<I
> *request
= CloseImageRequest
<I
>::create(
689 m_local_image_ctx
, ctx
);
693 template <typename I
>
694 void BootstrapRequest
<I
>::handle_close_local_image(int r
) {
695 dout(20) << ": r=" << r
<< dendl
;
698 derr
<< ": error encountered closing local image: " << cpp_strerror(r
)
702 close_remote_image();
705 template <typename I
>
706 void BootstrapRequest
<I
>::close_remote_image() {
709 update_progress("CLOSE_REMOTE_IMAGE");
711 Context
*ctx
= create_context_callback
<
712 BootstrapRequest
<I
>, &BootstrapRequest
<I
>::handle_close_remote_image
>(
714 CloseImageRequest
<I
> *request
= CloseImageRequest
<I
>::create(
715 &m_remote_image_ctx
, ctx
);
719 template <typename I
>
720 void BootstrapRequest
<I
>::handle_close_remote_image(int r
) {
721 dout(20) << ": r=" << r
<< dendl
;
724 derr
<< ": error encountered closing remote image: " << cpp_strerror(r
)
731 template <typename I
>
732 bool BootstrapRequest
<I
>::decode_client_meta() {
735 librbd::journal::ClientData client_data
;
736 bufferlist::iterator it
= m_client
.data
.begin();
738 ::decode(client_data
, it
);
739 } catch (const buffer::error
&err
) {
740 derr
<< ": failed to decode client meta data: " << err
.what() << dendl
;
744 librbd::journal::MirrorPeerClientMeta
*client_meta
=
745 boost::get
<librbd::journal::MirrorPeerClientMeta
>(&client_data
.client_meta
);
746 if (client_meta
== nullptr) {
747 derr
<< ": unknown peer registration" << dendl
;
749 } else if (!client_meta
->image_id
.empty()) {
750 // have an image id -- use that to open the image
751 m_local_image_id
= client_meta
->image_id
;
754 *m_client_meta
= *client_meta
;
756 dout(20) << ": client found: image_id=" << m_local_image_id
757 << ", client_meta=" << *m_client_meta
<< dendl
;
761 template <typename I
>
762 void BootstrapRequest
<I
>::update_progress(const std::string
&description
) {
763 dout(20) << ": " << description
<< dendl
;
765 if (m_progress_ctx
) {
766 m_progress_ctx
->update_progress(description
);
770 } // namespace image_replayer
771 } // namespace mirror
774 template class rbd::mirror::image_replayer::BootstrapRequest
<librbd::ImageCtx
>;