1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
5 #include "common/debug.h"
6 #include "common/errno.h"
7 #include "include/stringify.h"
8 #include "common/Timer.h"
9 #include "common/WorkQueue.h"
10 #include "cls/rbd/cls_rbd_client.h"
11 #include "json_spirit/json_spirit.h"
12 #include "librbd/ImageCtx.h"
13 #include "librbd/ImageState.h"
14 #include "librbd/Operations.h"
15 #include "librbd/Utils.h"
16 #include "librbd/deep_copy/Handler.h"
17 #include "librbd/deep_copy/ImageCopyRequest.h"
18 #include "librbd/deep_copy/SnapshotCopyRequest.h"
19 #include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h"
20 #include "librbd/mirror/snapshot/GetImageStateRequest.h"
21 #include "librbd/mirror/snapshot/ImageMeta.h"
22 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
23 #include "tools/rbd_mirror/InstanceWatcher.h"
24 #include "tools/rbd_mirror/PoolMetaCache.h"
25 #include "tools/rbd_mirror/Threads.h"
26 #include "tools/rbd_mirror/Types.h"
27 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
28 #include "tools/rbd_mirror/image_replayer/ReplayerListener.h"
29 #include "tools/rbd_mirror/image_replayer/Utils.h"
30 #include "tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h"
31 #include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h"
32 #include "tools/rbd_mirror/image_replayer/snapshot/Utils.h"
35 #define dout_context g_ceph_context
36 #define dout_subsys ceph_subsys_rbd_mirror
38 #define dout_prefix *_dout << "rbd::mirror::image_replayer::snapshot::" \
39 << "Replayer: " << this << " " << __func__ << ": "
41 extern PerfCounters
*g_perf_counters
;
45 namespace image_replayer
{
50 double round_to_two_places(double value
) {
51 return abs(round(value
* 100) / 100);
55 std::pair
<uint64_t, librbd::SnapInfo
*> get_newest_mirror_snapshot(
57 for (auto snap_info_it
= image_ctx
->snap_info
.rbegin();
58 snap_info_it
!= image_ctx
->snap_info
.rend(); ++snap_info_it
) {
59 const auto& snap_ns
= snap_info_it
->second
.snap_namespace
;
60 auto mirror_ns
= boost::get
<
61 cls::rbd::MirrorSnapshotNamespace
>(&snap_ns
);
62 if (mirror_ns
== nullptr || !mirror_ns
->complete
) {
66 return {snap_info_it
->first
, &snap_info_it
->second
};
69 return {CEPH_NOSNAP
, nullptr};
72 } // anonymous namespace
74 using librbd::util::create_async_context_callback
;
75 using librbd::util::create_context_callback
;
76 using librbd::util::create_rados_callback
;
79 struct Replayer
<I
>::C_UpdateWatchCtx
: public librbd::UpdateWatchCtx
{
80 Replayer
<I
>* replayer
;
82 C_UpdateWatchCtx(Replayer
<I
>* replayer
) : replayer(replayer
) {
85 void handle_notify() override
{
86 replayer
->handle_image_update_notify();
91 struct Replayer
<I
>::DeepCopyHandler
: public librbd::deep_copy::Handler
{
94 DeepCopyHandler(Replayer
* replayer
) : replayer(replayer
) {
97 void handle_read(uint64_t bytes_read
) override
{
98 replayer
->handle_copy_image_read(bytes_read
);
101 int update_progress(uint64_t object_number
, uint64_t object_count
) override
{
102 replayer
->handle_copy_image_progress(object_number
, object_count
);
107 template <typename I
>
108 Replayer
<I
>::Replayer(
110 InstanceWatcher
<I
>* instance_watcher
,
111 const std::string
& local_mirror_uuid
,
112 PoolMetaCache
* pool_meta_cache
,
113 StateBuilder
<I
>* state_builder
,
114 ReplayerListener
* replayer_listener
)
115 : m_threads(threads
),
116 m_instance_watcher(instance_watcher
),
117 m_local_mirror_uuid(local_mirror_uuid
),
118 m_pool_meta_cache(pool_meta_cache
),
119 m_state_builder(state_builder
),
120 m_replayer_listener(replayer_listener
),
121 m_lock(ceph::make_mutex(librbd::util::unique_lock_name(
122 "rbd::mirror::image_replayer::snapshot::Replayer", this))) {
126 template <typename I
>
127 Replayer
<I
>::~Replayer() {
129 ceph_assert(m_state
== STATE_COMPLETE
);
130 ceph_assert(m_update_watch_ctx
== nullptr);
131 ceph_assert(m_deep_copy_handler
== nullptr);
134 template <typename I
>
135 void Replayer
<I
>::init(Context
* on_finish
) {
138 ceph_assert(m_state
== STATE_INIT
);
140 RemotePoolMeta remote_pool_meta
;
141 int r
= m_pool_meta_cache
->get_remote_pool_meta(
142 m_state_builder
->remote_image_ctx
->md_ctx
.get_id(), &remote_pool_meta
);
143 if (r
< 0 || remote_pool_meta
.mirror_peer_uuid
.empty()) {
144 derr
<< "failed to retrieve mirror peer uuid from remote pool" << dendl
;
145 m_state
= STATE_COMPLETE
;
146 m_threads
->work_queue
->queue(on_finish
, r
);
150 m_remote_mirror_peer_uuid
= remote_pool_meta
.mirror_peer_uuid
;
151 dout(10) << "remote_mirror_peer_uuid=" << m_remote_mirror_peer_uuid
<< dendl
;
153 ceph_assert(m_on_init_shutdown
== nullptr);
154 m_on_init_shutdown
= on_finish
;
156 register_local_update_watcher();
159 template <typename I
>
160 void Replayer
<I
>::shut_down(Context
* on_finish
) {
163 std::unique_lock locker
{m_lock
};
164 ceph_assert(m_on_init_shutdown
== nullptr);
165 m_on_init_shutdown
= on_finish
;
167 m_error_description
= "";
169 ceph_assert(m_state
!= STATE_INIT
);
170 auto state
= STATE_COMPLETE
;
171 std::swap(m_state
, state
);
173 if (state
== STATE_REPLAYING
) {
174 // if a sync request was pending, request a cancelation
175 m_instance_watcher
->cancel_sync_request(
176 m_state_builder
->local_image_ctx
->id
);
178 // TODO interrupt snapshot copy and image copy state machines even if remote
179 // cluster is unreachable
180 dout(10) << "shut down pending on completion of snapshot replay" << dendl
;
185 unregister_remote_update_watcher();
188 template <typename I
>
189 void Replayer
<I
>::flush(Context
* on_finish
) {
193 m_threads
->work_queue
->queue(on_finish
, 0);
196 template <typename I
>
197 bool Replayer
<I
>::get_replay_status(std::string
* description
,
198 Context
* on_finish
) {
201 std::unique_lock locker
{m_lock
};
202 if (m_state
!= STATE_REPLAYING
&& m_state
!= STATE_IDLE
) {
205 derr
<< "replay not running" << dendl
;
206 on_finish
->complete(-EAGAIN
);
210 std::shared_lock local_image_locker
{
211 m_state_builder
->local_image_ctx
->image_lock
};
212 auto [local_snap_id
, local_snap_info
] = get_newest_mirror_snapshot(
213 m_state_builder
->local_image_ctx
);
215 std::shared_lock remote_image_locker
{
216 m_state_builder
->remote_image_ctx
->image_lock
};
217 auto [remote_snap_id
, remote_snap_info
] = get_newest_mirror_snapshot(
218 m_state_builder
->remote_image_ctx
);
220 if (remote_snap_info
== nullptr) {
221 remote_image_locker
.unlock();
222 local_image_locker
.unlock();
225 derr
<< "remote image does not contain mirror snapshots" << dendl
;
226 on_finish
->complete(-EAGAIN
);
230 std::string replay_state
= "idle";
231 if (m_remote_snap_id_end
!= CEPH_NOSNAP
) {
232 replay_state
= "syncing";
235 json_spirit::mObject root_obj
;
236 root_obj
["replay_state"] = replay_state
;
237 root_obj
["remote_snapshot_timestamp"] = remote_snap_info
->timestamp
.sec();
239 auto matching_remote_snap_id
= util::compute_remote_snap_id(
240 m_state_builder
->local_image_ctx
->image_lock
,
241 m_state_builder
->local_image_ctx
->snap_info
,
242 local_snap_id
, m_state_builder
->remote_mirror_uuid
);
243 auto matching_remote_snap_it
=
244 m_state_builder
->remote_image_ctx
->snap_info
.find(matching_remote_snap_id
);
245 if (matching_remote_snap_id
!= CEPH_NOSNAP
&&
246 matching_remote_snap_it
!=
247 m_state_builder
->remote_image_ctx
->snap_info
.end()) {
248 // use the timestamp from the matching remote image since
249 // the local snapshot would just be the time the snapshot was
250 // synced and not the consistency point in time.
251 root_obj
["local_snapshot_timestamp"] =
252 matching_remote_snap_it
->second
.timestamp
.sec();
255 matching_remote_snap_it
= m_state_builder
->remote_image_ctx
->snap_info
.find(
256 m_remote_snap_id_end
);
257 if (m_remote_snap_id_end
!= CEPH_NOSNAP
&&
258 matching_remote_snap_it
!=
259 m_state_builder
->remote_image_ctx
->snap_info
.end()) {
260 root_obj
["syncing_snapshot_timestamp"] = remote_snap_info
->timestamp
.sec();
261 root_obj
["syncing_percent"] = static_cast<uint64_t>(
262 100 * m_local_mirror_snap_ns
.last_copied_object_number
/
263 static_cast<float>(std::max
<uint64_t>(1U, m_local_object_count
)));
266 m_bytes_per_second(0);
267 auto bytes_per_second
= m_bytes_per_second
.get_average();
268 root_obj
["bytes_per_second"] = round_to_two_places(bytes_per_second
);
270 auto bytes_per_snapshot
= boost::accumulators::rolling_mean(
271 m_bytes_per_snapshot
);
272 root_obj
["bytes_per_snapshot"] = round_to_two_places(bytes_per_snapshot
);
274 auto pending_bytes
= bytes_per_snapshot
* m_pending_snapshots
;
275 if (bytes_per_second
> 0 && m_pending_snapshots
> 0) {
276 auto seconds_until_synced
= round_to_two_places(
277 pending_bytes
/ bytes_per_second
);
278 if (seconds_until_synced
>= std::numeric_limits
<uint64_t>::max()) {
279 seconds_until_synced
= std::numeric_limits
<uint64_t>::max();
282 root_obj
["seconds_until_synced"] = static_cast<uint64_t>(
283 seconds_until_synced
);
286 *description
= json_spirit::write(
287 root_obj
, json_spirit::remove_trailing_zeros
);
289 local_image_locker
.unlock();
290 remote_image_locker
.unlock();
292 on_finish
->complete(-EEXIST
);
296 template <typename I
>
297 void Replayer
<I
>::load_local_image_meta() {
301 // reset state in case new snapshot is added while we are scanning
302 std::unique_lock locker
{m_lock
};
303 m_image_updated
= false;
306 ceph_assert(m_state_builder
->local_image_meta
!= nullptr);
307 auto ctx
= create_context_callback
<
308 Replayer
<I
>, &Replayer
<I
>::handle_load_local_image_meta
>(this);
309 m_state_builder
->local_image_meta
->load(ctx
);
312 template <typename I
>
313 void Replayer
<I
>::handle_load_local_image_meta(int r
) {
314 dout(10) << "r=" << r
<< dendl
;
316 if (r
< 0 && r
!= -ENOENT
) {
317 derr
<< "failed to load local image-meta: " << cpp_strerror(r
) << dendl
;
318 handle_replay_complete(r
, "failed to load local image-meta");
322 if (r
>= 0 && m_state_builder
->local_image_meta
->resync_requested
) {
323 m_resync_requested
= true;
325 dout(10) << "local image resync requested" << dendl
;
326 handle_replay_complete(0, "resync requested");
330 refresh_local_image();
333 template <typename I
>
334 void Replayer
<I
>::refresh_local_image() {
335 if (!m_state_builder
->local_image_ctx
->state
->is_refresh_required()) {
336 refresh_remote_image();
341 auto ctx
= create_context_callback
<
342 Replayer
<I
>, &Replayer
<I
>::handle_refresh_local_image
>(this);
343 m_state_builder
->local_image_ctx
->state
->refresh(ctx
);
346 template <typename I
>
347 void Replayer
<I
>::handle_refresh_local_image(int r
) {
348 dout(10) << "r=" << r
<< dendl
;
351 derr
<< "failed to refresh local image: " << cpp_strerror(r
) << dendl
;
352 handle_replay_complete(r
, "failed to refresh local image");
356 refresh_remote_image();
359 template <typename I
>
360 void Replayer
<I
>::refresh_remote_image() {
361 if (!m_state_builder
->remote_image_ctx
->state
->is_refresh_required()) {
362 std::unique_lock locker
{m_lock
};
363 scan_local_mirror_snapshots(&locker
);
368 auto ctx
= create_context_callback
<
369 Replayer
<I
>, &Replayer
<I
>::handle_refresh_remote_image
>(this);
370 m_state_builder
->remote_image_ctx
->state
->refresh(ctx
);
373 template <typename I
>
374 void Replayer
<I
>::handle_refresh_remote_image(int r
) {
375 dout(10) << "r=" << r
<< dendl
;
378 derr
<< "failed to refresh remote image: " << cpp_strerror(r
) << dendl
;
379 handle_replay_complete(r
, "failed to refresh remote image");
383 std::unique_lock locker
{m_lock
};
384 scan_local_mirror_snapshots(&locker
);
387 template <typename I
>
388 void Replayer
<I
>::scan_local_mirror_snapshots(
389 std::unique_lock
<ceph::mutex
>* locker
) {
390 if (is_replay_interrupted(locker
)) {
396 m_local_snap_id_start
= 0;
397 m_local_snap_id_end
= CEPH_NOSNAP
;
398 m_local_mirror_snap_ns
= {};
399 m_local_object_count
= 0;
401 m_remote_snap_id_start
= 0;
402 m_remote_snap_id_end
= CEPH_NOSNAP
;
403 m_remote_mirror_snap_ns
= {};
405 std::set
<uint64_t> prune_snap_ids
;
407 auto local_image_ctx
= m_state_builder
->local_image_ctx
;
408 std::shared_lock image_locker
{local_image_ctx
->image_lock
};
409 for (auto snap_info_it
= local_image_ctx
->snap_info
.begin();
410 snap_info_it
!= local_image_ctx
->snap_info
.end(); ++snap_info_it
) {
411 const auto& snap_ns
= snap_info_it
->second
.snap_namespace
;
412 auto mirror_ns
= boost::get
<
413 cls::rbd::MirrorSnapshotNamespace
>(&snap_ns
);
414 if (mirror_ns
== nullptr) {
418 dout(15) << "local mirror snapshot: id=" << snap_info_it
->first
<< ", "
419 << "mirror_ns=" << *mirror_ns
<< dendl
;
420 m_local_mirror_snap_ns
= *mirror_ns
;
422 auto local_snap_id
= snap_info_it
->first
;
423 if (mirror_ns
->is_non_primary()) {
424 if (mirror_ns
->complete
) {
425 // if remote has new snapshots, we would sync from here
426 m_local_snap_id_start
= local_snap_id
;
427 m_local_snap_id_end
= CEPH_NOSNAP
;
429 if (mirror_ns
->mirror_peer_uuids
.empty()) {
430 // no other peer will attempt to sync to this snapshot so store as
431 // a candidate for removal
432 prune_snap_ids
.insert(local_snap_id
);
435 if (mirror_ns
->last_copied_object_number
== 0) {
436 // snapshot might be missing image state, object-map, etc, so just
437 // delete and re-create it if we haven't started copying data
438 // objects. Also only prune this snapshot since we will need the
439 // previous mirror snapshot for syncing.
440 prune_snap_ids
.clear();
441 prune_snap_ids
.insert(local_snap_id
);
445 // start snap will be last complete mirror snapshot or initial
447 m_local_snap_id_end
= local_snap_id
;
449 } else if (mirror_ns
->is_primary()) {
450 if (mirror_ns
->complete
) {
451 m_local_snap_id_start
= local_snap_id
;
452 m_local_snap_id_end
= CEPH_NOSNAP
;
454 derr
<< "incomplete local primary snapshot" << dendl
;
455 handle_replay_complete(locker
, -EINVAL
,
456 "incomplete local primary snapshot");
460 derr
<< "unknown local mirror snapshot state" << dendl
;
461 handle_replay_complete(locker
, -EINVAL
,
462 "invalid local mirror snapshot state");
466 image_locker
.unlock();
468 if (m_local_snap_id_start
> 0 && m_local_snap_id_end
== CEPH_NOSNAP
) {
469 // remove candidate that is required for delta snapshot sync
470 prune_snap_ids
.erase(m_local_snap_id_start
);
472 if (!prune_snap_ids
.empty()) {
475 auto prune_snap_id
= *prune_snap_ids
.begin();
476 dout(5) << "pruning unused non-primary snapshot " << prune_snap_id
<< dendl
;
477 prune_non_primary_snapshot(prune_snap_id
);
481 if (m_local_snap_id_start
> 0 || m_local_snap_id_end
!= CEPH_NOSNAP
) {
482 if (m_local_mirror_snap_ns
.is_non_primary() &&
483 m_local_mirror_snap_ns
.primary_mirror_uuid
!=
484 m_state_builder
->remote_mirror_uuid
) {
485 // TODO support multiple peers
486 derr
<< "local image linked to unknown peer: "
487 << m_local_mirror_snap_ns
.primary_mirror_uuid
<< dendl
;
488 handle_replay_complete(locker
, -EEXIST
,
489 "local image linked to unknown peer");
491 } else if (m_local_mirror_snap_ns
.state
==
492 cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY
) {
493 dout(5) << "local image promoted" << dendl
;
494 handle_replay_complete(locker
, 0, "force promoted");
498 dout(10) << "found local mirror snapshot: "
499 << "local_snap_id_start=" << m_local_snap_id_start
<< ", "
500 << "local_snap_id_end=" << m_local_snap_id_end
<< ", "
501 << "local_snap_ns=" << m_local_mirror_snap_ns
<< dendl
;
502 if (!m_local_mirror_snap_ns
.is_primary() &&
503 m_local_mirror_snap_ns
.complete
) {
504 // our remote sync should start after this completed snapshot
505 m_remote_snap_id_start
= m_local_mirror_snap_ns
.primary_snap_id
;
509 // we don't have any mirror snapshots or only completed non-primary
511 scan_remote_mirror_snapshots(locker
);
514 template <typename I
>
515 void Replayer
<I
>::scan_remote_mirror_snapshots(
516 std::unique_lock
<ceph::mutex
>* locker
) {
519 m_pending_snapshots
= 0;
521 std::set
<uint64_t> unlink_snap_ids
;
522 bool split_brain
= false;
523 bool remote_demoted
= false;
524 auto remote_image_ctx
= m_state_builder
->remote_image_ctx
;
525 std::shared_lock image_locker
{remote_image_ctx
->image_lock
};
526 for (auto snap_info_it
= remote_image_ctx
->snap_info
.begin();
527 snap_info_it
!= remote_image_ctx
->snap_info
.end(); ++snap_info_it
) {
528 const auto& snap_ns
= snap_info_it
->second
.snap_namespace
;
529 auto mirror_ns
= boost::get
<
530 cls::rbd::MirrorSnapshotNamespace
>(&snap_ns
);
531 if (mirror_ns
== nullptr) {
535 dout(15) << "remote mirror snapshot: id=" << snap_info_it
->first
<< ", "
536 << "mirror_ns=" << *mirror_ns
<< dendl
;
537 remote_demoted
= mirror_ns
->is_demoted();
538 if (!mirror_ns
->is_primary() && !mirror_ns
->is_non_primary()) {
539 derr
<< "unknown remote mirror snapshot state" << dendl
;
540 handle_replay_complete(locker
, -EINVAL
,
541 "invalid remote mirror snapshot state");
543 } else if (mirror_ns
->mirror_peer_uuids
.count(m_remote_mirror_peer_uuid
) ==
545 dout(15) << "skipping remote snapshot due to missing mirror peer"
550 auto remote_snap_id
= snap_info_it
->first
;
551 if (m_local_snap_id_start
> 0 || m_local_snap_id_end
!= CEPH_NOSNAP
) {
552 // we have a local mirror snapshot
553 if (m_local_mirror_snap_ns
.is_non_primary()) {
554 // previously validated that it was linked to remote
555 ceph_assert(m_local_mirror_snap_ns
.primary_mirror_uuid
==
556 m_state_builder
->remote_mirror_uuid
);
558 unlink_snap_ids
.insert(remote_snap_id
);
559 if (m_local_mirror_snap_ns
.complete
&&
560 m_local_mirror_snap_ns
.primary_snap_id
>= remote_snap_id
) {
561 // skip past completed remote snapshot
562 m_remote_snap_id_start
= remote_snap_id
;
563 m_remote_mirror_snap_ns
= *mirror_ns
;
564 dout(15) << "skipping synced remote snapshot " << remote_snap_id
567 } else if (!m_local_mirror_snap_ns
.complete
&&
568 m_local_mirror_snap_ns
.primary_snap_id
> remote_snap_id
) {
569 // skip until we get to the in-progress remote snapshot
570 dout(15) << "skipping synced remote snapshot " << remote_snap_id
571 << " while search for in-progress sync" << dendl
;
572 m_remote_snap_id_start
= remote_snap_id
;
573 m_remote_mirror_snap_ns
= *mirror_ns
;
576 } else if (m_local_mirror_snap_ns
.state
==
577 cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED
) {
578 // find the matching demotion snapshot in remote image
579 ceph_assert(m_local_snap_id_start
> 0);
580 if (mirror_ns
->state
==
581 cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED
&&
582 mirror_ns
->primary_mirror_uuid
== m_local_mirror_uuid
&&
583 mirror_ns
->primary_snap_id
== m_local_snap_id_start
) {
584 dout(10) << "located matching demotion snapshot: "
585 << "remote_snap_id=" << remote_snap_id
<< ", "
586 << "local_snap_id=" << m_local_snap_id_start
<< dendl
;
587 m_remote_snap_id_start
= remote_snap_id
;
590 } else if (m_remote_snap_id_start
== 0) {
591 // still looking for our matching demotion snapshot
592 dout(15) << "skipping remote snapshot " << remote_snap_id
<< " "
593 << "while searching for demotion" << dendl
;
598 // should not have been able to reach this
601 } else if (!mirror_ns
->is_primary()) {
602 dout(15) << "skipping non-primary remote snapshot" << dendl
;
606 // found candidate snapshot to sync
607 ++m_pending_snapshots
;
608 if (m_remote_snap_id_end
!= CEPH_NOSNAP
) {
612 // first primary snapshot where were are listed as a peer
613 m_remote_snap_id_end
= remote_snap_id
;
614 m_remote_mirror_snap_ns
= *mirror_ns
;
616 image_locker
.unlock();
618 unlink_snap_ids
.erase(m_remote_snap_id_start
);
619 unlink_snap_ids
.erase(m_remote_snap_id_end
);
620 if (!unlink_snap_ids
.empty()) {
623 // retry the unlinking process for a remote snapshot that we do not
625 auto remote_snap_id
= *unlink_snap_ids
.begin();
626 dout(10) << "unlinking from remote snapshot " << remote_snap_id
<< dendl
;
627 unlink_peer(remote_snap_id
);
631 if (m_remote_snap_id_end
!= CEPH_NOSNAP
) {
632 dout(10) << "found remote mirror snapshot: "
633 << "remote_snap_id_start=" << m_remote_snap_id_start
<< ", "
634 << "remote_snap_id_end=" << m_remote_snap_id_end
<< ", "
635 << "remote_snap_ns=" << m_remote_mirror_snap_ns
<< dendl
;
636 if (m_remote_mirror_snap_ns
.complete
) {
639 if (m_local_snap_id_end
!= CEPH_NOSNAP
&&
640 !m_local_mirror_snap_ns
.complete
) {
641 // attempt to resume image-sync
642 dout(10) << "local image contains in-progress mirror snapshot"
644 get_local_image_state();
650 // might have raced with the creation of a remote mirror snapshot
651 // so we will need to refresh and rescan once it completes
652 dout(15) << "remote mirror snapshot not complete" << dendl
;
656 if (m_image_updated
) {
657 // received update notification while scanning image, restart ...
658 m_image_updated
= false;
661 dout(10) << "restarting snapshot scan due to remote update notification"
663 load_local_image_meta();
667 if (is_replay_interrupted(locker
)) {
669 } else if (split_brain
) {
670 derr
<< "split-brain detected: failed to find matching non-primary "
671 << "snapshot in remote image: "
672 << "local_snap_id_start=" << m_local_snap_id_start
<< ", "
673 << "local_snap_ns=" << m_local_mirror_snap_ns
<< dendl
;
674 handle_replay_complete(locker
, -EEXIST
, "split-brain");
676 } else if (remote_demoted
) {
677 dout(10) << "remote image demoted" << dendl
;
678 handle_replay_complete(locker
, -EREMOTEIO
, "remote image demoted");
682 dout(10) << "all remote snapshots synced: idling waiting for new snapshot"
684 ceph_assert(m_state
== STATE_REPLAYING
);
685 m_state
= STATE_IDLE
;
687 notify_status_updated();
690 template <typename I
>
691 void Replayer
<I
>::prune_non_primary_snapshot(uint64_t snap_id
) {
692 dout(10) << "snap_id=" << snap_id
<< dendl
;
694 auto local_image_ctx
= m_state_builder
->local_image_ctx
;
695 bool snap_valid
= false;
696 cls::rbd::SnapshotNamespace snap_namespace
;
697 std::string snap_name
;
700 std::shared_lock image_locker
{local_image_ctx
->image_lock
};
701 auto snap_info
= local_image_ctx
->get_snap_info(snap_id
);
702 if (snap_info
!= nullptr) {
704 snap_namespace
= snap_info
->snap_namespace
;
705 snap_name
= snap_info
->name
;
707 ceph_assert(boost::get
<cls::rbd::MirrorSnapshotNamespace
>(
708 &snap_namespace
) != nullptr);
713 load_local_image_meta();
717 auto ctx
= create_context_callback
<
718 Replayer
<I
>, &Replayer
<I
>::handle_prune_non_primary_snapshot
>(this);
719 local_image_ctx
->operations
->snap_remove(snap_namespace
, snap_name
, ctx
);
722 template <typename I
>
723 void Replayer
<I
>::handle_prune_non_primary_snapshot(int r
) {
724 dout(10) << "r=" << r
<< dendl
;
726 if (r
< 0 && r
!= -ENOENT
) {
727 derr
<< "failed to prune non-primary snapshot: " << cpp_strerror(r
)
729 handle_replay_complete(r
, "failed to prune non-primary snapshot");
733 if (is_replay_interrupted()) {
737 load_local_image_meta();
740 template <typename I
>
741 void Replayer
<I
>::copy_snapshots() {
742 dout(10) << "remote_snap_id_start=" << m_remote_snap_id_start
<< ", "
743 << "remote_snap_id_end=" << m_remote_snap_id_end
<< ", "
744 << "local_snap_id_start=" << m_local_snap_id_start
<< dendl
;
746 ceph_assert(m_remote_snap_id_start
!= CEPH_NOSNAP
);
747 ceph_assert(m_remote_snap_id_end
> 0 &&
748 m_remote_snap_id_end
!= CEPH_NOSNAP
);
749 ceph_assert(m_local_snap_id_start
!= CEPH_NOSNAP
);
751 m_local_mirror_snap_ns
= {};
752 auto ctx
= create_context_callback
<
753 Replayer
<I
>, &Replayer
<I
>::handle_copy_snapshots
>(this);
754 auto req
= librbd::deep_copy::SnapshotCopyRequest
<I
>::create(
755 m_state_builder
->remote_image_ctx
, m_state_builder
->local_image_ctx
,
756 m_remote_snap_id_start
, m_remote_snap_id_end
, m_local_snap_id_start
,
757 false, m_threads
->work_queue
, &m_local_mirror_snap_ns
.snap_seqs
,
762 template <typename I
>
763 void Replayer
<I
>::handle_copy_snapshots(int r
) {
764 dout(10) << "r=" << r
<< dendl
;
767 derr
<< "failed to copy snapshots from remote to local image: "
768 << cpp_strerror(r
) << dendl
;
769 handle_replay_complete(
770 r
, "failed to copy snapshots from remote to local image");
774 dout(10) << "remote_snap_id_start=" << m_remote_snap_id_start
<< ", "
775 << "remote_snap_id_end=" << m_remote_snap_id_end
<< ", "
776 << "local_snap_id_start=" << m_local_snap_id_start
<< ", "
777 << "snap_seqs=" << m_local_mirror_snap_ns
.snap_seqs
<< dendl
;
778 get_remote_image_state();
781 template <typename I
>
782 void Replayer
<I
>::get_remote_image_state() {
785 auto ctx
= create_context_callback
<
786 Replayer
<I
>, &Replayer
<I
>::handle_get_remote_image_state
>(this);
787 auto req
= librbd::mirror::snapshot::GetImageStateRequest
<I
>::create(
788 m_state_builder
->remote_image_ctx
, m_remote_snap_id_end
,
789 &m_image_state
, ctx
);
793 template <typename I
>
794 void Replayer
<I
>::handle_get_remote_image_state(int r
) {
795 dout(10) << "r=" << r
<< dendl
;
798 derr
<< "failed to retrieve remote snapshot image state: "
799 << cpp_strerror(r
) << dendl
;
800 handle_replay_complete(r
, "failed to retrieve remote snapshot image state");
804 create_non_primary_snapshot();
807 template <typename I
>
808 void Replayer
<I
>::get_local_image_state() {
811 ceph_assert(m_local_snap_id_end
!= CEPH_NOSNAP
);
812 auto ctx
= create_context_callback
<
813 Replayer
<I
>, &Replayer
<I
>::handle_get_local_image_state
>(this);
814 auto req
= librbd::mirror::snapshot::GetImageStateRequest
<I
>::create(
815 m_state_builder
->local_image_ctx
, m_local_snap_id_end
,
816 &m_image_state
, ctx
);
820 template <typename I
>
821 void Replayer
<I
>::handle_get_local_image_state(int r
) {
822 dout(10) << "r=" << r
<< dendl
;
825 derr
<< "failed to retrieve local snapshot image state: "
826 << cpp_strerror(r
) << dendl
;
827 handle_replay_complete(r
, "failed to retrieve local snapshot image state");
834 template <typename I
>
835 void Replayer
<I
>::create_non_primary_snapshot() {
836 auto local_image_ctx
= m_state_builder
->local_image_ctx
;
838 if (m_local_snap_id_start
> 0) {
839 std::shared_lock local_image_locker
{local_image_ctx
->image_lock
};
841 auto local_snap_info_it
= local_image_ctx
->snap_info
.find(
842 m_local_snap_id_start
);
843 if (local_snap_info_it
== local_image_ctx
->snap_info
.end()) {
844 local_image_locker
.unlock();
846 derr
<< "failed to locate local snapshot " << m_local_snap_id_start
848 handle_replay_complete(-ENOENT
, "failed to locate local start snapshot");
852 auto mirror_ns
= boost::get
<cls::rbd::MirrorSnapshotNamespace
>(
853 &local_snap_info_it
->second
.snap_namespace
);
854 ceph_assert(mirror_ns
!= nullptr);
856 auto remote_image_ctx
= m_state_builder
->remote_image_ctx
;
857 std::shared_lock remote_image_locker
{remote_image_ctx
->image_lock
};
859 // (re)build a full mapping from remote to local snap ids for all user
860 // snapshots to support applying image state in the future
861 for (auto& [remote_snap_id
, remote_snap_info
] :
862 remote_image_ctx
->snap_info
) {
863 if (remote_snap_id
>= m_remote_snap_id_end
) {
867 // we can ignore all non-user snapshots since image state only includes
869 if (boost::get
<cls::rbd::UserSnapshotNamespace
>(
870 &remote_snap_info
.snap_namespace
) == nullptr) {
874 uint64_t local_snap_id
= CEPH_NOSNAP
;
875 if (mirror_ns
->is_demoted() && !m_remote_mirror_snap_ns
.is_demoted()) {
876 // if we are creating a non-primary snapshot following a demotion,
877 // re-build the full snapshot sequence since we don't have a valid
879 auto local_snap_id_it
= local_image_ctx
->snap_ids
.find(
880 {remote_snap_info
.snap_namespace
, remote_snap_info
.name
});
881 if (local_snap_id_it
!= local_image_ctx
->snap_ids
.end()) {
882 local_snap_id
= local_snap_id_it
->second
;
885 auto snap_seq_it
= mirror_ns
->snap_seqs
.find(remote_snap_id
);
886 if (snap_seq_it
!= mirror_ns
->snap_seqs
.end()) {
887 local_snap_id
= snap_seq_it
->second
;
891 if (m_local_mirror_snap_ns
.snap_seqs
.count(remote_snap_id
) == 0 &&
892 local_snap_id
!= CEPH_NOSNAP
) {
893 dout(15) << "mapping remote snapshot " << remote_snap_id
<< " to "
894 << "local snapshot " << local_snap_id
<< dendl
;
895 m_local_mirror_snap_ns
.snap_seqs
[remote_snap_id
] = local_snap_id
;
900 dout(10) << "demoted=" << m_remote_mirror_snap_ns
.is_demoted() << ", "
901 << "primary_mirror_uuid="
902 << m_state_builder
->remote_mirror_uuid
<< ", "
903 << "primary_snap_id=" << m_remote_snap_id_end
<< ", "
904 << "snap_seqs=" << m_local_mirror_snap_ns
.snap_seqs
<< dendl
;
906 auto ctx
= create_context_callback
<
907 Replayer
<I
>, &Replayer
<I
>::handle_create_non_primary_snapshot
>(this);
908 auto req
= librbd::mirror::snapshot::CreateNonPrimaryRequest
<I
>::create(
909 local_image_ctx
, m_remote_mirror_snap_ns
.is_demoted(),
910 m_state_builder
->remote_mirror_uuid
, m_remote_snap_id_end
,
911 m_local_mirror_snap_ns
.snap_seqs
, m_image_state
, &m_local_snap_id_end
, ctx
);
915 template <typename I
>
916 void Replayer
<I
>::handle_create_non_primary_snapshot(int r
) {
917 dout(10) << "r=" << r
<< dendl
;
920 derr
<< "failed to create local mirror snapshot: " << cpp_strerror(r
)
922 handle_replay_complete(r
, "failed to create local mirror snapshot");
926 dout(15) << "local_snap_id_end=" << m_local_snap_id_end
<< dendl
;
931 template <typename I
>
932 void Replayer
<I
>::request_sync() {
933 if (m_remote_mirror_snap_ns
.clean_since_snap_id
== m_remote_snap_id_start
) {
934 dout(10) << "skipping unnecessary image copy: "
935 << "remote_snap_id_start=" << m_remote_snap_id_start
<< ", "
936 << "remote_mirror_snap_ns=" << m_remote_mirror_snap_ns
<< dendl
;
942 std::unique_lock locker
{m_lock
};
943 if (is_replay_interrupted(&locker
)) {
947 auto ctx
= create_async_context_callback(
948 m_threads
->work_queue
, create_context_callback
<
949 Replayer
<I
>, &Replayer
<I
>::handle_request_sync
>(this));
950 m_instance_watcher
->notify_sync_request(m_state_builder
->local_image_ctx
->id
,
954 template <typename I
>
955 void Replayer
<I
>::handle_request_sync(int r
) {
956 dout(10) << "r=" << r
<< dendl
;
958 std::unique_lock locker
{m_lock
};
959 if (is_replay_interrupted(&locker
)) {
961 } else if (r
== -ECANCELED
) {
962 dout(5) << "image-sync canceled" << dendl
;
963 handle_replay_complete(&locker
, r
, "image-sync canceled");
966 derr
<< "failed to request image-sync: " << cpp_strerror(r
) << dendl
;
967 handle_replay_complete(&locker
, r
, "failed to request image-sync");
971 m_sync_in_progress
= true;
977 template <typename I
>
978 void Replayer
<I
>::copy_image() {
979 dout(10) << "remote_snap_id_start=" << m_remote_snap_id_start
<< ", "
980 << "remote_snap_id_end=" << m_remote_snap_id_end
<< ", "
981 << "local_snap_id_start=" << m_local_snap_id_start
<< ", "
982 << "last_copied_object_number="
983 << m_local_mirror_snap_ns
.last_copied_object_number
<< ", "
984 << "snap_seqs=" << m_local_mirror_snap_ns
.snap_seqs
<< dendl
;
986 m_snapshot_bytes
= 0;
987 m_deep_copy_handler
= new DeepCopyHandler(this);
988 auto ctx
= create_context_callback
<
989 Replayer
<I
>, &Replayer
<I
>::handle_copy_image
>(this);
990 auto req
= librbd::deep_copy::ImageCopyRequest
<I
>::create(
991 m_state_builder
->remote_image_ctx
, m_state_builder
->local_image_ctx
,
992 m_remote_snap_id_start
, m_remote_snap_id_end
, m_local_snap_id_start
, false,
993 (m_local_mirror_snap_ns
.last_copied_object_number
> 0 ?
994 librbd::deep_copy::ObjectNumber
{
995 m_local_mirror_snap_ns
.last_copied_object_number
} :
996 librbd::deep_copy::ObjectNumber
{}),
997 m_local_mirror_snap_ns
.snap_seqs
, m_deep_copy_handler
, ctx
);
1001 template <typename I
>
1002 void Replayer
<I
>::handle_copy_image(int r
) {
1003 dout(10) << "r=" << r
<< dendl
;
1005 delete m_deep_copy_handler
;
1006 m_deep_copy_handler
= nullptr;
1009 derr
<< "failed to copy remote image to local image: " << cpp_strerror(r
)
1011 handle_replay_complete(r
, "failed to copy remote image");
1016 std::unique_lock locker
{m_lock
};
1017 m_bytes_per_snapshot(m_snapshot_bytes
);
1018 m_snapshot_bytes
= 0;
1021 apply_image_state();
1024 template <typename I
>
1025 void Replayer
<I
>::handle_copy_image_progress(uint64_t object_number
,
1026 uint64_t object_count
) {
1027 dout(10) << "object_number=" << object_number
<< ", "
1028 << "object_count=" << object_count
<< dendl
;
1030 std::unique_lock locker
{m_lock
};
1031 m_local_mirror_snap_ns
.last_copied_object_number
= std::min(
1032 object_number
, object_count
);
1033 m_local_object_count
= object_count
;
1035 update_non_primary_snapshot(false);
1038 template <typename I
>
1039 void Replayer
<I
>::handle_copy_image_read(uint64_t bytes_read
) {
1040 dout(20) << "bytes_read=" << bytes_read
<< dendl
;
1042 std::unique_lock locker
{m_lock
};
1043 m_bytes_per_second(bytes_read
);
1044 m_snapshot_bytes
+= bytes_read
;
1047 template <typename I
>
1048 void Replayer
<I
>::apply_image_state() {
1051 auto ctx
= create_context_callback
<
1052 Replayer
<I
>, &Replayer
<I
>::handle_apply_image_state
>(this);
1053 auto req
= ApplyImageStateRequest
<I
>::create(
1054 m_local_mirror_uuid
,
1055 m_state_builder
->remote_mirror_uuid
,
1056 m_state_builder
->local_image_ctx
,
1057 m_state_builder
->remote_image_ctx
,
1058 m_image_state
, ctx
);
1062 template <typename I
>
1063 void Replayer
<I
>::handle_apply_image_state(int r
) {
1064 dout(10) << "r=" << r
<< dendl
;
1066 if (r
< 0 && r
!= -ENOENT
) {
1067 derr
<< "failed to apply remote image state to local image: "
1068 << cpp_strerror(r
) << dendl
;
1069 handle_replay_complete(r
, "failed to apply remote image state");
1073 std::unique_lock locker
{m_lock
};
1074 update_non_primary_snapshot(true);
1077 template <typename I
>
1078 void Replayer
<I
>::update_non_primary_snapshot(bool complete
) {
1079 ceph_assert(ceph_mutex_is_locked_by_me(m_lock
));
1081 // disallow two in-flight updates if this isn't the completion of the sync
1082 if (m_updating_sync_point
) {
1085 m_updating_sync_point
= true;
1087 m_local_mirror_snap_ns
.complete
= true;
1092 librados::ObjectWriteOperation op
;
1093 librbd::cls_client::mirror_image_snapshot_set_copy_progress(
1094 &op
, m_local_snap_id_end
, m_local_mirror_snap_ns
.complete
,
1095 m_local_mirror_snap_ns
.last_copied_object_number
);
1097 auto ctx
= new C_TrackedOp(
1098 m_in_flight_op_tracker
, new LambdaContext([this, complete
](int r
) {
1099 handle_update_non_primary_snapshot(complete
, r
);
1101 auto aio_comp
= create_rados_callback(ctx
);
1102 int r
= m_state_builder
->local_image_ctx
->md_ctx
.aio_operate(
1103 m_state_builder
->local_image_ctx
->header_oid
, aio_comp
, &op
);
1104 ceph_assert(r
== 0);
1105 aio_comp
->release();
1108 template <typename I
>
1109 void Replayer
<I
>::handle_update_non_primary_snapshot(bool complete
, int r
) {
1110 dout(10) << "r=" << r
<< dendl
;
1113 derr
<< "failed to update local snapshot progress: " << cpp_strerror(r
)
1116 // only fail if this was the final update
1117 handle_replay_complete(r
, "failed to update local snapshot progress");
1123 // periodic sync-point update -- do not advance state machine
1124 std::unique_lock locker
{m_lock
};
1126 ceph_assert(m_updating_sync_point
);
1127 m_updating_sync_point
= false;
1131 notify_image_update();
1134 template <typename I
>
1135 void Replayer
<I
>::notify_image_update() {
1138 auto ctx
= create_context_callback
<
1139 Replayer
<I
>, &Replayer
<I
>::handle_notify_image_update
>(this);
1140 m_state_builder
->local_image_ctx
->notify_update(ctx
);
1143 template <typename I
>
1144 void Replayer
<I
>::handle_notify_image_update(int r
) {
1145 dout(10) << "r=" << r
<< dendl
;
1148 derr
<< "failed to notify local image update: " << cpp_strerror(r
) << dendl
;
1151 unlink_peer(m_remote_snap_id_start
);
1154 template <typename I
>
1155 void Replayer
<I
>::unlink_peer(uint64_t remote_snap_id
) {
1156 if (remote_snap_id
== 0) {
1161 // local snapshot fully synced -- we no longer depend on the sync
1162 // start snapshot in the remote image
1163 dout(10) << "remote_snap_id=" << remote_snap_id
<< dendl
;
1165 auto ctx
= create_context_callback
<
1166 Replayer
<I
>, &Replayer
<I
>::handle_unlink_peer
>(this);
1167 auto req
= librbd::mirror::snapshot::UnlinkPeerRequest
<I
>::create(
1168 m_state_builder
->remote_image_ctx
, remote_snap_id
,
1169 m_remote_mirror_peer_uuid
, ctx
);
1173 template <typename I
>
1174 void Replayer
<I
>::handle_unlink_peer(int r
) {
1175 dout(10) << "r=" << r
<< dendl
;
1177 if (r
< 0 && r
!= -ENOENT
) {
1178 derr
<< "failed to unlink local peer from remote image: " << cpp_strerror(r
)
1180 handle_replay_complete(r
, "failed to unlink local peer from remote image");
1187 template <typename I
>
1188 void Replayer
<I
>::finish_sync() {
1192 std::unique_lock locker
{m_lock
};
1193 notify_status_updated();
1195 if (m_sync_in_progress
) {
1196 m_sync_in_progress
= false;
1197 m_instance_watcher
->notify_sync_complete(
1198 m_state_builder
->local_image_ctx
->id
);
1202 if (is_replay_interrupted()) {
1206 load_local_image_meta();
1209 template <typename I
>
1210 void Replayer
<I
>::register_local_update_watcher() {
1213 m_update_watch_ctx
= new C_UpdateWatchCtx(this);
1215 int r
= m_state_builder
->local_image_ctx
->state
->register_update_watcher(
1216 m_update_watch_ctx
, &m_local_update_watcher_handle
);
1217 auto ctx
= create_context_callback
<
1218 Replayer
<I
>, &Replayer
<I
>::handle_register_local_update_watcher
>(this);
1219 m_threads
->work_queue
->queue(ctx
, r
);
1222 template <typename I
>
1223 void Replayer
<I
>::handle_register_local_update_watcher(int r
) {
1224 dout(10) << "r=" << r
<< dendl
;
1227 derr
<< "failed to register local update watcher: " << cpp_strerror(r
)
1229 handle_replay_complete(r
, "failed to register local image update watcher");
1230 m_state
= STATE_COMPLETE
;
1232 delete m_update_watch_ctx
;
1233 m_update_watch_ctx
= nullptr;
1235 Context
* on_init
= nullptr;
1236 std::swap(on_init
, m_on_init_shutdown
);
1237 on_init
->complete(r
);
1241 register_remote_update_watcher();
1244 template <typename I
>
1245 void Replayer
<I
>::register_remote_update_watcher() {
1248 int r
= m_state_builder
->remote_image_ctx
->state
->register_update_watcher(
1249 m_update_watch_ctx
, &m_remote_update_watcher_handle
);
1250 auto ctx
= create_context_callback
<
1251 Replayer
<I
>, &Replayer
<I
>::handle_register_remote_update_watcher
>(this);
1252 m_threads
->work_queue
->queue(ctx
, r
);
1255 template <typename I
>
1256 void Replayer
<I
>::handle_register_remote_update_watcher(int r
) {
1257 dout(10) << "r=" << r
<< dendl
;
1260 derr
<< "failed to register remote update watcher: " << cpp_strerror(r
)
1262 handle_replay_complete(r
, "failed to register remote image update watcher");
1263 m_state
= STATE_COMPLETE
;
1265 unregister_local_update_watcher();
1269 m_state
= STATE_REPLAYING
;
1271 Context
* on_init
= nullptr;
1272 std::swap(on_init
, m_on_init_shutdown
);
1273 on_init
->complete(0);
1275 // delay initial snapshot scan until after we have alerted
1276 // image replayer that we have initialized in case an error
1279 std::unique_lock locker
{m_lock
};
1280 notify_status_updated();
1283 load_local_image_meta();
1286 template <typename I
>
1287 void Replayer
<I
>::unregister_remote_update_watcher() {
1290 auto ctx
= create_context_callback
<
1292 &Replayer
<I
>::handle_unregister_remote_update_watcher
>(this);
1293 m_state_builder
->remote_image_ctx
->state
->unregister_update_watcher(
1294 m_remote_update_watcher_handle
, ctx
);
1297 template <typename I
>
1298 void Replayer
<I
>::handle_unregister_remote_update_watcher(int r
) {
1299 dout(10) << "r=" << r
<< dendl
;
1302 derr
<< "failed to unregister remote update watcher: " << cpp_strerror(r
)
1304 handle_replay_complete(
1305 r
, "failed to unregister remote image update watcher");
1308 unregister_local_update_watcher();
1311 template <typename I
>
1312 void Replayer
<I
>::unregister_local_update_watcher() {
1315 auto ctx
= create_context_callback
<
1317 &Replayer
<I
>::handle_unregister_local_update_watcher
>(this);
1318 m_state_builder
->local_image_ctx
->state
->unregister_update_watcher(
1319 m_local_update_watcher_handle
, ctx
);
1322 template <typename I
>
1323 void Replayer
<I
>::handle_unregister_local_update_watcher(int r
) {
1324 dout(10) << "r=" << r
<< dendl
;
1327 derr
<< "failed to unregister local update watcher: " << cpp_strerror(r
)
1329 handle_replay_complete(
1330 r
, "failed to unregister local image update watcher");
1333 delete m_update_watch_ctx
;
1334 m_update_watch_ctx
= nullptr;
1336 wait_for_in_flight_ops();
1339 template <typename I
>
1340 void Replayer
<I
>::wait_for_in_flight_ops() {
1343 auto ctx
= create_async_context_callback(
1344 m_threads
->work_queue
, create_context_callback
<
1345 Replayer
<I
>, &Replayer
<I
>::handle_wait_for_in_flight_ops
>(this));
1346 m_in_flight_op_tracker
.wait_for_ops(ctx
);
1349 template <typename I
>
1350 void Replayer
<I
>::handle_wait_for_in_flight_ops(int r
) {
1351 dout(10) << "r=" << r
<< dendl
;
1353 Context
* on_shutdown
= nullptr;
1355 std::unique_lock locker
{m_lock
};
1356 ceph_assert(m_on_init_shutdown
!= nullptr);
1357 std::swap(on_shutdown
, m_on_init_shutdown
);
1359 on_shutdown
->complete(m_error_code
);
1362 template <typename I
>
1363 void Replayer
<I
>::handle_image_update_notify() {
1366 std::unique_lock locker
{m_lock
};
1367 if (m_state
== STATE_REPLAYING
) {
1368 dout(15) << "flagging snapshot rescan required" << dendl
;
1369 m_image_updated
= true;
1370 } else if (m_state
== STATE_IDLE
) {
1371 m_state
= STATE_REPLAYING
;
1374 dout(15) << "restarting idle replayer" << dendl
;
1375 load_local_image_meta();
1379 template <typename I
>
1380 void Replayer
<I
>::handle_replay_complete(int r
,
1381 const std::string
& description
) {
1382 std::unique_lock locker
{m_lock
};
1383 handle_replay_complete(&locker
, r
, description
);
1386 template <typename I
>
1387 void Replayer
<I
>::handle_replay_complete(std::unique_lock
<ceph::mutex
>* locker
,
1389 const std::string
& description
) {
1390 ceph_assert(ceph_mutex_is_locked_by_me(m_lock
));
1392 if (m_error_code
== 0) {
1394 m_error_description
= description
;
1397 if (m_sync_in_progress
) {
1398 m_sync_in_progress
= false;
1399 m_instance_watcher
->notify_sync_complete(
1400 m_state_builder
->local_image_ctx
->id
);
1403 if (m_state
!= STATE_REPLAYING
&& m_state
!= STATE_IDLE
) {
1407 m_state
= STATE_COMPLETE
;
1408 notify_status_updated();
1411 template <typename I
>
1412 void Replayer
<I
>::notify_status_updated() {
1413 ceph_assert(ceph_mutex_is_locked_by_me(m_lock
));
1416 auto ctx
= new C_TrackedOp(m_in_flight_op_tracker
, new LambdaContext(
1418 m_replayer_listener
->handle_notification();
1420 m_threads
->work_queue
->queue(ctx
, 0);
1423 template <typename I
>
1424 bool Replayer
<I
>::is_replay_interrupted() {
1425 std::unique_lock locker
{m_lock
};
1426 return is_replay_interrupted(&locker
);
1429 template <typename I
>
1430 bool Replayer
<I
>::is_replay_interrupted(std::unique_lock
<ceph::mutex
>* locker
) {
1431 if (m_state
== STATE_COMPLETE
) {
1434 dout(10) << "resuming pending shut down" << dendl
;
1435 unregister_remote_update_watcher();
1441 } // namespace snapshot
1442 } // namespace image_replayer
1443 } // namespace mirror
1446 template class rbd::mirror::image_replayer::snapshot::Replayer
<librbd::ImageCtx
>;