X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=ceph%2Fsrc%2Ftest%2Frbd_mirror%2Fimage_replayer%2Fsnapshot%2Ftest_mock_Replayer.cc;h=26b4d32cc4ffd6bdaef6588d61b55d7f62669c9d;hb=2a845540123ad00df2e55947b8080306ebdcf410;hp=b2e95bad097e146e0d99d4d3d39afa0774638c38;hpb=9f95a23ce0ce15b5cd2b85dc8aaf3906fbf7f463;p=ceph.git diff --git a/ceph/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc b/ceph/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc index b2e95bad0..26b4d32cc 100644 --- a/ceph/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc +++ b/ceph/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc @@ -4,6 +4,7 @@ #include "test/rbd_mirror/test_mock_fixture.h" #include "librbd/deep_copy/ImageCopyRequest.h" #include "librbd/deep_copy/SnapshotCopyRequest.h" +#include "librbd/mirror/ImageStateUpdateRequest.h" #include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" #include "librbd/mirror/snapshot/GetImageStateRequest.h" #include "librbd/mirror/snapshot/ImageMeta.h" @@ -18,9 +19,12 @@ #include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockOperations.h" #include "test/rbd_mirror/mock/MockContextWQ.h" #include "test/rbd_mirror/mock/MockSafeTimer.h" +using namespace std::chrono_literals; + namespace librbd { namespace { @@ -51,7 +55,7 @@ struct ImageCopyRequest { bool flatten, const ObjectNumber &object_number, const SnapSeqs &snap_seqs, - ProgressContext *prog_ctx, + Handler *handler, Context *on_finish) { ceph_assert(s_instance != nullptr); s_instance->src_snap_id_start = src_snap_id_start; @@ -113,6 +117,34 @@ SnapshotCopyRequest* SnapshotCopyRequest::s_ } // namespace deep_copy namespace mirror { + +template <> +struct ImageStateUpdateRequest { + static ImageStateUpdateRequest* s_instance; + static ImageStateUpdateRequest* create( + librados::IoCtx& io_ctx, + const std::string& image_id, + cls::rbd::MirrorImageState mirror_image_state, + const cls::rbd::MirrorImage& mirror_image, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_EQ(cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + mirror_image_state); + EXPECT_EQ(cls::rbd::MirrorImage{}, mirror_image); + s_instance->on_finish = on_finish; + return s_instance; + } + + Context* on_finish = nullptr; + ImageStateUpdateRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +ImageStateUpdateRequest* ImageStateUpdateRequest::s_instance = nullptr; + namespace snapshot { template <> @@ -363,6 +395,7 @@ public: typedef CloseImageRequest MockCloseImageRequest; typedef librbd::deep_copy::ImageCopyRequest MockImageCopyRequest; typedef librbd::deep_copy::SnapshotCopyRequest MockSnapshotCopyRequest; + typedef librbd::mirror::ImageStateUpdateRequest MockImageStateUpdateRequest; typedef librbd::mirror::snapshot::CreateNonPrimaryRequest MockCreateNonPrimaryRequest; typedef librbd::mirror::snapshot::GetImageStateRequest MockGetImageStateRequest; typedef librbd::mirror::snapshot::ImageMeta MockImageMeta; @@ -456,6 +489,22 @@ public: })); } + void expect_prune_non_primary_snapshot(librbd::MockTestImageCtx& mock_image_ctx, + uint64_t snap_id, int r) { + EXPECT_CALL(mock_image_ctx, get_snap_info(snap_id)) + .WillOnce(Invoke([&mock_image_ctx](uint64_t snap_id) -> librbd::SnapInfo* { + auto it = mock_image_ctx.snap_info.find(snap_id); + if (it == mock_image_ctx.snap_info.end()) { + return nullptr; + } + return &it->second; + })); + EXPECT_CALL(*mock_image_ctx.operations, snap_remove(_, _, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + void expect_snapshot_copy(MockSnapshotCopyRequest& mock_snapshot_copy_request, uint64_t src_snap_id_start, uint64_t src_snap_id_end, @@ -501,6 +550,14 @@ public: })); } + void expect_update_mirror_image_state(MockImageStateUpdateRequest& mock_image_state_update_request, + int r) { + EXPECT_CALL(mock_image_state_update_request, send()) + .WillOnce(Invoke([this, &req=mock_image_state_update_request, r]() { + m_threads->work_queue->queue(req.on_finish, r); + })); + } + void expect_notify_sync_request(MockInstanceWatcher& mock_instance_watcher, const std::string& image_id, int r) { EXPECT_CALL(mock_instance_watcher, notify_sync_request(image_id, _)) @@ -514,6 +571,11 @@ public: EXPECT_CALL(mock_instance_watcher, notify_sync_complete(image_id)); } + void expect_cancel_sync_request(MockInstanceWatcher& mock_instance_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_instance_watcher, cancel_sync_request(image_id)); + } + void expect_image_copy(MockImageCopyRequest& mock_image_copy_request, uint64_t src_snap_id_start, uint64_t src_snap_id_end, uint64_t dst_snap_id_start, @@ -563,7 +625,7 @@ public: EXPECT_CALL(get_mock_io_ctx(mock_test_image_ctx.md_ctx), exec(mock_test_image_ctx.header_oid, _, StrEq("rbd"), StrEq("mirror_image_snapshot_set_copy_progress"), - ContentsEqual(bl), _, _)) + ContentsEqual(bl), _, _, _)) .WillOnce(Return(r)); } @@ -691,17 +753,17 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { mock_remote_image_ctx.snap_info = { {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, - "", 0U, true, 0, {}}, + "", CEPH_NOSNAP, true, 0, {}}, 0, {}, 0, 0, {}}}, {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, 0, {}, 0, 0, {}}}, {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""}, - "", 0U, true, 0, {}}, + "", CEPH_NOSNAP, true, 0, {}}, 0, {}, 0, 0, {}}}, {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, - "", 0U, true, 0, {}}, + "", CEPH_NOSNAP, true, 0, {}}, 0, {}, 0, 0, {}}}}; MockThreads mock_threads(m_threads); @@ -745,6 +807,8 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, @@ -763,19 +827,41 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { mock_local_image_ctx, { {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", - 1, true, 0, {}}, + 1, true, 0, {{1, CEPH_NOSNAP}}}, 0, {}, 0, 0, {}}}, }, 0); - expect_is_refresh_required(mock_remote_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {5U, librbd::SnapInfo{"snap5", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); expect_snapshot_copy(mock_snapshot_copy_request, 1, 4, 11, - {{1, 11}, {4, CEPH_NOSNAP}}, 0); + {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 0); expect_get_image_state(mock_get_image_state_request, 4, 0); expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 4, - {{1, 11}, {4, CEPH_NOSNAP}}, 14, 0); + {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 14, + 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); expect_image_copy(mock_image_copy_request, 1, 4, 11, {}, - {{1, 11}, {4, CEPH_NOSNAP}}, 0); + {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 0); expect_apply_image_state(mock_apply_state_request, 0); expect_mirror_image_snapshot_set_copy_progress( mock_local_image_ctx, 14, true, 0, 0); @@ -785,7 +871,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { 0); expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); - // idle + // prune non-primary snap1 expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, true); expect_refresh( @@ -794,12 +880,51 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", 1, true, 0, {}}, 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", 4, true, 0, {}}, 0, {}, 0, 0, {}}}, }, 0); - expect_is_refresh_required(mock_remote_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 4, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); // fire init C_SaferCond init_ctx; @@ -815,7 +940,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) { +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncInitial) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -847,11 +972,11 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) { mock_image_meta, &update_watch_ctx)); - // inject a incomplete sync snapshot + // inject an incomplete sync snapshot with last_copied_object_number > 0 mock_remote_image_ctx.snap_info = { {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, - "", 0U, true, 0, {}}, + "", CEPH_NOSNAP, true, 0, {}}, 0, {}, 0, 0, {}}}}; mock_local_image_ctx.snap_info = { {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ @@ -875,6 +1000,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) { expect_mirror_image_snapshot_set_copy_progress( mock_local_image_ctx, 11, true, 123, 0); expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); // idle expect_load_image_meta(mock_image_meta, false, 0); @@ -899,7 +1025,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) { mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) { +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncDelta) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -931,37 +1057,49 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) { mock_image_meta, &update_watch_ctx)); - // inject a demotion snapshot + // inject an incomplete sync snapshot with last_copied_object_number > 0 + // after a complete snapshot mock_remote_image_ctx.snap_info = { {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, - {"remote mirror peer uuid"}, "", 0U, true, 0, {}}, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, false, 123, {{2, CEPH_NOSNAP}}}, 0, {}, 0, 0, {}}}}; - // sync snap1 + // re-sync snap2 expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, false); expect_is_refresh_required(mock_remote_image_ctx, false); - MockSnapshotCopyRequest mock_snapshot_copy_request; - expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, - 0); MockGetImageStateRequest mock_get_image_state_request; - expect_get_image_state(mock_get_image_state_request, 1, 0); - MockCreateNonPrimaryRequest mock_create_non_primary_request; - expect_create_non_primary_request(mock_create_non_primary_request, - true, "remote mirror uuid", 1, - {{1, CEPH_NOSNAP}}, 11, 0); + expect_get_image_state(mock_get_image_state_request, 12, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; - expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, - {{1, CEPH_NOSNAP}}, 0); + expect_image_copy(mock_image_copy_request, 1, 2, 11, + librbd::deep_copy::ObjectNumber{123U}, + {{2, CEPH_NOSNAP}}, 0); MockApplyImageStateRequest mock_apply_state_request; expect_apply_image_state(mock_apply_state_request, 0); expect_mirror_image_snapshot_set_copy_progress( - mock_local_image_ctx, 11, true, 0, 0); + mock_local_image_ctx, 12, true, 123, 0); expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); - // idle + // prune non-primary snap1 expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, true); expect_refresh( @@ -970,22 +1108,45 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) { cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", 1, true, 0, {}}, 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, }, 0); expect_is_refresh_required(mock_remote_image_ctx, false); // wake-up replayer update_watch_ctx->handle_notify(); - // wait for sync to complete and expect replay complete + // wait for sync to complete ASSERT_EQ(0, wait_for_notification(2)); - ASSERT_FALSE(mock_replayer.is_replaying()); ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, LocalImagePromoted) { +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncDeltaDemote) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1017,31 +1178,84 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, LocalImagePromoted) { mock_image_meta, &update_watch_ctx)); - // inject a promotion snapshot - mock_local_image_ctx.snap_info = { + // inject an incomplete sync snapshot with last_copied_object_number > 0 + // after a primary demotion snapshot + mock_remote_image_ctx.snap_info = { {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, - {"remote mirror peer uuid"}, "", 0U, true, 0, {}}, + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "local mirror uuid", 11, true, 0, + {{11, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, false, 123, {{2, CEPH_NOSNAP}}}, 0, {}, 0, 0, {}}}}; - // idle + // re-sync snap2 expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, false); expect_is_refresh_required(mock_remote_image_ctx, false); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 12, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, + librbd::deep_copy::ObjectNumber{123U}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 12, true, 123, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); // wake-up replayer update_watch_ctx->handle_notify(); - // wait for sync to complete and expect replay complete - ASSERT_EQ(0, wait_for_notification(1)); - ASSERT_FALSE(mock_replayer.is_replaying()); + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, ResyncRequested) { +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncInitial) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1073,60 +1287,67 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, ResyncRequested) { mock_image_meta, &update_watch_ctx)); + // inject an incomplete sync snapshot with last_copied_object_number == 0 + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, false, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // re-sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + // idle - expect_load_image_meta(mock_image_meta, true, 0); + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); // wake-up replayer update_watch_ctx->handle_notify(); - // wait for sync to complete and expect replay complete - ASSERT_EQ(0, wait_for_notification(1)); - ASSERT_FALSE(mock_replayer.is_replaying()); + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterLocalUpdateWatcherError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncDelta) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; MockThreads mock_threads(m_threads); expect_work_queue_repeatedly(mock_threads); - InSequence seq; - - MockInstanceWatcher mock_instance_watcher; - MockImageMeta mock_image_meta; - MockStateBuilder mock_state_builder(mock_local_image_ctx, - mock_remote_image_ctx, - mock_image_meta); MockReplayerListener mock_replayer_listener; - MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, - "local mirror uuid", &m_pool_meta_cache, - &mock_state_builder, &mock_replayer_listener}; - m_pool_meta_cache.set_remote_pool_meta( - m_remote_io_ctx.get_id(), - {"remote mirror uuid", "remote mirror peer uuid"}); - - // init - librbd::UpdateWatchCtx* update_watch_ctx = nullptr; - expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, - -EINVAL); - - // fire init - C_SaferCond init_ctx; - mock_replayer.init(&init_ctx); - ASSERT_EQ(-EINVAL, init_ctx.wait()); -} - -TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError) { - librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; - librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; - - MockThreads mock_threads(m_threads); - expect_work_queue_repeatedly(mock_threads); + expect_notification(mock_threads, mock_replayer_listener); InSequence seq; @@ -1135,7 +1356,6 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError) MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayerListener mock_replayer_listener; MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, "local mirror uuid", &m_pool_meta_cache, &mock_state_builder, &mock_replayer_listener}; @@ -1143,22 +1363,1002 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError) m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); - // init librbd::UpdateWatchCtx* update_watch_ctx = nullptr; - expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, - 0); - expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234, - -EINVAL); - - expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); - // fire init + // inject an incomplete sync snapshot with last_copied_object_number == 0 + // after a complete snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, false, 0, {{2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // prune non-primary snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 12, 0); + + // sync snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, + {{2, CEPH_NOSNAP}}, 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 2, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 2, + {{2, CEPH_NOSNAP}}, 13, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, {}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 13, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // prune non-primary snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncDeltaDemote) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject an incomplete sync snapshot with last_copied_object_number == 0 + // after a primary demotion snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "local mirror uuid", 11, true, 0, + {{11, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, false, 0, {{2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // prune non-primary snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 12, 0); + + // sync snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, + {{2, CEPH_NOSNAP}}, 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 2, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 2, + {{2, CEPH_NOSNAP}}, 13, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, {}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 13, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject a demotion snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + true, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(2)); + ASSERT_FALSE(mock_replayer.is_replaying()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, LocalImagePromoted) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject a promotion snapshot + mock_local_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, ResyncRequested) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // idle + expect_load_image_meta(mock_image_meta, true, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterLocalUpdateWatcherError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayerListener mock_replayer_listener; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + // init + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + -EINVAL); + + // fire init + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-EINVAL, init_ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayerListener mock_replayer_listener; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + // init + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + 0); + expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234, + -EINVAL); + + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + + // fire init C_SaferCond init_ctx; mock_replayer.init(&init_ctx); ASSERT_EQ(-EINVAL, init_ctx.wait()); } -TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterRemoteUpdateWatcherError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterRemoteUpdateWatcherError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + + // shut down + expect_unregister_update_watcher(mock_remote_image_ctx, 234, -EINVAL); + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterLocalUpdateWatcherError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + + // shut down + expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0); + expect_unregister_update_watcher(mock_local_image_ctx, 123, -EINVAL); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // sync + expect_load_image_meta(mock_image_meta, false, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // sync + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh(mock_local_image_ctx, {}, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // sync + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh(mock_remote_image_ctx, {}, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, GetImageStateError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateMirrorImageStateError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, -EIO); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EIO, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1190,17 +2390,45 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterRemoteUpdateWatcherError mock_image_meta, &update_watch_ctx)); + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; - // shut down - expect_unregister_update_watcher(mock_remote_image_ctx, 234, -EINVAL); - expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, + -ECANCELED); - C_SaferCond shutdown_ctx; - mock_replayer.shut_down(&shutdown_ctx); - ASSERT_EQ(-EINVAL, shutdown_ctx.wait()); + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-ECANCELED, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterLocalUpdateWatcherError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1232,17 +2460,48 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterLocalUpdateWatcherError) mock_image_meta, &update_watch_ctx)); + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP,true, 0, {}}, + 0, {}, 0, 0, {}}}}; - // shut down - expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0); - expect_unregister_update_watcher(mock_local_image_ctx, 123, -EINVAL); + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, -EINVAL); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); - C_SaferCond shutdown_ctx; - mock_replayer.shut_down(&shutdown_ctx); - ASSERT_EQ(-EINVAL, shutdown_ctx.wait()); + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1274,8 +2533,37 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) { mock_image_meta, &update_watch_ctx)); - // sync - expect_load_image_meta(mock_image_meta, false, -EINVAL); + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, -EINVAL); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); // wake-up replayer update_watch_ctx->handle_notify(); @@ -1290,7 +2578,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) { mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1322,10 +2610,48 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) { mock_image_meta, &update_watch_ctx)); - // sync + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap2 expect_load_image_meta(mock_image_meta, false, 0); - expect_is_refresh_required(mock_local_image_ctx, true); - expect_refresh(mock_local_image_ctx, {}, -EINVAL); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, {{2, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 2, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 2, + {{2, CEPH_NOSNAP}}, 12, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, {}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 12, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + -EINVAL); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); // wake-up replayer update_watch_ctx->handle_notify(); @@ -1340,7 +2666,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) { mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1372,11 +2698,22 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) { mock_image_meta, &update_watch_ctx)); - // sync + // inject a primary demote to local image + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP, + true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // detect split-brain expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, false); - expect_is_refresh_required(mock_remote_image_ctx, true); - expect_refresh(mock_remote_image_ctx, {}, -EINVAL); + expect_is_refresh_required(mock_remote_image_ctx, false); // wake-up replayer update_watch_ctx->handle_notify(); @@ -1384,14 +2721,15 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) { // wait for sync to complete and expect replay complete ASSERT_EQ(0, wait_for_notification(1)); ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + ASSERT_EQ(-EEXIST, mock_replayer.get_error_code()); + ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description()); ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteSnapshotMissingSplitBrain) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1423,20 +2761,27 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) { mock_image_meta, &update_watch_ctx)); - // inject snapshot + // inject a missing remote start snap (deleted) + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, + "remote mirror uuid", 1, true, 0, + {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; mock_remote_image_ctx.snap_info = { - {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", - 0U, true, 0, {}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, 0, {}, 0, 0, {}}}}; - // sync snap1 + // split-brain due to missing snapshot 1 expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, false); expect_is_refresh_required(mock_remote_image_ctx, false); - MockSnapshotCopyRequest mock_snapshot_copy_request; - expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, - -EINVAL); // wake-up replayer update_watch_ctx->handle_notify(); @@ -1444,14 +2789,15 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) { // wait for sync to complete and expect replay complete ASSERT_EQ(0, wait_for_notification(1)); ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + ASSERT_EQ(-EEXIST, mock_replayer.get_error_code()); + ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description()); ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, GetImageStateError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteFailover) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1483,40 +2829,119 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, GetImageStateError) { mock_image_meta, &update_watch_ctx)); - // inject snapshot + // inject a primary demote to local image mock_remote_image_ctx.snap_info = { - {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", - 0U, true, 0, {}}, + {1U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "local mirror uuid", 12U, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_ids = { + {{cls::rbd::UserSnapshotNamespace{}, "snap1"}, 11}, + {{cls::rbd::MirrorSnapshotNamespace{}, "snap2"}, 12}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP, + true, 0, {}}, 0, {}, 0, 0, {}}}}; - // sync snap1 + // attach to promoted remote image expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, false); expect_is_refresh_required(mock_remote_image_ctx, false); MockSnapshotCopyRequest mock_snapshot_copy_request; - expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, - 0); + expect_snapshot_copy(mock_snapshot_copy_request, 2, 3, 12, + {{2, 12}, {3, CEPH_NOSNAP}}, 0); MockGetImageStateRequest mock_get_image_state_request; - expect_get_image_state(mock_get_image_state_request, 1, -EINVAL); + expect_get_image_state(mock_get_image_state_request, 3, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 3, + {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}, 13, + 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 2, 3, 12, {}, + {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 13, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 2, "remote mirror peer uuid", 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP, + true, 0, {}}, + 0, {}, 0, 0, {}}}, + {13U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, + "remote mirror uuid", 3, true, 0, + {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {1U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "local mirror uuid", 12U, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {}, "", CEPH_NOSNAP, true, 0, + {}}, + 0, {}, 0, 0, {}}} + }, 0); // wake-up replayer update_watch_ctx->handle_notify(); // wait for sync to complete and expect replay complete - ASSERT_EQ(0, wait_for_notification(1)); - ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); - + ASSERT_EQ(0, wait_for_notification(2)); ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkRemoteSnapshot) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + // it should attempt to unlink from remote snap1 since we don't need it + // anymore + mock_local_image_ctx.snap_info = { + {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 4, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + MockThreads mock_threads(m_threads); expect_work_queue_repeatedly(mock_threads); @@ -1538,51 +2963,63 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) { {"remote mirror uuid", "remote mirror peer uuid"}); librbd::UpdateWatchCtx* update_watch_ctx = nullptr; - ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, - mock_local_image_ctx, - mock_remote_image_ctx, - mock_replayer_listener, - mock_image_meta, - &update_watch_ctx)); - // inject snapshot - mock_remote_image_ctx.snap_info = { - {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", - 0U, true, 0, {}}, - 0, {}, 0, 0, {}}}}; + // init + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + 0); + expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234, + 0); - // sync snap1 + // unlink snap1 expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, false); expect_is_refresh_required(mock_remote_image_ctx, false); - MockSnapshotCopyRequest mock_snapshot_copy_request; - expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, - 0); - MockGetImageStateRequest mock_get_image_state_request; - expect_get_image_state(mock_get_image_state_request, 1, 0); - MockCreateNonPrimaryRequest mock_create_non_primary_request; - expect_create_non_primary_request(mock_create_non_primary_request, - false, "remote mirror uuid", 1, - {{1, CEPH_NOSNAP}}, 11, -EINVAL); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); - // wake-up replayer - update_watch_ctx->handle_notify(); + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); - // wait for sync to complete and expect replay complete - ASSERT_EQ(0, wait_for_notification(1)); - ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + // fire init + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(0, init_ctx.wait()); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(3)); + // shut down ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, SkipImageSync) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", 0U, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + MockThreads mock_threads(m_threads); expect_work_queue_repeatedly(mock_threads); @@ -1604,19 +3041,12 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { {"remote mirror uuid", "remote mirror peer uuid"}); librbd::UpdateWatchCtx* update_watch_ctx = nullptr; - ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, - mock_local_image_ctx, - mock_remote_image_ctx, - mock_replayer_listener, - mock_image_meta, - &update_watch_ctx)); - // inject snapshot - mock_remote_image_ctx.snap_info = { - {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", - 0U, true, 0, {}}, - 0, {}, 0, 0, {}}}}; + // init + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + 0); + expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234, + 0); // sync snap1 expect_load_image_meta(mock_image_meta, false, 0); @@ -1631,24 +3061,41 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); - expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, - -ECANCELED); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); - // wake-up replayer - update_watch_ctx->handle_notify(); + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); - // wait for sync to complete and expect replay complete - ASSERT_EQ(0, wait_for_notification(1)); - ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-ECANCELED, mock_replayer.get_error_code()); + // fire init + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(0, init_ctx.wait()); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(3)); + // shut down ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); - } -TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, ImageNameUpdated) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1680,45 +3127,29 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { mock_image_meta, &update_watch_ctx)); - // inject snapshot - mock_remote_image_ctx.snap_info = { - {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", - 0U, true, 0, {}}, - 0, {}, 0, 0, {}}}}; + // change the name of the image + mock_local_image_ctx.name = "NEW NAME"; - // sync snap1 - expect_load_image_meta(mock_image_meta, false, 0); - expect_is_refresh_required(mock_local_image_ctx, false); - expect_is_refresh_required(mock_remote_image_ctx, false); - MockSnapshotCopyRequest mock_snapshot_copy_request; - expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, - 0); - MockGetImageStateRequest mock_get_image_state_request; - expect_get_image_state(mock_get_image_state_request, 1, 0); - MockCreateNonPrimaryRequest mock_create_non_primary_request; - expect_create_non_primary_request(mock_create_non_primary_request, - false, "remote mirror uuid", 1, - {{1, CEPH_NOSNAP}}, 11, 0); - expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); - MockImageCopyRequest mock_image_copy_request; - expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, - {{1, CEPH_NOSNAP}}, -EINVAL); + // idle + expect_load_image_meta(mock_image_meta, true, 0); // wake-up replayer update_watch_ctx->handle_notify(); // wait for sync to complete and expect replay complete - ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_EQ(0, wait_for_notification(2)); + auto image_spec = image_replayer::util::compute_image_spec(m_local_io_ctx, + "NEW NAME"); + ASSERT_EQ(image_spec, mock_replayer.get_image_spec()); ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + // shut down ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); } -TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, ApplyImageStatePendingShutdown) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1738,6 +3169,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, "local mirror uuid", &m_pool_meta_cache, &mock_state_builder, &mock_replayer_listener}; + C_SaferCond shutdown_ctx; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1754,7 +3186,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { mock_remote_image_ctx.snap_info = { {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", - 0U, true, 0, {}}, + CEPH_NOSNAP, true, 0, {}}, 0, {}, 0, 0, {}}}}; // sync snap1 @@ -1770,29 +3202,41 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, {{1, CEPH_NOSNAP}}, 0); MockApplyImageStateRequest mock_apply_state_request; - expect_apply_image_state(mock_apply_state_request, 0); + EXPECT_CALL(mock_apply_state_request, send()) + .WillOnce(Invoke([this, &req=mock_apply_state_request, + &replayer=mock_replayer, &ctx=shutdown_ctx]() { + // inject a shutdown, to be pended due to STATE_REPLAYING + replayer.shut_down(&ctx); + m_threads->work_queue->queue(req.on_finish, 0); + })); + expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id); expect_mirror_image_snapshot_set_copy_progress( - mock_local_image_ctx, 11, true, 0, -EINVAL); + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // shutdown should be resumed + expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0); + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); // wake-up replayer update_watch_ctx->handle_notify(); - // wait for sync to complete and expect replay complete ASSERT_EQ(0, wait_for_notification(1)); ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + ASSERT_EQ(0, mock_replayer.get_error_code()); - ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, - mock_local_image_ctx, - mock_remote_image_ctx)); + ASSERT_EQ(0, shutdown_ctx.wait()); } -TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) { +TEST_F(TestMockImageReplayerSnapshotReplayer, ApplyImageStateErrorPendingShutdown) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1812,6 +3256,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) { MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, "local mirror uuid", &m_pool_meta_cache, &mock_state_builder, &mock_replayer_listener}; + C_SaferCond shutdown_ctx; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1828,119 +3273,47 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) { mock_remote_image_ctx.snap_info = { {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", - 0U, true, 0, {}}, - 0, {}, 0, 0, {}}}, - {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, - "", 0U, true, 0, {}}, - 0, {}, 0, 0, {}}}}; - mock_local_image_ctx.snap_info = { - {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", - 1, true, 0, {}}, + CEPH_NOSNAP, true, 0, {}}, 0, {}, 0, 0, {}}}}; - // sync snap2 + // sync snap1 expect_load_image_meta(mock_image_meta, false, 0); expect_is_refresh_required(mock_local_image_ctx, false); expect_is_refresh_required(mock_remote_image_ctx, false); MockSnapshotCopyRequest mock_snapshot_copy_request; - expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, {{2, CEPH_NOSNAP}}, + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, 0); MockGetImageStateRequest mock_get_image_state_request; - expect_get_image_state(mock_get_image_state_request, 2, 0); + expect_get_image_state(mock_get_image_state_request, 1, 0); MockCreateNonPrimaryRequest mock_create_non_primary_request; expect_create_non_primary_request(mock_create_non_primary_request, - false, "remote mirror uuid", 2, - {{2, CEPH_NOSNAP}}, 12, 0); + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; - expect_image_copy(mock_image_copy_request, 1, 2, 11, {}, - {{2, CEPH_NOSNAP}}, 0); + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); MockApplyImageStateRequest mock_apply_state_request; - expect_apply_image_state(mock_apply_state_request, 0); - expect_mirror_image_snapshot_set_copy_progress( - mock_local_image_ctx, 12, true, 0, 0); - expect_notify_update(mock_local_image_ctx); - MockUnlinkPeerRequest mock_unlink_peer_request; - expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", - -EINVAL); + EXPECT_CALL(mock_apply_state_request, send()) + .WillOnce(Invoke([this, &req=mock_apply_state_request, + &replayer=mock_replayer, &ctx=shutdown_ctx]() { + // inject a shutdown, to be pended due to STATE_REPLAYING + replayer.shut_down(&ctx); + m_threads->work_queue->queue(req.on_finish, -EINVAL); + })); + expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id); expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); - // wake-up replayer - update_watch_ctx->handle_notify(); - - // wait for sync to complete and expect replay complete - ASSERT_EQ(0, wait_for_notification(1)); - ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); - - ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, - mock_local_image_ctx, - mock_remote_image_ctx)); -} - -TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) { - librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; - librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; - - MockThreads mock_threads(m_threads); - expect_work_queue_repeatedly(mock_threads); - - MockReplayerListener mock_replayer_listener; - expect_notification(mock_threads, mock_replayer_listener); - - InSequence seq; - - MockInstanceWatcher mock_instance_watcher; - MockImageMeta mock_image_meta; - MockStateBuilder mock_state_builder(mock_local_image_ctx, - mock_remote_image_ctx, - mock_image_meta); - MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, - "local mirror uuid", &m_pool_meta_cache, - &mock_state_builder, &mock_replayer_listener}; - m_pool_meta_cache.set_remote_pool_meta( - m_remote_io_ctx.get_id(), - {"remote mirror uuid", "remote mirror peer uuid"}); - - librbd::UpdateWatchCtx* update_watch_ctx = nullptr; - ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, - mock_local_image_ctx, - mock_remote_image_ctx, - mock_replayer_listener, - mock_image_meta, - &update_watch_ctx)); - - // inject a primary demote to local image - mock_remote_image_ctx.snap_info = { - {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, - "", 0U, true, 0, {}}, - 0, {}, 0, 0, {}}}}; - mock_local_image_ctx.snap_info = { - {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", 0U, true, 0, - {}}, - 0, {}, 0, 0, {}}}}; - - // detect split-brain - expect_load_image_meta(mock_image_meta, false, 0); - expect_is_refresh_required(mock_local_image_ctx, false); - expect_is_refresh_required(mock_remote_image_ctx, false); + // shutdown should be resumed + expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0); + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); // wake-up replayer update_watch_ctx->handle_notify(); - // wait for sync to complete and expect replay complete - ASSERT_EQ(0, wait_for_notification(1)); - ASSERT_FALSE(mock_replayer.is_replaying()); - ASSERT_EQ(-EEXIST, mock_replayer.get_error_code()); - ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description()); - - ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, - mock_local_image_ctx, - mock_remote_image_ctx)); + ASSERT_EQ(0, shutdown_ctx.wait()); } } // namespace snapshot