ceph/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "test/rbd_mirror/test_mock_fixture.h"
   5 #include "librbd/deep_copy/ImageCopyRequest.h"
   6 #include "librbd/deep_copy/SnapshotCopyRequest.h"
   7 #include "librbd/mirror/ImageStateUpdateRequest.h"
   8 #include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h"
   9 #include "librbd/mirror/snapshot/GetImageStateRequest.h"
  10 #include "librbd/mirror/snapshot/ImageMeta.h"
  11 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
  12 #include "tools/rbd_mirror/InstanceWatcher.h"
  13 #include "tools/rbd_mirror/Threads.h"
  14 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
  15 #include "tools/rbd_mirror/image_replayer/ReplayerListener.h"
  16 #include "tools/rbd_mirror/image_replayer/Utils.h"
  17 #include "tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h"
  18 #include "tools/rbd_mirror/image_replayer/snapshot/Replayer.h"
  19 #include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h"
  20 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
  21 #include "test/librbd/mock/MockImageCtx.h"
  22 #include "test/librbd/mock/MockOperations.h"
  23 #include "test/rbd_mirror/mock/MockContextWQ.h"
  24 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  25
  26 using namespace std::chrono_literals;
  27
  28 namespace librbd {
  29 namespace {
  30
  31 struct MockTestImageCtx : public librbd::MockImageCtx {
  32   explicit MockTestImageCtx(librbd::ImageCtx &image_ctx)
  33     : librbd::MockImageCtx(image_ctx) {
  34   }
  35 };
  36
  37 } // anonymous namespace
  38
  39 namespace deep_copy {
  40
  41 template <>
  42 struct ImageCopyRequest<MockTestImageCtx> {
  43   uint64_t src_snap_id_start;
  44   uint64_t src_snap_id_end;
  45   uint64_t dst_snap_id_start;
  46   librbd::deep_copy::ObjectNumber object_number;
  47   librbd::SnapSeqs snap_seqs;
  48
  49   static ImageCopyRequest* s_instance;
  50   static ImageCopyRequest* create(MockTestImageCtx *src_image_ctx,
  51                                   MockTestImageCtx *dst_image_ctx,
  52                                   librados::snap_t src_snap_id_start,
  53                                   librados::snap_t src_snap_id_end,
  54                                   librados::snap_t dst_snap_id_start,
  55                                   bool flatten,
  56                                   const ObjectNumber &object_number,
  57                                   const SnapSeqs &snap_seqs,
  58                                   Handler *handler,
  59                                   Context *on_finish) {
  60     ceph_assert(s_instance != nullptr);
  61     s_instance->src_snap_id_start = src_snap_id_start;
  62     s_instance->src_snap_id_end = src_snap_id_end;
  63     s_instance->dst_snap_id_start = dst_snap_id_start;
  64     s_instance->object_number = object_number;
  65     s_instance->snap_seqs = snap_seqs;
  66     s_instance->on_finish = on_finish;
  67     return s_instance;
  68   }
  69
  70   Context* on_finish = nullptr;
  71
  72   ImageCopyRequest() {
  73     s_instance = this;
  74   }
  75
  76   MOCK_METHOD0(send, void());
  77 };
  78
  79 template <>
  80 struct SnapshotCopyRequest<MockTestImageCtx> {
  81   librados::snap_t src_snap_id_start;
  82   librados::snap_t src_snap_id_end;
  83   librados::snap_t dst_snap_id_start;
  84   SnapSeqs* snap_seqs = nullptr;
  85
  86   static SnapshotCopyRequest* s_instance;
  87   static SnapshotCopyRequest* create(MockTestImageCtx *src_image_ctx,
  88                                      MockTestImageCtx *dst_image_ctx,
  89                                      librados::snap_t src_snap_id_start,
  90                                      librados::snap_t src_snap_id_end,
  91                                      librados::snap_t dst_snap_id_start,
  92                                      bool flatten,
  93                                      ::MockContextWQ *work_queue,
  94                                      SnapSeqs *snap_seqs,
  95                                      Context *on_finish) {
  96     ceph_assert(s_instance != nullptr);
  97     s_instance->src_snap_id_start = src_snap_id_start;
  98     s_instance->src_snap_id_end = src_snap_id_end;
  99     s_instance->dst_snap_id_start = dst_snap_id_start;
 100     s_instance->snap_seqs = snap_seqs;
 101     s_instance->on_finish = on_finish;
 102     return s_instance;
 103   }
 104
 105   Context* on_finish = nullptr;
 106
 107   SnapshotCopyRequest() {
 108     s_instance = this;
 109   }
 110
 111   MOCK_METHOD0(send, void());
 112 };
 113
 114 ImageCopyRequest<MockTestImageCtx>* ImageCopyRequest<MockTestImageCtx>::s_instance = nullptr;
 115 SnapshotCopyRequest<MockTestImageCtx>* SnapshotCopyRequest<MockTestImageCtx>::s_instance = nullptr;
 116
 117 } // namespace deep_copy
 118
 119 namespace mirror {
 120
 121 template <>
 122 struct ImageStateUpdateRequest<MockTestImageCtx> {
 123   static ImageStateUpdateRequest* s_instance;
 124   static ImageStateUpdateRequest* create(
 125       librados::IoCtx& io_ctx,
 126       const std::string& image_id,
 127       cls::rbd::MirrorImageState mirror_image_state,
 128       const cls::rbd::MirrorImage& mirror_image,
 129       Context* on_finish) {
 130     ceph_assert(s_instance != nullptr);
 131     EXPECT_EQ(cls::rbd::MIRROR_IMAGE_STATE_ENABLED,
 132               mirror_image_state);
 133     EXPECT_EQ(cls::rbd::MirrorImage{}, mirror_image);
 134     s_instance->on_finish = on_finish;
 135     return s_instance;
 136   }
 137
 138   Context* on_finish = nullptr;
 139   ImageStateUpdateRequest() {
 140     s_instance = this;
 141   }
 142
 143   MOCK_METHOD0(send, void());
 144 };
 145
 146 ImageStateUpdateRequest<MockTestImageCtx>* ImageStateUpdateRequest<MockTestImageCtx>::s_instance = nullptr;
 147
 148 namespace snapshot {
 149
 150 template <>
 151 struct CreateNonPrimaryRequest<MockTestImageCtx> {
 152   bool demoted = false;
 153   std::string primary_mirror_uuid;
 154   uint64_t primary_snap_id;
 155   SnapSeqs snap_seqs;
 156   uint64_t* snap_id = nullptr;
 157
 158   static CreateNonPrimaryRequest* s_instance;
 159   static CreateNonPrimaryRequest* create(MockTestImageCtx *image_ctx,
 160                                          bool demoted,
 161                                          const std::string &primary_mirror_uuid,
 162                                          uint64_t primary_snap_id,
 163                                          const SnapSeqs& snap_seqs,
 164                                          const ImageState &image_state,
 165                                          uint64_t *snap_id,
 166                                          Context *on_finish) {
 167     ceph_assert(s_instance != nullptr);
 168     s_instance->demoted = demoted;
 169     s_instance->primary_mirror_uuid = primary_mirror_uuid;
 170     s_instance->primary_snap_id = primary_snap_id;
 171     s_instance->snap_seqs = snap_seqs;
 172     s_instance->snap_id = snap_id;
 173     s_instance->on_finish = on_finish;
 174     return s_instance;
 175   }
 176
 177   Context* on_finish = nullptr;
 178
 179   CreateNonPrimaryRequest() {
 180     s_instance = this;
 181   }
 182
 183   MOCK_METHOD0(send, void());
 184 };
 185
 186 template <>
 187 struct GetImageStateRequest<MockTestImageCtx> {
 188   uint64_t snap_id = CEPH_NOSNAP;
 189
 190   static GetImageStateRequest* s_instance;
 191   static GetImageStateRequest* create(MockTestImageCtx *image_ctx,
 192                                       uint64_t snap_id,
 193                                       ImageState *image_state,
 194                                       Context *on_finish) {
 195     ceph_assert(s_instance != nullptr);
 196     s_instance->snap_id = snap_id;
 197     s_instance->on_finish = on_finish;
 198     return s_instance;
 199   }
 200
 201   Context* on_finish = nullptr;
 202
 203   GetImageStateRequest() {
 204     s_instance = this;
 205   }
 206
 207   MOCK_METHOD0(send, void());
 208 };
 209
 210 template <>
 211 struct ImageMeta<MockTestImageCtx> {
 212   MOCK_METHOD1(load, void(Context*));
 213
 214   bool resync_requested = false;
 215 };
 216
 217 template <>
 218 struct UnlinkPeerRequest<MockTestImageCtx> {
 219   uint64_t snap_id;
 220   std::string mirror_peer_uuid;
 221
 222   static UnlinkPeerRequest* s_instance;
 223   static UnlinkPeerRequest*create (MockTestImageCtx *image_ctx,
 224                                    uint64_t snap_id,
 225                                    const std::string &mirror_peer_uuid,
 226                                    Context *on_finish) {
 227     ceph_assert(s_instance != nullptr);
 228     s_instance->snap_id = snap_id;
 229     s_instance->mirror_peer_uuid = mirror_peer_uuid;
 230     s_instance->on_finish = on_finish;
 231     return s_instance;
 232   }
 233
 234   Context* on_finish = nullptr;
 235
 236   UnlinkPeerRequest() {
 237     s_instance = this;
 238   }
 239
 240   MOCK_METHOD0(send, void());
 241 };
 242
 243 CreateNonPrimaryRequest<MockTestImageCtx>* CreateNonPrimaryRequest<MockTestImageCtx>::s_instance = nullptr;
 244 GetImageStateRequest<MockTestImageCtx>* GetImageStateRequest<MockTestImageCtx>::s_instance = nullptr;
 245 UnlinkPeerRequest<MockTestImageCtx>* UnlinkPeerRequest<MockTestImageCtx>::s_instance = nullptr;
 246
 247 } // namespace snapshot
 248 } // namespace mirror
 249 } // namespace librbd
 250
 251 namespace rbd {
 252 namespace mirror {
 253
 254 template <>
 255 struct InstanceWatcher<librbd::MockTestImageCtx> {
 256   MOCK_METHOD1(cancel_sync_request, void(const std::string&));
 257   MOCK_METHOD2(notify_sync_request, void(const std::string&,
 258                                          Context*));
 259   MOCK_METHOD1(notify_sync_complete, void(const std::string&));
 260 };
 261
 262 template <>
 263 struct Threads<librbd::MockTestImageCtx> {
 264   MockSafeTimer *timer;
 265   ceph::mutex &timer_lock;
 266
 267   MockContextWQ *work_queue;
 268
 269   Threads(Threads<librbd::ImageCtx>* threads)
 270     : timer(new MockSafeTimer()),
 271       timer_lock(threads->timer_lock),
 272       work_queue(new MockContextWQ()) {
 273   }
 274   ~Threads() {
 275     delete timer;
 276     delete work_queue;
 277   }
 278 };
 279
 280 namespace {
 281
 282 struct MockReplayerListener : public image_replayer::ReplayerListener {
 283   MOCK_METHOD0(handle_notification, void());
 284 };
 285
 286 } // anonymous namespace
 287
 288 namespace image_replayer {
 289
 290 template<>
 291 struct CloseImageRequest<librbd::MockTestImageCtx> {
 292   static CloseImageRequest* s_instance;
 293   librbd::MockTestImageCtx **image_ctx = nullptr;
 294   Context *on_finish = nullptr;
 295
 296   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
 297                                    Context *on_finish) {
 298     ceph_assert(s_instance != nullptr);
 299     s_instance->image_ctx = image_ctx;
 300     s_instance->on_finish = on_finish;
 301     return s_instance;
 302   }
 303
 304   CloseImageRequest() {
 305     ceph_assert(s_instance == nullptr);
 306     s_instance = this;
 307   }
 308
 309   ~CloseImageRequest() {
 310     ceph_assert(s_instance == this);
 311     s_instance = nullptr;
 312   }
 313
 314   MOCK_METHOD0(send, void());
 315 };
 316
 317 CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 318
 319 namespace snapshot {
 320
 321 template <>
 322 struct ApplyImageStateRequest<librbd::MockTestImageCtx> {
 323   Context* on_finish = nullptr;
 324
 325   static ApplyImageStateRequest* s_instance;
 326   static ApplyImageStateRequest* create(
 327       const std::string& local_mirror_uuid,
 328       const std::string& remote_mirror_uuid,
 329       librbd::MockTestImageCtx* local_image_ctx,
 330       librbd::MockTestImageCtx* remote_image_ctx,
 331       const librbd::mirror::snapshot::ImageState& image_state,
 332       Context* on_finish) {
 333     ceph_assert(s_instance != nullptr);
 334     s_instance->on_finish = on_finish;
 335     return s_instance;
 336   }
 337
 338   ApplyImageStateRequest() {
 339     s_instance = this;
 340   }
 341
 342   MOCK_METHOD0(send, void());
 343 };
 344
 345 template<>
 346 struct StateBuilder<librbd::MockTestImageCtx> {
 347   StateBuilder(librbd::MockTestImageCtx& local_image_ctx,
 348                librbd::MockTestImageCtx& remote_image_ctx,
 349                librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx>&
 350                  local_image_meta)
 351     : local_image_ctx(&local_image_ctx),
 352       remote_image_ctx(&remote_image_ctx),
 353       local_image_meta(&local_image_meta) {
 354   }
 355
 356   librbd::MockTestImageCtx* local_image_ctx;
 357   librbd::MockTestImageCtx* remote_image_ctx;
 358
 359   std::string remote_mirror_uuid = "remote mirror uuid";
 360
 361   librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx>*
 362     local_image_meta = nullptr;
 363 };
 364
 365 ApplyImageStateRequest<librbd::MockTestImageCtx>* ApplyImageStateRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 366
 367 } // namespace snapshot
 368 } // namespace image_replayer
 369 } // namespace mirror
 370 } // namespace rbd
 371
 372 #include "tools/rbd_mirror/image_replayer/snapshot/Replayer.cc"
 373
 374 namespace rbd {
 375 namespace mirror {
 376 namespace image_replayer {
 377 namespace snapshot {
 378
 379 using ::testing::_;
 380 using ::testing::DoAll;
 381 using ::testing::InSequence;
 382 using ::testing::Invoke;
 383 using ::testing::Return;
 384 using ::testing::ReturnArg;
 385 using ::testing::StrEq;
 386 using ::testing::WithArg;
 387
 388 class TestMockImageReplayerSnapshotReplayer : public TestMockFixture {
 389 public:
 390   typedef Replayer<librbd::MockTestImageCtx> MockReplayer;
 391   typedef ApplyImageStateRequest<librbd::MockTestImageCtx> MockApplyImageStateRequest;
 392   typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder;
 393   typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
 394   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 395   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
 396   typedef librbd::deep_copy::ImageCopyRequest<librbd::MockTestImageCtx> MockImageCopyRequest;
 397   typedef librbd::deep_copy::SnapshotCopyRequest<librbd::MockTestImageCtx> MockSnapshotCopyRequest;
 398   typedef librbd::mirror::ImageStateUpdateRequest<librbd::MockTestImageCtx> MockImageStateUpdateRequest;
 399   typedef librbd::mirror::snapshot::CreateNonPrimaryRequest<librbd::MockTestImageCtx> MockCreateNonPrimaryRequest;
 400   typedef librbd::mirror::snapshot::GetImageStateRequest<librbd::MockTestImageCtx> MockGetImageStateRequest;
 401   typedef librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx> MockImageMeta;
 402   typedef librbd::mirror::snapshot::UnlinkPeerRequest<librbd::MockTestImageCtx> MockUnlinkPeerRequest;
 403
 404   void SetUp() override {
 405     TestMockFixture::SetUp();
 406
 407     librbd::RBD rbd;
 408     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 409     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 410
 411     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name,
 412                               m_image_size));
 413     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name,
 414                             &m_remote_image_ctx));
 415   }
 416
 417   void expect_work_queue_repeatedly(MockThreads &mock_threads) {
 418     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 419       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 420           m_threads->work_queue->queue(ctx, r);
 421         }));
 422   }
 423
 424   void expect_add_event_after_repeatedly(MockThreads &mock_threads) {
 425     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 426       .WillRepeatedly(
 427         DoAll(Invoke([this](double seconds, Context *ctx) {
 428                        m_threads->timer->add_event_after(seconds, ctx);
 429                      }),
 430           ReturnArg<1>()));
 431     EXPECT_CALL(*mock_threads.timer, cancel_event(_))
 432       .WillRepeatedly(
 433         Invoke([this](Context *ctx) {
 434           return m_threads->timer->cancel_event(ctx);
 435         }));
 436   }
 437
 438   void expect_register_update_watcher(librbd::MockTestImageCtx& mock_image_ctx,
 439                                       librbd::UpdateWatchCtx** update_watch_ctx,
 440                                       uint64_t watch_handle, int r) {
 441     EXPECT_CALL(*mock_image_ctx.state, register_update_watcher(_, _))
 442       .WillOnce(Invoke([update_watch_ctx, watch_handle, r]
 443                        (librbd::UpdateWatchCtx* ctx, uint64_t* handle) {
 444           if (r >= 0) {
 445             *update_watch_ctx = ctx;
 446             *handle = watch_handle;
 447           }
 448           return r;
 449         }));
 450   }
 451
 452   void expect_unregister_update_watcher(librbd::MockTestImageCtx& mock_image_ctx,
 453                                         uint64_t watch_handle, int r) {
 454     EXPECT_CALL(*mock_image_ctx.state, unregister_update_watcher(watch_handle, _))
 455       .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) {
 456           m_threads->work_queue->queue(ctx, r);
 457         })));
 458   }
 459
 460   void expect_load_image_meta(MockImageMeta& mock_image_meta,
 461                               bool resync_requested, int r) {
 462     EXPECT_CALL(mock_image_meta, load(_))
 463       .WillOnce(Invoke([this, &mock_image_meta, resync_requested, r](Context* ctx) {
 464           mock_image_meta.resync_requested = resync_requested;
 465           m_threads->work_queue->queue(ctx, r);
 466         }));
 467   }
 468
 469   void expect_is_refresh_required(librbd::MockTestImageCtx& mock_image_ctx,
 470                                   bool is_required) {
 471     EXPECT_CALL(*mock_image_ctx.state, is_refresh_required())
 472       .WillOnce(Return(is_required));
 473   }
 474
 475   void expect_refresh(librbd::MockTestImageCtx& mock_image_ctx,
 476                       const std::map<uint64_t, librbd::SnapInfo>& snaps,
 477                       int r) {
 478     EXPECT_CALL(*mock_image_ctx.state, refresh(_))
 479       .WillOnce(Invoke([this, &mock_image_ctx, snaps, r](Context* ctx) {
 480         mock_image_ctx.snap_info = snaps;
 481         m_threads->work_queue->queue(ctx, r);
 482       }));
 483   }
 484
 485   void expect_notify_update(librbd::MockTestImageCtx& mock_image_ctx) {
 486     EXPECT_CALL(mock_image_ctx, notify_update(_))
 487       .WillOnce(Invoke([this](Context* ctx) {
 488         m_threads->work_queue->queue(ctx, 0);
 489       }));
 490   }
 491
 492   void expect_prune_non_primary_snapshot(librbd::MockTestImageCtx& mock_image_ctx,
 493                                          uint64_t snap_id, int r) {
 494     EXPECT_CALL(mock_image_ctx, get_snap_info(snap_id))
 495       .WillOnce(Invoke([&mock_image_ctx](uint64_t snap_id) -> librbd::SnapInfo* {
 496         auto it = mock_image_ctx.snap_info.find(snap_id);
 497         if (it == mock_image_ctx.snap_info.end()) {
 498           return nullptr;
 499         }
 500         return &it->second;
 501       }));
 502     EXPECT_CALL(*mock_image_ctx.operations, snap_remove(_, _, _))
 503       .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) {
 504         m_threads->work_queue->queue(ctx, r);
 505       })));
 506   }
 507
 508   void expect_snapshot_copy(MockSnapshotCopyRequest& mock_snapshot_copy_request,
 509                             uint64_t src_snap_id_start,
 510                             uint64_t src_snap_id_end,
 511                             uint64_t dst_snap_id_start,
 512                             const librbd::SnapSeqs& snap_seqs, int r) {
 513     EXPECT_CALL(mock_snapshot_copy_request, send())
 514       .WillOnce(Invoke([this, &req=mock_snapshot_copy_request,
 515                         src_snap_id_start, src_snap_id_end, dst_snap_id_start,
 516                         snap_seqs, r]() {
 517         ASSERT_EQ(src_snap_id_start, req.src_snap_id_start);
 518         ASSERT_EQ(src_snap_id_end, req.src_snap_id_end);
 519         ASSERT_EQ(dst_snap_id_start, req.dst_snap_id_start);
 520         *req.snap_seqs = snap_seqs;
 521         m_threads->work_queue->queue(req.on_finish, r);
 522       }));
 523   }
 524
 525   void expect_get_image_state(MockGetImageStateRequest& mock_get_image_state_request,
 526                               uint64_t snap_id, int r) {
 527     EXPECT_CALL(mock_get_image_state_request, send())
 528       .WillOnce(Invoke([this, &req=mock_get_image_state_request, snap_id, r]() {
 529         ASSERT_EQ(snap_id, req.snap_id);
 530         m_threads->work_queue->queue(req.on_finish, r);
 531       }));
 532   }
 533
 534   void expect_create_non_primary_request(MockCreateNonPrimaryRequest& mock_create_non_primary_request,
 535                                          bool demoted,
 536                                          const std::string& primary_mirror_uuid,
 537                                          uint64_t primary_snap_id,
 538                                          const librbd::SnapSeqs& snap_seqs,
 539                                          uint64_t snap_id, int r) {
 540     EXPECT_CALL(mock_create_non_primary_request, send())
 541       .WillOnce(Invoke([this, &req=mock_create_non_primary_request, demoted,
 542                         primary_mirror_uuid, primary_snap_id, snap_seqs,
 543                         snap_id, r]() {
 544         ASSERT_EQ(demoted, req.demoted);
 545         ASSERT_EQ(primary_mirror_uuid, req.primary_mirror_uuid);
 546         ASSERT_EQ(primary_snap_id, req.primary_snap_id);
 547         ASSERT_EQ(snap_seqs, req.snap_seqs);
 548         *req.snap_id = snap_id;
 549         m_threads->work_queue->queue(req.on_finish, r);
 550       }));
 551   }
 552
 553   void expect_update_mirror_image_state(MockImageStateUpdateRequest& mock_image_state_update_request,
 554                                         int r) {
 555     EXPECT_CALL(mock_image_state_update_request, send())
 556       .WillOnce(Invoke([this, &req=mock_image_state_update_request, r]() {
 557         m_threads->work_queue->queue(req.on_finish, r);
 558       }));
 559   }
 560
 561   void expect_notify_sync_request(MockInstanceWatcher& mock_instance_watcher,
 562                                   const std::string& image_id, int r) {
 563     EXPECT_CALL(mock_instance_watcher, notify_sync_request(image_id, _))
 564       .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) {
 565         m_threads->work_queue->queue(ctx, r);
 566       })));
 567   }
 568
 569   void expect_notify_sync_complete(MockInstanceWatcher& mock_instance_watcher,
 570                                    const std::string& image_id) {
 571     EXPECT_CALL(mock_instance_watcher, notify_sync_complete(image_id));
 572   }
 573
 574   void expect_cancel_sync_request(MockInstanceWatcher& mock_instance_watcher,
 575                                   const std::string& image_id) {
 576     EXPECT_CALL(mock_instance_watcher, cancel_sync_request(image_id));
 577   }
 578
 579   void expect_image_copy(MockImageCopyRequest& mock_image_copy_request,
 580                          uint64_t src_snap_id_start, uint64_t src_snap_id_end,
 581                          uint64_t dst_snap_id_start,
 582                          const librbd::deep_copy::ObjectNumber& object_number,
 583                          const librbd::SnapSeqs& snap_seqs, int r) {
 584     EXPECT_CALL(mock_image_copy_request, send())
 585       .WillOnce(Invoke([this, &req=mock_image_copy_request, src_snap_id_start,
 586                         src_snap_id_end, dst_snap_id_start, object_number,
 587                         snap_seqs, r]() {
 588         ASSERT_EQ(src_snap_id_start, req.src_snap_id_start);
 589         ASSERT_EQ(src_snap_id_end, req.src_snap_id_end);
 590         ASSERT_EQ(dst_snap_id_start, req.dst_snap_id_start);
 591         ASSERT_EQ(object_number, req.object_number);
 592         ASSERT_EQ(snap_seqs, req.snap_seqs);
 593         m_threads->work_queue->queue(req.on_finish, r);
 594       }));
 595   }
 596
 597   void expect_unlink_peer(MockUnlinkPeerRequest& mock_unlink_peer_request,
 598                           uint64_t snap_id, const std::string& mirror_peer_uuid,
 599                           int r) {
 600     EXPECT_CALL(mock_unlink_peer_request, send())
 601       .WillOnce(Invoke([this, &req=mock_unlink_peer_request, snap_id,
 602                         mirror_peer_uuid, r]() {
 603         ASSERT_EQ(snap_id, req.snap_id);
 604         ASSERT_EQ(mirror_peer_uuid, req.mirror_peer_uuid);
 605         m_threads->work_queue->queue(req.on_finish, r);
 606       }));
 607   }
 608
 609   void expect_apply_image_state(
 610       MockApplyImageStateRequest& mock_request, int r) {
 611     EXPECT_CALL(mock_request, send())
 612       .WillOnce(Invoke([this, &req=mock_request, r]() {
 613         m_threads->work_queue->queue(req.on_finish, r);
 614       }));
 615   }
 616
 617   void expect_mirror_image_snapshot_set_copy_progress(
 618       librbd::MockTestImageCtx& mock_test_image_ctx, uint64_t snap_id,
 619       bool completed, uint64_t last_copied_object, int r) {
 620     bufferlist bl;
 621     encode(snap_id, bl);
 622     encode(completed, bl);
 623     encode(last_copied_object, bl);
 624
 625     EXPECT_CALL(get_mock_io_ctx(mock_test_image_ctx.md_ctx),
 626                 exec(mock_test_image_ctx.header_oid, _, StrEq("rbd"),
 627                      StrEq("mirror_image_snapshot_set_copy_progress"),
 628                      ContentsEqual(bl), _, _, _))
 629       .WillOnce(Return(r));
 630   }
 631
 632   void expect_send(MockCloseImageRequest &mock_close_image_request, int r) {
 633     EXPECT_CALL(mock_close_image_request, send())
 634       .WillOnce(Invoke([this, &mock_close_image_request, r]() {
 635             *mock_close_image_request.image_ctx = nullptr;
 636             m_threads->work_queue->queue(mock_close_image_request.on_finish, r);
 637           }));
 638   }
 639
 640   void expect_notification(MockThreads& mock_threads,
 641                            MockReplayerListener& mock_replayer_listener) {
 642     EXPECT_CALL(mock_replayer_listener, handle_notification())
 643       .WillRepeatedly(Invoke([this]() {
 644           std::unique_lock locker{m_lock};
 645           ++m_notifications;
 646           m_cond.notify_all();
 647         }));
 648   }
 649
 650   int wait_for_notification(uint32_t count) {
 651     std::unique_lock locker{m_lock};
 652     for (uint32_t idx = 0; idx < count; ++idx) {
 653       while (m_notifications == 0) {
 654         if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) {
 655           return -ETIMEDOUT;
 656         }
 657       }
 658       --m_notifications;
 659     }
 660     return 0;
 661   }
 662
 663   int init_entry_replayer(MockReplayer& mock_replayer,
 664                           MockThreads& mock_threads,
 665                           librbd::MockTestImageCtx& mock_local_image_ctx,
 666                           librbd::MockTestImageCtx& mock_remote_image_ctx,
 667                           MockReplayerListener& mock_replayer_listener,
 668                           MockImageMeta& mock_image_meta,
 669                           librbd::UpdateWatchCtx** update_watch_ctx) {
 670     expect_register_update_watcher(mock_local_image_ctx, update_watch_ctx, 123,
 671                                    0);
 672     expect_register_update_watcher(mock_remote_image_ctx, update_watch_ctx, 234,
 673                                    0);
 674     expect_load_image_meta(mock_image_meta, false, 0);
 675     expect_is_refresh_required(mock_local_image_ctx, false);
 676     expect_is_refresh_required(mock_remote_image_ctx, false);
 677
 678     C_SaferCond init_ctx;
 679     mock_replayer.init(&init_ctx);
 680     int r = init_ctx.wait();
 681     if (r < 0) {
 682       return r;
 683     }
 684
 685     return wait_for_notification(2);
 686   }
 687
 688   int shut_down_entry_replayer(MockReplayer& mock_replayer,
 689                                MockThreads& mock_threads,
 690                                librbd::MockTestImageCtx& mock_local_image_ctx,
 691                                librbd::MockTestImageCtx& mock_remote_image_ctx) {
 692     expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0);
 693     expect_unregister_update_watcher(mock_local_image_ctx, 123, 0);
 694
 695     C_SaferCond shutdown_ctx;
 696     mock_replayer.shut_down(&shutdown_ctx);
 697     return shutdown_ctx.wait();
 698   }
 699
 700   librbd::ImageCtx* m_local_image_ctx = nullptr;
 701   librbd::ImageCtx* m_remote_image_ctx = nullptr;
 702
 703   PoolMetaCache m_pool_meta_cache{g_ceph_context};
 704
 705   ceph::mutex m_lock = ceph::make_mutex(
 706     "TestMockImageReplayerSnapshotReplayer");
 707   ceph::condition_variable m_cond;
 708   uint32_t m_notifications = 0;
 709 };
 710
 711 TEST_F(TestMockImageReplayerSnapshotReplayer, InitShutDown) {
 712   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
 713   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
 714
 715   MockThreads mock_threads(m_threads);
 716   expect_work_queue_repeatedly(mock_threads);
 717
 718   MockReplayerListener mock_replayer_listener;
 719   expect_notification(mock_threads, mock_replayer_listener);
 720
 721   InSequence seq;
 722
 723   MockInstanceWatcher mock_instance_watcher;
 724   MockImageMeta mock_image_meta;
 725   MockStateBuilder mock_state_builder(mock_local_image_ctx,
 726                                       mock_remote_image_ctx,
 727                                       mock_image_meta);
 728   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
 729                              "local mirror uuid", &m_pool_meta_cache,
 730                              &mock_state_builder, &mock_replayer_listener};
 731   m_pool_meta_cache.set_remote_pool_meta(
 732     m_remote_io_ctx.get_id(),
 733     {"remote mirror uuid", "remote mirror peer uuid"});
 734
 735   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
 736   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
 737                                    mock_local_image_ctx,
 738                                    mock_remote_image_ctx,
 739                                    mock_replayer_listener,
 740                                    mock_image_meta,
 741                                    &update_watch_ctx));
 742   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
 743                                         mock_local_image_ctx,
 744                                         mock_remote_image_ctx));
 745 }
 746
 747 TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) {
 748   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
 749   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
 750
 751   // it should sync two snapshots and skip two (user and mirror w/o matching
 752   // peer uuid)
 753   mock_remote_image_ctx.snap_info = {
 754     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 755        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 756        "", CEPH_NOSNAP, true, 0, {}},
 757      0, {}, 0, 0, {}}},
 758     {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{},
 759      0, {}, 0, 0, {}}},
 760     {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
 761        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""},
 762        "", CEPH_NOSNAP, true, 0, {}},
 763      0, {}, 0, 0, {}}},
 764     {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
 765        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 766        "", CEPH_NOSNAP, true, 0, {}},
 767      0, {}, 0, 0, {}}}};
 768
 769   MockThreads mock_threads(m_threads);
 770   expect_work_queue_repeatedly(mock_threads);
 771
 772   MockReplayerListener mock_replayer_listener;
 773   expect_notification(mock_threads, mock_replayer_listener);
 774
 775   InSequence seq;
 776
 777   MockInstanceWatcher mock_instance_watcher;
 778   MockImageMeta mock_image_meta;
 779   MockStateBuilder mock_state_builder(mock_local_image_ctx,
 780                                       mock_remote_image_ctx,
 781                                       mock_image_meta);
 782   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
 783                              "local mirror uuid", &m_pool_meta_cache,
 784                              &mock_state_builder, &mock_replayer_listener};
 785   m_pool_meta_cache.set_remote_pool_meta(
 786     m_remote_io_ctx.get_id(),
 787     {"remote mirror uuid", "remote mirror peer uuid"});
 788
 789   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
 790
 791   // init
 792   expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123,
 793                                  0);
 794   expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234,
 795                                  0);
 796
 797   // sync snap1
 798   expect_load_image_meta(mock_image_meta, false, 0);
 799   expect_is_refresh_required(mock_local_image_ctx, false);
 800   expect_is_refresh_required(mock_remote_image_ctx, false);
 801   MockSnapshotCopyRequest mock_snapshot_copy_request;
 802   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
 803                        0);
 804   MockGetImageStateRequest mock_get_image_state_request;
 805   expect_get_image_state(mock_get_image_state_request, 1, 0);
 806   MockCreateNonPrimaryRequest mock_create_non_primary_request;
 807   expect_create_non_primary_request(mock_create_non_primary_request,
 808                                     false, "remote mirror uuid", 1,
 809                                     {{1, CEPH_NOSNAP}}, 11, 0);
 810   MockImageStateUpdateRequest mock_image_state_update_request;
 811   expect_update_mirror_image_state(mock_image_state_update_request, 0);
 812   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
 813   MockImageCopyRequest mock_image_copy_request;
 814   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
 815                     {{1, CEPH_NOSNAP}}, 0);
 816   MockApplyImageStateRequest mock_apply_state_request;
 817   expect_apply_image_state(mock_apply_state_request, 0);
 818   expect_mirror_image_snapshot_set_copy_progress(
 819     mock_local_image_ctx, 11, true, 0, 0);
 820   expect_notify_update(mock_local_image_ctx);
 821   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
 822
 823   // sync snap4
 824   expect_load_image_meta(mock_image_meta, false, 0);
 825   expect_is_refresh_required(mock_local_image_ctx, true);
 826   expect_refresh(
 827     mock_local_image_ctx, {
 828       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 829          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 830          1, true, 0, {{1, CEPH_NOSNAP}}},
 831        0, {}, 0, 0, {}}},
 832     }, 0);
 833   expect_is_refresh_required(mock_remote_image_ctx, true);
 834   expect_refresh(
 835     mock_remote_image_ctx, {
 836       {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 837          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 838          "", CEPH_NOSNAP, true, 0, {}},
 839        0, {}, 0, 0, {}}},
 840       {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{},
 841        0, {}, 0, 0, {}}},
 842       {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
 843          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""},
 844          "", CEPH_NOSNAP, true, 0, {}},
 845        0, {}, 0, 0, {}}},
 846       {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
 847          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 848          "", CEPH_NOSNAP, true, 0, {}},
 849        0, {}, 0, 0, {}}},
 850       {5U, librbd::SnapInfo{"snap5", cls::rbd::MirrorSnapshotNamespace{
 851          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 852          "", CEPH_NOSNAP, true, 0, {}},
 853        0, {}, 0, 0, {}}}
 854     }, 0);
 855   expect_snapshot_copy(mock_snapshot_copy_request, 1, 4, 11,
 856                        {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 0);
 857   expect_get_image_state(mock_get_image_state_request, 4, 0);
 858   expect_create_non_primary_request(mock_create_non_primary_request,
 859                                     false, "remote mirror uuid", 4,
 860                                     {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 14,
 861                                     0);
 862   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
 863   expect_image_copy(mock_image_copy_request, 1, 4, 11, {},
 864                     {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 0);
 865   expect_apply_image_state(mock_apply_state_request, 0);
 866   expect_mirror_image_snapshot_set_copy_progress(
 867     mock_local_image_ctx, 14, true, 0, 0);
 868   expect_notify_update(mock_local_image_ctx);
 869   MockUnlinkPeerRequest mock_unlink_peer_request;
 870   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
 871                      0);
 872   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
 873
 874   // prune non-primary snap1
 875   expect_load_image_meta(mock_image_meta, false, 0);
 876   expect_is_refresh_required(mock_local_image_ctx, true);
 877   expect_refresh(
 878     mock_local_image_ctx, {
 879       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 880          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 881          1, true, 0, {}},
 882        0, {}, 0, 0, {}}},
 883       {12U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{},
 884        0, {}, 0, 0, {}}},
 885       {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
 886          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 887          4, true, 0, {}},
 888        0, {}, 0, 0, {}}},
 889     }, 0);
 890   expect_is_refresh_required(mock_remote_image_ctx, true);
 891   expect_refresh(
 892     mock_remote_image_ctx, {
 893       {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 894          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 895          "", CEPH_NOSNAP, true, 0, {}},
 896        0, {}, 0, 0, {}}},
 897       {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{},
 898        0, {}, 0, 0, {}}},
 899       {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
 900          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""},
 901          "", CEPH_NOSNAP, true, 0, {}},
 902        0, {}, 0, 0, {}}},
 903       {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
 904          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 905          "", CEPH_NOSNAP, true, 0, {}},
 906        0, {}, 0, 0, {}}}
 907     }, 0);
 908   expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0);
 909
 910   // idle
 911   expect_load_image_meta(mock_image_meta, false, 0);
 912   expect_is_refresh_required(mock_local_image_ctx, true);
 913   expect_refresh(
 914     mock_local_image_ctx, {
 915       {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
 916          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 917          4, true, 0, {}},
 918        0, {}, 0, 0, {}}},
 919     }, 0);
 920   expect_is_refresh_required(mock_remote_image_ctx, true);
 921   expect_refresh(
 922     mock_remote_image_ctx, {
 923       {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
 924          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 925          "", CEPH_NOSNAP, true, 0, {}},
 926        0, {}, 0, 0, {}}}
 927     }, 0);
 928
 929   // fire init
 930   C_SaferCond init_ctx;
 931   mock_replayer.init(&init_ctx);
 932   ASSERT_EQ(0, init_ctx.wait());
 933
 934   // wait for sync to complete
 935   ASSERT_EQ(0, wait_for_notification(4));
 936
 937   // shut down
 938   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
 939                                         mock_local_image_ctx,
 940                                         mock_remote_image_ctx));
 941 }
 942
 943 TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncInitial) {
 944   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
 945   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
 946
 947   MockThreads mock_threads(m_threads);
 948   expect_work_queue_repeatedly(mock_threads);
 949
 950   MockReplayerListener mock_replayer_listener;
 951   expect_notification(mock_threads, mock_replayer_listener);
 952
 953   InSequence seq;
 954
 955   MockInstanceWatcher mock_instance_watcher;
 956   MockImageMeta mock_image_meta;
 957   MockStateBuilder mock_state_builder(mock_local_image_ctx,
 958                                       mock_remote_image_ctx,
 959                                       mock_image_meta);
 960   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
 961                              "local mirror uuid", &m_pool_meta_cache,
 962                              &mock_state_builder, &mock_replayer_listener};
 963   m_pool_meta_cache.set_remote_pool_meta(
 964     m_remote_io_ctx.get_id(),
 965     {"remote mirror uuid", "remote mirror peer uuid"});
 966
 967   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
 968   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
 969                                    mock_local_image_ctx,
 970                                    mock_remote_image_ctx,
 971                                    mock_replayer_listener,
 972                                    mock_image_meta,
 973                                    &update_watch_ctx));
 974
 975   // inject an incomplete sync snapshot with last_copied_object_number > 0
 976   mock_remote_image_ctx.snap_info = {
 977     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 978        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 979        "", CEPH_NOSNAP, true, 0, {}},
 980      0, {}, 0, 0, {}}}};
 981   mock_local_image_ctx.snap_info = {
 982     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 983        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 984        1, false, 123, {{1, CEPH_NOSNAP}}},
 985      0, {}, 0, 0, {}}}};
 986
 987   // re-sync snap1
 988   expect_load_image_meta(mock_image_meta, false, 0);
 989   expect_is_refresh_required(mock_local_image_ctx, false);
 990   expect_is_refresh_required(mock_remote_image_ctx, false);
 991   MockGetImageStateRequest mock_get_image_state_request;
 992   expect_get_image_state(mock_get_image_state_request, 11, 0);
 993   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
 994   MockImageCopyRequest mock_image_copy_request;
 995   expect_image_copy(mock_image_copy_request, 0, 1, 0,
 996                     librbd::deep_copy::ObjectNumber{123U},
 997                     {{1, CEPH_NOSNAP}}, 0);
 998   MockApplyImageStateRequest mock_apply_state_request;
 999   expect_apply_image_state(mock_apply_state_request, 0);
1000   expect_mirror_image_snapshot_set_copy_progress(
1001     mock_local_image_ctx, 11, true, 123, 0);
1002   expect_notify_update(mock_local_image_ctx);
1003   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
1004
1005   // idle
1006   expect_load_image_meta(mock_image_meta, false, 0);
1007   expect_is_refresh_required(mock_local_image_ctx, true);
1008   expect_refresh(
1009     mock_local_image_ctx, {
1010       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1011          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1012          1, true, 0, {}},
1013        0, {}, 0, 0, {}}},
1014     }, 0);
1015   expect_is_refresh_required(mock_remote_image_ctx, false);
1016
1017   // wake-up replayer
1018   update_watch_ctx->handle_notify();
1019
1020   // wait for sync to complete
1021   ASSERT_EQ(0, wait_for_notification(2));
1022
1023   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1024                                         mock_local_image_ctx,
1025                                         mock_remote_image_ctx));
1026 }
1027
1028 TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncDelta) {
1029   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1030   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1031
1032   MockThreads mock_threads(m_threads);
1033   expect_work_queue_repeatedly(mock_threads);
1034
1035   MockReplayerListener mock_replayer_listener;
1036   expect_notification(mock_threads, mock_replayer_listener);
1037
1038   InSequence seq;
1039
1040   MockInstanceWatcher mock_instance_watcher;
1041   MockImageMeta mock_image_meta;
1042   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1043                                       mock_remote_image_ctx,
1044                                       mock_image_meta);
1045   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1046                              "local mirror uuid", &m_pool_meta_cache,
1047                              &mock_state_builder, &mock_replayer_listener};
1048   m_pool_meta_cache.set_remote_pool_meta(
1049     m_remote_io_ctx.get_id(),
1050     {"remote mirror uuid", "remote mirror peer uuid"});
1051
1052   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1053   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1054                                    mock_local_image_ctx,
1055                                    mock_remote_image_ctx,
1056                                    mock_replayer_listener,
1057                                    mock_image_meta,
1058                                    &update_watch_ctx));
1059
1060   // inject an incomplete sync snapshot with last_copied_object_number > 0
1061   // after a complete snapshot
1062   mock_remote_image_ctx.snap_info = {
1063     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1064        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1065        "", CEPH_NOSNAP, true, 0, {}},
1066      0, {}, 0, 0, {}}},
1067     {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1068        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1069        "", CEPH_NOSNAP, true, 0, {}},
1070      0, {}, 0, 0, {}}}};
1071   mock_local_image_ctx.snap_info = {
1072     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1073        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1074        1, true, 0, {{1, CEPH_NOSNAP}}},
1075      0, {}, 0, 0, {}}},
1076     {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1077        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1078        2, false, 123, {{2, CEPH_NOSNAP}}},
1079      0, {}, 0, 0, {}}}};
1080
1081   // re-sync snap2
1082   expect_load_image_meta(mock_image_meta, false, 0);
1083   expect_is_refresh_required(mock_local_image_ctx, false);
1084   expect_is_refresh_required(mock_remote_image_ctx, false);
1085   MockGetImageStateRequest mock_get_image_state_request;
1086   expect_get_image_state(mock_get_image_state_request, 12, 0);
1087   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1088   MockImageCopyRequest mock_image_copy_request;
1089   expect_image_copy(mock_image_copy_request, 1, 2, 11,
1090                     librbd::deep_copy::ObjectNumber{123U},
1091                     {{2, CEPH_NOSNAP}}, 0);
1092   MockApplyImageStateRequest mock_apply_state_request;
1093   expect_apply_image_state(mock_apply_state_request, 0);
1094   expect_mirror_image_snapshot_set_copy_progress(
1095     mock_local_image_ctx, 12, true, 123, 0);
1096   expect_notify_update(mock_local_image_ctx);
1097   MockUnlinkPeerRequest mock_unlink_peer_request;
1098   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
1099                      0);
1100   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
1101
1102   // prune non-primary snap1
1103   expect_load_image_meta(mock_image_meta, false, 0);
1104   expect_is_refresh_required(mock_local_image_ctx, true);
1105   expect_refresh(
1106     mock_local_image_ctx, {
1107       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1108          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1109          1, true, 0, {}},
1110        0, {}, 0, 0, {}}},
1111       {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1112          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1113          2, true, 0, {}},
1114        0, {}, 0, 0, {}}},
1115     }, 0);
1116   expect_is_refresh_required(mock_remote_image_ctx, true);
1117   expect_refresh(
1118     mock_remote_image_ctx, {
1119       {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1120          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1121          "", CEPH_NOSNAP, true, 0, {}},
1122        0, {}, 0, 0, {}}},
1123     }, 0);
1124   expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0);
1125
1126   // idle
1127   expect_load_image_meta(mock_image_meta, false, 0);
1128   expect_is_refresh_required(mock_local_image_ctx, true);
1129   expect_refresh(
1130     mock_local_image_ctx, {
1131       {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1132          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1133          2, true, 0, {}},
1134        0, {}, 0, 0, {}}},
1135     }, 0);
1136   expect_is_refresh_required(mock_remote_image_ctx, false);
1137
1138   // wake-up replayer
1139   update_watch_ctx->handle_notify();
1140
1141   // wait for sync to complete
1142   ASSERT_EQ(0, wait_for_notification(2));
1143
1144   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1145                                         mock_local_image_ctx,
1146                                         mock_remote_image_ctx));
1147 }
1148
1149 TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncDeltaDemote) {
1150   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1151   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1152
1153   MockThreads mock_threads(m_threads);
1154   expect_work_queue_repeatedly(mock_threads);
1155
1156   MockReplayerListener mock_replayer_listener;
1157   expect_notification(mock_threads, mock_replayer_listener);
1158
1159   InSequence seq;
1160
1161   MockInstanceWatcher mock_instance_watcher;
1162   MockImageMeta mock_image_meta;
1163   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1164                                       mock_remote_image_ctx,
1165                                       mock_image_meta);
1166   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1167                              "local mirror uuid", &m_pool_meta_cache,
1168                              &mock_state_builder, &mock_replayer_listener};
1169   m_pool_meta_cache.set_remote_pool_meta(
1170     m_remote_io_ctx.get_id(),
1171     {"remote mirror uuid", "remote mirror peer uuid"});
1172
1173   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1174   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1175                                    mock_local_image_ctx,
1176                                    mock_remote_image_ctx,
1177                                    mock_replayer_listener,
1178                                    mock_image_meta,
1179                                    &update_watch_ctx));
1180
1181   // inject an incomplete sync snapshot with last_copied_object_number > 0
1182   // after a primary demotion snapshot
1183   mock_remote_image_ctx.snap_info = {
1184     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1185        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED,
1186        {"remote mirror peer uuid"}, "local mirror uuid", 11, true, 0,
1187        {{11, CEPH_NOSNAP}}},
1188      0, {}, 0, 0, {}}},
1189     {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1190        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1191        "", CEPH_NOSNAP, true, 0, {}},
1192      0, {}, 0, 0, {}}}};
1193   mock_local_image_ctx.snap_info = {
1194     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1195        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
1196        {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}},
1197      0, {}, 0, 0, {}}},
1198     {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1199        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1200        2, false, 123, {{2, CEPH_NOSNAP}}},
1201      0, {}, 0, 0, {}}}};
1202
1203   // re-sync snap2
1204   expect_load_image_meta(mock_image_meta, false, 0);
1205   expect_is_refresh_required(mock_local_image_ctx, false);
1206   expect_is_refresh_required(mock_remote_image_ctx, false);
1207   MockGetImageStateRequest mock_get_image_state_request;
1208   expect_get_image_state(mock_get_image_state_request, 12, 0);
1209   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1210   MockImageCopyRequest mock_image_copy_request;
1211   expect_image_copy(mock_image_copy_request, 1, 2, 11,
1212                     librbd::deep_copy::ObjectNumber{123U},
1213                     {{2, CEPH_NOSNAP}}, 0);
1214   MockApplyImageStateRequest mock_apply_state_request;
1215   expect_apply_image_state(mock_apply_state_request, 0);
1216   expect_mirror_image_snapshot_set_copy_progress(
1217     mock_local_image_ctx, 12, true, 123, 0);
1218   expect_notify_update(mock_local_image_ctx);
1219   MockUnlinkPeerRequest mock_unlink_peer_request;
1220   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
1221                      0);
1222   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
1223
1224   // idle
1225   expect_load_image_meta(mock_image_meta, false, 0);
1226   expect_is_refresh_required(mock_local_image_ctx, true);
1227   expect_refresh(
1228     mock_local_image_ctx, {
1229       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1230          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
1231          {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}},
1232        0, {}, 0, 0, {}}},
1233       {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1234          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1235          2, true, 0, {}},
1236        0, {}, 0, 0, {}}},
1237     }, 0);
1238   expect_is_refresh_required(mock_remote_image_ctx, true);
1239   expect_refresh(
1240     mock_remote_image_ctx, {
1241       {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1242          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1243          "", CEPH_NOSNAP, true, 0, {}},
1244        0, {}, 0, 0, {}}},
1245     }, 0);
1246
1247   // wake-up replayer
1248   update_watch_ctx->handle_notify();
1249
1250   // wait for sync to complete
1251   ASSERT_EQ(0, wait_for_notification(2));
1252
1253   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1254                                         mock_local_image_ctx,
1255                                         mock_remote_image_ctx));
1256 }
1257
1258 TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncInitial) {
1259   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1260   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1261
1262   MockThreads mock_threads(m_threads);
1263   expect_work_queue_repeatedly(mock_threads);
1264
1265   MockReplayerListener mock_replayer_listener;
1266   expect_notification(mock_threads, mock_replayer_listener);
1267
1268   InSequence seq;
1269
1270   MockInstanceWatcher mock_instance_watcher;
1271   MockImageMeta mock_image_meta;
1272   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1273                                       mock_remote_image_ctx,
1274                                       mock_image_meta);
1275   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1276                              "local mirror uuid", &m_pool_meta_cache,
1277                              &mock_state_builder, &mock_replayer_listener};
1278   m_pool_meta_cache.set_remote_pool_meta(
1279     m_remote_io_ctx.get_id(),
1280     {"remote mirror uuid", "remote mirror peer uuid"});
1281
1282   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1283   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1284                                    mock_local_image_ctx,
1285                                    mock_remote_image_ctx,
1286                                    mock_replayer_listener,
1287                                    mock_image_meta,
1288                                    &update_watch_ctx));
1289
1290   // inject an incomplete sync snapshot with last_copied_object_number == 0
1291   mock_remote_image_ctx.snap_info = {
1292     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1293        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1294        "", CEPH_NOSNAP, true, 0, {}},
1295      0, {}, 0, 0, {}}}};
1296   mock_local_image_ctx.snap_info = {
1297     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1298        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1299        1, false, 0, {{1, CEPH_NOSNAP}}},
1300      0, {}, 0, 0, {}}}};
1301
1302   // re-sync snap1
1303   expect_load_image_meta(mock_image_meta, false, 0);
1304   expect_is_refresh_required(mock_local_image_ctx, false);
1305   expect_is_refresh_required(mock_remote_image_ctx, false);
1306   MockGetImageStateRequest mock_get_image_state_request;
1307   expect_get_image_state(mock_get_image_state_request, 11, 0);
1308   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1309   MockImageCopyRequest mock_image_copy_request;
1310   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
1311                     {{1, CEPH_NOSNAP}}, 0);
1312   MockApplyImageStateRequest mock_apply_state_request;
1313   expect_apply_image_state(mock_apply_state_request, 0);
1314   expect_mirror_image_snapshot_set_copy_progress(
1315     mock_local_image_ctx, 11, true, 0, 0);
1316   expect_notify_update(mock_local_image_ctx);
1317   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
1318
1319   // idle
1320   expect_load_image_meta(mock_image_meta, false, 0);
1321   expect_is_refresh_required(mock_local_image_ctx, true);
1322   expect_refresh(
1323     mock_local_image_ctx, {
1324       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1325          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1326          1, true, 0, {}},
1327        0, {}, 0, 0, {}}},
1328     }, 0);
1329   expect_is_refresh_required(mock_remote_image_ctx, false);
1330
1331   // wake-up replayer
1332   update_watch_ctx->handle_notify();
1333
1334   // wait for sync to complete
1335   ASSERT_EQ(0, wait_for_notification(2));
1336
1337   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1338                                         mock_local_image_ctx,
1339                                         mock_remote_image_ctx));
1340 }
1341
1342 TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncDelta) {
1343   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1344   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1345
1346   MockThreads mock_threads(m_threads);
1347   expect_work_queue_repeatedly(mock_threads);
1348
1349   MockReplayerListener mock_replayer_listener;
1350   expect_notification(mock_threads, mock_replayer_listener);
1351
1352   InSequence seq;
1353
1354   MockInstanceWatcher mock_instance_watcher;
1355   MockImageMeta mock_image_meta;
1356   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1357                                       mock_remote_image_ctx,
1358                                       mock_image_meta);
1359   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1360                              "local mirror uuid", &m_pool_meta_cache,
1361                              &mock_state_builder, &mock_replayer_listener};
1362   m_pool_meta_cache.set_remote_pool_meta(
1363     m_remote_io_ctx.get_id(),
1364     {"remote mirror uuid", "remote mirror peer uuid"});
1365
1366   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1367   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1368                                    mock_local_image_ctx,
1369                                    mock_remote_image_ctx,
1370                                    mock_replayer_listener,
1371                                    mock_image_meta,
1372                                    &update_watch_ctx));
1373
1374   // inject an incomplete sync snapshot with last_copied_object_number == 0
1375   // after a complete snapshot
1376   mock_remote_image_ctx.snap_info = {
1377     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1378        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1379        "", CEPH_NOSNAP, true, 0, {}},
1380      0, {}, 0, 0, {}}},
1381     {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1382        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1383        "", CEPH_NOSNAP, true, 0, {}},
1384      0, {}, 0, 0, {}}}};
1385   mock_local_image_ctx.snap_info = {
1386     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1387        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1388        1, true, 0, {{1, CEPH_NOSNAP}}},
1389      0, {}, 0, 0, {}}},
1390     {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1391        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1392        2, false, 0, {{2, CEPH_NOSNAP}}},
1393      0, {}, 0, 0, {}}}};
1394
1395   // prune non-primary snap2
1396   expect_load_image_meta(mock_image_meta, false, 0);
1397   expect_is_refresh_required(mock_local_image_ctx, false);
1398   expect_is_refresh_required(mock_remote_image_ctx, false);
1399   expect_prune_non_primary_snapshot(mock_local_image_ctx, 12, 0);
1400
1401   // sync snap2
1402   expect_load_image_meta(mock_image_meta, false, 0);
1403   expect_is_refresh_required(mock_local_image_ctx, true);
1404   expect_refresh(
1405     mock_local_image_ctx, {
1406       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1407          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1408          1, true, 0, {{1, CEPH_NOSNAP}}},
1409        0, {}, 0, 0, {}}},
1410     }, 0);
1411   expect_is_refresh_required(mock_remote_image_ctx, false);
1412   MockSnapshotCopyRequest mock_snapshot_copy_request;
1413   expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11,
1414                        {{2, CEPH_NOSNAP}}, 0);
1415   MockGetImageStateRequest mock_get_image_state_request;
1416   expect_get_image_state(mock_get_image_state_request, 2, 0);
1417   MockCreateNonPrimaryRequest mock_create_non_primary_request;
1418   expect_create_non_primary_request(mock_create_non_primary_request,
1419                                     false, "remote mirror uuid", 2,
1420                                     {{2, CEPH_NOSNAP}}, 13, 0);
1421   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1422   MockImageCopyRequest mock_image_copy_request;
1423   expect_image_copy(mock_image_copy_request, 1, 2, 11, {},
1424                     {{2, CEPH_NOSNAP}}, 0);
1425   MockApplyImageStateRequest mock_apply_state_request;
1426   expect_apply_image_state(mock_apply_state_request, 0);
1427   expect_mirror_image_snapshot_set_copy_progress(
1428     mock_local_image_ctx, 13, true, 0, 0);
1429   expect_notify_update(mock_local_image_ctx);
1430   MockUnlinkPeerRequest mock_unlink_peer_request;
1431   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
1432                      0);
1433   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
1434
1435   // prune non-primary snap1
1436   expect_load_image_meta(mock_image_meta, false, 0);
1437   expect_is_refresh_required(mock_local_image_ctx, true);
1438   expect_refresh(
1439     mock_local_image_ctx, {
1440       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1441          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1442          1, true, 0, {}},
1443        0, {}, 0, 0, {}}},
1444       {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1445          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1446          2, true, 0, {}},
1447        0, {}, 0, 0, {}}},
1448     }, 0);
1449   expect_is_refresh_required(mock_remote_image_ctx, true);
1450   expect_refresh(
1451     mock_remote_image_ctx, {
1452       {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1453          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1454          "", CEPH_NOSNAP, true, 0, {}},
1455        0, {}, 0, 0, {}}},
1456     }, 0);
1457   expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0);
1458
1459   // idle
1460   expect_load_image_meta(mock_image_meta, false, 0);
1461   expect_is_refresh_required(mock_local_image_ctx, true);
1462   expect_refresh(
1463     mock_local_image_ctx, {
1464       {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1465          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1466          2, true, 0, {}},
1467        0, {}, 0, 0, {}}},
1468     }, 0);
1469   expect_is_refresh_required(mock_remote_image_ctx, false);
1470
1471   // wake-up replayer
1472   update_watch_ctx->handle_notify();
1473
1474   // wait for sync to complete
1475   ASSERT_EQ(0, wait_for_notification(2));
1476
1477   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1478                                         mock_local_image_ctx,
1479                                         mock_remote_image_ctx));
1480 }
1481
1482 TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncDeltaDemote) {
1483   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1484   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1485
1486   MockThreads mock_threads(m_threads);
1487   expect_work_queue_repeatedly(mock_threads);
1488
1489   MockReplayerListener mock_replayer_listener;
1490   expect_notification(mock_threads, mock_replayer_listener);
1491
1492   InSequence seq;
1493
1494   MockInstanceWatcher mock_instance_watcher;
1495   MockImageMeta mock_image_meta;
1496   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1497                                       mock_remote_image_ctx,
1498                                       mock_image_meta);
1499   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1500                              "local mirror uuid", &m_pool_meta_cache,
1501                              &mock_state_builder, &mock_replayer_listener};
1502   m_pool_meta_cache.set_remote_pool_meta(
1503     m_remote_io_ctx.get_id(),
1504     {"remote mirror uuid", "remote mirror peer uuid"});
1505
1506   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1507   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1508                                    mock_local_image_ctx,
1509                                    mock_remote_image_ctx,
1510                                    mock_replayer_listener,
1511                                    mock_image_meta,
1512                                    &update_watch_ctx));
1513
1514   // inject an incomplete sync snapshot with last_copied_object_number == 0
1515   // after a primary demotion snapshot
1516   mock_remote_image_ctx.snap_info = {
1517     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1518        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED,
1519        {"remote mirror peer uuid"}, "local mirror uuid", 11, true, 0,
1520        {{11, CEPH_NOSNAP}}},
1521      0, {}, 0, 0, {}}},
1522     {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1523        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1524        "", CEPH_NOSNAP, true, 0, {}},
1525      0, {}, 0, 0, {}}}};
1526   mock_local_image_ctx.snap_info = {
1527     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1528        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
1529        {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}},
1530      0, {}, 0, 0, {}}},
1531     {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1532        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1533        2, false, 0, {{2, CEPH_NOSNAP}}},
1534      0, {}, 0, 0, {}}}};
1535
1536   // prune non-primary snap2
1537   expect_load_image_meta(mock_image_meta, false, 0);
1538   expect_is_refresh_required(mock_local_image_ctx, false);
1539   expect_is_refresh_required(mock_remote_image_ctx, false);
1540   expect_prune_non_primary_snapshot(mock_local_image_ctx, 12, 0);
1541
1542   // sync snap2
1543   expect_load_image_meta(mock_image_meta, false, 0);
1544   expect_is_refresh_required(mock_local_image_ctx, true);
1545   expect_refresh(
1546     mock_local_image_ctx, {
1547       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1548          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
1549          {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}},
1550        0, {}, 0, 0, {}}},
1551     }, 0);
1552   expect_is_refresh_required(mock_remote_image_ctx, false);
1553   MockSnapshotCopyRequest mock_snapshot_copy_request;
1554   expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11,
1555                        {{2, CEPH_NOSNAP}}, 0);
1556   MockGetImageStateRequest mock_get_image_state_request;
1557   expect_get_image_state(mock_get_image_state_request, 2, 0);
1558   MockCreateNonPrimaryRequest mock_create_non_primary_request;
1559   expect_create_non_primary_request(mock_create_non_primary_request,
1560                                     false, "remote mirror uuid", 2,
1561                                     {{2, CEPH_NOSNAP}}, 13, 0);
1562   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1563   MockImageCopyRequest mock_image_copy_request;
1564   expect_image_copy(mock_image_copy_request, 1, 2, 11, {},
1565                     {{2, CEPH_NOSNAP}}, 0);
1566   MockApplyImageStateRequest mock_apply_state_request;
1567   expect_apply_image_state(mock_apply_state_request, 0);
1568   expect_mirror_image_snapshot_set_copy_progress(
1569     mock_local_image_ctx, 13, true, 0, 0);
1570   expect_notify_update(mock_local_image_ctx);
1571   MockUnlinkPeerRequest mock_unlink_peer_request;
1572   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
1573                      0);
1574   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
1575
1576   // idle
1577   expect_load_image_meta(mock_image_meta, false, 0);
1578   expect_is_refresh_required(mock_local_image_ctx, true);
1579   expect_refresh(
1580     mock_local_image_ctx, {
1581       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1582          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
1583          {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}},
1584        0, {}, 0, 0, {}}},
1585       {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1586          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1587          2, true, 0, {}},
1588        0, {}, 0, 0, {}}},
1589     }, 0);
1590   expect_is_refresh_required(mock_remote_image_ctx, true);
1591   expect_refresh(
1592     mock_remote_image_ctx, {
1593       {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1594          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1595          "", CEPH_NOSNAP, true, 0, {}},
1596        0, {}, 0, 0, {}}},
1597     }, 0);
1598
1599   // wake-up replayer
1600   update_watch_ctx->handle_notify();
1601
1602   // wait for sync to complete
1603   ASSERT_EQ(0, wait_for_notification(2));
1604
1605   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1606                                         mock_local_image_ctx,
1607                                         mock_remote_image_ctx));
1608 }
1609
1610 TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) {
1611   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1612   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1613
1614   MockThreads mock_threads(m_threads);
1615   expect_work_queue_repeatedly(mock_threads);
1616
1617   MockReplayerListener mock_replayer_listener;
1618   expect_notification(mock_threads, mock_replayer_listener);
1619
1620   InSequence seq;
1621
1622   MockInstanceWatcher mock_instance_watcher;
1623   MockImageMeta mock_image_meta;
1624   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1625                                       mock_remote_image_ctx,
1626                                       mock_image_meta);
1627   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1628                              "local mirror uuid", &m_pool_meta_cache,
1629                              &mock_state_builder, &mock_replayer_listener};
1630   m_pool_meta_cache.set_remote_pool_meta(
1631     m_remote_io_ctx.get_id(),
1632     {"remote mirror uuid", "remote mirror peer uuid"});
1633
1634   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1635   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1636                                    mock_local_image_ctx,
1637                                    mock_remote_image_ctx,
1638                                    mock_replayer_listener,
1639                                    mock_image_meta,
1640                                    &update_watch_ctx));
1641
1642   // inject a demotion snapshot
1643   mock_remote_image_ctx.snap_info = {
1644     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1645        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
1646        {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}},
1647      0, {}, 0, 0, {}}}};
1648
1649   // sync snap1
1650   expect_load_image_meta(mock_image_meta, false, 0);
1651   expect_is_refresh_required(mock_local_image_ctx, false);
1652   expect_is_refresh_required(mock_remote_image_ctx, false);
1653   MockSnapshotCopyRequest mock_snapshot_copy_request;
1654   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
1655                        0);
1656   MockGetImageStateRequest mock_get_image_state_request;
1657   expect_get_image_state(mock_get_image_state_request, 1, 0);
1658   MockCreateNonPrimaryRequest mock_create_non_primary_request;
1659   expect_create_non_primary_request(mock_create_non_primary_request,
1660                                     true, "remote mirror uuid", 1,
1661                                     {{1, CEPH_NOSNAP}}, 11, 0);
1662   MockImageStateUpdateRequest mock_image_state_update_request;
1663   expect_update_mirror_image_state(mock_image_state_update_request, 0);
1664   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1665   MockImageCopyRequest mock_image_copy_request;
1666   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
1667                     {{1, CEPH_NOSNAP}}, 0);
1668   MockApplyImageStateRequest mock_apply_state_request;
1669   expect_apply_image_state(mock_apply_state_request, 0);
1670   expect_mirror_image_snapshot_set_copy_progress(
1671     mock_local_image_ctx, 11, true, 0, 0);
1672   expect_notify_update(mock_local_image_ctx);
1673   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
1674
1675   // idle
1676   expect_load_image_meta(mock_image_meta, false, 0);
1677   expect_is_refresh_required(mock_local_image_ctx, true);
1678   expect_refresh(
1679     mock_local_image_ctx, {
1680       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1681          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1682          1, true, 0, {}},
1683        0, {}, 0, 0, {}}},
1684     }, 0);
1685   expect_is_refresh_required(mock_remote_image_ctx, false);
1686
1687   // wake-up replayer
1688   update_watch_ctx->handle_notify();
1689
1690   // wait for sync to complete and expect replay complete
1691   ASSERT_EQ(0, wait_for_notification(2));
1692   ASSERT_FALSE(mock_replayer.is_replaying());
1693
1694   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1695                                         mock_local_image_ctx,
1696                                         mock_remote_image_ctx));
1697 }
1698
1699 TEST_F(TestMockImageReplayerSnapshotReplayer, LocalImagePromoted) {
1700   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1701   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1702
1703   MockThreads mock_threads(m_threads);
1704   expect_work_queue_repeatedly(mock_threads);
1705
1706   MockReplayerListener mock_replayer_listener;
1707   expect_notification(mock_threads, mock_replayer_listener);
1708
1709   InSequence seq;
1710
1711   MockInstanceWatcher mock_instance_watcher;
1712   MockImageMeta mock_image_meta;
1713   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1714                                       mock_remote_image_ctx,
1715                                       mock_image_meta);
1716   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1717                              "local mirror uuid", &m_pool_meta_cache,
1718                              &mock_state_builder, &mock_replayer_listener};
1719   m_pool_meta_cache.set_remote_pool_meta(
1720     m_remote_io_ctx.get_id(),
1721     {"remote mirror uuid", "remote mirror peer uuid"});
1722
1723   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1724   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1725                                    mock_local_image_ctx,
1726                                    mock_remote_image_ctx,
1727                                    mock_replayer_listener,
1728                                    mock_image_meta,
1729                                    &update_watch_ctx));
1730
1731   // inject a promotion snapshot
1732   mock_local_image_ctx.snap_info = {
1733     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1734        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
1735        {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}},
1736      0, {}, 0, 0, {}}}};
1737
1738   // idle
1739   expect_load_image_meta(mock_image_meta, false, 0);
1740   expect_is_refresh_required(mock_local_image_ctx, false);
1741   expect_is_refresh_required(mock_remote_image_ctx, false);
1742
1743   // wake-up replayer
1744   update_watch_ctx->handle_notify();
1745
1746   // wait for sync to complete and expect replay complete
1747   ASSERT_EQ(0, wait_for_notification(1));
1748   ASSERT_FALSE(mock_replayer.is_replaying());
1749
1750   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1751                                         mock_local_image_ctx,
1752                                         mock_remote_image_ctx));
1753 }
1754
1755 TEST_F(TestMockImageReplayerSnapshotReplayer, ResyncRequested) {
1756   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1757   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1758
1759   MockThreads mock_threads(m_threads);
1760   expect_work_queue_repeatedly(mock_threads);
1761
1762   MockReplayerListener mock_replayer_listener;
1763   expect_notification(mock_threads, mock_replayer_listener);
1764
1765   InSequence seq;
1766
1767   MockInstanceWatcher mock_instance_watcher;
1768   MockImageMeta mock_image_meta;
1769   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1770                                       mock_remote_image_ctx,
1771                                       mock_image_meta);
1772   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1773                              "local mirror uuid", &m_pool_meta_cache,
1774                              &mock_state_builder, &mock_replayer_listener};
1775   m_pool_meta_cache.set_remote_pool_meta(
1776     m_remote_io_ctx.get_id(),
1777     {"remote mirror uuid", "remote mirror peer uuid"});
1778
1779   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1780   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1781                                    mock_local_image_ctx,
1782                                    mock_remote_image_ctx,
1783                                    mock_replayer_listener,
1784                                    mock_image_meta,
1785                                    &update_watch_ctx));
1786
1787   // idle
1788   expect_load_image_meta(mock_image_meta, true, 0);
1789
1790   // wake-up replayer
1791   update_watch_ctx->handle_notify();
1792
1793   // wait for sync to complete and expect replay complete
1794   ASSERT_EQ(0, wait_for_notification(1));
1795   ASSERT_FALSE(mock_replayer.is_replaying());
1796
1797   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1798                                         mock_local_image_ctx,
1799                                         mock_remote_image_ctx));
1800 }
1801
1802 TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterLocalUpdateWatcherError) {
1803   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1804   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1805
1806   MockThreads mock_threads(m_threads);
1807   expect_work_queue_repeatedly(mock_threads);
1808
1809   InSequence seq;
1810
1811   MockInstanceWatcher mock_instance_watcher;
1812   MockImageMeta mock_image_meta;
1813   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1814                                       mock_remote_image_ctx,
1815                                       mock_image_meta);
1816   MockReplayerListener mock_replayer_listener;
1817   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1818                              "local mirror uuid", &m_pool_meta_cache,
1819                              &mock_state_builder, &mock_replayer_listener};
1820   m_pool_meta_cache.set_remote_pool_meta(
1821     m_remote_io_ctx.get_id(),
1822     {"remote mirror uuid", "remote mirror peer uuid"});
1823
1824   // init
1825   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1826   expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123,
1827                                  -EINVAL);
1828
1829   // fire init
1830   C_SaferCond init_ctx;
1831   mock_replayer.init(&init_ctx);
1832   ASSERT_EQ(-EINVAL, init_ctx.wait());
1833 }
1834
1835 TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError) {
1836   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1837   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1838
1839   MockThreads mock_threads(m_threads);
1840   expect_work_queue_repeatedly(mock_threads);
1841
1842   InSequence seq;
1843
1844   MockInstanceWatcher mock_instance_watcher;
1845   MockImageMeta mock_image_meta;
1846   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1847                                       mock_remote_image_ctx,
1848                                       mock_image_meta);
1849   MockReplayerListener mock_replayer_listener;
1850   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1851                              "local mirror uuid", &m_pool_meta_cache,
1852                              &mock_state_builder, &mock_replayer_listener};
1853   m_pool_meta_cache.set_remote_pool_meta(
1854     m_remote_io_ctx.get_id(),
1855     {"remote mirror uuid", "remote mirror peer uuid"});
1856
1857   // init
1858   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1859   expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123,
1860                                  0);
1861   expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234,
1862                                  -EINVAL);
1863
1864   expect_unregister_update_watcher(mock_local_image_ctx, 123, 0);
1865
1866   // fire init
1867   C_SaferCond init_ctx;
1868   mock_replayer.init(&init_ctx);
1869   ASSERT_EQ(-EINVAL, init_ctx.wait());
1870 }
1871
1872 TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterRemoteUpdateWatcherError) {
1873   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1874   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1875
1876   MockThreads mock_threads(m_threads);
1877   expect_work_queue_repeatedly(mock_threads);
1878
1879   MockReplayerListener mock_replayer_listener;
1880   expect_notification(mock_threads, mock_replayer_listener);
1881
1882   InSequence seq;
1883
1884   MockInstanceWatcher mock_instance_watcher;
1885   MockImageMeta mock_image_meta;
1886   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1887                                       mock_remote_image_ctx,
1888                                       mock_image_meta);
1889   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1890                              "local mirror uuid", &m_pool_meta_cache,
1891                              &mock_state_builder, &mock_replayer_listener};
1892   m_pool_meta_cache.set_remote_pool_meta(
1893     m_remote_io_ctx.get_id(),
1894     {"remote mirror uuid", "remote mirror peer uuid"});
1895
1896   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1897   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1898                                    mock_local_image_ctx,
1899                                    mock_remote_image_ctx,
1900                                    mock_replayer_listener,
1901                                    mock_image_meta,
1902                                    &update_watch_ctx));
1903
1904
1905   // shut down
1906   expect_unregister_update_watcher(mock_remote_image_ctx, 234, -EINVAL);
1907   expect_unregister_update_watcher(mock_local_image_ctx, 123, 0);
1908
1909   C_SaferCond shutdown_ctx;
1910   mock_replayer.shut_down(&shutdown_ctx);
1911   ASSERT_EQ(0, shutdown_ctx.wait());
1912 }
1913
1914 TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterLocalUpdateWatcherError) {
1915   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1916   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1917
1918   MockThreads mock_threads(m_threads);
1919   expect_work_queue_repeatedly(mock_threads);
1920
1921   MockReplayerListener mock_replayer_listener;
1922   expect_notification(mock_threads, mock_replayer_listener);
1923
1924   InSequence seq;
1925
1926   MockInstanceWatcher mock_instance_watcher;
1927   MockImageMeta mock_image_meta;
1928   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1929                                       mock_remote_image_ctx,
1930                                       mock_image_meta);
1931   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1932                              "local mirror uuid", &m_pool_meta_cache,
1933                              &mock_state_builder, &mock_replayer_listener};
1934   m_pool_meta_cache.set_remote_pool_meta(
1935     m_remote_io_ctx.get_id(),
1936     {"remote mirror uuid", "remote mirror peer uuid"});
1937
1938   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1939   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1940                                    mock_local_image_ctx,
1941                                    mock_remote_image_ctx,
1942                                    mock_replayer_listener,
1943                                    mock_image_meta,
1944                                    &update_watch_ctx));
1945
1946
1947   // shut down
1948   expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0);
1949   expect_unregister_update_watcher(mock_local_image_ctx, 123, -EINVAL);
1950
1951   C_SaferCond shutdown_ctx;
1952   mock_replayer.shut_down(&shutdown_ctx);
1953   ASSERT_EQ(0, shutdown_ctx.wait());
1954 }
1955
1956 TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) {
1957   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1958   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1959
1960   MockThreads mock_threads(m_threads);
1961   expect_work_queue_repeatedly(mock_threads);
1962
1963   MockReplayerListener mock_replayer_listener;
1964   expect_notification(mock_threads, mock_replayer_listener);
1965
1966   InSequence seq;
1967
1968   MockInstanceWatcher mock_instance_watcher;
1969   MockImageMeta mock_image_meta;
1970   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1971                                       mock_remote_image_ctx,
1972                                       mock_image_meta);
1973   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1974                              "local mirror uuid", &m_pool_meta_cache,
1975                              &mock_state_builder, &mock_replayer_listener};
1976   m_pool_meta_cache.set_remote_pool_meta(
1977     m_remote_io_ctx.get_id(),
1978     {"remote mirror uuid", "remote mirror peer uuid"});
1979
1980   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1981   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1982                                    mock_local_image_ctx,
1983                                    mock_remote_image_ctx,
1984                                    mock_replayer_listener,
1985                                    mock_image_meta,
1986                                    &update_watch_ctx));
1987
1988   // sync
1989   expect_load_image_meta(mock_image_meta, false, -EINVAL);
1990
1991   // wake-up replayer
1992   update_watch_ctx->handle_notify();
1993
1994   // wait for sync to complete and expect replay complete
1995   ASSERT_EQ(0, wait_for_notification(1));
1996   ASSERT_FALSE(mock_replayer.is_replaying());
1997   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1998
1999   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2000                                         mock_local_image_ctx,
2001                                         mock_remote_image_ctx));
2002 }
2003
2004 TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) {
2005   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2006   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2007
2008   MockThreads mock_threads(m_threads);
2009   expect_work_queue_repeatedly(mock_threads);
2010
2011   MockReplayerListener mock_replayer_listener;
2012   expect_notification(mock_threads, mock_replayer_listener);
2013
2014   InSequence seq;
2015
2016   MockInstanceWatcher mock_instance_watcher;
2017   MockImageMeta mock_image_meta;
2018   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2019                                       mock_remote_image_ctx,
2020                                       mock_image_meta);
2021   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2022                              "local mirror uuid", &m_pool_meta_cache,
2023                              &mock_state_builder, &mock_replayer_listener};
2024   m_pool_meta_cache.set_remote_pool_meta(
2025     m_remote_io_ctx.get_id(),
2026     {"remote mirror uuid", "remote mirror peer uuid"});
2027
2028   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2029   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2030                                    mock_local_image_ctx,
2031                                    mock_remote_image_ctx,
2032                                    mock_replayer_listener,
2033                                    mock_image_meta,
2034                                    &update_watch_ctx));
2035
2036   // sync
2037   expect_load_image_meta(mock_image_meta, false, 0);
2038   expect_is_refresh_required(mock_local_image_ctx, true);
2039   expect_refresh(mock_local_image_ctx, {}, -EINVAL);
2040
2041   // wake-up replayer
2042   update_watch_ctx->handle_notify();
2043
2044   // wait for sync to complete and expect replay complete
2045   ASSERT_EQ(0, wait_for_notification(1));
2046   ASSERT_FALSE(mock_replayer.is_replaying());
2047   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
2048
2049   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2050                                         mock_local_image_ctx,
2051                                         mock_remote_image_ctx));
2052 }
2053
2054 TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) {
2055   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2056   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2057
2058   MockThreads mock_threads(m_threads);
2059   expect_work_queue_repeatedly(mock_threads);
2060
2061   MockReplayerListener mock_replayer_listener;
2062   expect_notification(mock_threads, mock_replayer_listener);
2063
2064   InSequence seq;
2065
2066   MockInstanceWatcher mock_instance_watcher;
2067   MockImageMeta mock_image_meta;
2068   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2069                                       mock_remote_image_ctx,
2070                                       mock_image_meta);
2071   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2072                              "local mirror uuid", &m_pool_meta_cache,
2073                              &mock_state_builder, &mock_replayer_listener};
2074   m_pool_meta_cache.set_remote_pool_meta(
2075     m_remote_io_ctx.get_id(),
2076     {"remote mirror uuid", "remote mirror peer uuid"});
2077
2078   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2079   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2080                                    mock_local_image_ctx,
2081                                    mock_remote_image_ctx,
2082                                    mock_replayer_listener,
2083                                    mock_image_meta,
2084                                    &update_watch_ctx));
2085
2086   // sync
2087   expect_load_image_meta(mock_image_meta, false, 0);
2088   expect_is_refresh_required(mock_local_image_ctx, false);
2089   expect_is_refresh_required(mock_remote_image_ctx, true);
2090   expect_refresh(mock_remote_image_ctx, {}, -EINVAL);
2091
2092   // wake-up replayer
2093   update_watch_ctx->handle_notify();
2094
2095   // wait for sync to complete and expect replay complete
2096   ASSERT_EQ(0, wait_for_notification(1));
2097   ASSERT_FALSE(mock_replayer.is_replaying());
2098   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
2099
2100   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2101                                         mock_local_image_ctx,
2102                                         mock_remote_image_ctx));
2103 }
2104
2105 TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) {
2106   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2107   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2108
2109   MockThreads mock_threads(m_threads);
2110   expect_work_queue_repeatedly(mock_threads);
2111
2112   MockReplayerListener mock_replayer_listener;
2113   expect_notification(mock_threads, mock_replayer_listener);
2114
2115   InSequence seq;
2116
2117   MockInstanceWatcher mock_instance_watcher;
2118   MockImageMeta mock_image_meta;
2119   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2120                                       mock_remote_image_ctx,
2121                                       mock_image_meta);
2122   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2123                              "local mirror uuid", &m_pool_meta_cache,
2124                              &mock_state_builder, &mock_replayer_listener};
2125   m_pool_meta_cache.set_remote_pool_meta(
2126     m_remote_io_ctx.get_id(),
2127     {"remote mirror uuid", "remote mirror peer uuid"});
2128
2129   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2130   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2131                                    mock_local_image_ctx,
2132                                    mock_remote_image_ctx,
2133                                    mock_replayer_listener,
2134                                    mock_image_meta,
2135                                    &update_watch_ctx));
2136
2137   // inject snapshot
2138   mock_remote_image_ctx.snap_info = {
2139     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2140        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
2141        CEPH_NOSNAP, true, 0, {}},
2142      0, {}, 0, 0, {}}}};
2143
2144   // sync snap1
2145   expect_load_image_meta(mock_image_meta, false, 0);
2146   expect_is_refresh_required(mock_local_image_ctx, false);
2147   expect_is_refresh_required(mock_remote_image_ctx, false);
2148   MockSnapshotCopyRequest mock_snapshot_copy_request;
2149   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
2150                        -EINVAL);
2151
2152   // wake-up replayer
2153   update_watch_ctx->handle_notify();
2154
2155   // wait for sync to complete and expect replay complete
2156   ASSERT_EQ(0, wait_for_notification(1));
2157   ASSERT_FALSE(mock_replayer.is_replaying());
2158   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
2159
2160   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2161                                         mock_local_image_ctx,
2162                                         mock_remote_image_ctx));
2163 }
2164
2165 TEST_F(TestMockImageReplayerSnapshotReplayer, GetImageStateError) {
2166   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2167   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2168
2169   MockThreads mock_threads(m_threads);
2170   expect_work_queue_repeatedly(mock_threads);
2171
2172   MockReplayerListener mock_replayer_listener;
2173   expect_notification(mock_threads, mock_replayer_listener);
2174
2175   InSequence seq;
2176
2177   MockInstanceWatcher mock_instance_watcher;
2178   MockImageMeta mock_image_meta;
2179   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2180                                       mock_remote_image_ctx,
2181                                       mock_image_meta);
2182   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2183                              "local mirror uuid", &m_pool_meta_cache,
2184                              &mock_state_builder, &mock_replayer_listener};
2185   m_pool_meta_cache.set_remote_pool_meta(
2186     m_remote_io_ctx.get_id(),
2187     {"remote mirror uuid", "remote mirror peer uuid"});
2188
2189   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2190   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2191                                    mock_local_image_ctx,
2192                                    mock_remote_image_ctx,
2193                                    mock_replayer_listener,
2194                                    mock_image_meta,
2195                                    &update_watch_ctx));
2196
2197   // inject snapshot
2198   mock_remote_image_ctx.snap_info = {
2199     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2200        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
2201        CEPH_NOSNAP, true, 0, {}},
2202      0, {}, 0, 0, {}}}};
2203
2204   // sync snap1
2205   expect_load_image_meta(mock_image_meta, false, 0);
2206   expect_is_refresh_required(mock_local_image_ctx, false);
2207   expect_is_refresh_required(mock_remote_image_ctx, false);
2208   MockSnapshotCopyRequest mock_snapshot_copy_request;
2209   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
2210                        0);
2211   MockGetImageStateRequest mock_get_image_state_request;
2212   expect_get_image_state(mock_get_image_state_request, 1, -EINVAL);
2213
2214   // wake-up replayer
2215   update_watch_ctx->handle_notify();
2216
2217   // wait for sync to complete and expect replay complete
2218   ASSERT_EQ(0, wait_for_notification(1));
2219   ASSERT_FALSE(mock_replayer.is_replaying());
2220   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
2221
2222   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2223                                         mock_local_image_ctx,
2224                                         mock_remote_image_ctx));
2225 }
2226
2227 TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) {
2228   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2229   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2230
2231   MockThreads mock_threads(m_threads);
2232   expect_work_queue_repeatedly(mock_threads);
2233
2234   MockReplayerListener mock_replayer_listener;
2235   expect_notification(mock_threads, mock_replayer_listener);
2236
2237   InSequence seq;
2238
2239   MockInstanceWatcher mock_instance_watcher;
2240   MockImageMeta mock_image_meta;
2241   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2242                                       mock_remote_image_ctx,
2243                                       mock_image_meta);
2244   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2245                              "local mirror uuid", &m_pool_meta_cache,
2246                              &mock_state_builder, &mock_replayer_listener};
2247   m_pool_meta_cache.set_remote_pool_meta(
2248     m_remote_io_ctx.get_id(),
2249     {"remote mirror uuid", "remote mirror peer uuid"});
2250
2251   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2252   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2253                                    mock_local_image_ctx,
2254                                    mock_remote_image_ctx,
2255                                    mock_replayer_listener,
2256                                    mock_image_meta,
2257                                    &update_watch_ctx));
2258
2259   // inject snapshot
2260   mock_remote_image_ctx.snap_info = {
2261     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2262        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
2263        CEPH_NOSNAP, true, 0, {}},
2264      0, {}, 0, 0, {}}}};
2265
2266   // sync snap1
2267   expect_load_image_meta(mock_image_meta, false, 0);
2268   expect_is_refresh_required(mock_local_image_ctx, false);
2269   expect_is_refresh_required(mock_remote_image_ctx, false);
2270   MockSnapshotCopyRequest mock_snapshot_copy_request;
2271   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
2272                        0);
2273   MockGetImageStateRequest mock_get_image_state_request;
2274   expect_get_image_state(mock_get_image_state_request, 1, 0);
2275   MockCreateNonPrimaryRequest mock_create_non_primary_request;
2276   expect_create_non_primary_request(mock_create_non_primary_request,
2277                                     false, "remote mirror uuid", 1,
2278                                     {{1, CEPH_NOSNAP}}, 11, -EINVAL);
2279
2280   // wake-up replayer
2281   update_watch_ctx->handle_notify();
2282
2283   // wait for sync to complete and expect replay complete
2284   ASSERT_EQ(0, wait_for_notification(1));
2285   ASSERT_FALSE(mock_replayer.is_replaying());
2286   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
2287
2288   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2289                                         mock_local_image_ctx,
2290                                         mock_remote_image_ctx));
2291 }
2292
2293 TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateMirrorImageStateError) {
2294   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2295   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2296
2297   MockThreads mock_threads(m_threads);
2298   expect_work_queue_repeatedly(mock_threads);
2299
2300   MockReplayerListener mock_replayer_listener;
2301   expect_notification(mock_threads, mock_replayer_listener);
2302
2303   InSequence seq;
2304
2305   MockInstanceWatcher mock_instance_watcher;
2306   MockImageMeta mock_image_meta;
2307   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2308                                       mock_remote_image_ctx,
2309                                       mock_image_meta);
2310   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2311                              "local mirror uuid", &m_pool_meta_cache,
2312                              &mock_state_builder, &mock_replayer_listener};
2313   m_pool_meta_cache.set_remote_pool_meta(
2314     m_remote_io_ctx.get_id(),
2315     {"remote mirror uuid", "remote mirror peer uuid"});
2316
2317   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2318   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2319                                    mock_local_image_ctx,
2320                                    mock_remote_image_ctx,
2321                                    mock_replayer_listener,
2322                                    mock_image_meta,
2323                                    &update_watch_ctx));
2324
2325   // inject snapshot
2326   mock_remote_image_ctx.snap_info = {
2327     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2328        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
2329        CEPH_NOSNAP, true, 0, {}},
2330      0, {}, 0, 0, {}}}};
2331
2332   // sync snap1
2333   expect_load_image_meta(mock_image_meta, false, 0);
2334   expect_is_refresh_required(mock_local_image_ctx, false);
2335   expect_is_refresh_required(mock_remote_image_ctx, false);
2336   MockSnapshotCopyRequest mock_snapshot_copy_request;
2337   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
2338                        0);
2339   MockGetImageStateRequest mock_get_image_state_request;
2340   expect_get_image_state(mock_get_image_state_request, 1, 0);
2341   MockCreateNonPrimaryRequest mock_create_non_primary_request;
2342   expect_create_non_primary_request(mock_create_non_primary_request,
2343                                     false, "remote mirror uuid", 1,
2344                                     {{1, CEPH_NOSNAP}}, 11, 0);
2345   MockImageStateUpdateRequest mock_image_state_update_request;
2346   expect_update_mirror_image_state(mock_image_state_update_request, -EIO);
2347
2348   // wake-up replayer
2349   update_watch_ctx->handle_notify();
2350
2351   // wait for sync to complete and expect replay complete
2352   ASSERT_EQ(0, wait_for_notification(1));
2353   ASSERT_FALSE(mock_replayer.is_replaying());
2354   ASSERT_EQ(-EIO, mock_replayer.get_error_code());
2355
2356   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2357                                         mock_local_image_ctx,
2358                                         mock_remote_image_ctx));
2359 }
2360
2361 TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) {
2362   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2363   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2364
2365   MockThreads mock_threads(m_threads);
2366   expect_work_queue_repeatedly(mock_threads);
2367
2368   MockReplayerListener mock_replayer_listener;
2369   expect_notification(mock_threads, mock_replayer_listener);
2370
2371   InSequence seq;
2372
2373   MockInstanceWatcher mock_instance_watcher;
2374   MockImageMeta mock_image_meta;
2375   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2376                                       mock_remote_image_ctx,
2377                                       mock_image_meta);
2378   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2379                              "local mirror uuid", &m_pool_meta_cache,
2380                              &mock_state_builder, &mock_replayer_listener};
2381   m_pool_meta_cache.set_remote_pool_meta(
2382     m_remote_io_ctx.get_id(),
2383     {"remote mirror uuid", "remote mirror peer uuid"});
2384
2385   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2386   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2387                                    mock_local_image_ctx,
2388                                    mock_remote_image_ctx,
2389                                    mock_replayer_listener,
2390                                    mock_image_meta,
2391                                    &update_watch_ctx));
2392
2393   // inject snapshot
2394   mock_remote_image_ctx.snap_info = {
2395     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2396        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
2397        CEPH_NOSNAP, true, 0, {}},
2398      0, {}, 0, 0, {}}}};
2399
2400   // sync snap1
2401   expect_load_image_meta(mock_image_meta, false, 0);
2402   expect_is_refresh_required(mock_local_image_ctx, false);
2403   expect_is_refresh_required(mock_remote_image_ctx, false);
2404   MockSnapshotCopyRequest mock_snapshot_copy_request;
2405   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
2406                        0);
2407   MockGetImageStateRequest mock_get_image_state_request;
2408   expect_get_image_state(mock_get_image_state_request, 1, 0);
2409   MockCreateNonPrimaryRequest mock_create_non_primary_request;
2410   expect_create_non_primary_request(mock_create_non_primary_request,
2411                                     false, "remote mirror uuid", 1,
2412                                     {{1, CEPH_NOSNAP}}, 11, 0);
2413   MockImageStateUpdateRequest mock_image_state_update_request;
2414   expect_update_mirror_image_state(mock_image_state_update_request, 0);
2415   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id,
2416                              -ECANCELED);
2417
2418   // wake-up replayer
2419   update_watch_ctx->handle_notify();
2420
2421   // wait for sync to complete and expect replay complete
2422   ASSERT_EQ(0, wait_for_notification(1));
2423   ASSERT_FALSE(mock_replayer.is_replaying());
2424   ASSERT_EQ(-ECANCELED, mock_replayer.get_error_code());
2425
2426   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2427                                         mock_local_image_ctx,
2428                                         mock_remote_image_ctx));
2429 }
2430
2431 TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) {
2432   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2433   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2434
2435   MockThreads mock_threads(m_threads);
2436   expect_work_queue_repeatedly(mock_threads);
2437
2438   MockReplayerListener mock_replayer_listener;
2439   expect_notification(mock_threads, mock_replayer_listener);
2440
2441   InSequence seq;
2442
2443   MockInstanceWatcher mock_instance_watcher;
2444   MockImageMeta mock_image_meta;
2445   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2446                                       mock_remote_image_ctx,
2447                                       mock_image_meta);
2448   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2449                              "local mirror uuid", &m_pool_meta_cache,
2450                              &mock_state_builder, &mock_replayer_listener};
2451   m_pool_meta_cache.set_remote_pool_meta(
2452     m_remote_io_ctx.get_id(),
2453     {"remote mirror uuid", "remote mirror peer uuid"});
2454
2455   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2456   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2457                                    mock_local_image_ctx,
2458                                    mock_remote_image_ctx,
2459                                    mock_replayer_listener,
2460                                    mock_image_meta,
2461                                    &update_watch_ctx));
2462
2463   // inject snapshot
2464   mock_remote_image_ctx.snap_info = {
2465     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2466        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
2467        CEPH_NOSNAP,true, 0, {}},
2468      0, {}, 0, 0, {}}}};
2469
2470   // sync snap1
2471   expect_load_image_meta(mock_image_meta, false, 0);
2472   expect_is_refresh_required(mock_local_image_ctx, false);
2473   expect_is_refresh_required(mock_remote_image_ctx, false);
2474   MockSnapshotCopyRequest mock_snapshot_copy_request;
2475   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
2476                        0);
2477   MockGetImageStateRequest mock_get_image_state_request;
2478   expect_get_image_state(mock_get_image_state_request, 1, 0);
2479   MockCreateNonPrimaryRequest mock_create_non_primary_request;
2480   expect_create_non_primary_request(mock_create_non_primary_request,
2481                                     false, "remote mirror uuid", 1,
2482                                     {{1, CEPH_NOSNAP}}, 11, 0);
2483   MockImageStateUpdateRequest mock_image_state_update_request;
2484   expect_update_mirror_image_state(mock_image_state_update_request, 0);
2485   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
2486   MockImageCopyRequest mock_image_copy_request;
2487   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
2488                     {{1, CEPH_NOSNAP}}, -EINVAL);
2489   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
2490
2491   // wake-up replayer
2492   update_watch_ctx->handle_notify();
2493
2494   // wait for sync to complete and expect replay complete
2495   ASSERT_EQ(0, wait_for_notification(1));
2496   ASSERT_FALSE(mock_replayer.is_replaying());
2497   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
2498
2499   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2500                                         mock_local_image_ctx,
2501                                         mock_remote_image_ctx));
2502 }
2503
2504 TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) {
2505   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2506   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2507
2508   MockThreads mock_threads(m_threads);
2509   expect_work_queue_repeatedly(mock_threads);
2510
2511   MockReplayerListener mock_replayer_listener;
2512   expect_notification(mock_threads, mock_replayer_listener);
2513
2514   InSequence seq;
2515
2516   MockInstanceWatcher mock_instance_watcher;
2517   MockImageMeta mock_image_meta;
2518   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2519                                       mock_remote_image_ctx,
2520                                       mock_image_meta);
2521   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2522                              "local mirror uuid", &m_pool_meta_cache,
2523                              &mock_state_builder, &mock_replayer_listener};
2524   m_pool_meta_cache.set_remote_pool_meta(
2525     m_remote_io_ctx.get_id(),
2526     {"remote mirror uuid", "remote mirror peer uuid"});
2527
2528   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2529   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2530                                    mock_local_image_ctx,
2531                                    mock_remote_image_ctx,
2532                                    mock_replayer_listener,
2533                                    mock_image_meta,
2534                                    &update_watch_ctx));
2535
2536   // inject snapshot
2537   mock_remote_image_ctx.snap_info = {
2538     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2539        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
2540        CEPH_NOSNAP, true, 0, {}},
2541      0, {}, 0, 0, {}}}};
2542
2543   // sync snap1
2544   expect_load_image_meta(mock_image_meta, false, 0);
2545   expect_is_refresh_required(mock_local_image_ctx, false);
2546   expect_is_refresh_required(mock_remote_image_ctx, false);
2547   MockSnapshotCopyRequest mock_snapshot_copy_request;
2548   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
2549                        0);
2550   MockGetImageStateRequest mock_get_image_state_request;
2551   expect_get_image_state(mock_get_image_state_request, 1, 0);
2552   MockCreateNonPrimaryRequest mock_create_non_primary_request;
2553   expect_create_non_primary_request(mock_create_non_primary_request,
2554                                     false, "remote mirror uuid", 1,
2555                                     {{1, CEPH_NOSNAP}}, 11, 0);
2556   MockImageStateUpdateRequest mock_image_state_update_request;
2557   expect_update_mirror_image_state(mock_image_state_update_request, 0);
2558   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
2559   MockImageCopyRequest mock_image_copy_request;
2560   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
2561                     {{1, CEPH_NOSNAP}}, 0);
2562   MockApplyImageStateRequest mock_apply_state_request;
2563   expect_apply_image_state(mock_apply_state_request, 0);
2564   expect_mirror_image_snapshot_set_copy_progress(
2565     mock_local_image_ctx, 11, true, 0, -EINVAL);
2566   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
2567
2568   // wake-up replayer
2569   update_watch_ctx->handle_notify();
2570
2571   // wait for sync to complete and expect replay complete
2572   ASSERT_EQ(0, wait_for_notification(1));
2573   ASSERT_FALSE(mock_replayer.is_replaying());
2574   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
2575
2576   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2577                                         mock_local_image_ctx,
2578                                         mock_remote_image_ctx));
2579 }
2580
2581 TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) {
2582   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2583   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2584
2585   MockThreads mock_threads(m_threads);
2586   expect_work_queue_repeatedly(mock_threads);
2587
2588   MockReplayerListener mock_replayer_listener;
2589   expect_notification(mock_threads, mock_replayer_listener);
2590
2591   InSequence seq;
2592
2593   MockInstanceWatcher mock_instance_watcher;
2594   MockImageMeta mock_image_meta;
2595   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2596                                       mock_remote_image_ctx,
2597                                       mock_image_meta);
2598   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2599                              "local mirror uuid", &m_pool_meta_cache,
2600                              &mock_state_builder, &mock_replayer_listener};
2601   m_pool_meta_cache.set_remote_pool_meta(
2602     m_remote_io_ctx.get_id(),
2603     {"remote mirror uuid", "remote mirror peer uuid"});
2604
2605   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2606   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2607                                    mock_local_image_ctx,
2608                                    mock_remote_image_ctx,
2609                                    mock_replayer_listener,
2610                                    mock_image_meta,
2611                                    &update_watch_ctx));
2612
2613   // inject snapshot
2614   mock_remote_image_ctx.snap_info = {
2615     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2616        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
2617        CEPH_NOSNAP, true, 0, {}},
2618      0, {}, 0, 0, {}}},
2619     {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
2620        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
2621        "", CEPH_NOSNAP, true, 0, {}},
2622      0, {}, 0, 0, {}}}};
2623   mock_local_image_ctx.snap_info = {
2624     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2625        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
2626        1, true, 0, {}},
2627      0, {}, 0, 0, {}}}};
2628
2629   // sync snap2
2630   expect_load_image_meta(mock_image_meta, false, 0);
2631   expect_is_refresh_required(mock_local_image_ctx, false);
2632   expect_is_refresh_required(mock_remote_image_ctx, false);
2633   MockSnapshotCopyRequest mock_snapshot_copy_request;
2634   expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, {{2, CEPH_NOSNAP}},
2635                        0);
2636   MockGetImageStateRequest mock_get_image_state_request;
2637   expect_get_image_state(mock_get_image_state_request, 2, 0);
2638   MockCreateNonPrimaryRequest mock_create_non_primary_request;
2639   expect_create_non_primary_request(mock_create_non_primary_request,
2640                                     false, "remote mirror uuid", 2,
2641                                     {{2, CEPH_NOSNAP}}, 12, 0);
2642   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
2643   MockImageCopyRequest mock_image_copy_request;
2644   expect_image_copy(mock_image_copy_request, 1, 2, 11, {},
2645                     {{2, CEPH_NOSNAP}}, 0);
2646   MockApplyImageStateRequest mock_apply_state_request;
2647   expect_apply_image_state(mock_apply_state_request, 0);
2648   expect_mirror_image_snapshot_set_copy_progress(
2649     mock_local_image_ctx, 12, true, 0, 0);
2650   expect_notify_update(mock_local_image_ctx);
2651   MockUnlinkPeerRequest mock_unlink_peer_request;
2652   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
2653                      -EINVAL);
2654   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
2655
2656   // wake-up replayer
2657   update_watch_ctx->handle_notify();
2658
2659   // wait for sync to complete and expect replay complete
2660   ASSERT_EQ(0, wait_for_notification(1));
2661   ASSERT_FALSE(mock_replayer.is_replaying());
2662   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
2663
2664   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2665                                         mock_local_image_ctx,
2666                                         mock_remote_image_ctx));
2667 }
2668
2669 TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) {
2670   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2671   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2672
2673   MockThreads mock_threads(m_threads);
2674   expect_work_queue_repeatedly(mock_threads);
2675
2676   MockReplayerListener mock_replayer_listener;
2677   expect_notification(mock_threads, mock_replayer_listener);
2678
2679   InSequence seq;
2680
2681   MockInstanceWatcher mock_instance_watcher;
2682   MockImageMeta mock_image_meta;
2683   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2684                                       mock_remote_image_ctx,
2685                                       mock_image_meta);
2686   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2687                              "local mirror uuid", &m_pool_meta_cache,
2688                              &mock_state_builder, &mock_replayer_listener};
2689   m_pool_meta_cache.set_remote_pool_meta(
2690     m_remote_io_ctx.get_id(),
2691     {"remote mirror uuid", "remote mirror peer uuid"});
2692
2693   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2694   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2695                                    mock_local_image_ctx,
2696                                    mock_remote_image_ctx,
2697                                    mock_replayer_listener,
2698                                    mock_image_meta,
2699                                    &update_watch_ctx));
2700
2701   // inject a primary demote to local image
2702   mock_remote_image_ctx.snap_info = {
2703     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2704        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
2705        "", CEPH_NOSNAP, true, 0, {}},
2706      0, {}, 0, 0, {}}}};
2707   mock_local_image_ctx.snap_info = {
2708     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2709        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP,
2710        true, 0, {}},
2711      0, {}, 0, 0, {}}}};
2712
2713   // detect split-brain
2714   expect_load_image_meta(mock_image_meta, false, 0);
2715   expect_is_refresh_required(mock_local_image_ctx, false);
2716   expect_is_refresh_required(mock_remote_image_ctx, false);
2717
2718   // wake-up replayer
2719   update_watch_ctx->handle_notify();
2720
2721   // wait for sync to complete and expect replay complete
2722   ASSERT_EQ(0, wait_for_notification(1));
2723   ASSERT_FALSE(mock_replayer.is_replaying());
2724   ASSERT_EQ(-EEXIST, mock_replayer.get_error_code());
2725   ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description());
2726
2727   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2728                                         mock_local_image_ctx,
2729                                         mock_remote_image_ctx));
2730 }
2731
2732 TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteSnapshotMissingSplitBrain) {
2733   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2734   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2735
2736   MockThreads mock_threads(m_threads);
2737   expect_work_queue_repeatedly(mock_threads);
2738
2739   MockReplayerListener mock_replayer_listener;
2740   expect_notification(mock_threads, mock_replayer_listener);
2741
2742   InSequence seq;
2743
2744   MockInstanceWatcher mock_instance_watcher;
2745   MockImageMeta mock_image_meta;
2746   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2747                                       mock_remote_image_ctx,
2748                                       mock_image_meta);
2749   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2750                              "local mirror uuid", &m_pool_meta_cache,
2751                              &mock_state_builder, &mock_replayer_listener};
2752   m_pool_meta_cache.set_remote_pool_meta(
2753     m_remote_io_ctx.get_id(),
2754     {"remote mirror uuid", "remote mirror peer uuid"});
2755
2756   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2757   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2758                                    mock_local_image_ctx,
2759                                    mock_remote_image_ctx,
2760                                    mock_replayer_listener,
2761                                    mock_image_meta,
2762                                    &update_watch_ctx));
2763
2764   // inject a missing remote start snap (deleted)
2765   mock_local_image_ctx.snap_info = {
2766     {11U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
2767        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {},
2768        "remote mirror uuid", 1, true, 0,
2769        {{1, CEPH_NOSNAP}}},
2770      0, {}, 0, 0, {}}}};
2771   mock_remote_image_ctx.snap_info = {
2772     {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
2773        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
2774        "", CEPH_NOSNAP, true, 0, {}},
2775      0, {}, 0, 0, {}}},
2776     {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
2777        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
2778        "", CEPH_NOSNAP, true, 0, {}},
2779      0, {}, 0, 0, {}}}};
2780
2781   // split-brain due to missing snapshot 1
2782   expect_load_image_meta(mock_image_meta, false, 0);
2783   expect_is_refresh_required(mock_local_image_ctx, false);
2784   expect_is_refresh_required(mock_remote_image_ctx, false);
2785
2786   // wake-up replayer
2787   update_watch_ctx->handle_notify();
2788
2789   // wait for sync to complete and expect replay complete
2790   ASSERT_EQ(0, wait_for_notification(1));
2791   ASSERT_FALSE(mock_replayer.is_replaying());
2792   ASSERT_EQ(-EEXIST, mock_replayer.get_error_code());
2793   ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description());
2794
2795   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2796                                         mock_local_image_ctx,
2797                                         mock_remote_image_ctx));
2798 }
2799
2800 TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteFailover) {
2801   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2802   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2803
2804   MockThreads mock_threads(m_threads);
2805   expect_work_queue_repeatedly(mock_threads);
2806
2807   MockReplayerListener mock_replayer_listener;
2808   expect_notification(mock_threads, mock_replayer_listener);
2809
2810   InSequence seq;
2811
2812   MockInstanceWatcher mock_instance_watcher;
2813   MockImageMeta mock_image_meta;
2814   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2815                                       mock_remote_image_ctx,
2816                                       mock_image_meta);
2817   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2818                              "local mirror uuid", &m_pool_meta_cache,
2819                              &mock_state_builder, &mock_replayer_listener};
2820   m_pool_meta_cache.set_remote_pool_meta(
2821     m_remote_io_ctx.get_id(),
2822     {"remote mirror uuid", "remote mirror peer uuid"});
2823
2824   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2825   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
2826                                    mock_local_image_ctx,
2827                                    mock_remote_image_ctx,
2828                                    mock_replayer_listener,
2829                                    mock_image_meta,
2830                                    &update_watch_ctx));
2831
2832   // inject a primary demote to local image
2833   mock_remote_image_ctx.snap_info = {
2834     {1U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{},
2835      0, {}, 0, 0, {}}},
2836     {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
2837        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED,
2838        {"remote mirror peer uuid"}, "local mirror uuid", 12U, true, 0, {}},
2839      0, {}, 0, 0, {}}},
2840     {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
2841        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
2842        "", CEPH_NOSNAP, true, 0, {}},
2843      0, {}, 0, 0, {}}}};
2844   mock_local_image_ctx.snap_ids = {
2845     {{cls::rbd::UserSnapshotNamespace{}, "snap1"}, 11},
2846     {{cls::rbd::MirrorSnapshotNamespace{}, "snap2"}, 12}};
2847   mock_local_image_ctx.snap_info = {
2848     {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{},
2849      0, {}, 0, 0, {}}},
2850     {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
2851        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP,
2852        true, 0, {}},
2853      0, {}, 0, 0, {}}}};
2854
2855   // attach to promoted remote image
2856   expect_load_image_meta(mock_image_meta, false, 0);
2857   expect_is_refresh_required(mock_local_image_ctx, false);
2858   expect_is_refresh_required(mock_remote_image_ctx, false);
2859   MockSnapshotCopyRequest mock_snapshot_copy_request;
2860   expect_snapshot_copy(mock_snapshot_copy_request, 2, 3, 12,
2861                        {{2, 12}, {3, CEPH_NOSNAP}}, 0);
2862   MockGetImageStateRequest mock_get_image_state_request;
2863   expect_get_image_state(mock_get_image_state_request, 3, 0);
2864   MockCreateNonPrimaryRequest mock_create_non_primary_request;
2865   expect_create_non_primary_request(mock_create_non_primary_request,
2866                                     false, "remote mirror uuid", 3,
2867                                     {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}, 13,
2868                                     0);
2869   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
2870   MockImageCopyRequest mock_image_copy_request;
2871   expect_image_copy(mock_image_copy_request, 2, 3, 12, {},
2872                     {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}, 0);
2873   MockApplyImageStateRequest mock_apply_state_request;
2874   expect_apply_image_state(mock_apply_state_request, 0);
2875   expect_mirror_image_snapshot_set_copy_progress(
2876     mock_local_image_ctx, 13, true, 0, 0);
2877   expect_notify_update(mock_local_image_ctx);
2878   MockUnlinkPeerRequest mock_unlink_peer_request;
2879   expect_unlink_peer(mock_unlink_peer_request, 2, "remote mirror peer uuid", 0);
2880   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
2881
2882   // idle
2883   expect_load_image_meta(mock_image_meta, false, 0);
2884   expect_is_refresh_required(mock_local_image_ctx, true);
2885   expect_refresh(
2886     mock_local_image_ctx, {
2887       {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{},
2888          0, {}, 0, 0, {}}},
2889       {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
2890          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP,
2891          true, 0, {}},
2892        0, {}, 0, 0, {}}},
2893       {13U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
2894          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {},
2895          "remote mirror uuid", 3, true, 0,
2896          {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}},
2897        0, {}, 0, 0, {}}},
2898     }, 0);
2899   expect_is_refresh_required(mock_remote_image_ctx, true);
2900   expect_refresh(
2901     mock_remote_image_ctx, {
2902       {1U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{},
2903          0, {}, 0, 0, {}}},
2904       {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
2905          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED,
2906          {"remote mirror peer uuid"}, "local mirror uuid", 12U, true, 0, {}},
2907        0, {}, 0, 0, {}}},
2908       {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
2909          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {}, "", CEPH_NOSNAP, true, 0,
2910          {}},
2911        0, {}, 0, 0, {}}}
2912     }, 0);
2913
2914   // wake-up replayer
2915   update_watch_ctx->handle_notify();
2916
2917   // wait for sync to complete and expect replay complete
2918   ASSERT_EQ(0, wait_for_notification(2));
2919   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
2920                                         mock_local_image_ctx,
2921                                         mock_remote_image_ctx));
2922 }
2923
2924 TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkRemoteSnapshot) {
2925   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
2926   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
2927
2928   // it should attempt to unlink from remote snap1 since we don't need it
2929   // anymore
2930   mock_local_image_ctx.snap_info = {
2931     {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
2932        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
2933        4, true, 0, {}},
2934      0, {}, 0, 0, {}}}};
2935   mock_remote_image_ctx.snap_info = {
2936     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
2937        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
2938        "", CEPH_NOSNAP, true, 0, {}},
2939      0, {}, 0, 0, {}}},
2940     {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
2941        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
2942        "", CEPH_NOSNAP, true, 0, {}},
2943      0, {}, 0, 0, {}}}};
2944
2945   MockThreads mock_threads(m_threads);
2946   expect_work_queue_repeatedly(mock_threads);
2947
2948   MockReplayerListener mock_replayer_listener;
2949   expect_notification(mock_threads, mock_replayer_listener);
2950
2951   InSequence seq;
2952
2953   MockInstanceWatcher mock_instance_watcher;
2954   MockImageMeta mock_image_meta;
2955   MockStateBuilder mock_state_builder(mock_local_image_ctx,
2956                                       mock_remote_image_ctx,
2957                                       mock_image_meta);
2958   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
2959                              "local mirror uuid", &m_pool_meta_cache,
2960                              &mock_state_builder, &mock_replayer_listener};
2961   m_pool_meta_cache.set_remote_pool_meta(
2962     m_remote_io_ctx.get_id(),
2963     {"remote mirror uuid", "remote mirror peer uuid"});
2964
2965   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
2966
2967   // init
2968   expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123,
2969                                  0);
2970   expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234,
2971                                  0);
2972
2973   // unlink snap1
2974   expect_load_image_meta(mock_image_meta, false, 0);
2975   expect_is_refresh_required(mock_local_image_ctx, false);
2976   expect_is_refresh_required(mock_remote_image_ctx, false);
2977   MockUnlinkPeerRequest mock_unlink_peer_request;
2978   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
2979                      0);
2980
2981   // idle
2982   expect_load_image_meta(mock_image_meta, false, 0);
2983   expect_is_refresh_required(mock_local_image_ctx, false);
2984   expect_is_refresh_required(mock_remote_image_ctx, true);
2985   expect_refresh(
2986     mock_remote_image_ctx, {
2987       {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{},
2988        0, {}, 0, 0, {}}},
2989       {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
2990          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""},
2991          "", CEPH_NOSNAP, true, 0, {}},
2992        0, {}, 0, 0, {}}},
2993       {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
2994          cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
2995          "", CEPH_NOSNAP, true, 0, {}},
2996        0, {}, 0, 0, {}}}
2997     }, 0);
2998
2999   // fire init
3000   C_SaferCond init_ctx;
3001   mock_replayer.init(&init_ctx);
3002   ASSERT_EQ(0, init_ctx.wait());
3003
3004   // wait for sync to complete
3005   ASSERT_EQ(0, wait_for_notification(3));
3006
3007   // shut down
3008   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
3009                                         mock_local_image_ctx,
3010                                         mock_remote_image_ctx));
3011 }
3012
3013 TEST_F(TestMockImageReplayerSnapshotReplayer, SkipImageSync) {
3014   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
3015   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
3016
3017   mock_remote_image_ctx.snap_info = {
3018     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
3019        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
3020        "", 0U, true, 0, {}},
3021      0, {}, 0, 0, {}}}};
3022
3023   MockThreads mock_threads(m_threads);
3024   expect_work_queue_repeatedly(mock_threads);
3025
3026   MockReplayerListener mock_replayer_listener;
3027   expect_notification(mock_threads, mock_replayer_listener);
3028
3029   InSequence seq;
3030
3031   MockInstanceWatcher mock_instance_watcher;
3032   MockImageMeta mock_image_meta;
3033   MockStateBuilder mock_state_builder(mock_local_image_ctx,
3034                                       mock_remote_image_ctx,
3035                                       mock_image_meta);
3036   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
3037                              "local mirror uuid", &m_pool_meta_cache,
3038                              &mock_state_builder, &mock_replayer_listener};
3039   m_pool_meta_cache.set_remote_pool_meta(
3040     m_remote_io_ctx.get_id(),
3041     {"remote mirror uuid", "remote mirror peer uuid"});
3042
3043   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
3044
3045   // init
3046   expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123,
3047                                  0);
3048   expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234,
3049                                  0);
3050
3051   // sync snap1
3052   expect_load_image_meta(mock_image_meta, false, 0);
3053   expect_is_refresh_required(mock_local_image_ctx, false);
3054   expect_is_refresh_required(mock_remote_image_ctx, false);
3055   MockSnapshotCopyRequest mock_snapshot_copy_request;
3056   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
3057                        0);
3058   MockGetImageStateRequest mock_get_image_state_request;
3059   expect_get_image_state(mock_get_image_state_request, 1, 0);
3060   MockCreateNonPrimaryRequest mock_create_non_primary_request;
3061   expect_create_non_primary_request(mock_create_non_primary_request,
3062                                     false, "remote mirror uuid", 1,
3063                                     {{1, CEPH_NOSNAP}}, 11, 0);
3064   MockImageStateUpdateRequest mock_image_state_update_request;
3065   expect_update_mirror_image_state(mock_image_state_update_request, 0);
3066   MockApplyImageStateRequest mock_apply_state_request;
3067   expect_apply_image_state(mock_apply_state_request, 0);
3068   expect_mirror_image_snapshot_set_copy_progress(
3069     mock_local_image_ctx, 11, true, 0, 0);
3070   expect_notify_update(mock_local_image_ctx);
3071
3072   // idle
3073   expect_load_image_meta(mock_image_meta, false, 0);
3074   expect_is_refresh_required(mock_local_image_ctx, true);
3075   expect_refresh(
3076     mock_local_image_ctx, {
3077       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
3078          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
3079          1, true, 0, {{1, CEPH_NOSNAP}}},
3080        0, {}, 0, 0, {}}},
3081     }, 0);
3082   expect_is_refresh_required(mock_remote_image_ctx, false);
3083
3084   // fire init
3085   C_SaferCond init_ctx;
3086   mock_replayer.init(&init_ctx);
3087   ASSERT_EQ(0, init_ctx.wait());
3088
3089   // wait for sync to complete
3090   ASSERT_EQ(0, wait_for_notification(3));
3091
3092   // shut down
3093   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
3094                                         mock_local_image_ctx,
3095                                         mock_remote_image_ctx));
3096 }
3097
3098 TEST_F(TestMockImageReplayerSnapshotReplayer, ImageNameUpdated) {
3099   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
3100   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
3101
3102   MockThreads mock_threads(m_threads);
3103   expect_work_queue_repeatedly(mock_threads);
3104
3105   MockReplayerListener mock_replayer_listener;
3106   expect_notification(mock_threads, mock_replayer_listener);
3107
3108   InSequence seq;
3109
3110   MockInstanceWatcher mock_instance_watcher;
3111   MockImageMeta mock_image_meta;
3112   MockStateBuilder mock_state_builder(mock_local_image_ctx,
3113                                       mock_remote_image_ctx,
3114                                       mock_image_meta);
3115   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
3116                              "local mirror uuid", &m_pool_meta_cache,
3117                              &mock_state_builder, &mock_replayer_listener};
3118   m_pool_meta_cache.set_remote_pool_meta(
3119     m_remote_io_ctx.get_id(),
3120     {"remote mirror uuid", "remote mirror peer uuid"});
3121
3122   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
3123   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
3124                                    mock_local_image_ctx,
3125                                    mock_remote_image_ctx,
3126                                    mock_replayer_listener,
3127                                    mock_image_meta,
3128                                    &update_watch_ctx));
3129
3130   // change the name of the image
3131   mock_local_image_ctx.name = "NEW NAME";
3132
3133   // idle
3134   expect_load_image_meta(mock_image_meta, true, 0);
3135
3136   // wake-up replayer
3137   update_watch_ctx->handle_notify();
3138
3139   // wait for sync to complete and expect replay complete
3140   ASSERT_EQ(0, wait_for_notification(2));
3141   auto image_spec = image_replayer::util::compute_image_spec(m_local_io_ctx,
3142                                                              "NEW NAME");
3143   ASSERT_EQ(image_spec, mock_replayer.get_image_spec());
3144   ASSERT_FALSE(mock_replayer.is_replaying());
3145
3146   // shut down
3147   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
3148                                         mock_local_image_ctx,
3149                                         mock_remote_image_ctx));
3150 }
3151
3152 TEST_F(TestMockImageReplayerSnapshotReplayer, ApplyImageStatePendingShutdown) {
3153   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
3154   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
3155
3156   MockThreads mock_threads(m_threads);
3157   expect_work_queue_repeatedly(mock_threads);
3158
3159   MockReplayerListener mock_replayer_listener;
3160   expect_notification(mock_threads, mock_replayer_listener);
3161
3162   InSequence seq;
3163
3164   MockInstanceWatcher mock_instance_watcher;
3165   MockImageMeta mock_image_meta;
3166   MockStateBuilder mock_state_builder(mock_local_image_ctx,
3167                                       mock_remote_image_ctx,
3168                                       mock_image_meta);
3169   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
3170                              "local mirror uuid", &m_pool_meta_cache,
3171                              &mock_state_builder, &mock_replayer_listener};
3172   C_SaferCond shutdown_ctx;
3173   m_pool_meta_cache.set_remote_pool_meta(
3174     m_remote_io_ctx.get_id(),
3175     {"remote mirror uuid", "remote mirror peer uuid"});
3176
3177   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
3178   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
3179                                    mock_local_image_ctx,
3180                                    mock_remote_image_ctx,
3181                                    mock_replayer_listener,
3182                                    mock_image_meta,
3183                                    &update_watch_ctx));
3184
3185   // inject snapshot
3186   mock_remote_image_ctx.snap_info = {
3187     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
3188        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
3189        CEPH_NOSNAP, true, 0, {}},
3190      0, {}, 0, 0, {}}}};
3191
3192   // sync snap1
3193   expect_load_image_meta(mock_image_meta, false, 0);
3194   expect_is_refresh_required(mock_local_image_ctx, false);
3195   expect_is_refresh_required(mock_remote_image_ctx, false);
3196   MockSnapshotCopyRequest mock_snapshot_copy_request;
3197   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
3198                        0);
3199   MockGetImageStateRequest mock_get_image_state_request;
3200   expect_get_image_state(mock_get_image_state_request, 1, 0);
3201   MockCreateNonPrimaryRequest mock_create_non_primary_request;
3202   expect_create_non_primary_request(mock_create_non_primary_request,
3203                                     false, "remote mirror uuid", 1,
3204                                     {{1, CEPH_NOSNAP}}, 11, 0);
3205   MockImageStateUpdateRequest mock_image_state_update_request;
3206   expect_update_mirror_image_state(mock_image_state_update_request, 0);
3207   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
3208   MockImageCopyRequest mock_image_copy_request;
3209   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
3210                     {{1, CEPH_NOSNAP}}, 0);
3211   MockApplyImageStateRequest mock_apply_state_request;
3212   EXPECT_CALL(mock_apply_state_request, send())
3213     .WillOnce(Invoke([this, &req=mock_apply_state_request,
3214                       &replayer=mock_replayer, &ctx=shutdown_ctx]() {
3215       // inject a shutdown, to be pended due to STATE_REPLAYING
3216       replayer.shut_down(&ctx);
3217       m_threads->work_queue->queue(req.on_finish, 0);
3218     }));
3219   expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id);
3220   expect_mirror_image_snapshot_set_copy_progress(
3221       mock_local_image_ctx, 11, true, 0, 0);
3222   expect_notify_update(mock_local_image_ctx);
3223   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
3224
3225   // shutdown should be resumed
3226   expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0);
3227   expect_unregister_update_watcher(mock_local_image_ctx, 123, 0);
3228
3229   // wake-up replayer
3230   update_watch_ctx->handle_notify();
3231
3232   ASSERT_EQ(0, wait_for_notification(1));
3233   ASSERT_FALSE(mock_replayer.is_replaying());
3234   ASSERT_EQ(0, mock_replayer.get_error_code());
3235
3236   ASSERT_EQ(0, shutdown_ctx.wait());
3237 }
3238
3239 TEST_F(TestMockImageReplayerSnapshotReplayer, ApplyImageStateErrorPendingShutdown) {
3240   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
3241   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
3242
3243   MockThreads mock_threads(m_threads);
3244   expect_work_queue_repeatedly(mock_threads);
3245
3246   MockReplayerListener mock_replayer_listener;
3247   expect_notification(mock_threads, mock_replayer_listener);
3248
3249   InSequence seq;
3250
3251   MockInstanceWatcher mock_instance_watcher;
3252   MockImageMeta mock_image_meta;
3253   MockStateBuilder mock_state_builder(mock_local_image_ctx,
3254                                       mock_remote_image_ctx,
3255                                       mock_image_meta);
3256   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
3257                              "local mirror uuid", &m_pool_meta_cache,
3258                              &mock_state_builder, &mock_replayer_listener};
3259   C_SaferCond shutdown_ctx;
3260   m_pool_meta_cache.set_remote_pool_meta(
3261     m_remote_io_ctx.get_id(),
3262     {"remote mirror uuid", "remote mirror peer uuid"});
3263
3264   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
3265   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
3266                                    mock_local_image_ctx,
3267                                    mock_remote_image_ctx,
3268                                    mock_replayer_listener,
3269                                    mock_image_meta,
3270                                    &update_watch_ctx));
3271
3272   // inject snapshot
3273   mock_remote_image_ctx.snap_info = {
3274     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
3275        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
3276        CEPH_NOSNAP, true, 0, {}},
3277      0, {}, 0, 0, {}}}};
3278
3279   // sync snap1
3280   expect_load_image_meta(mock_image_meta, false, 0);
3281   expect_is_refresh_required(mock_local_image_ctx, false);
3282   expect_is_refresh_required(mock_remote_image_ctx, false);
3283   MockSnapshotCopyRequest mock_snapshot_copy_request;
3284   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
3285                        0);
3286   MockGetImageStateRequest mock_get_image_state_request;
3287   expect_get_image_state(mock_get_image_state_request, 1, 0);
3288   MockCreateNonPrimaryRequest mock_create_non_primary_request;
3289   expect_create_non_primary_request(mock_create_non_primary_request,
3290                                     false, "remote mirror uuid", 1,
3291                                     {{1, CEPH_NOSNAP}}, 11, 0);
3292   MockImageStateUpdateRequest mock_image_state_update_request;
3293   expect_update_mirror_image_state(mock_image_state_update_request, 0);
3294   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
3295   MockImageCopyRequest mock_image_copy_request;
3296   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
3297                     {{1, CEPH_NOSNAP}}, 0);
3298   MockApplyImageStateRequest mock_apply_state_request;
3299   EXPECT_CALL(mock_apply_state_request, send())
3300     .WillOnce(Invoke([this, &req=mock_apply_state_request,
3301                       &replayer=mock_replayer, &ctx=shutdown_ctx]() {
3302       // inject a shutdown, to be pended due to STATE_REPLAYING
3303       replayer.shut_down(&ctx);
3304       m_threads->work_queue->queue(req.on_finish, -EINVAL);
3305     }));
3306   expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id);
3307   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
3308
3309   // shutdown should be resumed
3310   expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0);
3311   expect_unregister_update_watcher(mock_local_image_ctx, 123, 0);
3312
3313   // wake-up replayer
3314   update_watch_ctx->handle_notify();
3315
3316   ASSERT_EQ(0, shutdown_ctx.wait());
3317 }
3318
3319 } // namespace snapshot
3320 } // namespace image_replayer
3321 } // namespace mirror
3322 } // namespace rbd