ceph/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "test/rbd_mirror/test_mock_fixture.h"
   5 #include "librbd/deep_copy/ImageCopyRequest.h"
   6 #include "librbd/deep_copy/SnapshotCopyRequest.h"
   7 #include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h"
   8 #include "librbd/mirror/snapshot/GetImageStateRequest.h"
   9 #include "librbd/mirror/snapshot/ImageMeta.h"
  10 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
  11 #include "tools/rbd_mirror/InstanceWatcher.h"
  12 #include "tools/rbd_mirror/Threads.h"
  13 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
  14 #include "tools/rbd_mirror/image_replayer/ReplayerListener.h"
  15 #include "tools/rbd_mirror/image_replayer/Utils.h"
  16 #include "tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h"
  17 #include "tools/rbd_mirror/image_replayer/snapshot/Replayer.h"
  18 #include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h"
  19 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
  20 #include "test/librbd/mock/MockImageCtx.h"
  21 #include "test/rbd_mirror/mock/MockContextWQ.h"
  22 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  23
  24 namespace librbd {
  25 namespace {
  26
  27 struct MockTestImageCtx : public librbd::MockImageCtx {
  28   explicit MockTestImageCtx(librbd::ImageCtx &image_ctx)
  29     : librbd::MockImageCtx(image_ctx) {
  30   }
  31 };
  32
  33 } // anonymous namespace
  34
  35 namespace deep_copy {
  36
  37 template <>
  38 struct ImageCopyRequest<MockTestImageCtx> {
  39   uint64_t src_snap_id_start;
  40   uint64_t src_snap_id_end;
  41   uint64_t dst_snap_id_start;
  42   librbd::deep_copy::ObjectNumber object_number;
  43   librbd::SnapSeqs snap_seqs;
  44
  45   static ImageCopyRequest* s_instance;
  46   static ImageCopyRequest* create(MockTestImageCtx *src_image_ctx,
  47                                   MockTestImageCtx *dst_image_ctx,
  48                                   librados::snap_t src_snap_id_start,
  49                                   librados::snap_t src_snap_id_end,
  50                                   librados::snap_t dst_snap_id_start,
  51                                   bool flatten,
  52                                   const ObjectNumber &object_number,
  53                                   const SnapSeqs &snap_seqs,
  54                                   ProgressContext *prog_ctx,
  55                                   Context *on_finish) {
  56     ceph_assert(s_instance != nullptr);
  57     s_instance->src_snap_id_start = src_snap_id_start;
  58     s_instance->src_snap_id_end = src_snap_id_end;
  59     s_instance->dst_snap_id_start = dst_snap_id_start;
  60     s_instance->object_number = object_number;
  61     s_instance->snap_seqs = snap_seqs;
  62     s_instance->on_finish = on_finish;
  63     return s_instance;
  64   }
  65
  66   Context* on_finish = nullptr;
  67
  68   ImageCopyRequest() {
  69     s_instance = this;
  70   }
  71
  72   MOCK_METHOD0(send, void());
  73 };
  74
  75 template <>
  76 struct SnapshotCopyRequest<MockTestImageCtx> {
  77   librados::snap_t src_snap_id_start;
  78   librados::snap_t src_snap_id_end;
  79   librados::snap_t dst_snap_id_start;
  80   SnapSeqs* snap_seqs = nullptr;
  81
  82   static SnapshotCopyRequest* s_instance;
  83   static SnapshotCopyRequest* create(MockTestImageCtx *src_image_ctx,
  84                                      MockTestImageCtx *dst_image_ctx,
  85                                      librados::snap_t src_snap_id_start,
  86                                      librados::snap_t src_snap_id_end,
  87                                      librados::snap_t dst_snap_id_start,
  88                                      bool flatten,
  89                                      ::MockContextWQ *work_queue,
  90                                      SnapSeqs *snap_seqs,
  91                                      Context *on_finish) {
  92     ceph_assert(s_instance != nullptr);
  93     s_instance->src_snap_id_start = src_snap_id_start;
  94     s_instance->src_snap_id_end = src_snap_id_end;
  95     s_instance->dst_snap_id_start = dst_snap_id_start;
  96     s_instance->snap_seqs = snap_seqs;
  97     s_instance->on_finish = on_finish;
  98     return s_instance;
  99   }
 100
 101   Context* on_finish = nullptr;
 102
 103   SnapshotCopyRequest() {
 104     s_instance = this;
 105   }
 106
 107   MOCK_METHOD0(send, void());
 108 };
 109
 110 ImageCopyRequest<MockTestImageCtx>* ImageCopyRequest<MockTestImageCtx>::s_instance = nullptr;
 111 SnapshotCopyRequest<MockTestImageCtx>* SnapshotCopyRequest<MockTestImageCtx>::s_instance = nullptr;
 112
 113 } // namespace deep_copy
 114
 115 namespace mirror {
 116 namespace snapshot {
 117
 118 template <>
 119 struct CreateNonPrimaryRequest<MockTestImageCtx> {
 120   bool demoted = false;
 121   std::string primary_mirror_uuid;
 122   uint64_t primary_snap_id;
 123   SnapSeqs snap_seqs;
 124   uint64_t* snap_id = nullptr;
 125
 126   static CreateNonPrimaryRequest* s_instance;
 127   static CreateNonPrimaryRequest* create(MockTestImageCtx *image_ctx,
 128                                          bool demoted,
 129                                          const std::string &primary_mirror_uuid,
 130                                          uint64_t primary_snap_id,
 131                                          const SnapSeqs& snap_seqs,
 132                                          const ImageState &image_state,
 133                                          uint64_t *snap_id,
 134                                          Context *on_finish) {
 135     ceph_assert(s_instance != nullptr);
 136     s_instance->demoted = demoted;
 137     s_instance->primary_mirror_uuid = primary_mirror_uuid;
 138     s_instance->primary_snap_id = primary_snap_id;
 139     s_instance->snap_seqs = snap_seqs;
 140     s_instance->snap_id = snap_id;
 141     s_instance->on_finish = on_finish;
 142     return s_instance;
 143   }
 144
 145   Context* on_finish = nullptr;
 146
 147   CreateNonPrimaryRequest() {
 148     s_instance = this;
 149   }
 150
 151   MOCK_METHOD0(send, void());
 152 };
 153
 154 template <>
 155 struct GetImageStateRequest<MockTestImageCtx> {
 156   uint64_t snap_id = CEPH_NOSNAP;
 157
 158   static GetImageStateRequest* s_instance;
 159   static GetImageStateRequest* create(MockTestImageCtx *image_ctx,
 160                                       uint64_t snap_id,
 161                                       ImageState *image_state,
 162                                       Context *on_finish) {
 163     ceph_assert(s_instance != nullptr);
 164     s_instance->snap_id = snap_id;
 165     s_instance->on_finish = on_finish;
 166     return s_instance;
 167   }
 168
 169   Context* on_finish = nullptr;
 170
 171   GetImageStateRequest() {
 172     s_instance = this;
 173   }
 174
 175   MOCK_METHOD0(send, void());
 176 };
 177
 178 template <>
 179 struct ImageMeta<MockTestImageCtx> {
 180   MOCK_METHOD1(load, void(Context*));
 181
 182   bool resync_requested = false;
 183 };
 184
 185 template <>
 186 struct UnlinkPeerRequest<MockTestImageCtx> {
 187   uint64_t snap_id;
 188   std::string mirror_peer_uuid;
 189
 190   static UnlinkPeerRequest* s_instance;
 191   static UnlinkPeerRequest*create (MockTestImageCtx *image_ctx,
 192                                    uint64_t snap_id,
 193                                    const std::string &mirror_peer_uuid,
 194                                    Context *on_finish) {
 195     ceph_assert(s_instance != nullptr);
 196     s_instance->snap_id = snap_id;
 197     s_instance->mirror_peer_uuid = mirror_peer_uuid;
 198     s_instance->on_finish = on_finish;
 199     return s_instance;
 200   }
 201
 202   Context* on_finish = nullptr;
 203
 204   UnlinkPeerRequest() {
 205     s_instance = this;
 206   }
 207
 208   MOCK_METHOD0(send, void());
 209 };
 210
 211 CreateNonPrimaryRequest<MockTestImageCtx>* CreateNonPrimaryRequest<MockTestImageCtx>::s_instance = nullptr;
 212 GetImageStateRequest<MockTestImageCtx>* GetImageStateRequest<MockTestImageCtx>::s_instance = nullptr;
 213 UnlinkPeerRequest<MockTestImageCtx>* UnlinkPeerRequest<MockTestImageCtx>::s_instance = nullptr;
 214
 215 } // namespace snapshot
 216 } // namespace mirror
 217 } // namespace librbd
 218
 219 namespace rbd {
 220 namespace mirror {
 221
 222 template <>
 223 struct InstanceWatcher<librbd::MockTestImageCtx> {
 224   MOCK_METHOD1(cancel_sync_request, void(const std::string&));
 225   MOCK_METHOD2(notify_sync_request, void(const std::string&,
 226                                          Context*));
 227   MOCK_METHOD1(notify_sync_complete, void(const std::string&));
 228 };
 229
 230 template <>
 231 struct Threads<librbd::MockTestImageCtx> {
 232   MockSafeTimer *timer;
 233   ceph::mutex &timer_lock;
 234
 235   MockContextWQ *work_queue;
 236
 237   Threads(Threads<librbd::ImageCtx>* threads)
 238     : timer(new MockSafeTimer()),
 239       timer_lock(threads->timer_lock),
 240       work_queue(new MockContextWQ()) {
 241   }
 242   ~Threads() {
 243     delete timer;
 244     delete work_queue;
 245   }
 246 };
 247
 248 namespace {
 249
 250 struct MockReplayerListener : public image_replayer::ReplayerListener {
 251   MOCK_METHOD0(handle_notification, void());
 252 };
 253
 254 } // anonymous namespace
 255
 256 namespace image_replayer {
 257
 258 template<>
 259 struct CloseImageRequest<librbd::MockTestImageCtx> {
 260   static CloseImageRequest* s_instance;
 261   librbd::MockTestImageCtx **image_ctx = nullptr;
 262   Context *on_finish = nullptr;
 263
 264   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
 265                                    Context *on_finish) {
 266     ceph_assert(s_instance != nullptr);
 267     s_instance->image_ctx = image_ctx;
 268     s_instance->on_finish = on_finish;
 269     return s_instance;
 270   }
 271
 272   CloseImageRequest() {
 273     ceph_assert(s_instance == nullptr);
 274     s_instance = this;
 275   }
 276
 277   ~CloseImageRequest() {
 278     ceph_assert(s_instance == this);
 279     s_instance = nullptr;
 280   }
 281
 282   MOCK_METHOD0(send, void());
 283 };
 284
 285 CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 286
 287 namespace snapshot {
 288
 289 template <>
 290 struct ApplyImageStateRequest<librbd::MockTestImageCtx> {
 291   Context* on_finish = nullptr;
 292
 293   static ApplyImageStateRequest* s_instance;
 294   static ApplyImageStateRequest* create(
 295       const std::string& local_mirror_uuid,
 296       const std::string& remote_mirror_uuid,
 297       librbd::MockTestImageCtx* local_image_ctx,
 298       librbd::MockTestImageCtx* remote_image_ctx,
 299       const librbd::mirror::snapshot::ImageState& image_state,
 300       Context* on_finish) {
 301     ceph_assert(s_instance != nullptr);
 302     s_instance->on_finish = on_finish;
 303     return s_instance;
 304   }
 305
 306   ApplyImageStateRequest() {
 307     s_instance = this;
 308   }
 309
 310   MOCK_METHOD0(send, void());
 311 };
 312
 313 template<>
 314 struct StateBuilder<librbd::MockTestImageCtx> {
 315   StateBuilder(librbd::MockTestImageCtx& local_image_ctx,
 316                librbd::MockTestImageCtx& remote_image_ctx,
 317                librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx>&
 318                  local_image_meta)
 319     : local_image_ctx(&local_image_ctx),
 320       remote_image_ctx(&remote_image_ctx),
 321       local_image_meta(&local_image_meta) {
 322   }
 323
 324   librbd::MockTestImageCtx* local_image_ctx;
 325   librbd::MockTestImageCtx* remote_image_ctx;
 326
 327   std::string remote_mirror_uuid = "remote mirror uuid";
 328
 329   librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx>*
 330     local_image_meta = nullptr;
 331 };
 332
 333 ApplyImageStateRequest<librbd::MockTestImageCtx>* ApplyImageStateRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 334
 335 } // namespace snapshot
 336 } // namespace image_replayer
 337 } // namespace mirror
 338 } // namespace rbd
 339
 340 #include "tools/rbd_mirror/image_replayer/snapshot/Replayer.cc"
 341
 342 namespace rbd {
 343 namespace mirror {
 344 namespace image_replayer {
 345 namespace snapshot {
 346
 347 using ::testing::_;
 348 using ::testing::DoAll;
 349 using ::testing::InSequence;
 350 using ::testing::Invoke;
 351 using ::testing::Return;
 352 using ::testing::ReturnArg;
 353 using ::testing::StrEq;
 354 using ::testing::WithArg;
 355
 356 class TestMockImageReplayerSnapshotReplayer : public TestMockFixture {
 357 public:
 358   typedef Replayer<librbd::MockTestImageCtx> MockReplayer;
 359   typedef ApplyImageStateRequest<librbd::MockTestImageCtx> MockApplyImageStateRequest;
 360   typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder;
 361   typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
 362   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 363   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
 364   typedef librbd::deep_copy::ImageCopyRequest<librbd::MockTestImageCtx> MockImageCopyRequest;
 365   typedef librbd::deep_copy::SnapshotCopyRequest<librbd::MockTestImageCtx> MockSnapshotCopyRequest;
 366   typedef librbd::mirror::snapshot::CreateNonPrimaryRequest<librbd::MockTestImageCtx> MockCreateNonPrimaryRequest;
 367   typedef librbd::mirror::snapshot::GetImageStateRequest<librbd::MockTestImageCtx> MockGetImageStateRequest;
 368   typedef librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx> MockImageMeta;
 369   typedef librbd::mirror::snapshot::UnlinkPeerRequest<librbd::MockTestImageCtx> MockUnlinkPeerRequest;
 370
 371   void SetUp() override {
 372     TestMockFixture::SetUp();
 373
 374     librbd::RBD rbd;
 375     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 376     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 377
 378     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name,
 379                               m_image_size));
 380     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name,
 381                             &m_remote_image_ctx));
 382   }
 383
 384   void expect_work_queue_repeatedly(MockThreads &mock_threads) {
 385     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 386       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 387           m_threads->work_queue->queue(ctx, r);
 388         }));
 389   }
 390
 391   void expect_add_event_after_repeatedly(MockThreads &mock_threads) {
 392     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 393       .WillRepeatedly(
 394         DoAll(Invoke([this](double seconds, Context *ctx) {
 395                        m_threads->timer->add_event_after(seconds, ctx);
 396                      }),
 397           ReturnArg<1>()));
 398     EXPECT_CALL(*mock_threads.timer, cancel_event(_))
 399       .WillRepeatedly(
 400         Invoke([this](Context *ctx) {
 401           return m_threads->timer->cancel_event(ctx);
 402         }));
 403   }
 404
 405   void expect_register_update_watcher(librbd::MockTestImageCtx& mock_image_ctx,
 406                                       librbd::UpdateWatchCtx** update_watch_ctx,
 407                                       uint64_t watch_handle, int r) {
 408     EXPECT_CALL(*mock_image_ctx.state, register_update_watcher(_, _))
 409       .WillOnce(Invoke([update_watch_ctx, watch_handle, r]
 410                        (librbd::UpdateWatchCtx* ctx, uint64_t* handle) {
 411           if (r >= 0) {
 412             *update_watch_ctx = ctx;
 413             *handle = watch_handle;
 414           }
 415           return r;
 416         }));
 417   }
 418
 419   void expect_unregister_update_watcher(librbd::MockTestImageCtx& mock_image_ctx,
 420                                         uint64_t watch_handle, int r) {
 421     EXPECT_CALL(*mock_image_ctx.state, unregister_update_watcher(watch_handle, _))
 422       .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) {
 423           m_threads->work_queue->queue(ctx, r);
 424         })));
 425   }
 426
 427   void expect_load_image_meta(MockImageMeta& mock_image_meta,
 428                               bool resync_requested, int r) {
 429     EXPECT_CALL(mock_image_meta, load(_))
 430       .WillOnce(Invoke([this, &mock_image_meta, resync_requested, r](Context* ctx) {
 431           mock_image_meta.resync_requested = resync_requested;
 432           m_threads->work_queue->queue(ctx, r);
 433         }));
 434   }
 435
 436   void expect_is_refresh_required(librbd::MockTestImageCtx& mock_image_ctx,
 437                                   bool is_required) {
 438     EXPECT_CALL(*mock_image_ctx.state, is_refresh_required())
 439       .WillOnce(Return(is_required));
 440   }
 441
 442   void expect_refresh(librbd::MockTestImageCtx& mock_image_ctx,
 443                       const std::map<uint64_t, librbd::SnapInfo>& snaps,
 444                       int r) {
 445     EXPECT_CALL(*mock_image_ctx.state, refresh(_))
 446       .WillOnce(Invoke([this, &mock_image_ctx, snaps, r](Context* ctx) {
 447         mock_image_ctx.snap_info = snaps;
 448         m_threads->work_queue->queue(ctx, r);
 449       }));
 450   }
 451
 452   void expect_notify_update(librbd::MockTestImageCtx& mock_image_ctx) {
 453     EXPECT_CALL(mock_image_ctx, notify_update(_))
 454       .WillOnce(Invoke([this](Context* ctx) {
 455         m_threads->work_queue->queue(ctx, 0);
 456       }));
 457   }
 458
 459   void expect_snapshot_copy(MockSnapshotCopyRequest& mock_snapshot_copy_request,
 460                             uint64_t src_snap_id_start,
 461                             uint64_t src_snap_id_end,
 462                             uint64_t dst_snap_id_start,
 463                             const librbd::SnapSeqs& snap_seqs, int r) {
 464     EXPECT_CALL(mock_snapshot_copy_request, send())
 465       .WillOnce(Invoke([this, &req=mock_snapshot_copy_request,
 466                         src_snap_id_start, src_snap_id_end, dst_snap_id_start,
 467                         snap_seqs, r]() {
 468         ASSERT_EQ(src_snap_id_start, req.src_snap_id_start);
 469         ASSERT_EQ(src_snap_id_end, req.src_snap_id_end);
 470         ASSERT_EQ(dst_snap_id_start, req.dst_snap_id_start);
 471         *req.snap_seqs = snap_seqs;
 472         m_threads->work_queue->queue(req.on_finish, r);
 473       }));
 474   }
 475
 476   void expect_get_image_state(MockGetImageStateRequest& mock_get_image_state_request,
 477                               uint64_t snap_id, int r) {
 478     EXPECT_CALL(mock_get_image_state_request, send())
 479       .WillOnce(Invoke([this, &req=mock_get_image_state_request, snap_id, r]() {
 480         ASSERT_EQ(snap_id, req.snap_id);
 481         m_threads->work_queue->queue(req.on_finish, r);
 482       }));
 483   }
 484
 485   void expect_create_non_primary_request(MockCreateNonPrimaryRequest& mock_create_non_primary_request,
 486                                          bool demoted,
 487                                          const std::string& primary_mirror_uuid,
 488                                          uint64_t primary_snap_id,
 489                                          const librbd::SnapSeqs& snap_seqs,
 490                                          uint64_t snap_id, int r) {
 491     EXPECT_CALL(mock_create_non_primary_request, send())
 492       .WillOnce(Invoke([this, &req=mock_create_non_primary_request, demoted,
 493                         primary_mirror_uuid, primary_snap_id, snap_seqs,
 494                         snap_id, r]() {
 495         ASSERT_EQ(demoted, req.demoted);
 496         ASSERT_EQ(primary_mirror_uuid, req.primary_mirror_uuid);
 497         ASSERT_EQ(primary_snap_id, req.primary_snap_id);
 498         ASSERT_EQ(snap_seqs, req.snap_seqs);
 499         *req.snap_id = snap_id;
 500         m_threads->work_queue->queue(req.on_finish, r);
 501       }));
 502   }
 503
 504   void expect_notify_sync_request(MockInstanceWatcher& mock_instance_watcher,
 505                                   const std::string& image_id, int r) {
 506     EXPECT_CALL(mock_instance_watcher, notify_sync_request(image_id, _))
 507       .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) {
 508         m_threads->work_queue->queue(ctx, r);
 509       })));
 510   }
 511
 512   void expect_notify_sync_complete(MockInstanceWatcher& mock_instance_watcher,
 513                                    const std::string& image_id) {
 514     EXPECT_CALL(mock_instance_watcher, notify_sync_complete(image_id));
 515   }
 516
 517   void expect_image_copy(MockImageCopyRequest& mock_image_copy_request,
 518                          uint64_t src_snap_id_start, uint64_t src_snap_id_end,
 519                          uint64_t dst_snap_id_start,
 520                          const librbd::deep_copy::ObjectNumber& object_number,
 521                          const librbd::SnapSeqs& snap_seqs, int r) {
 522     EXPECT_CALL(mock_image_copy_request, send())
 523       .WillOnce(Invoke([this, &req=mock_image_copy_request, src_snap_id_start,
 524                         src_snap_id_end, dst_snap_id_start, object_number,
 525                         snap_seqs, r]() {
 526         ASSERT_EQ(src_snap_id_start, req.src_snap_id_start);
 527         ASSERT_EQ(src_snap_id_end, req.src_snap_id_end);
 528         ASSERT_EQ(dst_snap_id_start, req.dst_snap_id_start);
 529         ASSERT_EQ(object_number, req.object_number);
 530         ASSERT_EQ(snap_seqs, req.snap_seqs);
 531         m_threads->work_queue->queue(req.on_finish, r);
 532       }));
 533   }
 534
 535   void expect_unlink_peer(MockUnlinkPeerRequest& mock_unlink_peer_request,
 536                           uint64_t snap_id, const std::string& mirror_peer_uuid,
 537                           int r) {
 538     EXPECT_CALL(mock_unlink_peer_request, send())
 539       .WillOnce(Invoke([this, &req=mock_unlink_peer_request, snap_id,
 540                         mirror_peer_uuid, r]() {
 541         ASSERT_EQ(snap_id, req.snap_id);
 542         ASSERT_EQ(mirror_peer_uuid, req.mirror_peer_uuid);
 543         m_threads->work_queue->queue(req.on_finish, r);
 544       }));
 545   }
 546
 547   void expect_apply_image_state(
 548       MockApplyImageStateRequest& mock_request, int r) {
 549     EXPECT_CALL(mock_request, send())
 550       .WillOnce(Invoke([this, &req=mock_request, r]() {
 551         m_threads->work_queue->queue(req.on_finish, r);
 552       }));
 553   }
 554
 555   void expect_mirror_image_snapshot_set_copy_progress(
 556       librbd::MockTestImageCtx& mock_test_image_ctx, uint64_t snap_id,
 557       bool completed, uint64_t last_copied_object, int r) {
 558     bufferlist bl;
 559     encode(snap_id, bl);
 560     encode(completed, bl);
 561     encode(last_copied_object, bl);
 562
 563     EXPECT_CALL(get_mock_io_ctx(mock_test_image_ctx.md_ctx),
 564                 exec(mock_test_image_ctx.header_oid, _, StrEq("rbd"),
 565                      StrEq("mirror_image_snapshot_set_copy_progress"),
 566                      ContentsEqual(bl), _, _))
 567       .WillOnce(Return(r));
 568   }
 569
 570   void expect_send(MockCloseImageRequest &mock_close_image_request, int r) {
 571     EXPECT_CALL(mock_close_image_request, send())
 572       .WillOnce(Invoke([this, &mock_close_image_request, r]() {
 573             *mock_close_image_request.image_ctx = nullptr;
 574             m_threads->work_queue->queue(mock_close_image_request.on_finish, r);
 575           }));
 576   }
 577
 578   void expect_notification(MockThreads& mock_threads,
 579                            MockReplayerListener& mock_replayer_listener) {
 580     EXPECT_CALL(mock_replayer_listener, handle_notification())
 581       .WillRepeatedly(Invoke([this]() {
 582           std::unique_lock locker{m_lock};
 583           ++m_notifications;
 584           m_cond.notify_all();
 585         }));
 586   }
 587
 588   int wait_for_notification(uint32_t count) {
 589     std::unique_lock locker{m_lock};
 590     for (uint32_t idx = 0; idx < count; ++idx) {
 591       while (m_notifications == 0) {
 592         if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) {
 593           return -ETIMEDOUT;
 594         }
 595       }
 596       --m_notifications;
 597     }
 598     return 0;
 599   }
 600
 601   int init_entry_replayer(MockReplayer& mock_replayer,
 602                           MockThreads& mock_threads,
 603                           librbd::MockTestImageCtx& mock_local_image_ctx,
 604                           librbd::MockTestImageCtx& mock_remote_image_ctx,
 605                           MockReplayerListener& mock_replayer_listener,
 606                           MockImageMeta& mock_image_meta,
 607                           librbd::UpdateWatchCtx** update_watch_ctx) {
 608     expect_register_update_watcher(mock_local_image_ctx, update_watch_ctx, 123,
 609                                    0);
 610     expect_register_update_watcher(mock_remote_image_ctx, update_watch_ctx, 234,
 611                                    0);
 612     expect_load_image_meta(mock_image_meta, false, 0);
 613     expect_is_refresh_required(mock_local_image_ctx, false);
 614     expect_is_refresh_required(mock_remote_image_ctx, false);
 615
 616     C_SaferCond init_ctx;
 617     mock_replayer.init(&init_ctx);
 618     int r = init_ctx.wait();
 619     if (r < 0) {
 620       return r;
 621     }
 622
 623     return wait_for_notification(2);
 624   }
 625
 626   int shut_down_entry_replayer(MockReplayer& mock_replayer,
 627                                MockThreads& mock_threads,
 628                                librbd::MockTestImageCtx& mock_local_image_ctx,
 629                                librbd::MockTestImageCtx& mock_remote_image_ctx) {
 630     expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0);
 631     expect_unregister_update_watcher(mock_local_image_ctx, 123, 0);
 632
 633     C_SaferCond shutdown_ctx;
 634     mock_replayer.shut_down(&shutdown_ctx);
 635     return shutdown_ctx.wait();
 636   }
 637
 638   librbd::ImageCtx* m_local_image_ctx = nullptr;
 639   librbd::ImageCtx* m_remote_image_ctx = nullptr;
 640
 641   PoolMetaCache m_pool_meta_cache{g_ceph_context};
 642
 643   ceph::mutex m_lock = ceph::make_mutex(
 644     "TestMockImageReplayerSnapshotReplayer");
 645   ceph::condition_variable m_cond;
 646   uint32_t m_notifications = 0;
 647 };
 648
 649 TEST_F(TestMockImageReplayerSnapshotReplayer, InitShutDown) {
 650   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
 651   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
 652
 653   MockThreads mock_threads(m_threads);
 654   expect_work_queue_repeatedly(mock_threads);
 655
 656   MockReplayerListener mock_replayer_listener;
 657   expect_notification(mock_threads, mock_replayer_listener);
 658
 659   InSequence seq;
 660
 661   MockInstanceWatcher mock_instance_watcher;
 662   MockImageMeta mock_image_meta;
 663   MockStateBuilder mock_state_builder(mock_local_image_ctx,
 664                                       mock_remote_image_ctx,
 665                                       mock_image_meta);
 666   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
 667                              "local mirror uuid", &m_pool_meta_cache,
 668                              &mock_state_builder, &mock_replayer_listener};
 669   m_pool_meta_cache.set_remote_pool_meta(
 670     m_remote_io_ctx.get_id(),
 671     {"remote mirror uuid", "remote mirror peer uuid"});
 672
 673   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
 674   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
 675                                    mock_local_image_ctx,
 676                                    mock_remote_image_ctx,
 677                                    mock_replayer_listener,
 678                                    mock_image_meta,
 679                                    &update_watch_ctx));
 680   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
 681                                         mock_local_image_ctx,
 682                                         mock_remote_image_ctx));
 683 }
 684
 685 TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) {
 686   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
 687   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
 688
 689   // it should sync two snapshots and skip two (user and mirror w/o matching
 690   // peer uuid)
 691   mock_remote_image_ctx.snap_info = {
 692     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 693        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 694        "", 0U, true, 0, {}},
 695      0, {}, 0, 0, {}}},
 696     {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{},
 697      0, {}, 0, 0, {}}},
 698     {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{
 699        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""},
 700        "", 0U, true, 0, {}},
 701      0, {}, 0, 0, {}}},
 702     {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
 703        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 704        "", 0U, true, 0, {}},
 705      0, {}, 0, 0, {}}}};
 706
 707   MockThreads mock_threads(m_threads);
 708   expect_work_queue_repeatedly(mock_threads);
 709
 710   MockReplayerListener mock_replayer_listener;
 711   expect_notification(mock_threads, mock_replayer_listener);
 712
 713   InSequence seq;
 714
 715   MockInstanceWatcher mock_instance_watcher;
 716   MockImageMeta mock_image_meta;
 717   MockStateBuilder mock_state_builder(mock_local_image_ctx,
 718                                       mock_remote_image_ctx,
 719                                       mock_image_meta);
 720   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
 721                              "local mirror uuid", &m_pool_meta_cache,
 722                              &mock_state_builder, &mock_replayer_listener};
 723   m_pool_meta_cache.set_remote_pool_meta(
 724     m_remote_io_ctx.get_id(),
 725     {"remote mirror uuid", "remote mirror peer uuid"});
 726
 727   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
 728
 729   // init
 730   expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123,
 731                                  0);
 732   expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234,
 733                                  0);
 734
 735   // sync snap1
 736   expect_load_image_meta(mock_image_meta, false, 0);
 737   expect_is_refresh_required(mock_local_image_ctx, false);
 738   expect_is_refresh_required(mock_remote_image_ctx, false);
 739   MockSnapshotCopyRequest mock_snapshot_copy_request;
 740   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
 741                        0);
 742   MockGetImageStateRequest mock_get_image_state_request;
 743   expect_get_image_state(mock_get_image_state_request, 1, 0);
 744   MockCreateNonPrimaryRequest mock_create_non_primary_request;
 745   expect_create_non_primary_request(mock_create_non_primary_request,
 746                                     false, "remote mirror uuid", 1,
 747                                     {{1, CEPH_NOSNAP}}, 11, 0);
 748   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
 749   MockImageCopyRequest mock_image_copy_request;
 750   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
 751                     {{1, CEPH_NOSNAP}}, 0);
 752   MockApplyImageStateRequest mock_apply_state_request;
 753   expect_apply_image_state(mock_apply_state_request, 0);
 754   expect_mirror_image_snapshot_set_copy_progress(
 755     mock_local_image_ctx, 11, true, 0, 0);
 756   expect_notify_update(mock_local_image_ctx);
 757   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
 758
 759   // sync snap4
 760   expect_load_image_meta(mock_image_meta, false, 0);
 761   expect_is_refresh_required(mock_local_image_ctx, true);
 762   expect_refresh(
 763     mock_local_image_ctx, {
 764       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 765          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 766          1, true, 0, {}},
 767        0, {}, 0, 0, {}}},
 768     }, 0);
 769   expect_is_refresh_required(mock_remote_image_ctx, false);
 770   expect_snapshot_copy(mock_snapshot_copy_request, 1, 4, 11,
 771                        {{1, 11}, {4, CEPH_NOSNAP}}, 0);
 772   expect_get_image_state(mock_get_image_state_request, 4, 0);
 773   expect_create_non_primary_request(mock_create_non_primary_request,
 774                                     false, "remote mirror uuid", 4,
 775                                     {{1, 11}, {4, CEPH_NOSNAP}}, 14, 0);
 776   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
 777   expect_image_copy(mock_image_copy_request, 1, 4, 11, {},
 778                     {{1, 11}, {4, CEPH_NOSNAP}}, 0);
 779   expect_apply_image_state(mock_apply_state_request, 0);
 780   expect_mirror_image_snapshot_set_copy_progress(
 781     mock_local_image_ctx, 14, true, 0, 0);
 782   expect_notify_update(mock_local_image_ctx);
 783   MockUnlinkPeerRequest mock_unlink_peer_request;
 784   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
 785                      0);
 786   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
 787
 788   // idle
 789   expect_load_image_meta(mock_image_meta, false, 0);
 790   expect_is_refresh_required(mock_local_image_ctx, true);
 791   expect_refresh(
 792     mock_local_image_ctx, {
 793       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 794          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 795          1, true, 0, {}},
 796        0, {}, 0, 0, {}}},
 797       {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{
 798          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 799          4, true, 0, {}},
 800        0, {}, 0, 0, {}}},
 801     }, 0);
 802   expect_is_refresh_required(mock_remote_image_ctx, false);
 803
 804   // fire init
 805   C_SaferCond init_ctx;
 806   mock_replayer.init(&init_ctx);
 807   ASSERT_EQ(0, init_ctx.wait());
 808
 809   // wait for sync to complete
 810   ASSERT_EQ(0, wait_for_notification(4));
 811
 812   // shut down
 813   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
 814                                         mock_local_image_ctx,
 815                                         mock_remote_image_ctx));
 816 }
 817
 818 TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) {
 819   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
 820   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
 821
 822   MockThreads mock_threads(m_threads);
 823   expect_work_queue_repeatedly(mock_threads);
 824
 825   MockReplayerListener mock_replayer_listener;
 826   expect_notification(mock_threads, mock_replayer_listener);
 827
 828   InSequence seq;
 829
 830   MockInstanceWatcher mock_instance_watcher;
 831   MockImageMeta mock_image_meta;
 832   MockStateBuilder mock_state_builder(mock_local_image_ctx,
 833                                       mock_remote_image_ctx,
 834                                       mock_image_meta);
 835   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
 836                              "local mirror uuid", &m_pool_meta_cache,
 837                              &mock_state_builder, &mock_replayer_listener};
 838   m_pool_meta_cache.set_remote_pool_meta(
 839     m_remote_io_ctx.get_id(),
 840     {"remote mirror uuid", "remote mirror peer uuid"});
 841
 842   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
 843   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
 844                                    mock_local_image_ctx,
 845                                    mock_remote_image_ctx,
 846                                    mock_replayer_listener,
 847                                    mock_image_meta,
 848                                    &update_watch_ctx));
 849
 850   // inject a incomplete sync snapshot
 851   mock_remote_image_ctx.snap_info = {
 852     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 853        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
 854        "", 0U, true, 0, {}},
 855      0, {}, 0, 0, {}}}};
 856   mock_local_image_ctx.snap_info = {
 857     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 858        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 859        1, false, 123, {{1, CEPH_NOSNAP}}},
 860      0, {}, 0, 0, {}}}};
 861
 862   // re-sync snap1
 863   expect_load_image_meta(mock_image_meta, false, 0);
 864   expect_is_refresh_required(mock_local_image_ctx, false);
 865   expect_is_refresh_required(mock_remote_image_ctx, false);
 866   MockGetImageStateRequest mock_get_image_state_request;
 867   expect_get_image_state(mock_get_image_state_request, 11, 0);
 868   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
 869   MockImageCopyRequest mock_image_copy_request;
 870   expect_image_copy(mock_image_copy_request, 0, 1, 0,
 871                     librbd::deep_copy::ObjectNumber{123U},
 872                     {{1, CEPH_NOSNAP}}, 0);
 873   MockApplyImageStateRequest mock_apply_state_request;
 874   expect_apply_image_state(mock_apply_state_request, 0);
 875   expect_mirror_image_snapshot_set_copy_progress(
 876     mock_local_image_ctx, 11, true, 123, 0);
 877   expect_notify_update(mock_local_image_ctx);
 878
 879   // idle
 880   expect_load_image_meta(mock_image_meta, false, 0);
 881   expect_is_refresh_required(mock_local_image_ctx, true);
 882   expect_refresh(
 883     mock_local_image_ctx, {
 884       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 885          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 886          1, true, 0, {}},
 887        0, {}, 0, 0, {}}},
 888     }, 0);
 889   expect_is_refresh_required(mock_remote_image_ctx, false);
 890
 891   // wake-up replayer
 892   update_watch_ctx->handle_notify();
 893
 894   // wait for sync to complete
 895   ASSERT_EQ(0, wait_for_notification(2));
 896
 897   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
 898                                         mock_local_image_ctx,
 899                                         mock_remote_image_ctx));
 900 }
 901
 902 TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) {
 903   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
 904   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
 905
 906   MockThreads mock_threads(m_threads);
 907   expect_work_queue_repeatedly(mock_threads);
 908
 909   MockReplayerListener mock_replayer_listener;
 910   expect_notification(mock_threads, mock_replayer_listener);
 911
 912   InSequence seq;
 913
 914   MockInstanceWatcher mock_instance_watcher;
 915   MockImageMeta mock_image_meta;
 916   MockStateBuilder mock_state_builder(mock_local_image_ctx,
 917                                       mock_remote_image_ctx,
 918                                       mock_image_meta);
 919   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
 920                              "local mirror uuid", &m_pool_meta_cache,
 921                              &mock_state_builder, &mock_replayer_listener};
 922   m_pool_meta_cache.set_remote_pool_meta(
 923     m_remote_io_ctx.get_id(),
 924     {"remote mirror uuid", "remote mirror peer uuid"});
 925
 926   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
 927   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
 928                                    mock_local_image_ctx,
 929                                    mock_remote_image_ctx,
 930                                    mock_replayer_listener,
 931                                    mock_image_meta,
 932                                    &update_watch_ctx));
 933
 934   // inject a demotion snapshot
 935   mock_remote_image_ctx.snap_info = {
 936     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 937        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
 938        {"remote mirror peer uuid"}, "", 0U, true, 0, {}},
 939      0, {}, 0, 0, {}}}};
 940
 941   // sync snap1
 942   expect_load_image_meta(mock_image_meta, false, 0);
 943   expect_is_refresh_required(mock_local_image_ctx, false);
 944   expect_is_refresh_required(mock_remote_image_ctx, false);
 945   MockSnapshotCopyRequest mock_snapshot_copy_request;
 946   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
 947                        0);
 948   MockGetImageStateRequest mock_get_image_state_request;
 949   expect_get_image_state(mock_get_image_state_request, 1, 0);
 950   MockCreateNonPrimaryRequest mock_create_non_primary_request;
 951   expect_create_non_primary_request(mock_create_non_primary_request,
 952                                     true, "remote mirror uuid", 1,
 953                                     {{1, CEPH_NOSNAP}}, 11, 0);
 954   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
 955   MockImageCopyRequest mock_image_copy_request;
 956   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
 957                     {{1, CEPH_NOSNAP}}, 0);
 958   MockApplyImageStateRequest mock_apply_state_request;
 959   expect_apply_image_state(mock_apply_state_request, 0);
 960   expect_mirror_image_snapshot_set_copy_progress(
 961     mock_local_image_ctx, 11, true, 0, 0);
 962   expect_notify_update(mock_local_image_ctx);
 963
 964   // idle
 965   expect_load_image_meta(mock_image_meta, false, 0);
 966   expect_is_refresh_required(mock_local_image_ctx, true);
 967   expect_refresh(
 968     mock_local_image_ctx, {
 969       {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
 970          cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
 971          1, true, 0, {}},
 972        0, {}, 0, 0, {}}},
 973     }, 0);
 974   expect_is_refresh_required(mock_remote_image_ctx, false);
 975
 976   // wake-up replayer
 977   update_watch_ctx->handle_notify();
 978
 979   // wait for sync to complete and expect replay complete
 980   ASSERT_EQ(0, wait_for_notification(2));
 981   ASSERT_FALSE(mock_replayer.is_replaying());
 982
 983   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
 984                                         mock_local_image_ctx,
 985                                         mock_remote_image_ctx));
 986 }
 987
 988 TEST_F(TestMockImageReplayerSnapshotReplayer, LocalImagePromoted) {
 989   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
 990   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
 991
 992   MockThreads mock_threads(m_threads);
 993   expect_work_queue_repeatedly(mock_threads);
 994
 995   MockReplayerListener mock_replayer_listener;
 996   expect_notification(mock_threads, mock_replayer_listener);
 997
 998   InSequence seq;
 999
1000   MockInstanceWatcher mock_instance_watcher;
1001   MockImageMeta mock_image_meta;
1002   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1003                                       mock_remote_image_ctx,
1004                                       mock_image_meta);
1005   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1006                              "local mirror uuid", &m_pool_meta_cache,
1007                              &mock_state_builder, &mock_replayer_listener};
1008   m_pool_meta_cache.set_remote_pool_meta(
1009     m_remote_io_ctx.get_id(),
1010     {"remote mirror uuid", "remote mirror peer uuid"});
1011
1012   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1013   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1014                                    mock_local_image_ctx,
1015                                    mock_remote_image_ctx,
1016                                    mock_replayer_listener,
1017                                    mock_image_meta,
1018                                    &update_watch_ctx));
1019
1020   // inject a promotion snapshot
1021   mock_local_image_ctx.snap_info = {
1022     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1023        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
1024        {"remote mirror peer uuid"}, "", 0U, true, 0, {}},
1025      0, {}, 0, 0, {}}}};
1026
1027   // idle
1028   expect_load_image_meta(mock_image_meta, false, 0);
1029   expect_is_refresh_required(mock_local_image_ctx, false);
1030   expect_is_refresh_required(mock_remote_image_ctx, false);
1031
1032   // wake-up replayer
1033   update_watch_ctx->handle_notify();
1034
1035   // wait for sync to complete and expect replay complete
1036   ASSERT_EQ(0, wait_for_notification(1));
1037   ASSERT_FALSE(mock_replayer.is_replaying());
1038
1039   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1040                                         mock_local_image_ctx,
1041                                         mock_remote_image_ctx));
1042 }
1043
1044 TEST_F(TestMockImageReplayerSnapshotReplayer, ResyncRequested) {
1045   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1046   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1047
1048   MockThreads mock_threads(m_threads);
1049   expect_work_queue_repeatedly(mock_threads);
1050
1051   MockReplayerListener mock_replayer_listener;
1052   expect_notification(mock_threads, mock_replayer_listener);
1053
1054   InSequence seq;
1055
1056   MockInstanceWatcher mock_instance_watcher;
1057   MockImageMeta mock_image_meta;
1058   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1059                                       mock_remote_image_ctx,
1060                                       mock_image_meta);
1061   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1062                              "local mirror uuid", &m_pool_meta_cache,
1063                              &mock_state_builder, &mock_replayer_listener};
1064   m_pool_meta_cache.set_remote_pool_meta(
1065     m_remote_io_ctx.get_id(),
1066     {"remote mirror uuid", "remote mirror peer uuid"});
1067
1068   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1069   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1070                                    mock_local_image_ctx,
1071                                    mock_remote_image_ctx,
1072                                    mock_replayer_listener,
1073                                    mock_image_meta,
1074                                    &update_watch_ctx));
1075
1076   // idle
1077   expect_load_image_meta(mock_image_meta, true, 0);
1078
1079   // wake-up replayer
1080   update_watch_ctx->handle_notify();
1081
1082   // wait for sync to complete and expect replay complete
1083   ASSERT_EQ(0, wait_for_notification(1));
1084   ASSERT_FALSE(mock_replayer.is_replaying());
1085
1086   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1087                                         mock_local_image_ctx,
1088                                         mock_remote_image_ctx));
1089 }
1090
1091 TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterLocalUpdateWatcherError) {
1092   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1093   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1094
1095   MockThreads mock_threads(m_threads);
1096   expect_work_queue_repeatedly(mock_threads);
1097
1098   InSequence seq;
1099
1100   MockInstanceWatcher mock_instance_watcher;
1101   MockImageMeta mock_image_meta;
1102   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1103                                       mock_remote_image_ctx,
1104                                       mock_image_meta);
1105   MockReplayerListener mock_replayer_listener;
1106   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1107                              "local mirror uuid", &m_pool_meta_cache,
1108                              &mock_state_builder, &mock_replayer_listener};
1109   m_pool_meta_cache.set_remote_pool_meta(
1110     m_remote_io_ctx.get_id(),
1111     {"remote mirror uuid", "remote mirror peer uuid"});
1112
1113   // init
1114   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1115   expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123,
1116                                  -EINVAL);
1117
1118   // fire init
1119   C_SaferCond init_ctx;
1120   mock_replayer.init(&init_ctx);
1121   ASSERT_EQ(-EINVAL, init_ctx.wait());
1122 }
1123
1124 TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError) {
1125   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1126   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1127
1128   MockThreads mock_threads(m_threads);
1129   expect_work_queue_repeatedly(mock_threads);
1130
1131   InSequence seq;
1132
1133   MockInstanceWatcher mock_instance_watcher;
1134   MockImageMeta mock_image_meta;
1135   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1136                                       mock_remote_image_ctx,
1137                                       mock_image_meta);
1138   MockReplayerListener mock_replayer_listener;
1139   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1140                              "local mirror uuid", &m_pool_meta_cache,
1141                              &mock_state_builder, &mock_replayer_listener};
1142   m_pool_meta_cache.set_remote_pool_meta(
1143     m_remote_io_ctx.get_id(),
1144     {"remote mirror uuid", "remote mirror peer uuid"});
1145
1146   // init
1147   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1148   expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123,
1149                                  0);
1150   expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234,
1151                                  -EINVAL);
1152
1153   expect_unregister_update_watcher(mock_local_image_ctx, 123, 0);
1154
1155   // fire init
1156   C_SaferCond init_ctx;
1157   mock_replayer.init(&init_ctx);
1158   ASSERT_EQ(-EINVAL, init_ctx.wait());
1159 }
1160
1161 TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterRemoteUpdateWatcherError) {
1162   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1163   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1164
1165   MockThreads mock_threads(m_threads);
1166   expect_work_queue_repeatedly(mock_threads);
1167
1168   MockReplayerListener mock_replayer_listener;
1169   expect_notification(mock_threads, mock_replayer_listener);
1170
1171   InSequence seq;
1172
1173   MockInstanceWatcher mock_instance_watcher;
1174   MockImageMeta mock_image_meta;
1175   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1176                                       mock_remote_image_ctx,
1177                                       mock_image_meta);
1178   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1179                              "local mirror uuid", &m_pool_meta_cache,
1180                              &mock_state_builder, &mock_replayer_listener};
1181   m_pool_meta_cache.set_remote_pool_meta(
1182     m_remote_io_ctx.get_id(),
1183     {"remote mirror uuid", "remote mirror peer uuid"});
1184
1185   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1186   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1187                                    mock_local_image_ctx,
1188                                    mock_remote_image_ctx,
1189                                    mock_replayer_listener,
1190                                    mock_image_meta,
1191                                    &update_watch_ctx));
1192
1193
1194   // shut down
1195   expect_unregister_update_watcher(mock_remote_image_ctx, 234, -EINVAL);
1196   expect_unregister_update_watcher(mock_local_image_ctx, 123, 0);
1197
1198   C_SaferCond shutdown_ctx;
1199   mock_replayer.shut_down(&shutdown_ctx);
1200   ASSERT_EQ(-EINVAL, shutdown_ctx.wait());
1201 }
1202
1203 TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterLocalUpdateWatcherError) {
1204   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1205   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1206
1207   MockThreads mock_threads(m_threads);
1208   expect_work_queue_repeatedly(mock_threads);
1209
1210   MockReplayerListener mock_replayer_listener;
1211   expect_notification(mock_threads, mock_replayer_listener);
1212
1213   InSequence seq;
1214
1215   MockInstanceWatcher mock_instance_watcher;
1216   MockImageMeta mock_image_meta;
1217   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1218                                       mock_remote_image_ctx,
1219                                       mock_image_meta);
1220   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1221                              "local mirror uuid", &m_pool_meta_cache,
1222                              &mock_state_builder, &mock_replayer_listener};
1223   m_pool_meta_cache.set_remote_pool_meta(
1224     m_remote_io_ctx.get_id(),
1225     {"remote mirror uuid", "remote mirror peer uuid"});
1226
1227   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1228   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1229                                    mock_local_image_ctx,
1230                                    mock_remote_image_ctx,
1231                                    mock_replayer_listener,
1232                                    mock_image_meta,
1233                                    &update_watch_ctx));
1234
1235
1236   // shut down
1237   expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0);
1238   expect_unregister_update_watcher(mock_local_image_ctx, 123, -EINVAL);
1239
1240   C_SaferCond shutdown_ctx;
1241   mock_replayer.shut_down(&shutdown_ctx);
1242   ASSERT_EQ(-EINVAL, shutdown_ctx.wait());
1243 }
1244
1245 TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) {
1246   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1247   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1248
1249   MockThreads mock_threads(m_threads);
1250   expect_work_queue_repeatedly(mock_threads);
1251
1252   MockReplayerListener mock_replayer_listener;
1253   expect_notification(mock_threads, mock_replayer_listener);
1254
1255   InSequence seq;
1256
1257   MockInstanceWatcher mock_instance_watcher;
1258   MockImageMeta mock_image_meta;
1259   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1260                                       mock_remote_image_ctx,
1261                                       mock_image_meta);
1262   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1263                              "local mirror uuid", &m_pool_meta_cache,
1264                              &mock_state_builder, &mock_replayer_listener};
1265   m_pool_meta_cache.set_remote_pool_meta(
1266     m_remote_io_ctx.get_id(),
1267     {"remote mirror uuid", "remote mirror peer uuid"});
1268
1269   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1270   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1271                                    mock_local_image_ctx,
1272                                    mock_remote_image_ctx,
1273                                    mock_replayer_listener,
1274                                    mock_image_meta,
1275                                    &update_watch_ctx));
1276
1277   // sync
1278   expect_load_image_meta(mock_image_meta, false, -EINVAL);
1279
1280   // wake-up replayer
1281   update_watch_ctx->handle_notify();
1282
1283   // wait for sync to complete and expect replay complete
1284   ASSERT_EQ(0, wait_for_notification(1));
1285   ASSERT_FALSE(mock_replayer.is_replaying());
1286   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1287
1288   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1289                                         mock_local_image_ctx,
1290                                         mock_remote_image_ctx));
1291 }
1292
1293 TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) {
1294   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1295   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1296
1297   MockThreads mock_threads(m_threads);
1298   expect_work_queue_repeatedly(mock_threads);
1299
1300   MockReplayerListener mock_replayer_listener;
1301   expect_notification(mock_threads, mock_replayer_listener);
1302
1303   InSequence seq;
1304
1305   MockInstanceWatcher mock_instance_watcher;
1306   MockImageMeta mock_image_meta;
1307   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1308                                       mock_remote_image_ctx,
1309                                       mock_image_meta);
1310   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1311                              "local mirror uuid", &m_pool_meta_cache,
1312                              &mock_state_builder, &mock_replayer_listener};
1313   m_pool_meta_cache.set_remote_pool_meta(
1314     m_remote_io_ctx.get_id(),
1315     {"remote mirror uuid", "remote mirror peer uuid"});
1316
1317   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1318   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1319                                    mock_local_image_ctx,
1320                                    mock_remote_image_ctx,
1321                                    mock_replayer_listener,
1322                                    mock_image_meta,
1323                                    &update_watch_ctx));
1324
1325   // sync
1326   expect_load_image_meta(mock_image_meta, false, 0);
1327   expect_is_refresh_required(mock_local_image_ctx, true);
1328   expect_refresh(mock_local_image_ctx, {}, -EINVAL);
1329
1330   // wake-up replayer
1331   update_watch_ctx->handle_notify();
1332
1333   // wait for sync to complete and expect replay complete
1334   ASSERT_EQ(0, wait_for_notification(1));
1335   ASSERT_FALSE(mock_replayer.is_replaying());
1336   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1337
1338   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1339                                         mock_local_image_ctx,
1340                                         mock_remote_image_ctx));
1341 }
1342
1343 TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) {
1344   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1345   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1346
1347   MockThreads mock_threads(m_threads);
1348   expect_work_queue_repeatedly(mock_threads);
1349
1350   MockReplayerListener mock_replayer_listener;
1351   expect_notification(mock_threads, mock_replayer_listener);
1352
1353   InSequence seq;
1354
1355   MockInstanceWatcher mock_instance_watcher;
1356   MockImageMeta mock_image_meta;
1357   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1358                                       mock_remote_image_ctx,
1359                                       mock_image_meta);
1360   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1361                              "local mirror uuid", &m_pool_meta_cache,
1362                              &mock_state_builder, &mock_replayer_listener};
1363   m_pool_meta_cache.set_remote_pool_meta(
1364     m_remote_io_ctx.get_id(),
1365     {"remote mirror uuid", "remote mirror peer uuid"});
1366
1367   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1368   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1369                                    mock_local_image_ctx,
1370                                    mock_remote_image_ctx,
1371                                    mock_replayer_listener,
1372                                    mock_image_meta,
1373                                    &update_watch_ctx));
1374
1375   // sync
1376   expect_load_image_meta(mock_image_meta, false, 0);
1377   expect_is_refresh_required(mock_local_image_ctx, false);
1378   expect_is_refresh_required(mock_remote_image_ctx, true);
1379   expect_refresh(mock_remote_image_ctx, {}, -EINVAL);
1380
1381   // wake-up replayer
1382   update_watch_ctx->handle_notify();
1383
1384   // wait for sync to complete and expect replay complete
1385   ASSERT_EQ(0, wait_for_notification(1));
1386   ASSERT_FALSE(mock_replayer.is_replaying());
1387   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1388
1389   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1390                                         mock_local_image_ctx,
1391                                         mock_remote_image_ctx));
1392 }
1393
1394 TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) {
1395   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1396   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1397
1398   MockThreads mock_threads(m_threads);
1399   expect_work_queue_repeatedly(mock_threads);
1400
1401   MockReplayerListener mock_replayer_listener;
1402   expect_notification(mock_threads, mock_replayer_listener);
1403
1404   InSequence seq;
1405
1406   MockInstanceWatcher mock_instance_watcher;
1407   MockImageMeta mock_image_meta;
1408   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1409                                       mock_remote_image_ctx,
1410                                       mock_image_meta);
1411   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1412                              "local mirror uuid", &m_pool_meta_cache,
1413                              &mock_state_builder, &mock_replayer_listener};
1414   m_pool_meta_cache.set_remote_pool_meta(
1415     m_remote_io_ctx.get_id(),
1416     {"remote mirror uuid", "remote mirror peer uuid"});
1417
1418   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1419   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1420                                    mock_local_image_ctx,
1421                                    mock_remote_image_ctx,
1422                                    mock_replayer_listener,
1423                                    mock_image_meta,
1424                                    &update_watch_ctx));
1425
1426   // inject snapshot
1427   mock_remote_image_ctx.snap_info = {
1428     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1429        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
1430        0U, true, 0, {}},
1431      0, {}, 0, 0, {}}}};
1432
1433   // sync snap1
1434   expect_load_image_meta(mock_image_meta, false, 0);
1435   expect_is_refresh_required(mock_local_image_ctx, false);
1436   expect_is_refresh_required(mock_remote_image_ctx, false);
1437   MockSnapshotCopyRequest mock_snapshot_copy_request;
1438   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
1439                        -EINVAL);
1440
1441   // wake-up replayer
1442   update_watch_ctx->handle_notify();
1443
1444   // wait for sync to complete and expect replay complete
1445   ASSERT_EQ(0, wait_for_notification(1));
1446   ASSERT_FALSE(mock_replayer.is_replaying());
1447   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1448
1449   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1450                                         mock_local_image_ctx,
1451                                         mock_remote_image_ctx));
1452 }
1453
1454 TEST_F(TestMockImageReplayerSnapshotReplayer, GetImageStateError) {
1455   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1456   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1457
1458   MockThreads mock_threads(m_threads);
1459   expect_work_queue_repeatedly(mock_threads);
1460
1461   MockReplayerListener mock_replayer_listener;
1462   expect_notification(mock_threads, mock_replayer_listener);
1463
1464   InSequence seq;
1465
1466   MockInstanceWatcher mock_instance_watcher;
1467   MockImageMeta mock_image_meta;
1468   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1469                                       mock_remote_image_ctx,
1470                                       mock_image_meta);
1471   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1472                              "local mirror uuid", &m_pool_meta_cache,
1473                              &mock_state_builder, &mock_replayer_listener};
1474   m_pool_meta_cache.set_remote_pool_meta(
1475     m_remote_io_ctx.get_id(),
1476     {"remote mirror uuid", "remote mirror peer uuid"});
1477
1478   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1479   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1480                                    mock_local_image_ctx,
1481                                    mock_remote_image_ctx,
1482                                    mock_replayer_listener,
1483                                    mock_image_meta,
1484                                    &update_watch_ctx));
1485
1486   // inject snapshot
1487   mock_remote_image_ctx.snap_info = {
1488     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1489        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
1490        0U, true, 0, {}},
1491      0, {}, 0, 0, {}}}};
1492
1493   // sync snap1
1494   expect_load_image_meta(mock_image_meta, false, 0);
1495   expect_is_refresh_required(mock_local_image_ctx, false);
1496   expect_is_refresh_required(mock_remote_image_ctx, false);
1497   MockSnapshotCopyRequest mock_snapshot_copy_request;
1498   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
1499                        0);
1500   MockGetImageStateRequest mock_get_image_state_request;
1501   expect_get_image_state(mock_get_image_state_request, 1, -EINVAL);
1502
1503   // wake-up replayer
1504   update_watch_ctx->handle_notify();
1505
1506   // wait for sync to complete and expect replay complete
1507   ASSERT_EQ(0, wait_for_notification(1));
1508   ASSERT_FALSE(mock_replayer.is_replaying());
1509   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1510
1511   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1512                                         mock_local_image_ctx,
1513                                         mock_remote_image_ctx));
1514 }
1515
1516 TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) {
1517   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1518   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1519
1520   MockThreads mock_threads(m_threads);
1521   expect_work_queue_repeatedly(mock_threads);
1522
1523   MockReplayerListener mock_replayer_listener;
1524   expect_notification(mock_threads, mock_replayer_listener);
1525
1526   InSequence seq;
1527
1528   MockInstanceWatcher mock_instance_watcher;
1529   MockImageMeta mock_image_meta;
1530   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1531                                       mock_remote_image_ctx,
1532                                       mock_image_meta);
1533   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1534                              "local mirror uuid", &m_pool_meta_cache,
1535                              &mock_state_builder, &mock_replayer_listener};
1536   m_pool_meta_cache.set_remote_pool_meta(
1537     m_remote_io_ctx.get_id(),
1538     {"remote mirror uuid", "remote mirror peer uuid"});
1539
1540   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1541   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1542                                    mock_local_image_ctx,
1543                                    mock_remote_image_ctx,
1544                                    mock_replayer_listener,
1545                                    mock_image_meta,
1546                                    &update_watch_ctx));
1547
1548   // inject snapshot
1549   mock_remote_image_ctx.snap_info = {
1550     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1551        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
1552        0U, true, 0, {}},
1553      0, {}, 0, 0, {}}}};
1554
1555   // sync snap1
1556   expect_load_image_meta(mock_image_meta, false, 0);
1557   expect_is_refresh_required(mock_local_image_ctx, false);
1558   expect_is_refresh_required(mock_remote_image_ctx, false);
1559   MockSnapshotCopyRequest mock_snapshot_copy_request;
1560   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
1561                        0);
1562   MockGetImageStateRequest mock_get_image_state_request;
1563   expect_get_image_state(mock_get_image_state_request, 1, 0);
1564   MockCreateNonPrimaryRequest mock_create_non_primary_request;
1565   expect_create_non_primary_request(mock_create_non_primary_request,
1566                                     false, "remote mirror uuid", 1,
1567                                     {{1, CEPH_NOSNAP}}, 11, -EINVAL);
1568
1569   // wake-up replayer
1570   update_watch_ctx->handle_notify();
1571
1572   // wait for sync to complete and expect replay complete
1573   ASSERT_EQ(0, wait_for_notification(1));
1574   ASSERT_FALSE(mock_replayer.is_replaying());
1575   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1576
1577   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1578                                         mock_local_image_ctx,
1579                                         mock_remote_image_ctx));
1580 }
1581
1582 TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) {
1583   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1584   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1585
1586   MockThreads mock_threads(m_threads);
1587   expect_work_queue_repeatedly(mock_threads);
1588
1589   MockReplayerListener mock_replayer_listener;
1590   expect_notification(mock_threads, mock_replayer_listener);
1591
1592   InSequence seq;
1593
1594   MockInstanceWatcher mock_instance_watcher;
1595   MockImageMeta mock_image_meta;
1596   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1597                                       mock_remote_image_ctx,
1598                                       mock_image_meta);
1599   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1600                              "local mirror uuid", &m_pool_meta_cache,
1601                              &mock_state_builder, &mock_replayer_listener};
1602   m_pool_meta_cache.set_remote_pool_meta(
1603     m_remote_io_ctx.get_id(),
1604     {"remote mirror uuid", "remote mirror peer uuid"});
1605
1606   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1607   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1608                                    mock_local_image_ctx,
1609                                    mock_remote_image_ctx,
1610                                    mock_replayer_listener,
1611                                    mock_image_meta,
1612                                    &update_watch_ctx));
1613
1614   // inject snapshot
1615   mock_remote_image_ctx.snap_info = {
1616     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1617        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
1618        0U, true, 0, {}},
1619      0, {}, 0, 0, {}}}};
1620
1621   // sync snap1
1622   expect_load_image_meta(mock_image_meta, false, 0);
1623   expect_is_refresh_required(mock_local_image_ctx, false);
1624   expect_is_refresh_required(mock_remote_image_ctx, false);
1625   MockSnapshotCopyRequest mock_snapshot_copy_request;
1626   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
1627                        0);
1628   MockGetImageStateRequest mock_get_image_state_request;
1629   expect_get_image_state(mock_get_image_state_request, 1, 0);
1630   MockCreateNonPrimaryRequest mock_create_non_primary_request;
1631   expect_create_non_primary_request(mock_create_non_primary_request,
1632                                     false, "remote mirror uuid", 1,
1633                                     {{1, CEPH_NOSNAP}}, 11, 0);
1634   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id,
1635                              -ECANCELED);
1636
1637   // wake-up replayer
1638   update_watch_ctx->handle_notify();
1639
1640   // wait for sync to complete and expect replay complete
1641   ASSERT_EQ(0, wait_for_notification(1));
1642   ASSERT_FALSE(mock_replayer.is_replaying());
1643   ASSERT_EQ(-ECANCELED, mock_replayer.get_error_code());
1644
1645   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1646                                         mock_local_image_ctx,
1647                                         mock_remote_image_ctx));
1648
1649 }
1650
1651 TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) {
1652   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1653   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1654
1655   MockThreads mock_threads(m_threads);
1656   expect_work_queue_repeatedly(mock_threads);
1657
1658   MockReplayerListener mock_replayer_listener;
1659   expect_notification(mock_threads, mock_replayer_listener);
1660
1661   InSequence seq;
1662
1663   MockInstanceWatcher mock_instance_watcher;
1664   MockImageMeta mock_image_meta;
1665   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1666                                       mock_remote_image_ctx,
1667                                       mock_image_meta);
1668   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1669                              "local mirror uuid", &m_pool_meta_cache,
1670                              &mock_state_builder, &mock_replayer_listener};
1671   m_pool_meta_cache.set_remote_pool_meta(
1672     m_remote_io_ctx.get_id(),
1673     {"remote mirror uuid", "remote mirror peer uuid"});
1674
1675   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1676   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1677                                    mock_local_image_ctx,
1678                                    mock_remote_image_ctx,
1679                                    mock_replayer_listener,
1680                                    mock_image_meta,
1681                                    &update_watch_ctx));
1682
1683   // inject snapshot
1684   mock_remote_image_ctx.snap_info = {
1685     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1686        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
1687        0U, true, 0, {}},
1688      0, {}, 0, 0, {}}}};
1689
1690   // sync snap1
1691   expect_load_image_meta(mock_image_meta, false, 0);
1692   expect_is_refresh_required(mock_local_image_ctx, false);
1693   expect_is_refresh_required(mock_remote_image_ctx, false);
1694   MockSnapshotCopyRequest mock_snapshot_copy_request;
1695   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
1696                        0);
1697   MockGetImageStateRequest mock_get_image_state_request;
1698   expect_get_image_state(mock_get_image_state_request, 1, 0);
1699   MockCreateNonPrimaryRequest mock_create_non_primary_request;
1700   expect_create_non_primary_request(mock_create_non_primary_request,
1701                                     false, "remote mirror uuid", 1,
1702                                     {{1, CEPH_NOSNAP}}, 11, 0);
1703   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1704   MockImageCopyRequest mock_image_copy_request;
1705   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
1706                     {{1, CEPH_NOSNAP}}, -EINVAL);
1707
1708   // wake-up replayer
1709   update_watch_ctx->handle_notify();
1710
1711   // wait for sync to complete and expect replay complete
1712   ASSERT_EQ(0, wait_for_notification(1));
1713   ASSERT_FALSE(mock_replayer.is_replaying());
1714   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1715
1716   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1717                                         mock_local_image_ctx,
1718                                         mock_remote_image_ctx));
1719 }
1720
1721 TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) {
1722   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1723   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1724
1725   MockThreads mock_threads(m_threads);
1726   expect_work_queue_repeatedly(mock_threads);
1727
1728   MockReplayerListener mock_replayer_listener;
1729   expect_notification(mock_threads, mock_replayer_listener);
1730
1731   InSequence seq;
1732
1733   MockInstanceWatcher mock_instance_watcher;
1734   MockImageMeta mock_image_meta;
1735   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1736                                       mock_remote_image_ctx,
1737                                       mock_image_meta);
1738   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1739                              "local mirror uuid", &m_pool_meta_cache,
1740                              &mock_state_builder, &mock_replayer_listener};
1741   m_pool_meta_cache.set_remote_pool_meta(
1742     m_remote_io_ctx.get_id(),
1743     {"remote mirror uuid", "remote mirror peer uuid"});
1744
1745   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1746   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1747                                    mock_local_image_ctx,
1748                                    mock_remote_image_ctx,
1749                                    mock_replayer_listener,
1750                                    mock_image_meta,
1751                                    &update_watch_ctx));
1752
1753   // inject snapshot
1754   mock_remote_image_ctx.snap_info = {
1755     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1756        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
1757        0U, true, 0, {}},
1758      0, {}, 0, 0, {}}}};
1759
1760   // sync snap1
1761   expect_load_image_meta(mock_image_meta, false, 0);
1762   expect_is_refresh_required(mock_local_image_ctx, false);
1763   expect_is_refresh_required(mock_remote_image_ctx, false);
1764   MockSnapshotCopyRequest mock_snapshot_copy_request;
1765   expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
1766                        0);
1767   MockGetImageStateRequest mock_get_image_state_request;
1768   expect_get_image_state(mock_get_image_state_request, 1, 0);
1769   MockCreateNonPrimaryRequest mock_create_non_primary_request;
1770   expect_create_non_primary_request(mock_create_non_primary_request,
1771                                     false, "remote mirror uuid", 1,
1772                                     {{1, CEPH_NOSNAP}}, 11, 0);
1773   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1774   MockImageCopyRequest mock_image_copy_request;
1775   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
1776                     {{1, CEPH_NOSNAP}}, 0);
1777   MockApplyImageStateRequest mock_apply_state_request;
1778   expect_apply_image_state(mock_apply_state_request, 0);
1779   expect_mirror_image_snapshot_set_copy_progress(
1780     mock_local_image_ctx, 11, true, 0, -EINVAL);
1781
1782   // wake-up replayer
1783   update_watch_ctx->handle_notify();
1784
1785   // wait for sync to complete and expect replay complete
1786   ASSERT_EQ(0, wait_for_notification(1));
1787   ASSERT_FALSE(mock_replayer.is_replaying());
1788   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1789
1790   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1791                                         mock_local_image_ctx,
1792                                         mock_remote_image_ctx));
1793 }
1794
1795 TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) {
1796   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1797   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1798
1799   MockThreads mock_threads(m_threads);
1800   expect_work_queue_repeatedly(mock_threads);
1801
1802   MockReplayerListener mock_replayer_listener;
1803   expect_notification(mock_threads, mock_replayer_listener);
1804
1805   InSequence seq;
1806
1807   MockInstanceWatcher mock_instance_watcher;
1808   MockImageMeta mock_image_meta;
1809   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1810                                       mock_remote_image_ctx,
1811                                       mock_image_meta);
1812   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1813                              "local mirror uuid", &m_pool_meta_cache,
1814                              &mock_state_builder, &mock_replayer_listener};
1815   m_pool_meta_cache.set_remote_pool_meta(
1816     m_remote_io_ctx.get_id(),
1817     {"remote mirror uuid", "remote mirror peer uuid"});
1818
1819   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1820   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1821                                    mock_local_image_ctx,
1822                                    mock_remote_image_ctx,
1823                                    mock_replayer_listener,
1824                                    mock_image_meta,
1825                                    &update_watch_ctx));
1826
1827   // inject snapshot
1828   mock_remote_image_ctx.snap_info = {
1829     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1830        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
1831        0U, true, 0, {}},
1832      0, {}, 0, 0, {}}},
1833     {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{
1834        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1835        "", 0U, true, 0, {}},
1836      0, {}, 0, 0, {}}}};
1837   mock_local_image_ctx.snap_info = {
1838     {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1839        cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid",
1840        1, true, 0, {}},
1841      0, {}, 0, 0, {}}}};
1842
1843   // sync snap2
1844   expect_load_image_meta(mock_image_meta, false, 0);
1845   expect_is_refresh_required(mock_local_image_ctx, false);
1846   expect_is_refresh_required(mock_remote_image_ctx, false);
1847   MockSnapshotCopyRequest mock_snapshot_copy_request;
1848   expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, {{2, CEPH_NOSNAP}},
1849                        0);
1850   MockGetImageStateRequest mock_get_image_state_request;
1851   expect_get_image_state(mock_get_image_state_request, 2, 0);
1852   MockCreateNonPrimaryRequest mock_create_non_primary_request;
1853   expect_create_non_primary_request(mock_create_non_primary_request,
1854                                     false, "remote mirror uuid", 2,
1855                                     {{2, CEPH_NOSNAP}}, 12, 0);
1856   expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
1857   MockImageCopyRequest mock_image_copy_request;
1858   expect_image_copy(mock_image_copy_request, 1, 2, 11, {},
1859                     {{2, CEPH_NOSNAP}}, 0);
1860   MockApplyImageStateRequest mock_apply_state_request;
1861   expect_apply_image_state(mock_apply_state_request, 0);
1862   expect_mirror_image_snapshot_set_copy_progress(
1863     mock_local_image_ctx, 12, true, 0, 0);
1864   expect_notify_update(mock_local_image_ctx);
1865   MockUnlinkPeerRequest mock_unlink_peer_request;
1866   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
1867                      -EINVAL);
1868   expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
1869
1870   // wake-up replayer
1871   update_watch_ctx->handle_notify();
1872
1873   // wait for sync to complete and expect replay complete
1874   ASSERT_EQ(0, wait_for_notification(1));
1875   ASSERT_FALSE(mock_replayer.is_replaying());
1876   ASSERT_EQ(-EINVAL, mock_replayer.get_error_code());
1877
1878   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1879                                         mock_local_image_ctx,
1880                                         mock_remote_image_ctx));
1881 }
1882
1883 TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) {
1884   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
1885   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
1886
1887   MockThreads mock_threads(m_threads);
1888   expect_work_queue_repeatedly(mock_threads);
1889
1890   MockReplayerListener mock_replayer_listener;
1891   expect_notification(mock_threads, mock_replayer_listener);
1892
1893   InSequence seq;
1894
1895   MockInstanceWatcher mock_instance_watcher;
1896   MockImageMeta mock_image_meta;
1897   MockStateBuilder mock_state_builder(mock_local_image_ctx,
1898                                       mock_remote_image_ctx,
1899                                       mock_image_meta);
1900   MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
1901                              "local mirror uuid", &m_pool_meta_cache,
1902                              &mock_state_builder, &mock_replayer_listener};
1903   m_pool_meta_cache.set_remote_pool_meta(
1904     m_remote_io_ctx.get_id(),
1905     {"remote mirror uuid", "remote mirror peer uuid"});
1906
1907   librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
1908   ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
1909                                    mock_local_image_ctx,
1910                                    mock_remote_image_ctx,
1911                                    mock_replayer_listener,
1912                                    mock_image_meta,
1913                                    &update_watch_ctx));
1914
1915   // inject a primary demote to local image
1916   mock_remote_image_ctx.snap_info = {
1917     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1918        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
1919        "", 0U, true, 0, {}},
1920      0, {}, 0, 0, {}}}};
1921   mock_local_image_ctx.snap_info = {
1922     {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
1923        cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", 0U, true, 0,
1924        {}},
1925      0, {}, 0, 0, {}}}};
1926
1927   // detect split-brain
1928   expect_load_image_meta(mock_image_meta, false, 0);
1929   expect_is_refresh_required(mock_local_image_ctx, false);
1930   expect_is_refresh_required(mock_remote_image_ctx, false);
1931
1932   // wake-up replayer
1933   update_watch_ctx->handle_notify();
1934
1935   // wait for sync to complete and expect replay complete
1936   ASSERT_EQ(0, wait_for_notification(1));
1937   ASSERT_FALSE(mock_replayer.is_replaying());
1938   ASSERT_EQ(-EEXIST, mock_replayer.get_error_code());
1939   ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description());
1940
1941   ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
1942                                         mock_local_image_ctx,
1943                                         mock_remote_image_ctx));
1944 }
1945
1946 } // namespace snapshot
1947 } // namespace image_replayer
1948 } // namespace mirror
1949 } // namespace rbd