ceph/src/test/rbd_mirror/test_mock_ImageReplayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "cls/journal/cls_journal_types.h"
   5 #include "librbd/journal/Replay.h"
   6 #include "librbd/journal/Types.h"
   7 #include "tools/rbd_mirror/ImageDeleter.h"
   8 #include "tools/rbd_mirror/ImageReplayer.h"
   9 #include "tools/rbd_mirror/InstanceWatcher.h"
  10 #include "tools/rbd_mirror/Threads.h"
  11 #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
  12 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
  13 #include "tools/rbd_mirror/image_replayer/EventPreprocessor.h"
  14 #include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h"
  15 #include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h"
  16 #include "test/rbd_mirror/test_mock_fixture.h"
  17 #include "test/journal/mock/MockJournaler.h"
  18 #include "test/librbd/mock/MockImageCtx.h"
  19 #include "test/librbd/mock/MockJournal.h"
  20 #include "test/rbd_mirror/mock/MockContextWQ.h"
  21 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  22
  23 namespace librbd {
  24
  25 namespace {
  26
  27 struct MockTestJournal;
  28
  29 struct MockTestImageCtx : public MockImageCtx {
  30   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  31     : librbd::MockImageCtx(image_ctx) {
  32   }
  33   MockTestJournal *journal = nullptr;
  34 };
  35
  36 struct MockTestJournal : public MockJournal {
  37   MOCK_METHOD2(start_external_replay, void(journal::Replay<MockTestImageCtx> **,
  38                                            Context *on_start));
  39   MOCK_METHOD0(stop_external_replay, void());
  40 };
  41
  42 } // anonymous namespace
  43
  44 namespace journal {
  45
  46 template<>
  47 struct Replay<MockTestImageCtx> {
  48   MOCK_METHOD2(decode, int(bufferlist::iterator *, EventEntry *));
  49   MOCK_METHOD3(process, void(const EventEntry &, Context *, Context *));
  50   MOCK_METHOD1(flush, void(Context*));
  51   MOCK_METHOD2(shut_down, void(bool, Context*));
  52 };
  53
  54 template <>
  55 struct TypeTraits<MockTestImageCtx> {
  56   typedef ::journal::MockJournalerProxy Journaler;
  57   typedef ::journal::MockReplayEntryProxy ReplayEntry;
  58 };
  59
  60 struct MirrorPeerClientMeta;
  61
  62 } // namespace journal
  63 } // namespace librbd
  64
  65 namespace rbd {
  66 namespace mirror {
  67
  68 template <>
  69 struct Threads<librbd::MockTestImageCtx> {
  70   MockSafeTimer *timer;
  71   Mutex &timer_lock;
  72
  73   MockContextWQ *work_queue;
  74
  75   Threads(Threads<librbd::ImageCtx> *threads)
  76     : timer(new MockSafeTimer()),
  77       timer_lock(threads->timer_lock),
  78       work_queue(new MockContextWQ()) {
  79   }
  80   ~Threads() {
  81     delete timer;
  82     delete work_queue;
  83   }
  84 };
  85
  86 template <>
  87 struct ImageDeleter<librbd::MockTestImageCtx> {
  88   MOCK_METHOD4(schedule_image_delete, void(RadosRef, int64_t,
  89                                            const std::string&, bool));
  90   MOCK_METHOD4(wait_for_scheduled_deletion,
  91                void(int64_t, const std::string&, Context*, bool));
  92   MOCK_METHOD2(cancel_waiter, void(int64_t, const std::string&));
  93 };
  94
  95 template<>
  96 class InstanceWatcher<librbd::MockTestImageCtx> {
  97 };
  98
  99 namespace image_replayer {
 100
 101 using ::testing::_;
 102 using ::testing::AtLeast;
 103 using ::testing::DoAll;
 104 using ::testing::InSequence;
 105 using ::testing::Invoke;
 106 using ::testing::MatcherCast;
 107 using ::testing::Return;
 108 using ::testing::ReturnArg;
 109 using ::testing::SetArgPointee;
 110 using ::testing::WithArg;
 111
 112 template<>
 113 struct PrepareLocalImageRequest<librbd::MockTestImageCtx> {
 114   static PrepareLocalImageRequest* s_instance;
 115   std::string *local_image_id = nullptr;
 116   std::string *tag_owner = nullptr;
 117   Context *on_finish = nullptr;
 118
 119   static PrepareLocalImageRequest* create(librados::IoCtx &,
 120                                           const std::string &global_image_id,
 121                                           std::string *local_image_id,
 122                                           std::string *tag_owner,
 123                                           MockContextWQ *work_queue,
 124                                           Context *on_finish) {
 125     assert(s_instance != nullptr);
 126     s_instance->local_image_id = local_image_id;
 127     s_instance->tag_owner = tag_owner;
 128     s_instance->on_finish = on_finish;
 129     return s_instance;
 130   }
 131
 132   PrepareLocalImageRequest() {
 133     s_instance = this;
 134   }
 135
 136   MOCK_METHOD0(send, void());
 137 };
 138
 139 template<>
 140 struct PrepareRemoteImageRequest<librbd::MockTestImageCtx> {
 141   static PrepareRemoteImageRequest* s_instance;
 142   std::string *remote_mirror_uuid = nullptr;
 143   std::string *remote_image_id = nullptr;
 144   cls::journal::ClientState *client_state;
 145   ::journal::MockJournalerProxy **remote_journaler = nullptr;
 146   librbd::journal::MirrorPeerClientMeta *client_meta = nullptr;
 147   Context *on_finish = nullptr;
 148
 149   static PrepareRemoteImageRequest* create(Threads<librbd::MockTestImageCtx> *threads,
 150                                            librados::IoCtx &,
 151                                            const std::string &global_image_id,
 152                                            const std::string &local_mirror_uuid,
 153                                            const std::string &local_image_id,
 154                                            std::string *remote_mirror_uuid,
 155                                            std::string *remote_image_id,
 156                                            ::journal::MockJournalerProxy **remote_journaler,
 157                                            cls::journal::ClientState *client_state,
 158                                            librbd::journal::MirrorPeerClientMeta *client_meta,
 159                                            Context *on_finish) {
 160     assert(s_instance != nullptr);
 161     s_instance->remote_mirror_uuid = remote_mirror_uuid;
 162     s_instance->remote_image_id = remote_image_id;
 163     s_instance->remote_journaler = remote_journaler;
 164     s_instance->client_state = client_state;
 165     s_instance->client_meta = client_meta;
 166     s_instance->on_finish = on_finish;
 167     return s_instance;
 168   }
 169
 170   PrepareRemoteImageRequest() {
 171     s_instance = this;
 172   }
 173
 174   MOCK_METHOD0(send, void());
 175 };
 176
 177 template<>
 178 struct BootstrapRequest<librbd::MockTestImageCtx> {
 179   static BootstrapRequest* s_instance;
 180   librbd::MockTestImageCtx **image_ctx = nullptr;
 181   Context *on_finish = nullptr;
 182   bool *do_resync = nullptr;
 183
 184   static BootstrapRequest* create(
 185       librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx,
 186       rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
 187       librbd::MockTestImageCtx **local_image_ctx,
 188       const std::string &local_image_name, const std::string &remote_image_id,
 189       const std::string &global_image_id, MockContextWQ *work_queue,
 190       MockSafeTimer *timer, Mutex *timer_lock,
 191       const std::string &local_mirror_uuid,
 192       const std::string &remote_mirror_uuid,
 193       ::journal::MockJournalerProxy *journaler,
 194       cls::journal::ClientState *client_state,
 195       librbd::journal::MirrorPeerClientMeta *client_meta,
 196       Context *on_finish, bool *do_resync,
 197       rbd::mirror::ProgressContext *progress_ctx = nullptr) {
 198     assert(s_instance != nullptr);
 199     s_instance->image_ctx = local_image_ctx;
 200     s_instance->on_finish = on_finish;
 201     s_instance->do_resync = do_resync;
 202     return s_instance;
 203   }
 204
 205   BootstrapRequest() {
 206     assert(s_instance == nullptr);
 207     s_instance = this;
 208   }
 209
 210   ~BootstrapRequest() {
 211     assert(s_instance == this);
 212     s_instance = nullptr;
 213   }
 214
 215   void put() {
 216   }
 217
 218   void get() {
 219   }
 220
 221   inline bool is_syncing() const {
 222     return false;
 223   }
 224
 225   MOCK_METHOD0(send, void());
 226   MOCK_METHOD0(cancel, void());
 227 };
 228
 229 template<>
 230 struct CloseImageRequest<librbd::MockTestImageCtx> {
 231   static CloseImageRequest* s_instance;
 232   librbd::MockTestImageCtx **image_ctx = nullptr;
 233   Context *on_finish = nullptr;
 234
 235   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
 236                                    Context *on_finish) {
 237     assert(s_instance != nullptr);
 238     s_instance->image_ctx = image_ctx;
 239     s_instance->on_finish = on_finish;
 240     return s_instance;
 241   }
 242
 243   CloseImageRequest() {
 244     assert(s_instance == nullptr);
 245     s_instance = this;
 246   }
 247
 248   ~CloseImageRequest() {
 249     assert(s_instance == this);
 250     s_instance = nullptr;
 251   }
 252
 253   MOCK_METHOD0(send, void());
 254 };
 255
 256 template<>
 257 struct EventPreprocessor<librbd::MockTestImageCtx> {
 258   static EventPreprocessor *s_instance;
 259
 260   static EventPreprocessor *create(librbd::MockTestImageCtx &local_image_ctx,
 261                                    ::journal::MockJournalerProxy &remote_journaler,
 262                                    const std::string &local_mirror_uuid,
 263                                    librbd::journal::MirrorPeerClientMeta *client_meta,
 264                                    MockContextWQ *work_queue) {
 265     assert(s_instance != nullptr);
 266     return s_instance;
 267   }
 268
 269   static void destroy(EventPreprocessor* processor) {
 270   }
 271
 272   EventPreprocessor() {
 273     assert(s_instance == nullptr);
 274     s_instance = this;
 275   }
 276
 277   ~EventPreprocessor() {
 278     assert(s_instance == this);
 279     s_instance = nullptr;
 280   }
 281
 282   MOCK_METHOD1(is_required, bool(const librbd::journal::EventEntry &));
 283   MOCK_METHOD2(preprocess, void(librbd::journal::EventEntry *, Context *));
 284 };
 285
 286 template<>
 287 struct ReplayStatusFormatter<librbd::MockTestImageCtx> {
 288   static ReplayStatusFormatter* s_instance;
 289
 290   static ReplayStatusFormatter* create(::journal::MockJournalerProxy *journaler,
 291                                        const std::string &mirror_uuid) {
 292     assert(s_instance != nullptr);
 293     return s_instance;
 294   }
 295
 296   static void destroy(ReplayStatusFormatter* formatter) {
 297   }
 298
 299   ReplayStatusFormatter() {
 300     assert(s_instance == nullptr);
 301     s_instance = this;
 302   }
 303
 304   ~ReplayStatusFormatter() {
 305     assert(s_instance == this);
 306     s_instance = nullptr;
 307   }
 308
 309   MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish));
 310 };
 311
 312 BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 313 CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 314 EventPreprocessor<librbd::MockTestImageCtx>* EventPreprocessor<librbd::MockTestImageCtx>::s_instance = nullptr;
 315 PrepareLocalImageRequest<librbd::MockTestImageCtx>* PrepareLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 316 PrepareRemoteImageRequest<librbd::MockTestImageCtx>* PrepareRemoteImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 317 ReplayStatusFormatter<librbd::MockTestImageCtx>* ReplayStatusFormatter<librbd::MockTestImageCtx>::s_instance = nullptr;
 318
 319 } // namespace image_replayer
 320 } // namespace mirror
 321 } // namespace rbd
 322
 323 // template definitions
 324 #include "tools/rbd_mirror/ImageReplayer.cc"
 325
 326 namespace rbd {
 327 namespace mirror {
 328
 329 class TestMockImageReplayer : public TestMockFixture {
 330 public:
 331   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 332   typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter;
 333   typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
 334   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
 335   typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor;
 336   typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest;
 337   typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest;
 338   typedef ReplayStatusFormatter<librbd::MockTestImageCtx> MockReplayStatusFormatter;
 339   typedef librbd::journal::Replay<librbd::MockTestImageCtx> MockReplay;
 340   typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer;
 341   typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
 342
 343   void SetUp() override {
 344     TestMockFixture::SetUp();
 345
 346     librbd::RBD rbd;
 347     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size));
 348     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx));
 349   }
 350
 351   void TearDown() override {
 352     delete m_image_replayer;
 353
 354     TestMockFixture::TearDown();
 355   }
 356
 357   void create_local_image() {
 358     librbd::RBD rbd;
 359     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 360     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 361   }
 362
 363   void expect_work_queue_repeatedly(MockThreads &mock_threads) {
 364     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 365       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 366           m_threads->work_queue->queue(ctx, r);
 367         }));
 368   }
 369
 370   void expect_add_event_after_repeatedly(MockThreads &mock_threads) {
 371     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 372       .WillRepeatedly(
 373         DoAll(Invoke([this](double seconds, Context *ctx) {
 374                        m_threads->timer->add_event_after(seconds, ctx);
 375                      }),
 376           ReturnArg<1>()));
 377     EXPECT_CALL(*mock_threads.timer, cancel_event(_))
 378       .WillRepeatedly(
 379         Invoke([this](Context *ctx) {
 380           return m_threads->timer->cancel_event(ctx);
 381         }));
 382   }
 383
 384   void expect_wait_for_scheduled_deletion(MockImageDeleter& mock_image_deleter,
 385                                           const std::string& global_image_id,
 386                                           int r) {
 387     EXPECT_CALL(mock_image_deleter,
 388                 wait_for_scheduled_deletion(_, global_image_id, _, false))
 389       .WillOnce(WithArg<2>(Invoke([this, r](Context *ctx) {
 390                              m_threads->work_queue->queue(ctx, r);
 391                            })));
 392   }
 393
 394   void expect_cancel_waiter(MockImageDeleter& mock_image_deleter) {
 395     EXPECT_CALL(mock_image_deleter, cancel_waiter(m_local_io_ctx.get_id(),
 396                                                   "global image id"));
 397   }
 398
 399   void expect_schedule_image_delete(MockImageDeleter& mock_image_deleter,
 400                                     const std::string& global_image_id,
 401                                     bool ignore_orphan) {
 402     EXPECT_CALL(mock_image_deleter,
 403                 schedule_image_delete(_, _, global_image_id, ignore_orphan));
 404   }
 405
 406   bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) {
 407     bufferlist bl;
 408     ::encode(tag_data, bl);
 409     return bl;
 410   }
 411
 412   void expect_get_or_send_update(
 413     MockReplayStatusFormatter &mock_replay_status_formatter) {
 414     EXPECT_CALL(mock_replay_status_formatter, get_or_send_update(_, _))
 415       .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)),
 416                             Return(true)));
 417   }
 418
 419   void expect_send(MockPrepareLocalImageRequest &mock_request,
 420                    const std::string &local_image_id,
 421                    const std::string &tag_owner,
 422                    int r) {
 423     EXPECT_CALL(mock_request, send())
 424       .WillOnce(Invoke([&mock_request, local_image_id, tag_owner, r]() {
 425           if (r == 0) {
 426             *mock_request.local_image_id = local_image_id;
 427             *mock_request.tag_owner = tag_owner;
 428           }
 429           mock_request.on_finish->complete(r);
 430         }));
 431   }
 432
 433   void expect_send(MockPrepareRemoteImageRequest& mock_request,
 434                    const std::string& mirror_uuid, const std::string& image_id,
 435                    int r) {
 436     EXPECT_CALL(mock_request, send())
 437       .WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() {
 438                   if (r >= 0) {
 439                     *mock_request.remote_journaler = new ::journal::MockJournalerProxy();
 440                   }
 441
 442                   *mock_request.remote_mirror_uuid = mirror_uuid;
 443                   *mock_request.remote_image_id = image_id;
 444                   mock_request.on_finish->complete(r);
 445                 }));
 446   }
 447
 448   void expect_send(MockBootstrapRequest &mock_bootstrap_request,
 449                    librbd::MockTestImageCtx &mock_local_image_ctx,
 450                    bool do_resync, int r) {
 451     EXPECT_CALL(mock_bootstrap_request, send())
 452       .WillOnce(Invoke([&mock_bootstrap_request, &mock_local_image_ctx,
 453                         do_resync, r]() {
 454             if (r == 0) {
 455               *mock_bootstrap_request.image_ctx = &mock_local_image_ctx;
 456               *mock_bootstrap_request.do_resync = do_resync;
 457             }
 458             mock_bootstrap_request.on_finish->complete(r);
 459           }));
 460   }
 461
 462   void expect_start_external_replay(librbd::MockTestJournal &mock_journal,
 463                                     MockReplay *mock_replay, int r) {
 464     EXPECT_CALL(mock_journal, start_external_replay(_, _))
 465       .WillOnce(DoAll(SetArgPointee<0>(mock_replay),
 466                       WithArg<1>(CompleteContext(r))));
 467   }
 468
 469   void expect_init(::journal::MockJournaler &mock_journaler, int r) {
 470     EXPECT_CALL(mock_journaler, init(_))
 471       .WillOnce(CompleteContext(r));
 472   }
 473
 474   void expect_get_cached_client(::journal::MockJournaler &mock_journaler,
 475                                 int r) {
 476     librbd::journal::ImageClientMeta image_client_meta;
 477     image_client_meta.tag_class = 0;
 478
 479     librbd::journal::ClientData client_data;
 480     client_data.client_meta = image_client_meta;
 481
 482     cls::journal::Client client;
 483     ::encode(client_data, client.data);
 484
 485     EXPECT_CALL(mock_journaler, get_cached_client("local_mirror_uuid", _))
 486       .WillOnce(DoAll(SetArgPointee<1>(client),
 487                       Return(r)));
 488   }
 489
 490   void expect_stop_replay(::journal::MockJournaler &mock_journaler, int r) {
 491     EXPECT_CALL(mock_journaler, stop_replay(_))
 492       .WillOnce(CompleteContext(r));
 493   }
 494
 495   void expect_flush(MockReplay &mock_replay, int r) {
 496     EXPECT_CALL(mock_replay, flush(_)).WillOnce(CompleteContext(r));
 497   }
 498
 499   void expect_shut_down(MockReplay &mock_replay, bool cancel_ops, int r) {
 500     EXPECT_CALL(mock_replay, shut_down(cancel_ops, _))
 501       .WillOnce(WithArg<1>(CompleteContext(r)));
 502   }
 503
 504   void expect_shut_down(journal::MockJournaler &mock_journaler, int r) {
 505     EXPECT_CALL(mock_journaler, shut_down(_))
 506       .WillOnce(CompleteContext(r));
 507   }
 508
 509   void expect_send(MockCloseImageRequest &mock_close_image_request, int r) {
 510     EXPECT_CALL(mock_close_image_request, send())
 511       .WillOnce(Invoke([&mock_close_image_request, r]() {
 512             *mock_close_image_request.image_ctx = nullptr;
 513             mock_close_image_request.on_finish->complete(r);
 514           }));
 515   }
 516
 517   void expect_get_commit_tid_in_debug(
 518     ::journal::MockReplayEntry &mock_replay_entry) {
 519     // It is used in debug messages and depends on debug level
 520     EXPECT_CALL(mock_replay_entry, get_commit_tid())
 521     .Times(AtLeast(0))
 522       .WillRepeatedly(Return(0));
 523   }
 524
 525   void expect_committed(::journal::MockJournaler &mock_journaler, int times) {
 526     EXPECT_CALL(mock_journaler, committed(
 527                   MatcherCast<const ::journal::MockReplayEntryProxy&>(_)))
 528       .Times(times);
 529   }
 530
 531   void expect_try_pop_front(::journal::MockJournaler &mock_journaler,
 532                             uint64_t replay_tag_tid, bool entries_available) {
 533     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 534       .WillOnce(DoAll(SetArgPointee<0>(::journal::MockReplayEntryProxy()),
 535                       SetArgPointee<1>(replay_tag_tid),
 536                       Return(entries_available)));
 537   }
 538
 539   void expect_try_pop_front_return_no_entries(
 540     ::journal::MockJournaler &mock_journaler, Context *on_finish) {
 541     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 542       .WillOnce(DoAll(Invoke([on_finish](::journal::MockReplayEntryProxy *e,
 543                                          uint64_t *t) {
 544                                on_finish->complete(0);
 545                              }),
 546                       Return(false)));
 547   }
 548
 549   void expect_get_tag(::journal::MockJournaler &mock_journaler,
 550                       const cls::journal::Tag &tag, int r) {
 551     EXPECT_CALL(mock_journaler, get_tag(_, _, _))
 552       .WillOnce(DoAll(SetArgPointee<1>(tag),
 553                       WithArg<2>(CompleteContext(r))));
 554   }
 555
 556   void expect_allocate_tag(librbd::MockTestJournal &mock_journal, int r) {
 557     EXPECT_CALL(mock_journal, allocate_tag(_, _, _))
 558       .WillOnce(WithArg<2>(CompleteContext(r)));
 559   }
 560
 561   void expect_preprocess(MockEventPreprocessor &mock_event_preprocessor,
 562                          bool required, int r) {
 563     EXPECT_CALL(mock_event_preprocessor, is_required(_))
 564       .WillOnce(Return(required));
 565     if (required) {
 566       EXPECT_CALL(mock_event_preprocessor, preprocess(_, _))
 567         .WillOnce(WithArg<1>(CompleteContext(r)));
 568     }
 569   }
 570
 571   void expect_process(MockReplay &mock_replay,
 572                       int on_ready_r, int on_commit_r) {
 573     EXPECT_CALL(mock_replay, process(_, _, _))
 574       .WillOnce(DoAll(WithArg<1>(CompleteContext(on_ready_r)),
 575                       WithArg<2>(CompleteContext(on_commit_r))));
 576   }
 577
 578   void create_image_replayer(MockThreads &mock_threads,
 579                              MockImageDeleter &mock_image_deleter) {
 580     m_image_replayer = new MockImageReplayer(
 581       &mock_threads, &mock_image_deleter, &m_instance_watcher,
 582       rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
 583       "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id");
 584     m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx);
 585   }
 586
 587   librbd::ImageCtx *m_remote_image_ctx;
 588   librbd::ImageCtx *m_local_image_ctx = nullptr;
 589   MockInstanceWatcher m_instance_watcher;
 590   MockImageReplayer *m_image_replayer = nullptr;
 591 };
 592
 593 TEST_F(TestMockImageReplayer, StartStop) {
 594   // START
 595
 596   create_local_image();
 597   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 598
 599   librbd::MockTestJournal mock_local_journal;
 600   mock_local_image_ctx.journal = &mock_local_journal;
 601
 602   journal::MockJournaler mock_remote_journaler;
 603   MockThreads mock_threads(m_threads);
 604   expect_work_queue_repeatedly(mock_threads);
 605   expect_add_event_after_repeatedly(mock_threads);
 606
 607   MockImageDeleter mock_image_deleter;
 608   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 609   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 610   MockBootstrapRequest mock_bootstrap_request;
 611   MockReplay mock_local_replay;
 612   MockEventPreprocessor mock_event_preprocessor;
 613   MockReplayStatusFormatter mock_replay_status_formatter;
 614
 615   expect_get_or_send_update(mock_replay_status_formatter);
 616
 617   InSequence seq;
 618   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 619   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 620               "remote mirror uuid", 0);
 621   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 622               m_remote_image_ctx->id, 0);
 623   EXPECT_CALL(mock_remote_journaler, construct());
 624   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 625
 626   EXPECT_CALL(mock_local_journal, add_listener(_));
 627
 628   expect_init(mock_remote_journaler, 0);
 629
 630   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 631   expect_get_cached_client(mock_remote_journaler, 0);
 632
 633   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 634
 635   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 636
 637   create_image_replayer(mock_threads, mock_image_deleter);
 638
 639   C_SaferCond start_ctx;
 640   m_image_replayer->start(&start_ctx);
 641   ASSERT_EQ(0, start_ctx.wait());
 642   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 643             m_image_replayer->get_health_state());
 644
 645   // STOP
 646
 647   MockCloseImageRequest mock_close_local_image_request;
 648
 649   expect_cancel_waiter(mock_image_deleter);
 650   expect_shut_down(mock_local_replay, true, 0);
 651   EXPECT_CALL(mock_local_journal, remove_listener(_));
 652   EXPECT_CALL(mock_local_journal, stop_external_replay());
 653   expect_send(mock_close_local_image_request, 0);
 654
 655   expect_stop_replay(mock_remote_journaler, 0);
 656   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 657   expect_shut_down(mock_remote_journaler, 0);
 658
 659   C_SaferCond stop_ctx;
 660   m_image_replayer->stop(&stop_ctx);
 661   ASSERT_EQ(0, stop_ctx.wait());
 662   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 663             m_image_replayer->get_health_state());
 664 }
 665
 666 TEST_F(TestMockImageReplayer, LocalImagePrimary) {
 667   create_local_image();
 668   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 669
 670   journal::MockJournaler mock_remote_journaler;
 671   MockThreads mock_threads(m_threads);
 672   expect_work_queue_repeatedly(mock_threads);
 673   expect_add_event_after_repeatedly(mock_threads);
 674
 675   MockImageDeleter mock_image_deleter;
 676   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 677   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 678   MockReplayStatusFormatter mock_replay_status_formatter;
 679
 680   expect_get_or_send_update(mock_replay_status_formatter);
 681
 682   InSequence seq;
 683   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 684   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 685               "", 0);
 686   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 687               "remote image id", 0);
 688   EXPECT_CALL(mock_remote_journaler, construct());
 689   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 690   expect_shut_down(mock_remote_journaler, 0);
 691
 692   create_image_replayer(mock_threads, mock_image_deleter);
 693
 694   C_SaferCond start_ctx;
 695   m_image_replayer->start(&start_ctx);
 696   ASSERT_EQ(0, start_ctx.wait());
 697 }
 698
 699 TEST_F(TestMockImageReplayer, LocalImageDNE) {
 700   create_local_image();
 701   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 702
 703   journal::MockJournaler mock_remote_journaler;
 704   MockThreads mock_threads(m_threads);
 705   expect_work_queue_repeatedly(mock_threads);
 706   expect_add_event_after_repeatedly(mock_threads);
 707
 708   MockImageDeleter mock_image_deleter;
 709   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 710   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 711   MockBootstrapRequest mock_bootstrap_request;
 712   MockReplayStatusFormatter mock_replay_status_formatter;
 713
 714   expect_get_or_send_update(mock_replay_status_formatter);
 715
 716   InSequence seq;
 717   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 718   expect_send(mock_prepare_local_image_request, "", "", -ENOENT);
 719   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 720               m_remote_image_ctx->id, 0);
 721   EXPECT_CALL(mock_remote_journaler, construct());
 722   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EREMOTEIO);
 723
 724   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 725   expect_shut_down(mock_remote_journaler, 0);
 726
 727   create_image_replayer(mock_threads, mock_image_deleter);
 728
 729   C_SaferCond start_ctx;
 730   m_image_replayer->start(&start_ctx);
 731   ASSERT_EQ(-EREMOTEIO, start_ctx.wait());
 732 }
 733
 734 TEST_F(TestMockImageReplayer, PrepareLocalImageError) {
 735   create_local_image();
 736   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 737
 738   MockThreads mock_threads(m_threads);
 739   expect_work_queue_repeatedly(mock_threads);
 740   expect_add_event_after_repeatedly(mock_threads);
 741
 742   MockImageDeleter mock_image_deleter;
 743   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 744   MockReplayStatusFormatter mock_replay_status_formatter;
 745
 746   expect_get_or_send_update(mock_replay_status_formatter);
 747
 748   InSequence seq;
 749   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 750   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 751               "remote mirror uuid", -EINVAL);
 752
 753   create_image_replayer(mock_threads, mock_image_deleter);
 754
 755   C_SaferCond start_ctx;
 756   m_image_replayer->start(&start_ctx);
 757   ASSERT_EQ(-EINVAL, start_ctx.wait());
 758 }
 759
 760 TEST_F(TestMockImageReplayer, GetRemoteImageIdDNE) {
 761   create_local_image();
 762   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 763
 764   MockThreads mock_threads(m_threads);
 765   expect_work_queue_repeatedly(mock_threads);
 766   expect_add_event_after_repeatedly(mock_threads);
 767
 768   MockImageDeleter mock_image_deleter;
 769   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 770   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 771   MockReplayStatusFormatter mock_replay_status_formatter;
 772
 773   expect_get_or_send_update(mock_replay_status_formatter);
 774
 775   InSequence seq;
 776   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 777   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 778               "remote mirror uuid", 0);
 779   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 780               "", -ENOENT);
 781   expect_schedule_image_delete(mock_image_deleter, "global image id", false);
 782
 783   create_image_replayer(mock_threads, mock_image_deleter);
 784
 785   C_SaferCond start_ctx;
 786   m_image_replayer->start(&start_ctx);
 787   ASSERT_EQ(0, start_ctx.wait());
 788 }
 789
 790 TEST_F(TestMockImageReplayer, GetRemoteImageIdNonLinkedDNE) {
 791   create_local_image();
 792   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 793
 794   MockThreads mock_threads(m_threads);
 795   expect_work_queue_repeatedly(mock_threads);
 796   expect_add_event_after_repeatedly(mock_threads);
 797
 798   MockImageDeleter mock_image_deleter;
 799   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 800   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 801   MockReplayStatusFormatter mock_replay_status_formatter;
 802
 803   expect_get_or_send_update(mock_replay_status_formatter);
 804
 805   InSequence seq;
 806   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 807   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 808               "some other mirror uuid", 0);
 809   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 810               "", -ENOENT);
 811
 812   create_image_replayer(mock_threads, mock_image_deleter);
 813
 814   C_SaferCond start_ctx;
 815   m_image_replayer->start(&start_ctx);
 816   ASSERT_EQ(-ENOENT, start_ctx.wait());
 817 }
 818
 819 TEST_F(TestMockImageReplayer, GetRemoteImageIdError) {
 820   create_local_image();
 821   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 822
 823   MockThreads mock_threads(m_threads);
 824   expect_work_queue_repeatedly(mock_threads);
 825   expect_add_event_after_repeatedly(mock_threads);
 826
 827   MockImageDeleter mock_image_deleter;
 828   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 829   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 830   MockReplayStatusFormatter mock_replay_status_formatter;
 831
 832   expect_get_or_send_update(mock_replay_status_formatter);
 833
 834   InSequence seq;
 835   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 836   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 837               "remote mirror uuid", 0);
 838   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 839               m_remote_image_ctx->id, -EINVAL);
 840
 841   create_image_replayer(mock_threads, mock_image_deleter);
 842
 843   C_SaferCond start_ctx;
 844   m_image_replayer->start(&start_ctx);
 845   ASSERT_EQ(-EINVAL, start_ctx.wait());
 846 }
 847
 848 TEST_F(TestMockImageReplayer, BootstrapError) {
 849
 850   create_local_image();
 851   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 852
 853   journal::MockJournaler mock_remote_journaler;
 854   MockThreads mock_threads(m_threads);
 855   expect_work_queue_repeatedly(mock_threads);
 856   expect_add_event_after_repeatedly(mock_threads);
 857
 858   MockImageDeleter mock_image_deleter;
 859   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 860   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 861   MockBootstrapRequest mock_bootstrap_request;
 862   MockReplayStatusFormatter mock_replay_status_formatter;
 863
 864   expect_get_or_send_update(mock_replay_status_formatter);
 865
 866   InSequence seq;
 867   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 868   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 869               "remote mirror uuid", 0);
 870   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 871               m_remote_image_ctx->id, 0);
 872   EXPECT_CALL(mock_remote_journaler, construct());
 873   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EINVAL);
 874
 875   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 876   expect_shut_down(mock_remote_journaler, 0);
 877
 878   create_image_replayer(mock_threads, mock_image_deleter);
 879
 880   C_SaferCond start_ctx;
 881   m_image_replayer->start(&start_ctx);
 882   ASSERT_EQ(-EINVAL, start_ctx.wait());
 883 }
 884
 885 TEST_F(TestMockImageReplayer, StartExternalReplayError) {
 886   // START
 887
 888   create_local_image();
 889   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 890
 891   librbd::MockTestJournal mock_local_journal;
 892   mock_local_image_ctx.journal = &mock_local_journal;
 893
 894   journal::MockJournaler mock_remote_journaler;
 895   MockThreads mock_threads(m_threads);
 896   expect_work_queue_repeatedly(mock_threads);
 897   expect_add_event_after_repeatedly(mock_threads);
 898
 899   MockImageDeleter mock_image_deleter;
 900   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 901   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 902   MockBootstrapRequest mock_bootstrap_request;
 903   MockReplay mock_local_replay;
 904   MockEventPreprocessor mock_event_preprocessor;
 905   MockReplayStatusFormatter mock_replay_status_formatter;
 906
 907   expect_get_or_send_update(mock_replay_status_formatter);
 908
 909   InSequence seq;
 910   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 911   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 912               "remote mirror uuid", 0);
 913   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 914               m_remote_image_ctx->id, 0);
 915   EXPECT_CALL(mock_remote_journaler, construct());
 916   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 917
 918   EXPECT_CALL(mock_local_journal, add_listener(_));
 919
 920   expect_init(mock_remote_journaler, 0);
 921
 922   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 923   expect_get_cached_client(mock_remote_journaler, 0);
 924
 925   expect_start_external_replay(mock_local_journal, nullptr, -EINVAL);
 926
 927   MockCloseImageRequest mock_close_local_image_request;
 928   EXPECT_CALL(mock_local_journal, remove_listener(_));
 929   expect_send(mock_close_local_image_request, 0);
 930
 931   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 932   expect_shut_down(mock_remote_journaler, 0);
 933
 934   create_image_replayer(mock_threads, mock_image_deleter);
 935
 936   C_SaferCond start_ctx;
 937   m_image_replayer->start(&start_ctx);
 938   ASSERT_EQ(-EINVAL, start_ctx.wait());
 939   ASSERT_EQ(image_replayer::HEALTH_STATE_ERROR,
 940             m_image_replayer->get_health_state());
 941 }
 942
 943 TEST_F(TestMockImageReplayer, StopError) {
 944   // START
 945
 946   create_local_image();
 947   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 948
 949   librbd::MockTestJournal mock_local_journal;
 950   mock_local_image_ctx.journal = &mock_local_journal;
 951
 952   journal::MockJournaler mock_remote_journaler;
 953   MockThreads mock_threads(m_threads);
 954   expect_work_queue_repeatedly(mock_threads);
 955   expect_add_event_after_repeatedly(mock_threads);
 956
 957   MockImageDeleter mock_image_deleter;
 958   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 959   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 960   MockBootstrapRequest mock_bootstrap_request;
 961   MockReplay mock_local_replay;
 962   MockEventPreprocessor mock_event_preprocessor;
 963   MockReplayStatusFormatter mock_replay_status_formatter;
 964
 965   expect_get_or_send_update(mock_replay_status_formatter);
 966
 967   InSequence seq;
 968   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 969   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 970               "remote mirror uuid", 0);
 971   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 972               m_remote_image_ctx->id, 0);
 973   EXPECT_CALL(mock_remote_journaler, construct());
 974   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 975
 976   EXPECT_CALL(mock_local_journal, add_listener(_));
 977
 978   expect_init(mock_remote_journaler, 0);
 979
 980   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 981   expect_get_cached_client(mock_remote_journaler, 0);
 982
 983   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 984
 985   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 986
 987   create_image_replayer(mock_threads, mock_image_deleter);
 988
 989   C_SaferCond start_ctx;
 990   m_image_replayer->start(&start_ctx);
 991   ASSERT_EQ(0, start_ctx.wait());
 992
 993   // STOP (errors are ignored)
 994
 995   MockCloseImageRequest mock_close_local_image_request;
 996
 997   expect_cancel_waiter(mock_image_deleter);
 998   expect_shut_down(mock_local_replay, true, -EINVAL);
 999   EXPECT_CALL(mock_local_journal, remove_listener(_));
1000   EXPECT_CALL(mock_local_journal, stop_external_replay());
1001   expect_send(mock_close_local_image_request, -EINVAL);
1002
1003   expect_stop_replay(mock_remote_journaler, -EINVAL);
1004   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1005   expect_shut_down(mock_remote_journaler, -EINVAL);
1006
1007   C_SaferCond stop_ctx;
1008   m_image_replayer->stop(&stop_ctx);
1009   ASSERT_EQ(0, stop_ctx.wait());
1010 }
1011
1012 TEST_F(TestMockImageReplayer, Replay) {
1013   // START
1014
1015   create_local_image();
1016   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1017
1018   librbd::MockTestJournal mock_local_journal;
1019   mock_local_image_ctx.journal = &mock_local_journal;
1020
1021   journal::MockJournaler mock_remote_journaler;
1022   MockThreads mock_threads(m_threads);
1023   expect_work_queue_repeatedly(mock_threads);
1024   expect_add_event_after_repeatedly(mock_threads);
1025
1026   MockImageDeleter mock_image_deleter;
1027   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1028   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1029   MockBootstrapRequest mock_bootstrap_request;
1030   MockReplay mock_local_replay;
1031   MockEventPreprocessor mock_event_preprocessor;
1032   MockReplayStatusFormatter mock_replay_status_formatter;
1033   ::journal::MockReplayEntry mock_replay_entry;
1034
1035   expect_get_or_send_update(mock_replay_status_formatter);
1036   expect_get_commit_tid_in_debug(mock_replay_entry);
1037   expect_committed(mock_remote_journaler, 2);
1038
1039   InSequence seq;
1040   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1041   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1042               "remote mirror uuid", 0);
1043   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1044               m_remote_image_ctx->id, 0);
1045   EXPECT_CALL(mock_remote_journaler, construct());
1046   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1047
1048   EXPECT_CALL(mock_local_journal, add_listener(_));
1049
1050   expect_init(mock_remote_journaler, 0);
1051
1052   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1053   expect_get_cached_client(mock_remote_journaler, 0);
1054
1055   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1056
1057   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1058
1059   create_image_replayer(mock_threads, mock_image_deleter);
1060
1061   C_SaferCond start_ctx;
1062   m_image_replayer->start(&start_ctx);
1063   ASSERT_EQ(0, start_ctx.wait());
1064
1065   // REPLAY
1066
1067   cls::journal::Tag tag =
1068     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1069                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1070                             true, 0, 0})};
1071
1072   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1073
1074   // replay_flush
1075   expect_shut_down(mock_local_replay, false, 0);
1076   EXPECT_CALL(mock_local_journal, stop_external_replay());
1077   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1078   expect_get_tag(mock_remote_journaler, tag, 0);
1079   expect_allocate_tag(mock_local_journal, 0);
1080
1081   // process
1082   EXPECT_CALL(mock_replay_entry, get_data());
1083   EXPECT_CALL(mock_local_replay, decode(_, _))
1084     .WillOnce(Return(0));
1085   expect_preprocess(mock_event_preprocessor, false, 0);
1086   expect_process(mock_local_replay, 0, 0);
1087
1088   // the next event with preprocess
1089   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1090   EXPECT_CALL(mock_replay_entry, get_data());
1091   EXPECT_CALL(mock_local_replay, decode(_, _))
1092     .WillOnce(Return(0));
1093   expect_preprocess(mock_event_preprocessor, true, 0);
1094   expect_process(mock_local_replay, 0, 0);
1095
1096   // attempt to process the next event
1097   C_SaferCond replay_ctx;
1098   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1099
1100   // fire
1101   m_image_replayer->handle_replay_ready();
1102   ASSERT_EQ(0, replay_ctx.wait());
1103
1104   // STOP
1105
1106   MockCloseImageRequest mock_close_local_image_request;
1107   expect_cancel_waiter(mock_image_deleter);
1108   expect_shut_down(mock_local_replay, true, 0);
1109   EXPECT_CALL(mock_local_journal, remove_listener(_));
1110   EXPECT_CALL(mock_local_journal, stop_external_replay());
1111   expect_send(mock_close_local_image_request, 0);
1112
1113   expect_stop_replay(mock_remote_journaler, 0);
1114   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1115   expect_shut_down(mock_remote_journaler, 0);
1116
1117   C_SaferCond stop_ctx;
1118   m_image_replayer->stop(&stop_ctx);
1119   ASSERT_EQ(0, stop_ctx.wait());
1120 }
1121
1122 TEST_F(TestMockImageReplayer, DecodeError) {
1123   // START
1124
1125   create_local_image();
1126   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1127
1128   librbd::MockTestJournal mock_local_journal;
1129   mock_local_image_ctx.journal = &mock_local_journal;
1130
1131   journal::MockJournaler mock_remote_journaler;
1132   MockThreads mock_threads(m_threads);
1133   expect_work_queue_repeatedly(mock_threads);
1134   expect_add_event_after_repeatedly(mock_threads);
1135
1136   MockImageDeleter mock_image_deleter;
1137   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1138   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1139   MockBootstrapRequest mock_bootstrap_request;
1140   MockReplay mock_local_replay;
1141   MockEventPreprocessor mock_event_preprocessor;
1142   MockReplayStatusFormatter mock_replay_status_formatter;
1143   ::journal::MockReplayEntry mock_replay_entry;
1144
1145   expect_get_or_send_update(mock_replay_status_formatter);
1146   expect_get_commit_tid_in_debug(mock_replay_entry);
1147
1148   InSequence seq;
1149   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1150   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1151               "remote mirror uuid", 0);
1152   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1153               m_remote_image_ctx->id, 0);
1154   EXPECT_CALL(mock_remote_journaler, construct());
1155   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1156
1157   EXPECT_CALL(mock_local_journal, add_listener(_));
1158
1159   expect_init(mock_remote_journaler, 0);
1160
1161   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1162   expect_get_cached_client(mock_remote_journaler, 0);
1163
1164   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1165
1166   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1167
1168   create_image_replayer(mock_threads, mock_image_deleter);
1169
1170   C_SaferCond start_ctx;
1171   m_image_replayer->start(&start_ctx);
1172   ASSERT_EQ(0, start_ctx.wait());
1173
1174   // REPLAY
1175
1176   cls::journal::Tag tag =
1177     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1178                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1179                             true, 0, 0})};
1180
1181   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1182
1183   // replay_flush
1184   expect_shut_down(mock_local_replay, false, 0);
1185   EXPECT_CALL(mock_local_journal, stop_external_replay());
1186   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1187   expect_get_tag(mock_remote_journaler, tag, 0);
1188   expect_allocate_tag(mock_local_journal, 0);
1189
1190   // process
1191   EXPECT_CALL(mock_replay_entry, get_data());
1192   EXPECT_CALL(mock_local_replay, decode(_, _))
1193     .WillOnce(Return(-EINVAL));
1194
1195   // stop on error
1196   expect_shut_down(mock_local_replay, true, 0);
1197   EXPECT_CALL(mock_local_journal, remove_listener(_));
1198   EXPECT_CALL(mock_local_journal, stop_external_replay());
1199
1200   MockCloseImageRequest mock_close_local_image_request;
1201   C_SaferCond close_ctx;
1202   EXPECT_CALL(mock_close_local_image_request, send())
1203     .WillOnce(Invoke([&mock_close_local_image_request, &close_ctx]() {
1204           *mock_close_local_image_request.image_ctx = nullptr;
1205           mock_close_local_image_request.on_finish->complete(0);
1206           close_ctx.complete(0);
1207         }));
1208
1209   expect_stop_replay(mock_remote_journaler, 0);
1210   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1211   expect_shut_down(mock_remote_journaler, 0);
1212
1213   // fire
1214   m_image_replayer->handle_replay_ready();
1215   ASSERT_EQ(0, close_ctx.wait());
1216
1217   while (!m_image_replayer->is_stopped()) {
1218     usleep(1000);
1219   }
1220 }
1221
1222 TEST_F(TestMockImageReplayer, DelayedReplay) {
1223
1224   // START
1225
1226   create_local_image();
1227   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1228
1229   librbd::MockTestJournal mock_local_journal;
1230   mock_local_image_ctx.journal = &mock_local_journal;
1231
1232   journal::MockJournaler mock_remote_journaler;
1233   MockThreads mock_threads(m_threads);
1234   expect_work_queue_repeatedly(mock_threads);
1235   expect_add_event_after_repeatedly(mock_threads);
1236
1237   MockImageDeleter mock_image_deleter;
1238   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1239   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1240   MockBootstrapRequest mock_bootstrap_request;
1241   MockReplay mock_local_replay;
1242   MockEventPreprocessor mock_event_preprocessor;
1243   MockReplayStatusFormatter mock_replay_status_formatter;
1244   ::journal::MockReplayEntry mock_replay_entry;
1245
1246   expect_get_or_send_update(mock_replay_status_formatter);
1247   expect_get_commit_tid_in_debug(mock_replay_entry);
1248   expect_committed(mock_remote_journaler, 1);
1249
1250   InSequence seq;
1251   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1252   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1253               "remote mirror uuid", 0);
1254   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1255               m_remote_image_ctx->id, 0);
1256   EXPECT_CALL(mock_remote_journaler, construct());
1257   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1258
1259   EXPECT_CALL(mock_local_journal, add_listener(_));
1260
1261   expect_init(mock_remote_journaler, 0);
1262
1263   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1264   expect_get_cached_client(mock_remote_journaler, 0);
1265
1266   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1267
1268   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1269
1270   create_image_replayer(mock_threads, mock_image_deleter);
1271
1272   C_SaferCond start_ctx;
1273   m_image_replayer->start(&start_ctx);
1274   ASSERT_EQ(0, start_ctx.wait());
1275
1276   // REPLAY
1277
1278   cls::journal::Tag tag =
1279     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1280                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1281                             true, 0, 0})};
1282
1283   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1284
1285   // replay_flush
1286   expect_shut_down(mock_local_replay, false, 0);
1287   EXPECT_CALL(mock_local_journal, stop_external_replay());
1288   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1289   expect_get_tag(mock_remote_journaler, tag, 0);
1290   expect_allocate_tag(mock_local_journal, 0);
1291
1292   // process with delay
1293   EXPECT_CALL(mock_replay_entry, get_data());
1294   librbd::journal::EventEntry event_entry(
1295     librbd::journal::AioDiscardEvent(123, 345, false), ceph_clock_now());
1296   EXPECT_CALL(mock_local_replay, decode(_, _))
1297     .WillOnce(DoAll(SetArgPointee<1>(event_entry),
1298                     Return(0)));
1299   expect_preprocess(mock_event_preprocessor, false, 0);
1300   expect_process(mock_local_replay, 0, 0);
1301
1302   // attempt to process the next event
1303   C_SaferCond replay_ctx;
1304   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1305
1306   // fire
1307   mock_local_image_ctx.mirroring_replay_delay = 2;
1308   m_image_replayer->handle_replay_ready();
1309   ASSERT_EQ(0, replay_ctx.wait());
1310
1311   // add a pending (delayed) entry before stop
1312   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1313   EXPECT_CALL(mock_replay_entry, get_data());
1314   C_SaferCond decode_ctx;
1315   EXPECT_CALL(mock_local_replay, decode(_, _))
1316     .WillOnce(DoAll(Invoke([&decode_ctx](bufferlist::iterator* it,
1317                                          librbd::journal::EventEntry *e) {
1318                              decode_ctx.complete(0);
1319                            }),
1320                     Return(0)));
1321
1322   mock_local_image_ctx.mirroring_replay_delay = 10;
1323   m_image_replayer->handle_replay_ready();
1324   ASSERT_EQ(0, decode_ctx.wait());
1325
1326   // STOP
1327
1328   MockCloseImageRequest mock_close_local_image_request;
1329
1330   expect_cancel_waiter(mock_image_deleter);
1331   expect_shut_down(mock_local_replay, true, 0);
1332   EXPECT_CALL(mock_local_journal, remove_listener(_));
1333   EXPECT_CALL(mock_local_journal, stop_external_replay());
1334   expect_send(mock_close_local_image_request, 0);
1335
1336   expect_stop_replay(mock_remote_journaler, 0);
1337   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1338   expect_shut_down(mock_remote_journaler, 0);
1339
1340   C_SaferCond stop_ctx;
1341   m_image_replayer->stop(&stop_ctx);
1342   ASSERT_EQ(0, stop_ctx.wait());
1343 }
1344
1345
1346 } // namespace mirror
1347 } // namespace rbd