ceph/src/test/rbd_mirror/test_mock_ImageReplayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "cls/journal/cls_journal_types.h"
   5 #include "librbd/journal/Replay.h"
   6 #include "librbd/journal/Types.h"
   7 #include "tools/rbd_mirror/ImageDeleter.h"
   8 #include "tools/rbd_mirror/ImageReplayer.h"
   9 #include "tools/rbd_mirror/InstanceWatcher.h"
  10 #include "tools/rbd_mirror/Threads.h"
  11 #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
  12 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
  13 #include "tools/rbd_mirror/image_replayer/EventPreprocessor.h"
  14 #include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h"
  15 #include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h"
  16 #include "test/rbd_mirror/test_mock_fixture.h"
  17 #include "test/journal/mock/MockJournaler.h"
  18 #include "test/librbd/mock/MockImageCtx.h"
  19 #include "test/librbd/mock/MockJournal.h"
  20 #include "test/rbd_mirror/mock/MockContextWQ.h"
  21 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  22
  23 namespace librbd {
  24
  25 namespace {
  26
  27 struct MockTestJournal;
  28
  29 struct MockTestImageCtx : public MockImageCtx {
  30   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  31     : librbd::MockImageCtx(image_ctx) {
  32   }
  33   MockTestJournal *journal = nullptr;
  34 };
  35
  36 struct MockTestJournal : public MockJournal {
  37   MOCK_METHOD2(start_external_replay, void(journal::Replay<MockTestImageCtx> **,
  38                                            Context *on_start));
  39   MOCK_METHOD0(stop_external_replay, void());
  40 };
  41
  42 } // anonymous namespace
  43
  44 namespace journal {
  45
  46 template<>
  47 struct Replay<MockTestImageCtx> {
  48   MOCK_METHOD2(decode, int(bufferlist::iterator *, EventEntry *));
  49   MOCK_METHOD3(process, void(const EventEntry &, Context *, Context *));
  50   MOCK_METHOD1(flush, void(Context*));
  51   MOCK_METHOD2(shut_down, void(bool, Context*));
  52 };
  53
  54 template <>
  55 struct TypeTraits<MockTestImageCtx> {
  56   typedef ::journal::MockJournalerProxy Journaler;
  57   typedef ::journal::MockReplayEntryProxy ReplayEntry;
  58 };
  59
  60 struct MirrorPeerClientMeta;
  61
  62 } // namespace journal
  63 } // namespace librbd
  64
  65 namespace rbd {
  66 namespace mirror {
  67
  68 template <>
  69 struct Threads<librbd::MockTestImageCtx> {
  70   MockSafeTimer *timer;
  71   Mutex &timer_lock;
  72
  73   MockContextWQ *work_queue;
  74
  75   Threads(Threads<librbd::ImageCtx> *threads)
  76     : timer(new MockSafeTimer()),
  77       timer_lock(threads->timer_lock),
  78       work_queue(new MockContextWQ()) {
  79   }
  80   ~Threads() {
  81     delete timer;
  82     delete work_queue;
  83   }
  84 };
  85
  86 template <>
  87 struct ImageDeleter<librbd::MockTestImageCtx> {
  88   MOCK_METHOD4(schedule_image_delete, void(RadosRef, int64_t,
  89                                            const std::string&, bool));
  90   MOCK_METHOD4(wait_for_scheduled_deletion,
  91                void(int64_t, const std::string&, Context*, bool));
  92   MOCK_METHOD2(cancel_waiter, void(int64_t, const std::string&));
  93 };
  94
  95 template<>
  96 class InstanceWatcher<librbd::MockTestImageCtx> {
  97 };
  98
  99 namespace image_replayer {
 100
 101 using ::testing::_;
 102 using ::testing::AtLeast;
 103 using ::testing::DoAll;
 104 using ::testing::InSequence;
 105 using ::testing::Invoke;
 106 using ::testing::MatcherCast;
 107 using ::testing::Return;
 108 using ::testing::SetArgPointee;
 109 using ::testing::WithArg;
 110
 111 template<>
 112 struct PrepareLocalImageRequest<librbd::MockTestImageCtx> {
 113   static PrepareLocalImageRequest* s_instance;
 114   std::string *local_image_id = nullptr;
 115   std::string *tag_owner = nullptr;
 116   Context *on_finish = nullptr;
 117
 118   static PrepareLocalImageRequest* create(librados::IoCtx &,
 119                                           const std::string &global_image_id,
 120                                           std::string *local_image_id,
 121                                           std::string *tag_owner,
 122                                           MockContextWQ *work_queue,
 123                                           Context *on_finish) {
 124     assert(s_instance != nullptr);
 125     s_instance->local_image_id = local_image_id;
 126     s_instance->tag_owner = tag_owner;
 127     s_instance->on_finish = on_finish;
 128     return s_instance;
 129   }
 130
 131   PrepareLocalImageRequest() {
 132     s_instance = this;
 133   }
 134
 135   MOCK_METHOD0(send, void());
 136 };
 137
 138 template<>
 139 struct PrepareRemoteImageRequest<librbd::MockTestImageCtx> {
 140   static PrepareRemoteImageRequest* s_instance;
 141   std::string *remote_mirror_uuid = nullptr;
 142   std::string *remote_image_id = nullptr;
 143   Context *on_finish = nullptr;
 144
 145   static PrepareRemoteImageRequest* create(librados::IoCtx &,
 146                                            const std::string &global_image_id,
 147                                            std::string *remote_mirror_uuid,
 148                                            std::string *remote_image_id,
 149                                            Context *on_finish) {
 150     assert(s_instance != nullptr);
 151     s_instance->remote_mirror_uuid = remote_mirror_uuid;
 152     s_instance->remote_image_id = remote_image_id;
 153     s_instance->on_finish = on_finish;
 154     return s_instance;
 155   }
 156
 157   PrepareRemoteImageRequest() {
 158     s_instance = this;
 159   }
 160
 161   MOCK_METHOD0(send, void());
 162 };
 163
 164 template<>
 165 struct BootstrapRequest<librbd::MockTestImageCtx> {
 166   static BootstrapRequest* s_instance;
 167   librbd::MockTestImageCtx **image_ctx = nullptr;
 168   Context *on_finish = nullptr;
 169   bool *do_resync = nullptr;
 170
 171   static BootstrapRequest* create(
 172       librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx,
 173       rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
 174       librbd::MockTestImageCtx **local_image_ctx,
 175       const std::string &local_image_name, const std::string &remote_image_id,
 176       const std::string &global_image_id, MockContextWQ *work_queue,
 177       MockSafeTimer *timer, Mutex *timer_lock,
 178       const std::string &local_mirror_uuid,
 179       const std::string &remote_mirror_uuid,
 180       ::journal::MockJournalerProxy *journaler,
 181       librbd::journal::MirrorPeerClientMeta *client_meta,
 182       Context *on_finish, bool *do_resync,
 183       rbd::mirror::ProgressContext *progress_ctx = nullptr) {
 184     assert(s_instance != nullptr);
 185     s_instance->image_ctx = local_image_ctx;
 186     s_instance->on_finish = on_finish;
 187     s_instance->do_resync = do_resync;
 188     return s_instance;
 189   }
 190
 191   BootstrapRequest() {
 192     assert(s_instance == nullptr);
 193     s_instance = this;
 194   }
 195
 196   ~BootstrapRequest() {
 197     assert(s_instance == this);
 198     s_instance = nullptr;
 199   }
 200
 201   void put() {
 202   }
 203
 204   void get() {
 205   }
 206
 207   inline bool is_syncing() const {
 208     return false;
 209   }
 210
 211   MOCK_METHOD0(send, void());
 212   MOCK_METHOD0(cancel, void());
 213 };
 214
 215 template<>
 216 struct CloseImageRequest<librbd::MockTestImageCtx> {
 217   static CloseImageRequest* s_instance;
 218   librbd::MockTestImageCtx **image_ctx = nullptr;
 219   Context *on_finish = nullptr;
 220
 221   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
 222                                    Context *on_finish) {
 223     assert(s_instance != nullptr);
 224     s_instance->image_ctx = image_ctx;
 225     s_instance->on_finish = on_finish;
 226     return s_instance;
 227   }
 228
 229   CloseImageRequest() {
 230     assert(s_instance == nullptr);
 231     s_instance = this;
 232   }
 233
 234   ~CloseImageRequest() {
 235     assert(s_instance == this);
 236     s_instance = nullptr;
 237   }
 238
 239   MOCK_METHOD0(send, void());
 240 };
 241
 242 template<>
 243 struct EventPreprocessor<librbd::MockTestImageCtx> {
 244   static EventPreprocessor *s_instance;
 245
 246   static EventPreprocessor *create(librbd::MockTestImageCtx &local_image_ctx,
 247                                    ::journal::MockJournalerProxy &remote_journaler,
 248                                    const std::string &local_mirror_uuid,
 249                                    librbd::journal::MirrorPeerClientMeta *client_meta,
 250                                    MockContextWQ *work_queue) {
 251     assert(s_instance != nullptr);
 252     return s_instance;
 253   }
 254
 255   static void destroy(EventPreprocessor* processor) {
 256   }
 257
 258   EventPreprocessor() {
 259     assert(s_instance == nullptr);
 260     s_instance = this;
 261   }
 262
 263   ~EventPreprocessor() {
 264     assert(s_instance == this);
 265     s_instance = nullptr;
 266   }
 267
 268   MOCK_METHOD1(is_required, bool(const librbd::journal::EventEntry &));
 269   MOCK_METHOD2(preprocess, void(librbd::journal::EventEntry *, Context *));
 270 };
 271
 272 template<>
 273 struct ReplayStatusFormatter<librbd::MockTestImageCtx> {
 274   static ReplayStatusFormatter* s_instance;
 275
 276   static ReplayStatusFormatter* create(::journal::MockJournalerProxy *journaler,
 277                                        const std::string &mirror_uuid) {
 278     assert(s_instance != nullptr);
 279     return s_instance;
 280   }
 281
 282   static void destroy(ReplayStatusFormatter* formatter) {
 283   }
 284
 285   ReplayStatusFormatter() {
 286     assert(s_instance == nullptr);
 287     s_instance = this;
 288   }
 289
 290   ~ReplayStatusFormatter() {
 291     assert(s_instance == this);
 292     s_instance = nullptr;
 293   }
 294
 295   MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish));
 296 };
 297
 298 BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 299 CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 300 EventPreprocessor<librbd::MockTestImageCtx>* EventPreprocessor<librbd::MockTestImageCtx>::s_instance = nullptr;
 301 PrepareLocalImageRequest<librbd::MockTestImageCtx>* PrepareLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 302 PrepareRemoteImageRequest<librbd::MockTestImageCtx>* PrepareRemoteImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 303 ReplayStatusFormatter<librbd::MockTestImageCtx>* ReplayStatusFormatter<librbd::MockTestImageCtx>::s_instance = nullptr;
 304
 305 } // namespace image_replayer
 306 } // namespace mirror
 307 } // namespace rbd
 308
 309 // template definitions
 310 #include "tools/rbd_mirror/ImageReplayer.cc"
 311
 312 namespace rbd {
 313 namespace mirror {
 314
 315 class TestMockImageReplayer : public TestMockFixture {
 316 public:
 317   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 318   typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter;
 319   typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
 320   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
 321   typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor;
 322   typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest;
 323   typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest;
 324   typedef ReplayStatusFormatter<librbd::MockTestImageCtx> MockReplayStatusFormatter;
 325   typedef librbd::journal::Replay<librbd::MockTestImageCtx> MockReplay;
 326   typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer;
 327   typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
 328
 329   void SetUp() override {
 330     TestMockFixture::SetUp();
 331
 332     librbd::RBD rbd;
 333     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size));
 334     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx));
 335   }
 336
 337   void TearDown() override {
 338     delete m_image_replayer;
 339
 340     TestMockFixture::TearDown();
 341   }
 342
 343   void create_local_image() {
 344     librbd::RBD rbd;
 345     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 346     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 347   }
 348
 349   void expect_work_queue_repeatedly(MockThreads &mock_threads) {
 350     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 351       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 352           m_threads->work_queue->queue(ctx, r);
 353         }));
 354   }
 355
 356   void expect_add_event_after_repeatedly(MockThreads &mock_threads) {
 357     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 358       .WillRepeatedly(
 359         Invoke([this](double seconds, Context *ctx) {
 360           m_threads->timer->add_event_after(seconds, ctx);
 361         }));
 362     EXPECT_CALL(*mock_threads.timer, cancel_event(_))
 363       .WillRepeatedly(
 364         Invoke([this](Context *ctx) {
 365           return m_threads->timer->cancel_event(ctx);
 366         }));
 367   }
 368
 369   void expect_wait_for_scheduled_deletion(MockImageDeleter& mock_image_deleter,
 370                                           const std::string& global_image_id,
 371                                           int r) {
 372     EXPECT_CALL(mock_image_deleter,
 373                 wait_for_scheduled_deletion(_, global_image_id, _, false))
 374       .WillOnce(WithArg<2>(Invoke([this, r](Context *ctx) {
 375                              m_threads->work_queue->queue(ctx, r);
 376                            })));
 377   }
 378
 379   void expect_cancel_waiter(MockImageDeleter& mock_image_deleter) {
 380     EXPECT_CALL(mock_image_deleter, cancel_waiter(m_local_io_ctx.get_id(),
 381                                                   "global image id"));
 382   }
 383
 384   void expect_schedule_image_delete(MockImageDeleter& mock_image_deleter,
 385                                     const std::string& global_image_id,
 386                                     bool ignore_orphan) {
 387     EXPECT_CALL(mock_image_deleter,
 388                 schedule_image_delete(_, _, global_image_id, ignore_orphan));
 389   }
 390
 391   bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) {
 392     bufferlist bl;
 393     ::encode(tag_data, bl);
 394     return bl;
 395   }
 396
 397   void expect_get_or_send_update(
 398     MockReplayStatusFormatter &mock_replay_status_formatter) {
 399     EXPECT_CALL(mock_replay_status_formatter, get_or_send_update(_, _))
 400       .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)),
 401                             Return(true)));
 402   }
 403
 404   void expect_send(MockPrepareLocalImageRequest &mock_request,
 405                    const std::string &local_image_id,
 406                    const std::string &tag_owner,
 407                    int r) {
 408     EXPECT_CALL(mock_request, send())
 409       .WillOnce(Invoke([&mock_request, local_image_id, tag_owner, r]() {
 410           if (r == 0) {
 411             *mock_request.local_image_id = local_image_id;
 412             *mock_request.tag_owner = tag_owner;
 413           }
 414           mock_request.on_finish->complete(r);
 415         }));
 416   }
 417
 418   void expect_send(MockPrepareRemoteImageRequest& mock_request,
 419                    const std::string& mirror_uuid, const std::string& image_id,
 420                    int r) {
 421     EXPECT_CALL(mock_request, send())
 422       .WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() {
 423                   *mock_request.remote_mirror_uuid = mirror_uuid;
 424                   *mock_request.remote_image_id = image_id;
 425                   mock_request.on_finish->complete(r);
 426                 }));
 427   }
 428
 429   void expect_send(MockBootstrapRequest &mock_bootstrap_request,
 430                    librbd::MockTestImageCtx &mock_local_image_ctx,
 431                    bool do_resync, int r) {
 432     EXPECT_CALL(mock_bootstrap_request, send())
 433       .WillOnce(Invoke([&mock_bootstrap_request, &mock_local_image_ctx,
 434                         do_resync, r]() {
 435             if (r == 0) {
 436               *mock_bootstrap_request.image_ctx = &mock_local_image_ctx;
 437               *mock_bootstrap_request.do_resync = do_resync;
 438             }
 439             mock_bootstrap_request.on_finish->complete(r);
 440           }));
 441   }
 442
 443   void expect_start_external_replay(librbd::MockTestJournal &mock_journal,
 444                                     MockReplay *mock_replay, int r) {
 445     EXPECT_CALL(mock_journal, start_external_replay(_, _))
 446       .WillOnce(DoAll(SetArgPointee<0>(mock_replay),
 447                       WithArg<1>(CompleteContext(r))));
 448   }
 449
 450   void expect_init(::journal::MockJournaler &mock_journaler, int r) {
 451     EXPECT_CALL(mock_journaler, init(_))
 452       .WillOnce(CompleteContext(r));
 453   }
 454
 455   void expect_get_cached_client(::journal::MockJournaler &mock_journaler,
 456                                 int r) {
 457     librbd::journal::ImageClientMeta image_client_meta;
 458     image_client_meta.tag_class = 0;
 459
 460     librbd::journal::ClientData client_data;
 461     client_data.client_meta = image_client_meta;
 462
 463     cls::journal::Client client;
 464     ::encode(client_data, client.data);
 465
 466     EXPECT_CALL(mock_journaler, get_cached_client("local_mirror_uuid", _))
 467       .WillOnce(DoAll(SetArgPointee<1>(client),
 468                       Return(r)));
 469   }
 470
 471   void expect_stop_replay(::journal::MockJournaler &mock_journaler, int r) {
 472     EXPECT_CALL(mock_journaler, stop_replay(_))
 473       .WillOnce(CompleteContext(r));
 474   }
 475
 476   void expect_flush(MockReplay &mock_replay, int r) {
 477     EXPECT_CALL(mock_replay, flush(_)).WillOnce(CompleteContext(r));
 478   }
 479
 480   void expect_shut_down(MockReplay &mock_replay, bool cancel_ops, int r) {
 481     EXPECT_CALL(mock_replay, shut_down(cancel_ops, _))
 482       .WillOnce(WithArg<1>(CompleteContext(r)));
 483   }
 484
 485   void expect_shut_down(journal::MockJournaler &mock_journaler, int r) {
 486     EXPECT_CALL(mock_journaler, shut_down(_))
 487       .WillOnce(CompleteContext(r));
 488   }
 489
 490   void expect_send(MockCloseImageRequest &mock_close_image_request, int r) {
 491     EXPECT_CALL(mock_close_image_request, send())
 492       .WillOnce(Invoke([&mock_close_image_request, r]() {
 493             *mock_close_image_request.image_ctx = nullptr;
 494             mock_close_image_request.on_finish->complete(r);
 495           }));
 496   }
 497
 498   void expect_get_commit_tid_in_debug(
 499     ::journal::MockReplayEntry &mock_replay_entry) {
 500     // It is used in debug messages and depends on debug level
 501     EXPECT_CALL(mock_replay_entry, get_commit_tid())
 502     .Times(AtLeast(0))
 503       .WillRepeatedly(Return(0));
 504   }
 505
 506   void expect_committed(::journal::MockJournaler &mock_journaler, int times) {
 507     EXPECT_CALL(mock_journaler, committed(
 508                   MatcherCast<const ::journal::MockReplayEntryProxy&>(_)))
 509       .Times(times);
 510   }
 511
 512   void expect_try_pop_front(::journal::MockJournaler &mock_journaler,
 513                             uint64_t replay_tag_tid, bool entries_available) {
 514     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 515       .WillOnce(DoAll(SetArgPointee<0>(::journal::MockReplayEntryProxy()),
 516                       SetArgPointee<1>(replay_tag_tid),
 517                       Return(entries_available)));
 518   }
 519
 520   void expect_try_pop_front_return_no_entries(
 521     ::journal::MockJournaler &mock_journaler, Context *on_finish) {
 522     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 523       .WillOnce(DoAll(Invoke([on_finish](::journal::MockReplayEntryProxy *e,
 524                                          uint64_t *t) {
 525                                on_finish->complete(0);
 526                              }),
 527                       Return(false)));
 528   }
 529
 530   void expect_get_tag(::journal::MockJournaler &mock_journaler,
 531                       const cls::journal::Tag &tag, int r) {
 532     EXPECT_CALL(mock_journaler, get_tag(_, _, _))
 533       .WillOnce(DoAll(SetArgPointee<1>(tag),
 534                       WithArg<2>(CompleteContext(r))));
 535   }
 536
 537   void expect_allocate_tag(librbd::MockTestJournal &mock_journal, int r) {
 538     EXPECT_CALL(mock_journal, allocate_tag(_, _, _))
 539       .WillOnce(WithArg<2>(CompleteContext(r)));
 540   }
 541
 542   void expect_preprocess(MockEventPreprocessor &mock_event_preprocessor,
 543                          bool required, int r) {
 544     EXPECT_CALL(mock_event_preprocessor, is_required(_))
 545       .WillOnce(Return(required));
 546     if (required) {
 547       EXPECT_CALL(mock_event_preprocessor, preprocess(_, _))
 548         .WillOnce(WithArg<1>(CompleteContext(r)));
 549     }
 550   }
 551
 552   void expect_process(MockReplay &mock_replay,
 553                       int on_ready_r, int on_commit_r) {
 554     EXPECT_CALL(mock_replay, process(_, _, _))
 555       .WillOnce(DoAll(WithArg<1>(CompleteContext(on_ready_r)),
 556                       WithArg<2>(CompleteContext(on_commit_r))));
 557   }
 558
 559   void create_image_replayer(MockThreads &mock_threads,
 560                              MockImageDeleter &mock_image_deleter) {
 561     m_image_replayer = new MockImageReplayer(
 562       &mock_threads, &mock_image_deleter, &m_instance_watcher,
 563       rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
 564       "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id");
 565     m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx);
 566   }
 567
 568   librbd::ImageCtx *m_remote_image_ctx;
 569   librbd::ImageCtx *m_local_image_ctx = nullptr;
 570   MockInstanceWatcher m_instance_watcher;
 571   MockImageReplayer *m_image_replayer = nullptr;
 572 };
 573
 574 TEST_F(TestMockImageReplayer, StartStop) {
 575   // START
 576
 577   create_local_image();
 578   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 579
 580   librbd::MockTestJournal mock_local_journal;
 581   mock_local_image_ctx.journal = &mock_local_journal;
 582
 583   journal::MockJournaler mock_remote_journaler;
 584   MockThreads mock_threads(m_threads);
 585   expect_work_queue_repeatedly(mock_threads);
 586   expect_add_event_after_repeatedly(mock_threads);
 587
 588   MockImageDeleter mock_image_deleter;
 589   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 590   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 591   MockBootstrapRequest mock_bootstrap_request;
 592   MockReplay mock_local_replay;
 593   MockEventPreprocessor mock_event_preprocessor;
 594   MockReplayStatusFormatter mock_replay_status_formatter;
 595
 596   expect_get_or_send_update(mock_replay_status_formatter);
 597
 598   InSequence seq;
 599   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 600   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 601               "remote mirror uuid", 0);
 602   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 603               m_remote_image_ctx->id, 0);
 604   EXPECT_CALL(mock_remote_journaler, construct());
 605   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 606
 607   EXPECT_CALL(mock_local_journal, add_listener(_));
 608
 609   expect_init(mock_remote_journaler, 0);
 610
 611   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 612   expect_get_cached_client(mock_remote_journaler, 0);
 613
 614   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 615
 616   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 617
 618   create_image_replayer(mock_threads, mock_image_deleter);
 619
 620   C_SaferCond start_ctx;
 621   m_image_replayer->start(&start_ctx);
 622   ASSERT_EQ(0, start_ctx.wait());
 623   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 624             m_image_replayer->get_health_state());
 625
 626   // STOP
 627
 628   MockCloseImageRequest mock_close_local_image_request;
 629
 630   expect_cancel_waiter(mock_image_deleter);
 631   expect_shut_down(mock_local_replay, true, 0);
 632   EXPECT_CALL(mock_local_journal, remove_listener(_));
 633   EXPECT_CALL(mock_local_journal, stop_external_replay());
 634   expect_send(mock_close_local_image_request, 0);
 635
 636   expect_stop_replay(mock_remote_journaler, 0);
 637   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 638   expect_shut_down(mock_remote_journaler, 0);
 639
 640   C_SaferCond stop_ctx;
 641   m_image_replayer->stop(&stop_ctx);
 642   ASSERT_EQ(0, stop_ctx.wait());
 643   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 644             m_image_replayer->get_health_state());
 645 }
 646
 647 TEST_F(TestMockImageReplayer, LocalImagePrimary) {
 648   create_local_image();
 649   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 650
 651   MockThreads mock_threads(m_threads);
 652   expect_work_queue_repeatedly(mock_threads);
 653   expect_add_event_after_repeatedly(mock_threads);
 654
 655   MockImageDeleter mock_image_deleter;
 656   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 657   MockReplayStatusFormatter mock_replay_status_formatter;
 658
 659   expect_get_or_send_update(mock_replay_status_formatter);
 660
 661   InSequence seq;
 662   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 663   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 664               "", 0);
 665
 666   create_image_replayer(mock_threads, mock_image_deleter);
 667
 668   C_SaferCond start_ctx;
 669   m_image_replayer->start(&start_ctx);
 670   ASSERT_EQ(0, start_ctx.wait());
 671 }
 672
 673 TEST_F(TestMockImageReplayer, LocalImageDNE) {
 674   create_local_image();
 675   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 676
 677   journal::MockJournaler mock_remote_journaler;
 678   MockThreads mock_threads(m_threads);
 679   expect_work_queue_repeatedly(mock_threads);
 680   expect_add_event_after_repeatedly(mock_threads);
 681
 682   MockImageDeleter mock_image_deleter;
 683   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 684   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 685   MockBootstrapRequest mock_bootstrap_request;
 686   MockReplayStatusFormatter mock_replay_status_formatter;
 687
 688   expect_get_or_send_update(mock_replay_status_formatter);
 689
 690   InSequence seq;
 691   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 692   expect_send(mock_prepare_local_image_request, "", "", -ENOENT);
 693   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 694               m_remote_image_ctx->id, 0);
 695   EXPECT_CALL(mock_remote_journaler, construct());
 696   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EREMOTEIO);
 697
 698   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 699   expect_shut_down(mock_remote_journaler, 0);
 700
 701   create_image_replayer(mock_threads, mock_image_deleter);
 702
 703   C_SaferCond start_ctx;
 704   m_image_replayer->start(&start_ctx);
 705   ASSERT_EQ(-EREMOTEIO, start_ctx.wait());
 706 }
 707
 708 TEST_F(TestMockImageReplayer, PrepareLocalImageError) {
 709   create_local_image();
 710   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 711
 712   MockThreads mock_threads(m_threads);
 713   expect_work_queue_repeatedly(mock_threads);
 714   expect_add_event_after_repeatedly(mock_threads);
 715
 716   MockImageDeleter mock_image_deleter;
 717   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 718   MockReplayStatusFormatter mock_replay_status_formatter;
 719
 720   expect_get_or_send_update(mock_replay_status_formatter);
 721
 722   InSequence seq;
 723   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 724   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 725               "remote mirror uuid", -EINVAL);
 726
 727   create_image_replayer(mock_threads, mock_image_deleter);
 728
 729   C_SaferCond start_ctx;
 730   m_image_replayer->start(&start_ctx);
 731   ASSERT_EQ(-EINVAL, start_ctx.wait());
 732 }
 733
 734 TEST_F(TestMockImageReplayer, GetRemoteImageIdDNE) {
 735   create_local_image();
 736   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 737
 738   MockThreads mock_threads(m_threads);
 739   expect_work_queue_repeatedly(mock_threads);
 740   expect_add_event_after_repeatedly(mock_threads);
 741
 742   MockImageDeleter mock_image_deleter;
 743   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 744   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 745   MockReplayStatusFormatter mock_replay_status_formatter;
 746
 747   expect_get_or_send_update(mock_replay_status_formatter);
 748
 749   InSequence seq;
 750   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 751   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 752               "remote mirror uuid", 0);
 753   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 754               "", -ENOENT);
 755   expect_schedule_image_delete(mock_image_deleter, "global image id", false);
 756
 757   create_image_replayer(mock_threads, mock_image_deleter);
 758
 759   C_SaferCond start_ctx;
 760   m_image_replayer->start(&start_ctx);
 761   ASSERT_EQ(0, start_ctx.wait());
 762 }
 763
 764 TEST_F(TestMockImageReplayer, GetRemoteImageIdNonLinkedDNE) {
 765   create_local_image();
 766   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 767
 768   MockThreads mock_threads(m_threads);
 769   expect_work_queue_repeatedly(mock_threads);
 770   expect_add_event_after_repeatedly(mock_threads);
 771
 772   MockImageDeleter mock_image_deleter;
 773   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 774   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 775   MockReplayStatusFormatter mock_replay_status_formatter;
 776
 777   expect_get_or_send_update(mock_replay_status_formatter);
 778
 779   InSequence seq;
 780   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 781   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 782               "some other mirror uuid", 0);
 783   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 784               "", -ENOENT);
 785
 786   create_image_replayer(mock_threads, mock_image_deleter);
 787
 788   C_SaferCond start_ctx;
 789   m_image_replayer->start(&start_ctx);
 790   ASSERT_EQ(-ENOENT, start_ctx.wait());
 791 }
 792
 793 TEST_F(TestMockImageReplayer, GetRemoteImageIdError) {
 794   create_local_image();
 795   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 796
 797   MockThreads mock_threads(m_threads);
 798   expect_work_queue_repeatedly(mock_threads);
 799   expect_add_event_after_repeatedly(mock_threads);
 800
 801   MockImageDeleter mock_image_deleter;
 802   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 803   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 804   MockReplayStatusFormatter mock_replay_status_formatter;
 805
 806   expect_get_or_send_update(mock_replay_status_formatter);
 807
 808   InSequence seq;
 809   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 810   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 811               "remote mirror uuid", 0);
 812   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 813               m_remote_image_ctx->id, -EINVAL);
 814
 815   create_image_replayer(mock_threads, mock_image_deleter);
 816
 817   C_SaferCond start_ctx;
 818   m_image_replayer->start(&start_ctx);
 819   ASSERT_EQ(-EINVAL, start_ctx.wait());
 820 }
 821
 822 TEST_F(TestMockImageReplayer, BootstrapError) {
 823
 824   create_local_image();
 825   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 826
 827   journal::MockJournaler mock_remote_journaler;
 828   MockThreads mock_threads(m_threads);
 829   expect_work_queue_repeatedly(mock_threads);
 830   expect_add_event_after_repeatedly(mock_threads);
 831
 832   MockImageDeleter mock_image_deleter;
 833   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 834   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 835   MockBootstrapRequest mock_bootstrap_request;
 836   MockReplayStatusFormatter mock_replay_status_formatter;
 837
 838   expect_get_or_send_update(mock_replay_status_formatter);
 839
 840   InSequence seq;
 841   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 842   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 843               "remote mirror uuid", 0);
 844   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 845               m_remote_image_ctx->id, 0);
 846   EXPECT_CALL(mock_remote_journaler, construct());
 847   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EINVAL);
 848
 849   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 850   expect_shut_down(mock_remote_journaler, 0);
 851
 852   create_image_replayer(mock_threads, mock_image_deleter);
 853
 854   C_SaferCond start_ctx;
 855   m_image_replayer->start(&start_ctx);
 856   ASSERT_EQ(-EINVAL, start_ctx.wait());
 857 }
 858
 859 TEST_F(TestMockImageReplayer, StartExternalReplayError) {
 860   // START
 861
 862   create_local_image();
 863   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 864
 865   librbd::MockTestJournal mock_local_journal;
 866   mock_local_image_ctx.journal = &mock_local_journal;
 867
 868   journal::MockJournaler mock_remote_journaler;
 869   MockThreads mock_threads(m_threads);
 870   expect_work_queue_repeatedly(mock_threads);
 871   expect_add_event_after_repeatedly(mock_threads);
 872
 873   MockImageDeleter mock_image_deleter;
 874   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 875   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 876   MockBootstrapRequest mock_bootstrap_request;
 877   MockReplay mock_local_replay;
 878   MockEventPreprocessor mock_event_preprocessor;
 879   MockReplayStatusFormatter mock_replay_status_formatter;
 880
 881   expect_get_or_send_update(mock_replay_status_formatter);
 882
 883   InSequence seq;
 884   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 885   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 886               "remote mirror uuid", 0);
 887   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 888               m_remote_image_ctx->id, 0);
 889   EXPECT_CALL(mock_remote_journaler, construct());
 890   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 891
 892   EXPECT_CALL(mock_local_journal, add_listener(_));
 893
 894   expect_init(mock_remote_journaler, 0);
 895
 896   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 897   expect_get_cached_client(mock_remote_journaler, 0);
 898
 899   expect_start_external_replay(mock_local_journal, nullptr, -EINVAL);
 900
 901   MockCloseImageRequest mock_close_local_image_request;
 902   EXPECT_CALL(mock_local_journal, remove_listener(_));
 903   expect_send(mock_close_local_image_request, 0);
 904
 905   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 906   expect_shut_down(mock_remote_journaler, 0);
 907
 908   create_image_replayer(mock_threads, mock_image_deleter);
 909
 910   C_SaferCond start_ctx;
 911   m_image_replayer->start(&start_ctx);
 912   ASSERT_EQ(-EINVAL, start_ctx.wait());
 913   ASSERT_EQ(image_replayer::HEALTH_STATE_ERROR,
 914             m_image_replayer->get_health_state());
 915 }
 916
 917 TEST_F(TestMockImageReplayer, StopError) {
 918   // START
 919
 920   create_local_image();
 921   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 922
 923   librbd::MockTestJournal mock_local_journal;
 924   mock_local_image_ctx.journal = &mock_local_journal;
 925
 926   journal::MockJournaler mock_remote_journaler;
 927   MockThreads mock_threads(m_threads);
 928   expect_work_queue_repeatedly(mock_threads);
 929   expect_add_event_after_repeatedly(mock_threads);
 930
 931   MockImageDeleter mock_image_deleter;
 932   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 933   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 934   MockBootstrapRequest mock_bootstrap_request;
 935   MockReplay mock_local_replay;
 936   MockEventPreprocessor mock_event_preprocessor;
 937   MockReplayStatusFormatter mock_replay_status_formatter;
 938
 939   expect_get_or_send_update(mock_replay_status_formatter);
 940
 941   InSequence seq;
 942   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 943   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 944               "remote mirror uuid", 0);
 945   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 946               m_remote_image_ctx->id, 0);
 947   EXPECT_CALL(mock_remote_journaler, construct());
 948   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 949
 950   EXPECT_CALL(mock_local_journal, add_listener(_));
 951
 952   expect_init(mock_remote_journaler, 0);
 953
 954   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 955   expect_get_cached_client(mock_remote_journaler, 0);
 956
 957   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 958
 959   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 960
 961   create_image_replayer(mock_threads, mock_image_deleter);
 962
 963   C_SaferCond start_ctx;
 964   m_image_replayer->start(&start_ctx);
 965   ASSERT_EQ(0, start_ctx.wait());
 966
 967   // STOP (errors are ignored)
 968
 969   MockCloseImageRequest mock_close_local_image_request;
 970
 971   expect_cancel_waiter(mock_image_deleter);
 972   expect_shut_down(mock_local_replay, true, -EINVAL);
 973   EXPECT_CALL(mock_local_journal, remove_listener(_));
 974   EXPECT_CALL(mock_local_journal, stop_external_replay());
 975   expect_send(mock_close_local_image_request, -EINVAL);
 976
 977   expect_stop_replay(mock_remote_journaler, -EINVAL);
 978   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 979   expect_shut_down(mock_remote_journaler, -EINVAL);
 980
 981   C_SaferCond stop_ctx;
 982   m_image_replayer->stop(&stop_ctx);
 983   ASSERT_EQ(0, stop_ctx.wait());
 984 }
 985
 986 TEST_F(TestMockImageReplayer, Replay) {
 987   // START
 988
 989   create_local_image();
 990   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 991
 992   librbd::MockTestJournal mock_local_journal;
 993   mock_local_image_ctx.journal = &mock_local_journal;
 994
 995   journal::MockJournaler mock_remote_journaler;
 996   MockThreads mock_threads(m_threads);
 997   expect_work_queue_repeatedly(mock_threads);
 998   expect_add_event_after_repeatedly(mock_threads);
 999
1000   MockImageDeleter mock_image_deleter;
1001   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1002   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1003   MockBootstrapRequest mock_bootstrap_request;
1004   MockReplay mock_local_replay;
1005   MockEventPreprocessor mock_event_preprocessor;
1006   MockReplayStatusFormatter mock_replay_status_formatter;
1007   ::journal::MockReplayEntry mock_replay_entry;
1008
1009   expect_get_or_send_update(mock_replay_status_formatter);
1010   expect_get_commit_tid_in_debug(mock_replay_entry);
1011   expect_committed(mock_remote_journaler, 2);
1012
1013   InSequence seq;
1014   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1015   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1016               "remote mirror uuid", 0);
1017   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1018               m_remote_image_ctx->id, 0);
1019   EXPECT_CALL(mock_remote_journaler, construct());
1020   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1021
1022   EXPECT_CALL(mock_local_journal, add_listener(_));
1023
1024   expect_init(mock_remote_journaler, 0);
1025
1026   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1027   expect_get_cached_client(mock_remote_journaler, 0);
1028
1029   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1030
1031   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1032
1033   create_image_replayer(mock_threads, mock_image_deleter);
1034
1035   C_SaferCond start_ctx;
1036   m_image_replayer->start(&start_ctx);
1037   ASSERT_EQ(0, start_ctx.wait());
1038
1039   // REPLAY
1040
1041   cls::journal::Tag tag =
1042     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1043                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1044                             true, 0, 0})};
1045
1046   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1047
1048   // replay_flush
1049   expect_shut_down(mock_local_replay, false, 0);
1050   EXPECT_CALL(mock_local_journal, stop_external_replay());
1051   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1052   expect_get_tag(mock_remote_journaler, tag, 0);
1053   expect_allocate_tag(mock_local_journal, 0);
1054
1055   // process
1056   EXPECT_CALL(mock_replay_entry, get_data());
1057   EXPECT_CALL(mock_local_replay, decode(_, _))
1058     .WillOnce(Return(0));
1059   expect_preprocess(mock_event_preprocessor, false, 0);
1060   expect_process(mock_local_replay, 0, 0);
1061
1062   // the next event with preprocess
1063   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1064   EXPECT_CALL(mock_replay_entry, get_data());
1065   EXPECT_CALL(mock_local_replay, decode(_, _))
1066     .WillOnce(Return(0));
1067   expect_preprocess(mock_event_preprocessor, true, 0);
1068   expect_process(mock_local_replay, 0, 0);
1069
1070   // attempt to process the next event
1071   C_SaferCond replay_ctx;
1072   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1073
1074   // fire
1075   m_image_replayer->handle_replay_ready();
1076   ASSERT_EQ(0, replay_ctx.wait());
1077
1078   // STOP
1079
1080   MockCloseImageRequest mock_close_local_image_request;
1081   expect_cancel_waiter(mock_image_deleter);
1082   expect_shut_down(mock_local_replay, true, 0);
1083   EXPECT_CALL(mock_local_journal, remove_listener(_));
1084   EXPECT_CALL(mock_local_journal, stop_external_replay());
1085   expect_send(mock_close_local_image_request, 0);
1086
1087   expect_stop_replay(mock_remote_journaler, 0);
1088   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1089   expect_shut_down(mock_remote_journaler, 0);
1090
1091   C_SaferCond stop_ctx;
1092   m_image_replayer->stop(&stop_ctx);
1093   ASSERT_EQ(0, stop_ctx.wait());
1094 }
1095
1096 TEST_F(TestMockImageReplayer, DecodeError) {
1097   // START
1098
1099   create_local_image();
1100   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1101
1102   librbd::MockTestJournal mock_local_journal;
1103   mock_local_image_ctx.journal = &mock_local_journal;
1104
1105   journal::MockJournaler mock_remote_journaler;
1106   MockThreads mock_threads(m_threads);
1107   expect_work_queue_repeatedly(mock_threads);
1108   expect_add_event_after_repeatedly(mock_threads);
1109
1110   MockImageDeleter mock_image_deleter;
1111   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1112   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1113   MockBootstrapRequest mock_bootstrap_request;
1114   MockReplay mock_local_replay;
1115   MockEventPreprocessor mock_event_preprocessor;
1116   MockReplayStatusFormatter mock_replay_status_formatter;
1117   ::journal::MockReplayEntry mock_replay_entry;
1118
1119   expect_get_or_send_update(mock_replay_status_formatter);
1120   expect_get_commit_tid_in_debug(mock_replay_entry);
1121
1122   InSequence seq;
1123   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1124   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1125               "remote mirror uuid", 0);
1126   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1127               m_remote_image_ctx->id, 0);
1128   EXPECT_CALL(mock_remote_journaler, construct());
1129   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1130
1131   EXPECT_CALL(mock_local_journal, add_listener(_));
1132
1133   expect_init(mock_remote_journaler, 0);
1134
1135   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1136   expect_get_cached_client(mock_remote_journaler, 0);
1137
1138   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1139
1140   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1141
1142   create_image_replayer(mock_threads, mock_image_deleter);
1143
1144   C_SaferCond start_ctx;
1145   m_image_replayer->start(&start_ctx);
1146   ASSERT_EQ(0, start_ctx.wait());
1147
1148   // REPLAY
1149
1150   cls::journal::Tag tag =
1151     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1152                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1153                             true, 0, 0})};
1154
1155   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1156
1157   // replay_flush
1158   expect_shut_down(mock_local_replay, false, 0);
1159   EXPECT_CALL(mock_local_journal, stop_external_replay());
1160   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1161   expect_get_tag(mock_remote_journaler, tag, 0);
1162   expect_allocate_tag(mock_local_journal, 0);
1163
1164   // process
1165   EXPECT_CALL(mock_replay_entry, get_data());
1166   EXPECT_CALL(mock_local_replay, decode(_, _))
1167     .WillOnce(Return(-EINVAL));
1168
1169   // stop on error
1170   expect_shut_down(mock_local_replay, true, 0);
1171   EXPECT_CALL(mock_local_journal, remove_listener(_));
1172   EXPECT_CALL(mock_local_journal, stop_external_replay());
1173
1174   MockCloseImageRequest mock_close_local_image_request;
1175   C_SaferCond close_ctx;
1176   EXPECT_CALL(mock_close_local_image_request, send())
1177     .WillOnce(Invoke([&mock_close_local_image_request, &close_ctx]() {
1178           *mock_close_local_image_request.image_ctx = nullptr;
1179           mock_close_local_image_request.on_finish->complete(0);
1180           close_ctx.complete(0);
1181         }));
1182
1183   expect_stop_replay(mock_remote_journaler, 0);
1184   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1185   expect_shut_down(mock_remote_journaler, 0);
1186
1187   // fire
1188   m_image_replayer->handle_replay_ready();
1189   ASSERT_EQ(0, close_ctx.wait());
1190
1191   while (!m_image_replayer->is_stopped()) {
1192     usleep(1000);
1193   }
1194 }
1195
1196 TEST_F(TestMockImageReplayer, DelayedReplay) {
1197
1198   // START
1199
1200   create_local_image();
1201   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1202
1203   librbd::MockTestJournal mock_local_journal;
1204   mock_local_image_ctx.journal = &mock_local_journal;
1205
1206   journal::MockJournaler mock_remote_journaler;
1207   MockThreads mock_threads(m_threads);
1208   expect_work_queue_repeatedly(mock_threads);
1209   expect_add_event_after_repeatedly(mock_threads);
1210
1211   MockImageDeleter mock_image_deleter;
1212   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1213   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1214   MockBootstrapRequest mock_bootstrap_request;
1215   MockReplay mock_local_replay;
1216   MockEventPreprocessor mock_event_preprocessor;
1217   MockReplayStatusFormatter mock_replay_status_formatter;
1218   ::journal::MockReplayEntry mock_replay_entry;
1219
1220   expect_get_or_send_update(mock_replay_status_formatter);
1221   expect_get_commit_tid_in_debug(mock_replay_entry);
1222   expect_committed(mock_remote_journaler, 1);
1223
1224   InSequence seq;
1225   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1226   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1227               "remote mirror uuid", 0);
1228   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1229               m_remote_image_ctx->id, 0);
1230   EXPECT_CALL(mock_remote_journaler, construct());
1231   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1232
1233   EXPECT_CALL(mock_local_journal, add_listener(_));
1234
1235   expect_init(mock_remote_journaler, 0);
1236
1237   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1238   expect_get_cached_client(mock_remote_journaler, 0);
1239
1240   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1241
1242   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1243
1244   create_image_replayer(mock_threads, mock_image_deleter);
1245
1246   C_SaferCond start_ctx;
1247   m_image_replayer->start(&start_ctx);
1248   ASSERT_EQ(0, start_ctx.wait());
1249
1250   // REPLAY
1251
1252   cls::journal::Tag tag =
1253     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1254                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1255                             true, 0, 0})};
1256
1257   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1258
1259   // replay_flush
1260   expect_shut_down(mock_local_replay, false, 0);
1261   EXPECT_CALL(mock_local_journal, stop_external_replay());
1262   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1263   expect_get_tag(mock_remote_journaler, tag, 0);
1264   expect_allocate_tag(mock_local_journal, 0);
1265
1266   // process with delay
1267   EXPECT_CALL(mock_replay_entry, get_data());
1268   librbd::journal::EventEntry event_entry(
1269     librbd::journal::AioDiscardEvent(123, 345, false), ceph_clock_now());
1270   EXPECT_CALL(mock_local_replay, decode(_, _))
1271     .WillOnce(DoAll(SetArgPointee<1>(event_entry),
1272                     Return(0)));
1273   expect_preprocess(mock_event_preprocessor, false, 0);
1274   expect_process(mock_local_replay, 0, 0);
1275
1276   // attempt to process the next event
1277   C_SaferCond replay_ctx;
1278   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1279
1280   // fire
1281   mock_local_image_ctx.mirroring_replay_delay = 2;
1282   m_image_replayer->handle_replay_ready();
1283   ASSERT_EQ(0, replay_ctx.wait());
1284
1285   // add a pending (delayed) entry before stop
1286   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1287   EXPECT_CALL(mock_replay_entry, get_data());
1288   C_SaferCond decode_ctx;
1289   EXPECT_CALL(mock_local_replay, decode(_, _))
1290     .WillOnce(DoAll(Invoke([&decode_ctx](bufferlist::iterator* it,
1291                                          librbd::journal::EventEntry *e) {
1292                              decode_ctx.complete(0);
1293                            }),
1294                     Return(0)));
1295
1296   mock_local_image_ctx.mirroring_replay_delay = 10;
1297   m_image_replayer->handle_replay_ready();
1298   ASSERT_EQ(0, decode_ctx.wait());
1299
1300   // STOP
1301
1302   MockCloseImageRequest mock_close_local_image_request;
1303
1304   expect_cancel_waiter(mock_image_deleter);
1305   expect_shut_down(mock_local_replay, true, 0);
1306   EXPECT_CALL(mock_local_journal, remove_listener(_));
1307   EXPECT_CALL(mock_local_journal, stop_external_replay());
1308   expect_send(mock_close_local_image_request, 0);
1309
1310   expect_stop_replay(mock_remote_journaler, 0);
1311   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1312   expect_shut_down(mock_remote_journaler, 0);
1313
1314   C_SaferCond stop_ctx;
1315   m_image_replayer->stop(&stop_ctx);
1316   ASSERT_EQ(0, stop_ctx.wait());
1317 }
1318
1319
1320 } // namespace mirror
1321 } // namespace rbd