ceph/src/test/rbd_mirror/test_mock_ImageReplayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "cls/journal/cls_journal_types.h"
   5 #include "librbd/journal/Replay.h"
   6 #include "librbd/journal/Types.h"
   7 #include "tools/rbd_mirror/ImageDeleter.h"
   8 #include "tools/rbd_mirror/ImageReplayer.h"
   9 #include "tools/rbd_mirror/InstanceWatcher.h"
  10 #include "tools/rbd_mirror/Threads.h"
  11 #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
  12 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
  13 #include "tools/rbd_mirror/image_replayer/EventPreprocessor.h"
  14 #include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h"
  15 #include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h"
  16 #include "test/rbd_mirror/test_mock_fixture.h"
  17 #include "test/journal/mock/MockJournaler.h"
  18 #include "test/librbd/mock/MockImageCtx.h"
  19 #include "test/librbd/mock/MockJournal.h"
  20 #include "test/rbd_mirror/mock/MockContextWQ.h"
  21 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  22
  23 namespace librbd {
  24
  25 namespace {
  26
  27 struct MockTestJournal;
  28
  29 struct MockTestImageCtx : public MockImageCtx {
  30   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  31     : librbd::MockImageCtx(image_ctx) {
  32   }
  33   MockTestJournal *journal = nullptr;
  34 };
  35
  36 struct MockTestJournal : public MockJournal {
  37   MOCK_METHOD2(start_external_replay, void(journal::Replay<MockTestImageCtx> **,
  38                                            Context *on_start));
  39   MOCK_METHOD0(stop_external_replay, void());
  40 };
  41
  42 } // anonymous namespace
  43
  44 namespace journal {
  45
  46 template<>
  47 struct Replay<MockTestImageCtx> {
  48   MOCK_METHOD2(decode, int(bufferlist::iterator *, EventEntry *));
  49   MOCK_METHOD3(process, void(const EventEntry &, Context *, Context *));
  50   MOCK_METHOD1(flush, void(Context*));
  51   MOCK_METHOD2(shut_down, void(bool, Context*));
  52 };
  53
  54 template <>
  55 struct TypeTraits<MockTestImageCtx> {
  56   typedef ::journal::MockJournalerProxy Journaler;
  57   typedef ::journal::MockReplayEntryProxy ReplayEntry;
  58 };
  59
  60 struct MirrorPeerClientMeta;
  61
  62 } // namespace journal
  63 } // namespace librbd
  64
  65 namespace rbd {
  66 namespace mirror {
  67
  68 template <>
  69 struct Threads<librbd::MockTestImageCtx> {
  70   MockSafeTimer *timer;
  71   Mutex &timer_lock;
  72
  73   MockContextWQ *work_queue;
  74
  75   Threads(Threads<librbd::ImageCtx> *threads)
  76     : timer(new MockSafeTimer()),
  77       timer_lock(threads->timer_lock),
  78       work_queue(new MockContextWQ()) {
  79   }
  80   ~Threads() {
  81     delete timer;
  82     delete work_queue;
  83   }
  84 };
  85
  86 template <>
  87 struct ImageDeleter<librbd::MockTestImageCtx> {
  88   MOCK_METHOD4(schedule_image_delete, void(RadosRef, int64_t,
  89                                            const std::string&, bool));
  90   MOCK_METHOD4(wait_for_scheduled_deletion,
  91                void(int64_t, const std::string&, Context*, bool));
  92   MOCK_METHOD2(cancel_waiter, void(int64_t, const std::string&));
  93 };
  94
  95 template<>
  96 class InstanceWatcher<librbd::MockTestImageCtx> {
  97 };
  98
  99 namespace image_replayer {
 100
 101 using ::testing::_;
 102 using ::testing::AtLeast;
 103 using ::testing::DoAll;
 104 using ::testing::InSequence;
 105 using ::testing::Invoke;
 106 using ::testing::MatcherCast;
 107 using ::testing::Return;
 108 using ::testing::ReturnArg;
 109 using ::testing::SetArgPointee;
 110 using ::testing::WithArg;
 111
 112 template<>
 113 struct PrepareLocalImageRequest<librbd::MockTestImageCtx> {
 114   static PrepareLocalImageRequest* s_instance;
 115   std::string *local_image_id = nullptr;
 116   std::string *tag_owner = nullptr;
 117   Context *on_finish = nullptr;
 118
 119   static PrepareLocalImageRequest* create(librados::IoCtx &,
 120                                           const std::string &global_image_id,
 121                                           std::string *local_image_id,
 122                                           std::string *tag_owner,
 123                                           MockContextWQ *work_queue,
 124                                           Context *on_finish) {
 125     assert(s_instance != nullptr);
 126     s_instance->local_image_id = local_image_id;
 127     s_instance->tag_owner = tag_owner;
 128     s_instance->on_finish = on_finish;
 129     return s_instance;
 130   }
 131
 132   PrepareLocalImageRequest() {
 133     s_instance = this;
 134   }
 135
 136   MOCK_METHOD0(send, void());
 137 };
 138
 139 template<>
 140 struct PrepareRemoteImageRequest<librbd::MockTestImageCtx> {
 141   static PrepareRemoteImageRequest* s_instance;
 142   std::string *remote_mirror_uuid = nullptr;
 143   std::string *remote_image_id = nullptr;
 144   Context *on_finish = nullptr;
 145
 146   static PrepareRemoteImageRequest* create(librados::IoCtx &,
 147                                            const std::string &global_image_id,
 148                                            std::string *remote_mirror_uuid,
 149                                            std::string *remote_image_id,
 150                                            Context *on_finish) {
 151     assert(s_instance != nullptr);
 152     s_instance->remote_mirror_uuid = remote_mirror_uuid;
 153     s_instance->remote_image_id = remote_image_id;
 154     s_instance->on_finish = on_finish;
 155     return s_instance;
 156   }
 157
 158   PrepareRemoteImageRequest() {
 159     s_instance = this;
 160   }
 161
 162   MOCK_METHOD0(send, void());
 163 };
 164
 165 template<>
 166 struct BootstrapRequest<librbd::MockTestImageCtx> {
 167   static BootstrapRequest* s_instance;
 168   librbd::MockTestImageCtx **image_ctx = nullptr;
 169   Context *on_finish = nullptr;
 170   bool *do_resync = nullptr;
 171
 172   static BootstrapRequest* create(
 173       librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx,
 174       rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
 175       librbd::MockTestImageCtx **local_image_ctx,
 176       const std::string &local_image_name, const std::string &remote_image_id,
 177       const std::string &global_image_id, MockContextWQ *work_queue,
 178       MockSafeTimer *timer, Mutex *timer_lock,
 179       const std::string &local_mirror_uuid,
 180       const std::string &remote_mirror_uuid,
 181       ::journal::MockJournalerProxy *journaler,
 182       librbd::journal::MirrorPeerClientMeta *client_meta,
 183       Context *on_finish, bool *do_resync,
 184       rbd::mirror::ProgressContext *progress_ctx = nullptr) {
 185     assert(s_instance != nullptr);
 186     s_instance->image_ctx = local_image_ctx;
 187     s_instance->on_finish = on_finish;
 188     s_instance->do_resync = do_resync;
 189     return s_instance;
 190   }
 191
 192   BootstrapRequest() {
 193     assert(s_instance == nullptr);
 194     s_instance = this;
 195   }
 196
 197   ~BootstrapRequest() {
 198     assert(s_instance == this);
 199     s_instance = nullptr;
 200   }
 201
 202   void put() {
 203   }
 204
 205   void get() {
 206   }
 207
 208   inline bool is_syncing() const {
 209     return false;
 210   }
 211
 212   MOCK_METHOD0(send, void());
 213   MOCK_METHOD0(cancel, void());
 214 };
 215
 216 template<>
 217 struct CloseImageRequest<librbd::MockTestImageCtx> {
 218   static CloseImageRequest* s_instance;
 219   librbd::MockTestImageCtx **image_ctx = nullptr;
 220   Context *on_finish = nullptr;
 221
 222   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
 223                                    Context *on_finish) {
 224     assert(s_instance != nullptr);
 225     s_instance->image_ctx = image_ctx;
 226     s_instance->on_finish = on_finish;
 227     return s_instance;
 228   }
 229
 230   CloseImageRequest() {
 231     assert(s_instance == nullptr);
 232     s_instance = this;
 233   }
 234
 235   ~CloseImageRequest() {
 236     assert(s_instance == this);
 237     s_instance = nullptr;
 238   }
 239
 240   MOCK_METHOD0(send, void());
 241 };
 242
 243 template<>
 244 struct EventPreprocessor<librbd::MockTestImageCtx> {
 245   static EventPreprocessor *s_instance;
 246
 247   static EventPreprocessor *create(librbd::MockTestImageCtx &local_image_ctx,
 248                                    ::journal::MockJournalerProxy &remote_journaler,
 249                                    const std::string &local_mirror_uuid,
 250                                    librbd::journal::MirrorPeerClientMeta *client_meta,
 251                                    MockContextWQ *work_queue) {
 252     assert(s_instance != nullptr);
 253     return s_instance;
 254   }
 255
 256   static void destroy(EventPreprocessor* processor) {
 257   }
 258
 259   EventPreprocessor() {
 260     assert(s_instance == nullptr);
 261     s_instance = this;
 262   }
 263
 264   ~EventPreprocessor() {
 265     assert(s_instance == this);
 266     s_instance = nullptr;
 267   }
 268
 269   MOCK_METHOD1(is_required, bool(const librbd::journal::EventEntry &));
 270   MOCK_METHOD2(preprocess, void(librbd::journal::EventEntry *, Context *));
 271 };
 272
 273 template<>
 274 struct ReplayStatusFormatter<librbd::MockTestImageCtx> {
 275   static ReplayStatusFormatter* s_instance;
 276
 277   static ReplayStatusFormatter* create(::journal::MockJournalerProxy *journaler,
 278                                        const std::string &mirror_uuid) {
 279     assert(s_instance != nullptr);
 280     return s_instance;
 281   }
 282
 283   static void destroy(ReplayStatusFormatter* formatter) {
 284   }
 285
 286   ReplayStatusFormatter() {
 287     assert(s_instance == nullptr);
 288     s_instance = this;
 289   }
 290
 291   ~ReplayStatusFormatter() {
 292     assert(s_instance == this);
 293     s_instance = nullptr;
 294   }
 295
 296   MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish));
 297 };
 298
 299 BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 300 CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 301 EventPreprocessor<librbd::MockTestImageCtx>* EventPreprocessor<librbd::MockTestImageCtx>::s_instance = nullptr;
 302 PrepareLocalImageRequest<librbd::MockTestImageCtx>* PrepareLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 303 PrepareRemoteImageRequest<librbd::MockTestImageCtx>* PrepareRemoteImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 304 ReplayStatusFormatter<librbd::MockTestImageCtx>* ReplayStatusFormatter<librbd::MockTestImageCtx>::s_instance = nullptr;
 305
 306 } // namespace image_replayer
 307 } // namespace mirror
 308 } // namespace rbd
 309
 310 // template definitions
 311 #include "tools/rbd_mirror/ImageReplayer.cc"
 312
 313 namespace rbd {
 314 namespace mirror {
 315
 316 class TestMockImageReplayer : public TestMockFixture {
 317 public:
 318   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 319   typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter;
 320   typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
 321   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
 322   typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor;
 323   typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest;
 324   typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest;
 325   typedef ReplayStatusFormatter<librbd::MockTestImageCtx> MockReplayStatusFormatter;
 326   typedef librbd::journal::Replay<librbd::MockTestImageCtx> MockReplay;
 327   typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer;
 328   typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
 329
 330   void SetUp() override {
 331     TestMockFixture::SetUp();
 332
 333     librbd::RBD rbd;
 334     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size));
 335     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx));
 336   }
 337
 338   void TearDown() override {
 339     delete m_image_replayer;
 340
 341     TestMockFixture::TearDown();
 342   }
 343
 344   void create_local_image() {
 345     librbd::RBD rbd;
 346     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 347     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 348   }
 349
 350   void expect_work_queue_repeatedly(MockThreads &mock_threads) {
 351     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 352       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 353           m_threads->work_queue->queue(ctx, r);
 354         }));
 355   }
 356
 357   void expect_add_event_after_repeatedly(MockThreads &mock_threads) {
 358     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 359       .WillRepeatedly(
 360         DoAll(Invoke([this](double seconds, Context *ctx) {
 361                        m_threads->timer->add_event_after(seconds, ctx);
 362                      }),
 363           ReturnArg<1>()));
 364     EXPECT_CALL(*mock_threads.timer, cancel_event(_))
 365       .WillRepeatedly(
 366         Invoke([this](Context *ctx) {
 367           return m_threads->timer->cancel_event(ctx);
 368         }));
 369   }
 370
 371   void expect_wait_for_scheduled_deletion(MockImageDeleter& mock_image_deleter,
 372                                           const std::string& global_image_id,
 373                                           int r) {
 374     EXPECT_CALL(mock_image_deleter,
 375                 wait_for_scheduled_deletion(_, global_image_id, _, false))
 376       .WillOnce(WithArg<2>(Invoke([this, r](Context *ctx) {
 377                              m_threads->work_queue->queue(ctx, r);
 378                            })));
 379   }
 380
 381   void expect_cancel_waiter(MockImageDeleter& mock_image_deleter) {
 382     EXPECT_CALL(mock_image_deleter, cancel_waiter(m_local_io_ctx.get_id(),
 383                                                   "global image id"));
 384   }
 385
 386   void expect_schedule_image_delete(MockImageDeleter& mock_image_deleter,
 387                                     const std::string& global_image_id,
 388                                     bool ignore_orphan) {
 389     EXPECT_CALL(mock_image_deleter,
 390                 schedule_image_delete(_, _, global_image_id, ignore_orphan));
 391   }
 392
 393   bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) {
 394     bufferlist bl;
 395     ::encode(tag_data, bl);
 396     return bl;
 397   }
 398
 399   void expect_get_or_send_update(
 400     MockReplayStatusFormatter &mock_replay_status_formatter) {
 401     EXPECT_CALL(mock_replay_status_formatter, get_or_send_update(_, _))
 402       .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)),
 403                             Return(true)));
 404   }
 405
 406   void expect_send(MockPrepareLocalImageRequest &mock_request,
 407                    const std::string &local_image_id,
 408                    const std::string &tag_owner,
 409                    int r) {
 410     EXPECT_CALL(mock_request, send())
 411       .WillOnce(Invoke([&mock_request, local_image_id, tag_owner, r]() {
 412           if (r == 0) {
 413             *mock_request.local_image_id = local_image_id;
 414             *mock_request.tag_owner = tag_owner;
 415           }
 416           mock_request.on_finish->complete(r);
 417         }));
 418   }
 419
 420   void expect_send(MockPrepareRemoteImageRequest& mock_request,
 421                    const std::string& mirror_uuid, const std::string& image_id,
 422                    int r) {
 423     EXPECT_CALL(mock_request, send())
 424       .WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() {
 425                   *mock_request.remote_mirror_uuid = mirror_uuid;
 426                   *mock_request.remote_image_id = image_id;
 427                   mock_request.on_finish->complete(r);
 428                 }));
 429   }
 430
 431   void expect_send(MockBootstrapRequest &mock_bootstrap_request,
 432                    librbd::MockTestImageCtx &mock_local_image_ctx,
 433                    bool do_resync, int r) {
 434     EXPECT_CALL(mock_bootstrap_request, send())
 435       .WillOnce(Invoke([&mock_bootstrap_request, &mock_local_image_ctx,
 436                         do_resync, r]() {
 437             if (r == 0) {
 438               *mock_bootstrap_request.image_ctx = &mock_local_image_ctx;
 439               *mock_bootstrap_request.do_resync = do_resync;
 440             }
 441             mock_bootstrap_request.on_finish->complete(r);
 442           }));
 443   }
 444
 445   void expect_start_external_replay(librbd::MockTestJournal &mock_journal,
 446                                     MockReplay *mock_replay, int r) {
 447     EXPECT_CALL(mock_journal, start_external_replay(_, _))
 448       .WillOnce(DoAll(SetArgPointee<0>(mock_replay),
 449                       WithArg<1>(CompleteContext(r))));
 450   }
 451
 452   void expect_init(::journal::MockJournaler &mock_journaler, int r) {
 453     EXPECT_CALL(mock_journaler, init(_))
 454       .WillOnce(CompleteContext(r));
 455   }
 456
 457   void expect_get_cached_client(::journal::MockJournaler &mock_journaler,
 458                                 int r) {
 459     librbd::journal::ImageClientMeta image_client_meta;
 460     image_client_meta.tag_class = 0;
 461
 462     librbd::journal::ClientData client_data;
 463     client_data.client_meta = image_client_meta;
 464
 465     cls::journal::Client client;
 466     ::encode(client_data, client.data);
 467
 468     EXPECT_CALL(mock_journaler, get_cached_client("local_mirror_uuid", _))
 469       .WillOnce(DoAll(SetArgPointee<1>(client),
 470                       Return(r)));
 471   }
 472
 473   void expect_stop_replay(::journal::MockJournaler &mock_journaler, int r) {
 474     EXPECT_CALL(mock_journaler, stop_replay(_))
 475       .WillOnce(CompleteContext(r));
 476   }
 477
 478   void expect_flush(MockReplay &mock_replay, int r) {
 479     EXPECT_CALL(mock_replay, flush(_)).WillOnce(CompleteContext(r));
 480   }
 481
 482   void expect_shut_down(MockReplay &mock_replay, bool cancel_ops, int r) {
 483     EXPECT_CALL(mock_replay, shut_down(cancel_ops, _))
 484       .WillOnce(WithArg<1>(CompleteContext(r)));
 485   }
 486
 487   void expect_shut_down(journal::MockJournaler &mock_journaler, int r) {
 488     EXPECT_CALL(mock_journaler, shut_down(_))
 489       .WillOnce(CompleteContext(r));
 490   }
 491
 492   void expect_send(MockCloseImageRequest &mock_close_image_request, int r) {
 493     EXPECT_CALL(mock_close_image_request, send())
 494       .WillOnce(Invoke([&mock_close_image_request, r]() {
 495             *mock_close_image_request.image_ctx = nullptr;
 496             mock_close_image_request.on_finish->complete(r);
 497           }));
 498   }
 499
 500   void expect_get_commit_tid_in_debug(
 501     ::journal::MockReplayEntry &mock_replay_entry) {
 502     // It is used in debug messages and depends on debug level
 503     EXPECT_CALL(mock_replay_entry, get_commit_tid())
 504     .Times(AtLeast(0))
 505       .WillRepeatedly(Return(0));
 506   }
 507
 508   void expect_committed(::journal::MockJournaler &mock_journaler, int times) {
 509     EXPECT_CALL(mock_journaler, committed(
 510                   MatcherCast<const ::journal::MockReplayEntryProxy&>(_)))
 511       .Times(times);
 512   }
 513
 514   void expect_try_pop_front(::journal::MockJournaler &mock_journaler,
 515                             uint64_t replay_tag_tid, bool entries_available) {
 516     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 517       .WillOnce(DoAll(SetArgPointee<0>(::journal::MockReplayEntryProxy()),
 518                       SetArgPointee<1>(replay_tag_tid),
 519                       Return(entries_available)));
 520   }
 521
 522   void expect_try_pop_front_return_no_entries(
 523     ::journal::MockJournaler &mock_journaler, Context *on_finish) {
 524     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 525       .WillOnce(DoAll(Invoke([on_finish](::journal::MockReplayEntryProxy *e,
 526                                          uint64_t *t) {
 527                                on_finish->complete(0);
 528                              }),
 529                       Return(false)));
 530   }
 531
 532   void expect_get_tag(::journal::MockJournaler &mock_journaler,
 533                       const cls::journal::Tag &tag, int r) {
 534     EXPECT_CALL(mock_journaler, get_tag(_, _, _))
 535       .WillOnce(DoAll(SetArgPointee<1>(tag),
 536                       WithArg<2>(CompleteContext(r))));
 537   }
 538
 539   void expect_allocate_tag(librbd::MockTestJournal &mock_journal, int r) {
 540     EXPECT_CALL(mock_journal, allocate_tag(_, _, _))
 541       .WillOnce(WithArg<2>(CompleteContext(r)));
 542   }
 543
 544   void expect_preprocess(MockEventPreprocessor &mock_event_preprocessor,
 545                          bool required, int r) {
 546     EXPECT_CALL(mock_event_preprocessor, is_required(_))
 547       .WillOnce(Return(required));
 548     if (required) {
 549       EXPECT_CALL(mock_event_preprocessor, preprocess(_, _))
 550         .WillOnce(WithArg<1>(CompleteContext(r)));
 551     }
 552   }
 553
 554   void expect_process(MockReplay &mock_replay,
 555                       int on_ready_r, int on_commit_r) {
 556     EXPECT_CALL(mock_replay, process(_, _, _))
 557       .WillOnce(DoAll(WithArg<1>(CompleteContext(on_ready_r)),
 558                       WithArg<2>(CompleteContext(on_commit_r))));
 559   }
 560
 561   void create_image_replayer(MockThreads &mock_threads,
 562                              MockImageDeleter &mock_image_deleter) {
 563     m_image_replayer = new MockImageReplayer(
 564       &mock_threads, &mock_image_deleter, &m_instance_watcher,
 565       rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
 566       "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id");
 567     m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx);
 568   }
 569
 570   librbd::ImageCtx *m_remote_image_ctx;
 571   librbd::ImageCtx *m_local_image_ctx = nullptr;
 572   MockInstanceWatcher m_instance_watcher;
 573   MockImageReplayer *m_image_replayer = nullptr;
 574 };
 575
 576 TEST_F(TestMockImageReplayer, StartStop) {
 577   // START
 578
 579   create_local_image();
 580   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 581
 582   librbd::MockTestJournal mock_local_journal;
 583   mock_local_image_ctx.journal = &mock_local_journal;
 584
 585   journal::MockJournaler mock_remote_journaler;
 586   MockThreads mock_threads(m_threads);
 587   expect_work_queue_repeatedly(mock_threads);
 588   expect_add_event_after_repeatedly(mock_threads);
 589
 590   MockImageDeleter mock_image_deleter;
 591   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 592   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 593   MockBootstrapRequest mock_bootstrap_request;
 594   MockReplay mock_local_replay;
 595   MockEventPreprocessor mock_event_preprocessor;
 596   MockReplayStatusFormatter mock_replay_status_formatter;
 597
 598   expect_get_or_send_update(mock_replay_status_formatter);
 599
 600   InSequence seq;
 601   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 602   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 603               "remote mirror uuid", 0);
 604   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 605               m_remote_image_ctx->id, 0);
 606   EXPECT_CALL(mock_remote_journaler, construct());
 607   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 608
 609   EXPECT_CALL(mock_local_journal, add_listener(_));
 610
 611   expect_init(mock_remote_journaler, 0);
 612
 613   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 614   expect_get_cached_client(mock_remote_journaler, 0);
 615
 616   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 617
 618   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 619
 620   create_image_replayer(mock_threads, mock_image_deleter);
 621
 622   C_SaferCond start_ctx;
 623   m_image_replayer->start(&start_ctx);
 624   ASSERT_EQ(0, start_ctx.wait());
 625   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 626             m_image_replayer->get_health_state());
 627
 628   // STOP
 629
 630   MockCloseImageRequest mock_close_local_image_request;
 631
 632   expect_cancel_waiter(mock_image_deleter);
 633   expect_shut_down(mock_local_replay, true, 0);
 634   EXPECT_CALL(mock_local_journal, remove_listener(_));
 635   EXPECT_CALL(mock_local_journal, stop_external_replay());
 636   expect_send(mock_close_local_image_request, 0);
 637
 638   expect_stop_replay(mock_remote_journaler, 0);
 639   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 640   expect_shut_down(mock_remote_journaler, 0);
 641
 642   C_SaferCond stop_ctx;
 643   m_image_replayer->stop(&stop_ctx);
 644   ASSERT_EQ(0, stop_ctx.wait());
 645   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 646             m_image_replayer->get_health_state());
 647 }
 648
 649 TEST_F(TestMockImageReplayer, LocalImagePrimary) {
 650   create_local_image();
 651   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 652
 653   MockThreads mock_threads(m_threads);
 654   expect_work_queue_repeatedly(mock_threads);
 655   expect_add_event_after_repeatedly(mock_threads);
 656
 657   MockImageDeleter mock_image_deleter;
 658   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 659   MockReplayStatusFormatter mock_replay_status_formatter;
 660
 661   expect_get_or_send_update(mock_replay_status_formatter);
 662
 663   InSequence seq;
 664   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 665   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 666               "", 0);
 667
 668   create_image_replayer(mock_threads, mock_image_deleter);
 669
 670   C_SaferCond start_ctx;
 671   m_image_replayer->start(&start_ctx);
 672   ASSERT_EQ(0, start_ctx.wait());
 673 }
 674
 675 TEST_F(TestMockImageReplayer, LocalImageDNE) {
 676   create_local_image();
 677   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 678
 679   journal::MockJournaler mock_remote_journaler;
 680   MockThreads mock_threads(m_threads);
 681   expect_work_queue_repeatedly(mock_threads);
 682   expect_add_event_after_repeatedly(mock_threads);
 683
 684   MockImageDeleter mock_image_deleter;
 685   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 686   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 687   MockBootstrapRequest mock_bootstrap_request;
 688   MockReplayStatusFormatter mock_replay_status_formatter;
 689
 690   expect_get_or_send_update(mock_replay_status_formatter);
 691
 692   InSequence seq;
 693   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 694   expect_send(mock_prepare_local_image_request, "", "", -ENOENT);
 695   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 696               m_remote_image_ctx->id, 0);
 697   EXPECT_CALL(mock_remote_journaler, construct());
 698   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EREMOTEIO);
 699
 700   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 701   expect_shut_down(mock_remote_journaler, 0);
 702
 703   create_image_replayer(mock_threads, mock_image_deleter);
 704
 705   C_SaferCond start_ctx;
 706   m_image_replayer->start(&start_ctx);
 707   ASSERT_EQ(-EREMOTEIO, start_ctx.wait());
 708 }
 709
 710 TEST_F(TestMockImageReplayer, PrepareLocalImageError) {
 711   create_local_image();
 712   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 713
 714   MockThreads mock_threads(m_threads);
 715   expect_work_queue_repeatedly(mock_threads);
 716   expect_add_event_after_repeatedly(mock_threads);
 717
 718   MockImageDeleter mock_image_deleter;
 719   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 720   MockReplayStatusFormatter mock_replay_status_formatter;
 721
 722   expect_get_or_send_update(mock_replay_status_formatter);
 723
 724   InSequence seq;
 725   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 726   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 727               "remote mirror uuid", -EINVAL);
 728
 729   create_image_replayer(mock_threads, mock_image_deleter);
 730
 731   C_SaferCond start_ctx;
 732   m_image_replayer->start(&start_ctx);
 733   ASSERT_EQ(-EINVAL, start_ctx.wait());
 734 }
 735
 736 TEST_F(TestMockImageReplayer, GetRemoteImageIdDNE) {
 737   create_local_image();
 738   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 739
 740   MockThreads mock_threads(m_threads);
 741   expect_work_queue_repeatedly(mock_threads);
 742   expect_add_event_after_repeatedly(mock_threads);
 743
 744   MockImageDeleter mock_image_deleter;
 745   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 746   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 747   MockReplayStatusFormatter mock_replay_status_formatter;
 748
 749   expect_get_or_send_update(mock_replay_status_formatter);
 750
 751   InSequence seq;
 752   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 753   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 754               "remote mirror uuid", 0);
 755   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 756               "", -ENOENT);
 757   expect_schedule_image_delete(mock_image_deleter, "global image id", false);
 758
 759   create_image_replayer(mock_threads, mock_image_deleter);
 760
 761   C_SaferCond start_ctx;
 762   m_image_replayer->start(&start_ctx);
 763   ASSERT_EQ(0, start_ctx.wait());
 764 }
 765
 766 TEST_F(TestMockImageReplayer, GetRemoteImageIdNonLinkedDNE) {
 767   create_local_image();
 768   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 769
 770   MockThreads mock_threads(m_threads);
 771   expect_work_queue_repeatedly(mock_threads);
 772   expect_add_event_after_repeatedly(mock_threads);
 773
 774   MockImageDeleter mock_image_deleter;
 775   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 776   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 777   MockReplayStatusFormatter mock_replay_status_formatter;
 778
 779   expect_get_or_send_update(mock_replay_status_formatter);
 780
 781   InSequence seq;
 782   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 783   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 784               "some other mirror uuid", 0);
 785   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 786               "", -ENOENT);
 787
 788   create_image_replayer(mock_threads, mock_image_deleter);
 789
 790   C_SaferCond start_ctx;
 791   m_image_replayer->start(&start_ctx);
 792   ASSERT_EQ(-ENOENT, start_ctx.wait());
 793 }
 794
 795 TEST_F(TestMockImageReplayer, GetRemoteImageIdError) {
 796   create_local_image();
 797   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 798
 799   MockThreads mock_threads(m_threads);
 800   expect_work_queue_repeatedly(mock_threads);
 801   expect_add_event_after_repeatedly(mock_threads);
 802
 803   MockImageDeleter mock_image_deleter;
 804   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 805   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 806   MockReplayStatusFormatter mock_replay_status_formatter;
 807
 808   expect_get_or_send_update(mock_replay_status_formatter);
 809
 810   InSequence seq;
 811   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 812   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 813               "remote mirror uuid", 0);
 814   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 815               m_remote_image_ctx->id, -EINVAL);
 816
 817   create_image_replayer(mock_threads, mock_image_deleter);
 818
 819   C_SaferCond start_ctx;
 820   m_image_replayer->start(&start_ctx);
 821   ASSERT_EQ(-EINVAL, start_ctx.wait());
 822 }
 823
 824 TEST_F(TestMockImageReplayer, BootstrapError) {
 825
 826   create_local_image();
 827   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 828
 829   journal::MockJournaler mock_remote_journaler;
 830   MockThreads mock_threads(m_threads);
 831   expect_work_queue_repeatedly(mock_threads);
 832   expect_add_event_after_repeatedly(mock_threads);
 833
 834   MockImageDeleter mock_image_deleter;
 835   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 836   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 837   MockBootstrapRequest mock_bootstrap_request;
 838   MockReplayStatusFormatter mock_replay_status_formatter;
 839
 840   expect_get_or_send_update(mock_replay_status_formatter);
 841
 842   InSequence seq;
 843   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 844   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 845               "remote mirror uuid", 0);
 846   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 847               m_remote_image_ctx->id, 0);
 848   EXPECT_CALL(mock_remote_journaler, construct());
 849   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EINVAL);
 850
 851   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 852   expect_shut_down(mock_remote_journaler, 0);
 853
 854   create_image_replayer(mock_threads, mock_image_deleter);
 855
 856   C_SaferCond start_ctx;
 857   m_image_replayer->start(&start_ctx);
 858   ASSERT_EQ(-EINVAL, start_ctx.wait());
 859 }
 860
 861 TEST_F(TestMockImageReplayer, StartExternalReplayError) {
 862   // START
 863
 864   create_local_image();
 865   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 866
 867   librbd::MockTestJournal mock_local_journal;
 868   mock_local_image_ctx.journal = &mock_local_journal;
 869
 870   journal::MockJournaler mock_remote_journaler;
 871   MockThreads mock_threads(m_threads);
 872   expect_work_queue_repeatedly(mock_threads);
 873   expect_add_event_after_repeatedly(mock_threads);
 874
 875   MockImageDeleter mock_image_deleter;
 876   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 877   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 878   MockBootstrapRequest mock_bootstrap_request;
 879   MockReplay mock_local_replay;
 880   MockEventPreprocessor mock_event_preprocessor;
 881   MockReplayStatusFormatter mock_replay_status_formatter;
 882
 883   expect_get_or_send_update(mock_replay_status_formatter);
 884
 885   InSequence seq;
 886   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 887   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 888               "remote mirror uuid", 0);
 889   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 890               m_remote_image_ctx->id, 0);
 891   EXPECT_CALL(mock_remote_journaler, construct());
 892   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 893
 894   EXPECT_CALL(mock_local_journal, add_listener(_));
 895
 896   expect_init(mock_remote_journaler, 0);
 897
 898   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 899   expect_get_cached_client(mock_remote_journaler, 0);
 900
 901   expect_start_external_replay(mock_local_journal, nullptr, -EINVAL);
 902
 903   MockCloseImageRequest mock_close_local_image_request;
 904   EXPECT_CALL(mock_local_journal, remove_listener(_));
 905   expect_send(mock_close_local_image_request, 0);
 906
 907   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 908   expect_shut_down(mock_remote_journaler, 0);
 909
 910   create_image_replayer(mock_threads, mock_image_deleter);
 911
 912   C_SaferCond start_ctx;
 913   m_image_replayer->start(&start_ctx);
 914   ASSERT_EQ(-EINVAL, start_ctx.wait());
 915   ASSERT_EQ(image_replayer::HEALTH_STATE_ERROR,
 916             m_image_replayer->get_health_state());
 917 }
 918
 919 TEST_F(TestMockImageReplayer, StopError) {
 920   // START
 921
 922   create_local_image();
 923   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 924
 925   librbd::MockTestJournal mock_local_journal;
 926   mock_local_image_ctx.journal = &mock_local_journal;
 927
 928   journal::MockJournaler mock_remote_journaler;
 929   MockThreads mock_threads(m_threads);
 930   expect_work_queue_repeatedly(mock_threads);
 931   expect_add_event_after_repeatedly(mock_threads);
 932
 933   MockImageDeleter mock_image_deleter;
 934   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 935   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 936   MockBootstrapRequest mock_bootstrap_request;
 937   MockReplay mock_local_replay;
 938   MockEventPreprocessor mock_event_preprocessor;
 939   MockReplayStatusFormatter mock_replay_status_formatter;
 940
 941   expect_get_or_send_update(mock_replay_status_formatter);
 942
 943   InSequence seq;
 944   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
 945   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 946               "remote mirror uuid", 0);
 947   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 948               m_remote_image_ctx->id, 0);
 949   EXPECT_CALL(mock_remote_journaler, construct());
 950   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 951
 952   EXPECT_CALL(mock_local_journal, add_listener(_));
 953
 954   expect_init(mock_remote_journaler, 0);
 955
 956   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 957   expect_get_cached_client(mock_remote_journaler, 0);
 958
 959   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 960
 961   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 962
 963   create_image_replayer(mock_threads, mock_image_deleter);
 964
 965   C_SaferCond start_ctx;
 966   m_image_replayer->start(&start_ctx);
 967   ASSERT_EQ(0, start_ctx.wait());
 968
 969   // STOP (errors are ignored)
 970
 971   MockCloseImageRequest mock_close_local_image_request;
 972
 973   expect_cancel_waiter(mock_image_deleter);
 974   expect_shut_down(mock_local_replay, true, -EINVAL);
 975   EXPECT_CALL(mock_local_journal, remove_listener(_));
 976   EXPECT_CALL(mock_local_journal, stop_external_replay());
 977   expect_send(mock_close_local_image_request, -EINVAL);
 978
 979   expect_stop_replay(mock_remote_journaler, -EINVAL);
 980   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 981   expect_shut_down(mock_remote_journaler, -EINVAL);
 982
 983   C_SaferCond stop_ctx;
 984   m_image_replayer->stop(&stop_ctx);
 985   ASSERT_EQ(0, stop_ctx.wait());
 986 }
 987
 988 TEST_F(TestMockImageReplayer, Replay) {
 989   // START
 990
 991   create_local_image();
 992   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 993
 994   librbd::MockTestJournal mock_local_journal;
 995   mock_local_image_ctx.journal = &mock_local_journal;
 996
 997   journal::MockJournaler mock_remote_journaler;
 998   MockThreads mock_threads(m_threads);
 999   expect_work_queue_repeatedly(mock_threads);
1000   expect_add_event_after_repeatedly(mock_threads);
1001
1002   MockImageDeleter mock_image_deleter;
1003   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1004   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1005   MockBootstrapRequest mock_bootstrap_request;
1006   MockReplay mock_local_replay;
1007   MockEventPreprocessor mock_event_preprocessor;
1008   MockReplayStatusFormatter mock_replay_status_formatter;
1009   ::journal::MockReplayEntry mock_replay_entry;
1010
1011   expect_get_or_send_update(mock_replay_status_formatter);
1012   expect_get_commit_tid_in_debug(mock_replay_entry);
1013   expect_committed(mock_remote_journaler, 2);
1014
1015   InSequence seq;
1016   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1017   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1018               "remote mirror uuid", 0);
1019   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1020               m_remote_image_ctx->id, 0);
1021   EXPECT_CALL(mock_remote_journaler, construct());
1022   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1023
1024   EXPECT_CALL(mock_local_journal, add_listener(_));
1025
1026   expect_init(mock_remote_journaler, 0);
1027
1028   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1029   expect_get_cached_client(mock_remote_journaler, 0);
1030
1031   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1032
1033   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1034
1035   create_image_replayer(mock_threads, mock_image_deleter);
1036
1037   C_SaferCond start_ctx;
1038   m_image_replayer->start(&start_ctx);
1039   ASSERT_EQ(0, start_ctx.wait());
1040
1041   // REPLAY
1042
1043   cls::journal::Tag tag =
1044     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1045                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1046                             true, 0, 0})};
1047
1048   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1049
1050   // replay_flush
1051   expect_shut_down(mock_local_replay, false, 0);
1052   EXPECT_CALL(mock_local_journal, stop_external_replay());
1053   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1054   expect_get_tag(mock_remote_journaler, tag, 0);
1055   expect_allocate_tag(mock_local_journal, 0);
1056
1057   // process
1058   EXPECT_CALL(mock_replay_entry, get_data());
1059   EXPECT_CALL(mock_local_replay, decode(_, _))
1060     .WillOnce(Return(0));
1061   expect_preprocess(mock_event_preprocessor, false, 0);
1062   expect_process(mock_local_replay, 0, 0);
1063
1064   // the next event with preprocess
1065   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1066   EXPECT_CALL(mock_replay_entry, get_data());
1067   EXPECT_CALL(mock_local_replay, decode(_, _))
1068     .WillOnce(Return(0));
1069   expect_preprocess(mock_event_preprocessor, true, 0);
1070   expect_process(mock_local_replay, 0, 0);
1071
1072   // attempt to process the next event
1073   C_SaferCond replay_ctx;
1074   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1075
1076   // fire
1077   m_image_replayer->handle_replay_ready();
1078   ASSERT_EQ(0, replay_ctx.wait());
1079
1080   // STOP
1081
1082   MockCloseImageRequest mock_close_local_image_request;
1083   expect_cancel_waiter(mock_image_deleter);
1084   expect_shut_down(mock_local_replay, true, 0);
1085   EXPECT_CALL(mock_local_journal, remove_listener(_));
1086   EXPECT_CALL(mock_local_journal, stop_external_replay());
1087   expect_send(mock_close_local_image_request, 0);
1088
1089   expect_stop_replay(mock_remote_journaler, 0);
1090   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1091   expect_shut_down(mock_remote_journaler, 0);
1092
1093   C_SaferCond stop_ctx;
1094   m_image_replayer->stop(&stop_ctx);
1095   ASSERT_EQ(0, stop_ctx.wait());
1096 }
1097
1098 TEST_F(TestMockImageReplayer, DecodeError) {
1099   // START
1100
1101   create_local_image();
1102   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1103
1104   librbd::MockTestJournal mock_local_journal;
1105   mock_local_image_ctx.journal = &mock_local_journal;
1106
1107   journal::MockJournaler mock_remote_journaler;
1108   MockThreads mock_threads(m_threads);
1109   expect_work_queue_repeatedly(mock_threads);
1110   expect_add_event_after_repeatedly(mock_threads);
1111
1112   MockImageDeleter mock_image_deleter;
1113   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1114   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1115   MockBootstrapRequest mock_bootstrap_request;
1116   MockReplay mock_local_replay;
1117   MockEventPreprocessor mock_event_preprocessor;
1118   MockReplayStatusFormatter mock_replay_status_formatter;
1119   ::journal::MockReplayEntry mock_replay_entry;
1120
1121   expect_get_or_send_update(mock_replay_status_formatter);
1122   expect_get_commit_tid_in_debug(mock_replay_entry);
1123
1124   InSequence seq;
1125   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1126   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1127               "remote mirror uuid", 0);
1128   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1129               m_remote_image_ctx->id, 0);
1130   EXPECT_CALL(mock_remote_journaler, construct());
1131   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1132
1133   EXPECT_CALL(mock_local_journal, add_listener(_));
1134
1135   expect_init(mock_remote_journaler, 0);
1136
1137   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1138   expect_get_cached_client(mock_remote_journaler, 0);
1139
1140   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1141
1142   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1143
1144   create_image_replayer(mock_threads, mock_image_deleter);
1145
1146   C_SaferCond start_ctx;
1147   m_image_replayer->start(&start_ctx);
1148   ASSERT_EQ(0, start_ctx.wait());
1149
1150   // REPLAY
1151
1152   cls::journal::Tag tag =
1153     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1154                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1155                             true, 0, 0})};
1156
1157   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1158
1159   // replay_flush
1160   expect_shut_down(mock_local_replay, false, 0);
1161   EXPECT_CALL(mock_local_journal, stop_external_replay());
1162   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1163   expect_get_tag(mock_remote_journaler, tag, 0);
1164   expect_allocate_tag(mock_local_journal, 0);
1165
1166   // process
1167   EXPECT_CALL(mock_replay_entry, get_data());
1168   EXPECT_CALL(mock_local_replay, decode(_, _))
1169     .WillOnce(Return(-EINVAL));
1170
1171   // stop on error
1172   expect_shut_down(mock_local_replay, true, 0);
1173   EXPECT_CALL(mock_local_journal, remove_listener(_));
1174   EXPECT_CALL(mock_local_journal, stop_external_replay());
1175
1176   MockCloseImageRequest mock_close_local_image_request;
1177   C_SaferCond close_ctx;
1178   EXPECT_CALL(mock_close_local_image_request, send())
1179     .WillOnce(Invoke([&mock_close_local_image_request, &close_ctx]() {
1180           *mock_close_local_image_request.image_ctx = nullptr;
1181           mock_close_local_image_request.on_finish->complete(0);
1182           close_ctx.complete(0);
1183         }));
1184
1185   expect_stop_replay(mock_remote_journaler, 0);
1186   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1187   expect_shut_down(mock_remote_journaler, 0);
1188
1189   // fire
1190   m_image_replayer->handle_replay_ready();
1191   ASSERT_EQ(0, close_ctx.wait());
1192
1193   while (!m_image_replayer->is_stopped()) {
1194     usleep(1000);
1195   }
1196 }
1197
1198 TEST_F(TestMockImageReplayer, DelayedReplay) {
1199
1200   // START
1201
1202   create_local_image();
1203   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1204
1205   librbd::MockTestJournal mock_local_journal;
1206   mock_local_image_ctx.journal = &mock_local_journal;
1207
1208   journal::MockJournaler mock_remote_journaler;
1209   MockThreads mock_threads(m_threads);
1210   expect_work_queue_repeatedly(mock_threads);
1211   expect_add_event_after_repeatedly(mock_threads);
1212
1213   MockImageDeleter mock_image_deleter;
1214   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1215   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1216   MockBootstrapRequest mock_bootstrap_request;
1217   MockReplay mock_local_replay;
1218   MockEventPreprocessor mock_event_preprocessor;
1219   MockReplayStatusFormatter mock_replay_status_formatter;
1220   ::journal::MockReplayEntry mock_replay_entry;
1221
1222   expect_get_or_send_update(mock_replay_status_formatter);
1223   expect_get_commit_tid_in_debug(mock_replay_entry);
1224   expect_committed(mock_remote_journaler, 1);
1225
1226   InSequence seq;
1227   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
1228   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1229               "remote mirror uuid", 0);
1230   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1231               m_remote_image_ctx->id, 0);
1232   EXPECT_CALL(mock_remote_journaler, construct());
1233   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1234
1235   EXPECT_CALL(mock_local_journal, add_listener(_));
1236
1237   expect_init(mock_remote_journaler, 0);
1238
1239   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1240   expect_get_cached_client(mock_remote_journaler, 0);
1241
1242   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1243
1244   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1245
1246   create_image_replayer(mock_threads, mock_image_deleter);
1247
1248   C_SaferCond start_ctx;
1249   m_image_replayer->start(&start_ctx);
1250   ASSERT_EQ(0, start_ctx.wait());
1251
1252   // REPLAY
1253
1254   cls::journal::Tag tag =
1255     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1256                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1257                             true, 0, 0})};
1258
1259   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1260
1261   // replay_flush
1262   expect_shut_down(mock_local_replay, false, 0);
1263   EXPECT_CALL(mock_local_journal, stop_external_replay());
1264   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1265   expect_get_tag(mock_remote_journaler, tag, 0);
1266   expect_allocate_tag(mock_local_journal, 0);
1267
1268   // process with delay
1269   EXPECT_CALL(mock_replay_entry, get_data());
1270   librbd::journal::EventEntry event_entry(
1271     librbd::journal::AioDiscardEvent(123, 345, false), ceph_clock_now());
1272   EXPECT_CALL(mock_local_replay, decode(_, _))
1273     .WillOnce(DoAll(SetArgPointee<1>(event_entry),
1274                     Return(0)));
1275   expect_preprocess(mock_event_preprocessor, false, 0);
1276   expect_process(mock_local_replay, 0, 0);
1277
1278   // attempt to process the next event
1279   C_SaferCond replay_ctx;
1280   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1281
1282   // fire
1283   mock_local_image_ctx.mirroring_replay_delay = 2;
1284   m_image_replayer->handle_replay_ready();
1285   ASSERT_EQ(0, replay_ctx.wait());
1286
1287   // add a pending (delayed) entry before stop
1288   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1289   EXPECT_CALL(mock_replay_entry, get_data());
1290   C_SaferCond decode_ctx;
1291   EXPECT_CALL(mock_local_replay, decode(_, _))
1292     .WillOnce(DoAll(Invoke([&decode_ctx](bufferlist::iterator* it,
1293                                          librbd::journal::EventEntry *e) {
1294                              decode_ctx.complete(0);
1295                            }),
1296                     Return(0)));
1297
1298   mock_local_image_ctx.mirroring_replay_delay = 10;
1299   m_image_replayer->handle_replay_ready();
1300   ASSERT_EQ(0, decode_ctx.wait());
1301
1302   // STOP
1303
1304   MockCloseImageRequest mock_close_local_image_request;
1305
1306   expect_cancel_waiter(mock_image_deleter);
1307   expect_shut_down(mock_local_replay, true, 0);
1308   EXPECT_CALL(mock_local_journal, remove_listener(_));
1309   EXPECT_CALL(mock_local_journal, stop_external_replay());
1310   expect_send(mock_close_local_image_request, 0);
1311
1312   expect_stop_replay(mock_remote_journaler, 0);
1313   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1314   expect_shut_down(mock_remote_journaler, 0);
1315
1316   C_SaferCond stop_ctx;
1317   m_image_replayer->stop(&stop_ctx);
1318   ASSERT_EQ(0, stop_ctx.wait());
1319 }
1320
1321
1322 } // namespace mirror
1323 } // namespace rbd