ceph/src/test/rbd_mirror/test_mock_ImageReplayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "cls/journal/cls_journal_types.h"
   5 #include "librbd/journal/Replay.h"
   6 #include "librbd/journal/Types.h"
   7 #include "tools/rbd_mirror/ImageDeleter.h"
   8 #include "tools/rbd_mirror/ImageReplayer.h"
   9 #include "tools/rbd_mirror/InstanceWatcher.h"
  10 #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
  11 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
  12 #include "tools/rbd_mirror/image_replayer/EventPreprocessor.h"
  13 #include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h"
  14 #include "test/rbd_mirror/test_mock_fixture.h"
  15 #include "test/journal/mock/MockJournaler.h"
  16 #include "test/librbd/mock/MockImageCtx.h"
  17 #include "test/librbd/mock/MockJournal.h"
  18
  19 namespace librbd {
  20
  21 namespace {
  22
  23 struct MockTestJournal;
  24
  25 struct MockTestImageCtx : public MockImageCtx {
  26   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  27     : librbd::MockImageCtx(image_ctx) {
  28   }
  29   MockTestJournal *journal = nullptr;
  30 };
  31
  32 struct MockTestJournal : public MockJournal {
  33   MOCK_METHOD2(start_external_replay, void(journal::Replay<MockTestImageCtx> **,
  34                                            Context *on_start));
  35   MOCK_METHOD0(stop_external_replay, void());
  36 };
  37
  38 } // anonymous namespace
  39
  40 namespace journal {
  41
  42 template<>
  43 struct Replay<MockTestImageCtx> {
  44   MOCK_METHOD2(decode, int(bufferlist::iterator *, EventEntry *));
  45   MOCK_METHOD3(process, void(const EventEntry &, Context *, Context *));
  46   MOCK_METHOD1(flush, void(Context*));
  47   MOCK_METHOD2(shut_down, void(bool, Context*));
  48 };
  49
  50 template <>
  51 struct TypeTraits<MockTestImageCtx> {
  52   typedef ::journal::MockJournalerProxy Journaler;
  53   typedef ::journal::MockReplayEntryProxy ReplayEntry;
  54 };
  55
  56 struct MirrorPeerClientMeta;
  57
  58 } // namespace journal
  59 } // namespace librbd
  60
  61 namespace rbd {
  62 namespace mirror {
  63
  64 template <>
  65 struct ImageDeleter<librbd::MockTestImageCtx> {
  66   MOCK_METHOD4(schedule_image_delete, void(RadosRef, int64_t,
  67                                            const std::string&, bool));
  68 };
  69
  70 template<>
  71 class InstanceWatcher<librbd::MockTestImageCtx> {
  72 };
  73
  74 namespace image_replayer {
  75
  76 using ::testing::_;
  77 using ::testing::AtLeast;
  78 using ::testing::DoAll;
  79 using ::testing::InSequence;
  80 using ::testing::Invoke;
  81 using ::testing::MatcherCast;
  82 using ::testing::Return;
  83 using ::testing::SetArgPointee;
  84 using ::testing::WithArg;
  85
  86 template<>
  87 struct PrepareLocalImageRequest<librbd::MockTestImageCtx> {
  88   static PrepareLocalImageRequest* s_instance;
  89   std::string *local_image_id = nullptr;
  90   std::string *tag_owner = nullptr;
  91   Context *on_finish = nullptr;
  92
  93   static PrepareLocalImageRequest* create(librados::IoCtx &,
  94                                           const std::string &global_image_id,
  95                                           std::string *local_image_id,
  96                                           std::string *tag_owner,
  97                                           ContextWQ *work_queue,
  98                                           Context *on_finish) {
  99     assert(s_instance != nullptr);
 100     s_instance->local_image_id = local_image_id;
 101     s_instance->tag_owner = tag_owner;
 102     s_instance->on_finish = on_finish;
 103     return s_instance;
 104   }
 105
 106   PrepareLocalImageRequest() {
 107     s_instance = this;
 108   }
 109
 110   MOCK_METHOD0(send, void());
 111 };
 112
 113 template<>
 114 struct BootstrapRequest<librbd::MockTestImageCtx> {
 115   static BootstrapRequest* s_instance;
 116   librbd::MockTestImageCtx **image_ctx = nullptr;
 117   Context *on_finish = nullptr;
 118   bool *do_resync = nullptr;
 119
 120   static BootstrapRequest* create(
 121       librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx,
 122       rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
 123       librbd::MockTestImageCtx **local_image_ctx,
 124       const std::string &local_image_name, const std::string &remote_image_id,
 125       const std::string &global_image_id, ContextWQ *work_queue,
 126       SafeTimer *timer, Mutex *timer_lock, const std::string &local_mirror_uuid,
 127       const std::string &remote_mirror_uuid,
 128       ::journal::MockJournalerProxy *journaler,
 129       librbd::journal::MirrorPeerClientMeta *client_meta,
 130       Context *on_finish, bool *do_resync,
 131       rbd::mirror::ProgressContext *progress_ctx = nullptr) {
 132     assert(s_instance != nullptr);
 133     s_instance->image_ctx = local_image_ctx;
 134     s_instance->on_finish = on_finish;
 135     s_instance->do_resync = do_resync;
 136     return s_instance;
 137   }
 138
 139   BootstrapRequest() {
 140     assert(s_instance == nullptr);
 141     s_instance = this;
 142   }
 143
 144   ~BootstrapRequest() {
 145     assert(s_instance == this);
 146     s_instance = nullptr;
 147   }
 148
 149   void put() {
 150   }
 151
 152   void get() {
 153   }
 154
 155   inline bool is_syncing() const {
 156     return false;
 157   }
 158
 159   MOCK_METHOD0(send, void());
 160   MOCK_METHOD0(cancel, void());
 161 };
 162
 163 template<>
 164 struct CloseImageRequest<librbd::MockTestImageCtx> {
 165   static CloseImageRequest* s_instance;
 166   librbd::MockTestImageCtx **image_ctx = nullptr;
 167   Context *on_finish = nullptr;
 168
 169   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
 170                                    Context *on_finish) {
 171     assert(s_instance != nullptr);
 172     s_instance->image_ctx = image_ctx;
 173     s_instance->on_finish = on_finish;
 174     return s_instance;
 175   }
 176
 177   CloseImageRequest() {
 178     assert(s_instance == nullptr);
 179     s_instance = this;
 180   }
 181
 182   ~CloseImageRequest() {
 183     assert(s_instance == this);
 184     s_instance = nullptr;
 185   }
 186
 187   MOCK_METHOD0(send, void());
 188 };
 189
 190 template<>
 191 struct EventPreprocessor<librbd::MockTestImageCtx> {
 192   static EventPreprocessor *s_instance;
 193
 194   static EventPreprocessor *create(librbd::MockTestImageCtx &local_image_ctx,
 195                                    ::journal::MockJournalerProxy &remote_journaler,
 196                                    const std::string &local_mirror_uuid,
 197                                    librbd::journal::MirrorPeerClientMeta *client_meta,
 198                                    ContextWQ *work_queue) {
 199     assert(s_instance != nullptr);
 200     return s_instance;
 201   }
 202
 203   static void destroy(EventPreprocessor* processor) {
 204   }
 205
 206   EventPreprocessor() {
 207     assert(s_instance == nullptr);
 208     s_instance = this;
 209   }
 210
 211   ~EventPreprocessor() {
 212     assert(s_instance == this);
 213     s_instance = nullptr;
 214   }
 215
 216   MOCK_METHOD1(is_required, bool(const librbd::journal::EventEntry &));
 217   MOCK_METHOD2(preprocess, void(librbd::journal::EventEntry *, Context *));
 218 };
 219
 220 template<>
 221 struct ReplayStatusFormatter<librbd::MockTestImageCtx> {
 222   static ReplayStatusFormatter* s_instance;
 223
 224   static ReplayStatusFormatter* create(::journal::MockJournalerProxy *journaler,
 225                                        const std::string &mirror_uuid) {
 226     assert(s_instance != nullptr);
 227     return s_instance;
 228   }
 229
 230   static void destroy(ReplayStatusFormatter* formatter) {
 231   }
 232
 233   ReplayStatusFormatter() {
 234     assert(s_instance == nullptr);
 235     s_instance = this;
 236   }
 237
 238   ~ReplayStatusFormatter() {
 239     assert(s_instance == this);
 240     s_instance = nullptr;
 241   }
 242
 243   MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish));
 244 };
 245
 246 BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 247 CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 248 EventPreprocessor<librbd::MockTestImageCtx>* EventPreprocessor<librbd::MockTestImageCtx>::s_instance = nullptr;
 249 PrepareLocalImageRequest<librbd::MockTestImageCtx>* PrepareLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 250 ReplayStatusFormatter<librbd::MockTestImageCtx>* ReplayStatusFormatter<librbd::MockTestImageCtx>::s_instance = nullptr;
 251
 252 } // namespace image_replayer
 253 } // namespace mirror
 254 } // namespace rbd
 255
 256 // template definitions
 257 #include "tools/rbd_mirror/ImageReplayer.cc"
 258
 259 namespace rbd {
 260 namespace mirror {
 261
 262 class TestMockImageReplayer : public TestMockFixture {
 263 public:
 264   typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter;
 265   typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
 266   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
 267   typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor;
 268   typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest;
 269   typedef ReplayStatusFormatter<librbd::MockTestImageCtx> MockReplayStatusFormatter;
 270   typedef librbd::journal::Replay<librbd::MockTestImageCtx> MockReplay;
 271   typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer;
 272   typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
 273
 274   void SetUp() override {
 275     TestMockFixture::SetUp();
 276
 277     librbd::RBD rbd;
 278     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size));
 279     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx));
 280   }
 281
 282   void TearDown() override {
 283     delete m_image_replayer;
 284
 285     TestMockFixture::TearDown();
 286   }
 287
 288   void create_local_image() {
 289     librbd::RBD rbd;
 290     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 291     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 292   }
 293
 294   bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) {
 295     bufferlist bl;
 296     ::encode(tag_data, bl);
 297     return bl;
 298   }
 299
 300   void expect_get_or_send_update(
 301     MockReplayStatusFormatter &mock_replay_status_formatter) {
 302     EXPECT_CALL(mock_replay_status_formatter, get_or_send_update(_, _))
 303       .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)),
 304                             Return(true)));
 305   }
 306
 307   void expect_send(MockPrepareLocalImageRequest &mock_request,
 308                    const std::string &local_image_id,
 309                    const std::string &tag_owner,
 310                    int r) {
 311     EXPECT_CALL(mock_request, send())
 312       .WillOnce(Invoke([&mock_request, local_image_id, tag_owner, r]() {
 313           if (r == 0) {
 314             *mock_request.local_image_id = local_image_id;
 315             *mock_request.tag_owner = tag_owner;
 316           }
 317           mock_request.on_finish->complete(r);
 318         }));
 319   }
 320
 321   void expect_send(MockBootstrapRequest &mock_bootstrap_request,
 322                    librbd::MockTestImageCtx &mock_local_image_ctx,
 323                    bool do_resync, int r) {
 324     EXPECT_CALL(mock_bootstrap_request, send())
 325       .WillOnce(Invoke([&mock_bootstrap_request, &mock_local_image_ctx,
 326                         do_resync, r]() {
 327             if (r == 0) {
 328               *mock_bootstrap_request.image_ctx = &mock_local_image_ctx;
 329               *mock_bootstrap_request.do_resync = do_resync;
 330             }
 331             mock_bootstrap_request.on_finish->complete(r);
 332           }));
 333   }
 334
 335   void expect_start_external_replay(librbd::MockTestJournal &mock_journal,
 336                                     MockReplay *mock_replay, int r) {
 337     EXPECT_CALL(mock_journal, start_external_replay(_, _))
 338       .WillOnce(DoAll(SetArgPointee<0>(mock_replay),
 339                       WithArg<1>(CompleteContext(r))));
 340   }
 341
 342   void expect_init(::journal::MockJournaler &mock_journaler, int r) {
 343     EXPECT_CALL(mock_journaler, init(_))
 344       .WillOnce(CompleteContext(r));
 345   }
 346
 347   void expect_get_cached_client(::journal::MockJournaler &mock_journaler,
 348                                 int r) {
 349     librbd::journal::ImageClientMeta image_client_meta;
 350     image_client_meta.tag_class = 0;
 351
 352     librbd::journal::ClientData client_data;
 353     client_data.client_meta = image_client_meta;
 354
 355     cls::journal::Client client;
 356     ::encode(client_data, client.data);
 357
 358     EXPECT_CALL(mock_journaler, get_cached_client("local_mirror_uuid", _))
 359       .WillOnce(DoAll(SetArgPointee<1>(client),
 360                       Return(r)));
 361   }
 362
 363   void expect_stop_replay(::journal::MockJournaler &mock_journaler, int r) {
 364     EXPECT_CALL(mock_journaler, stop_replay(_))
 365       .WillOnce(CompleteContext(r));
 366   }
 367
 368   void expect_flush(MockReplay &mock_replay, int r) {
 369     EXPECT_CALL(mock_replay, flush(_)).WillOnce(CompleteContext(r));
 370   }
 371
 372   void expect_shut_down(MockReplay &mock_replay, bool cancel_ops, int r) {
 373     EXPECT_CALL(mock_replay, shut_down(cancel_ops, _))
 374       .WillOnce(WithArg<1>(CompleteContext(r)));
 375   }
 376
 377   void expect_shut_down(journal::MockJournaler &mock_journaler, int r) {
 378     EXPECT_CALL(mock_journaler, shut_down(_))
 379       .WillOnce(CompleteContext(r));
 380   }
 381
 382   void expect_send(MockCloseImageRequest &mock_close_image_request, int r) {
 383     EXPECT_CALL(mock_close_image_request, send())
 384       .WillOnce(Invoke([&mock_close_image_request, r]() {
 385             *mock_close_image_request.image_ctx = nullptr;
 386             mock_close_image_request.on_finish->complete(r);
 387           }));
 388   }
 389
 390   void expect_get_commit_tid_in_debug(
 391     ::journal::MockReplayEntry &mock_replay_entry) {
 392     // It is used in debug messages and depends on debug level
 393     EXPECT_CALL(mock_replay_entry, get_commit_tid())
 394     .Times(AtLeast(0))
 395       .WillRepeatedly(Return(0));
 396   }
 397
 398   void expect_committed(::journal::MockJournaler &mock_journaler, int times) {
 399     EXPECT_CALL(mock_journaler, committed(
 400                   MatcherCast<const ::journal::MockReplayEntryProxy&>(_)))
 401       .Times(times);
 402   }
 403
 404   void expect_try_pop_front(::journal::MockJournaler &mock_journaler,
 405                             uint64_t replay_tag_tid, bool entries_available) {
 406     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 407       .WillOnce(DoAll(SetArgPointee<0>(::journal::MockReplayEntryProxy()),
 408                       SetArgPointee<1>(replay_tag_tid),
 409                       Return(entries_available)));
 410   }
 411
 412   void expect_try_pop_front_return_no_entries(
 413     ::journal::MockJournaler &mock_journaler, Context *on_finish) {
 414     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 415       .WillOnce(DoAll(Invoke([on_finish](::journal::MockReplayEntryProxy *e,
 416                                          uint64_t *t) {
 417                                on_finish->complete(0);
 418                              }),
 419                       Return(false)));
 420   }
 421
 422   void expect_get_tag(::journal::MockJournaler &mock_journaler,
 423                       const cls::journal::Tag &tag, int r) {
 424     EXPECT_CALL(mock_journaler, get_tag(_, _, _))
 425       .WillOnce(DoAll(SetArgPointee<1>(tag),
 426                       WithArg<2>(CompleteContext(r))));
 427   }
 428
 429   void expect_allocate_tag(librbd::MockTestJournal &mock_journal, int r) {
 430     EXPECT_CALL(mock_journal, allocate_tag(_, _, _))
 431       .WillOnce(WithArg<2>(CompleteContext(r)));
 432   }
 433
 434   void expect_preprocess(MockEventPreprocessor &mock_event_preprocessor,
 435                          bool required, int r) {
 436     EXPECT_CALL(mock_event_preprocessor, is_required(_))
 437       .WillOnce(Return(required));
 438     if (required) {
 439       EXPECT_CALL(mock_event_preprocessor, preprocess(_, _))
 440         .WillOnce(WithArg<1>(CompleteContext(r)));
 441     }
 442   }
 443
 444   void expect_process(MockReplay &mock_replay,
 445                       int on_ready_r, int on_commit_r) {
 446     EXPECT_CALL(mock_replay, process(_, _, _))
 447       .WillOnce(DoAll(WithArg<1>(CompleteContext(on_ready_r)),
 448                       WithArg<2>(CompleteContext(on_commit_r))));
 449   }
 450
 451   void create_image_replayer(MockImageDeleter &mock_image_deleter) {
 452     m_image_replayer = new MockImageReplayer(
 453       m_threads, &mock_image_deleter, &m_instance_watcher,
 454       rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
 455       "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id");
 456     m_image_replayer->add_remote_image(
 457       "remote_mirror_uuid", m_remote_image_ctx->id, m_remote_io_ctx);
 458   }
 459
 460   librbd::ImageCtx *m_remote_image_ctx;
 461   librbd::ImageCtx *m_local_image_ctx = nullptr;
 462   MockInstanceWatcher m_instance_watcher;
 463   MockImageReplayer *m_image_replayer = nullptr;
 464 };
 465
 466 TEST_F(TestMockImageReplayer, StartStop) {
 467   // START
 468
 469   create_local_image();
 470   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 471
 472   librbd::MockTestJournal mock_local_journal;
 473   mock_local_image_ctx.journal = &mock_local_journal;
 474
 475   journal::MockJournaler mock_remote_journaler;
 476   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 477   MockBootstrapRequest mock_bootstrap_request;
 478   MockReplay mock_local_replay;
 479   MockEventPreprocessor mock_event_preprocessor;
 480   MockReplayStatusFormatter mock_replay_status_formatter;
 481
 482   expect_get_or_send_update(mock_replay_status_formatter);
 483
 484   InSequence seq;
 485   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 486               "remote mirror uuid", 0);
 487   EXPECT_CALL(mock_remote_journaler, construct());
 488   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 489
 490   EXPECT_CALL(mock_local_journal, add_listener(_));
 491
 492   expect_init(mock_remote_journaler, 0);
 493
 494   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 495   expect_get_cached_client(mock_remote_journaler, 0);
 496
 497   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 498
 499   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 500
 501   MockImageDeleter mock_image_deleter;
 502   create_image_replayer(mock_image_deleter);
 503
 504   C_SaferCond start_ctx;
 505   m_image_replayer->start(&start_ctx);
 506   ASSERT_EQ(0, start_ctx.wait());
 507   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 508             m_image_replayer->get_health_state());
 509
 510   // STOP
 511
 512   MockCloseImageRequest mock_close_local_image_request;
 513
 514   expect_shut_down(mock_local_replay, true, 0);
 515   EXPECT_CALL(mock_local_journal, remove_listener(_));
 516   EXPECT_CALL(mock_local_journal, stop_external_replay());
 517   expect_send(mock_close_local_image_request, 0);
 518
 519   expect_stop_replay(mock_remote_journaler, 0);
 520   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 521   expect_shut_down(mock_remote_journaler, 0);
 522
 523   C_SaferCond stop_ctx;
 524   m_image_replayer->stop(&stop_ctx);
 525   ASSERT_EQ(0, stop_ctx.wait());
 526   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 527             m_image_replayer->get_health_state());
 528 }
 529
 530 TEST_F(TestMockImageReplayer, LocalImagePrimary) {
 531   create_local_image();
 532   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 533
 534   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 535   MockReplayStatusFormatter mock_replay_status_formatter;
 536
 537   expect_get_or_send_update(mock_replay_status_formatter);
 538
 539   InSequence seq;
 540   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 541               "", 0);
 542
 543   MockImageDeleter mock_image_deleter;
 544   create_image_replayer(mock_image_deleter);
 545
 546   C_SaferCond start_ctx;
 547   m_image_replayer->start(&start_ctx);
 548   ASSERT_EQ(0, start_ctx.wait());
 549 }
 550
 551 TEST_F(TestMockImageReplayer, LocalImageDNE) {
 552   create_local_image();
 553   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 554
 555   journal::MockJournaler mock_remote_journaler;
 556   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 557   MockBootstrapRequest mock_bootstrap_request;
 558   MockReplayStatusFormatter mock_replay_status_formatter;
 559
 560   expect_get_or_send_update(mock_replay_status_formatter);
 561
 562   InSequence seq;
 563   expect_send(mock_prepare_local_image_request, "", "", -ENOENT);
 564   EXPECT_CALL(mock_remote_journaler, construct());
 565   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EREMOTEIO);
 566
 567   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 568   expect_shut_down(mock_remote_journaler, 0);
 569
 570   MockImageDeleter mock_image_deleter;
 571   create_image_replayer(mock_image_deleter);
 572
 573   C_SaferCond start_ctx;
 574   m_image_replayer->start(&start_ctx);
 575   ASSERT_EQ(-EREMOTEIO, start_ctx.wait());
 576 }
 577
 578 TEST_F(TestMockImageReplayer, PrepareLocalImageError) {
 579   create_local_image();
 580   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 581
 582   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 583   MockReplayStatusFormatter mock_replay_status_formatter;
 584
 585   expect_get_or_send_update(mock_replay_status_formatter);
 586
 587   InSequence seq;
 588   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 589               "remote mirror uuid", -EINVAL);
 590
 591   MockImageDeleter mock_image_deleter;
 592   create_image_replayer(mock_image_deleter);
 593
 594   C_SaferCond start_ctx;
 595   m_image_replayer->start(&start_ctx);
 596   ASSERT_EQ(-EINVAL, start_ctx.wait());
 597 }
 598
 599 TEST_F(TestMockImageReplayer, BootstrapError) {
 600
 601   create_local_image();
 602   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 603
 604   journal::MockJournaler mock_remote_journaler;
 605   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 606   MockBootstrapRequest mock_bootstrap_request;
 607   MockReplayStatusFormatter mock_replay_status_formatter;
 608
 609   expect_get_or_send_update(mock_replay_status_formatter);
 610
 611   InSequence seq;
 612   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 613               "remote mirror uuid", 0);
 614   EXPECT_CALL(mock_remote_journaler, construct());
 615   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EINVAL);
 616
 617   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 618   expect_shut_down(mock_remote_journaler, 0);
 619
 620   MockImageDeleter mock_image_deleter;
 621   create_image_replayer(mock_image_deleter);
 622
 623   C_SaferCond start_ctx;
 624   m_image_replayer->start(&start_ctx);
 625   ASSERT_EQ(-EINVAL, start_ctx.wait());
 626 }
 627
 628 TEST_F(TestMockImageReplayer, StartExternalReplayError) {
 629   // START
 630
 631   create_local_image();
 632   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 633
 634   librbd::MockTestJournal mock_local_journal;
 635   mock_local_image_ctx.journal = &mock_local_journal;
 636
 637   journal::MockJournaler mock_remote_journaler;
 638   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 639   MockBootstrapRequest mock_bootstrap_request;
 640   MockReplay mock_local_replay;
 641   MockEventPreprocessor mock_event_preprocessor;
 642   MockReplayStatusFormatter mock_replay_status_formatter;
 643
 644   expect_get_or_send_update(mock_replay_status_formatter);
 645
 646   InSequence seq;
 647   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 648               "remote mirror uuid", 0);
 649   EXPECT_CALL(mock_remote_journaler, construct());
 650   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 651
 652   EXPECT_CALL(mock_local_journal, add_listener(_));
 653
 654   expect_init(mock_remote_journaler, 0);
 655
 656   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 657   expect_get_cached_client(mock_remote_journaler, 0);
 658
 659   expect_start_external_replay(mock_local_journal, nullptr, -EINVAL);
 660
 661   MockCloseImageRequest mock_close_local_image_request;
 662   EXPECT_CALL(mock_local_journal, remove_listener(_));
 663   expect_send(mock_close_local_image_request, 0);
 664
 665   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 666   expect_shut_down(mock_remote_journaler, 0);
 667
 668   MockImageDeleter mock_image_deleter;
 669   create_image_replayer(mock_image_deleter);
 670
 671   C_SaferCond start_ctx;
 672   m_image_replayer->start(&start_ctx);
 673   ASSERT_EQ(-EINVAL, start_ctx.wait());
 674   ASSERT_EQ(image_replayer::HEALTH_STATE_ERROR,
 675             m_image_replayer->get_health_state());
 676 }
 677
 678 TEST_F(TestMockImageReplayer, StopError) {
 679   // START
 680
 681   create_local_image();
 682   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 683
 684   librbd::MockTestJournal mock_local_journal;
 685   mock_local_image_ctx.journal = &mock_local_journal;
 686
 687   journal::MockJournaler mock_remote_journaler;
 688   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 689   MockBootstrapRequest mock_bootstrap_request;
 690   MockReplay mock_local_replay;
 691   MockEventPreprocessor mock_event_preprocessor;
 692   MockReplayStatusFormatter mock_replay_status_formatter;
 693
 694   expect_get_or_send_update(mock_replay_status_formatter);
 695
 696   InSequence seq;
 697   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 698               "remote mirror uuid", 0);
 699   EXPECT_CALL(mock_remote_journaler, construct());
 700   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 701
 702   EXPECT_CALL(mock_local_journal, add_listener(_));
 703
 704   expect_init(mock_remote_journaler, 0);
 705
 706   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 707   expect_get_cached_client(mock_remote_journaler, 0);
 708
 709   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 710
 711   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 712
 713   MockImageDeleter mock_image_deleter;
 714   create_image_replayer(mock_image_deleter);
 715
 716   C_SaferCond start_ctx;
 717   m_image_replayer->start(&start_ctx);
 718   ASSERT_EQ(0, start_ctx.wait());
 719
 720   // STOP (errors are ignored)
 721
 722   MockCloseImageRequest mock_close_local_image_request;
 723
 724   expect_shut_down(mock_local_replay, true, -EINVAL);
 725   EXPECT_CALL(mock_local_journal, remove_listener(_));
 726   EXPECT_CALL(mock_local_journal, stop_external_replay());
 727   expect_send(mock_close_local_image_request, -EINVAL);
 728
 729   expect_stop_replay(mock_remote_journaler, -EINVAL);
 730   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 731   expect_shut_down(mock_remote_journaler, -EINVAL);
 732
 733   C_SaferCond stop_ctx;
 734   m_image_replayer->stop(&stop_ctx);
 735   ASSERT_EQ(0, stop_ctx.wait());
 736 }
 737
 738 TEST_F(TestMockImageReplayer, Replay) {
 739   // START
 740
 741   create_local_image();
 742   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 743
 744   librbd::MockTestJournal mock_local_journal;
 745   mock_local_image_ctx.journal = &mock_local_journal;
 746
 747   journal::MockJournaler mock_remote_journaler;
 748   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 749   MockBootstrapRequest mock_bootstrap_request;
 750   MockReplay mock_local_replay;
 751   MockEventPreprocessor mock_event_preprocessor;
 752   MockReplayStatusFormatter mock_replay_status_formatter;
 753   ::journal::MockReplayEntry mock_replay_entry;
 754
 755   expect_get_or_send_update(mock_replay_status_formatter);
 756   expect_get_commit_tid_in_debug(mock_replay_entry);
 757   expect_committed(mock_remote_journaler, 2);
 758
 759   InSequence seq;
 760   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 761               "remote mirror uuid", 0);
 762   EXPECT_CALL(mock_remote_journaler, construct());
 763   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 764
 765   EXPECT_CALL(mock_local_journal, add_listener(_));
 766
 767   expect_init(mock_remote_journaler, 0);
 768
 769   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 770   expect_get_cached_client(mock_remote_journaler, 0);
 771
 772   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 773
 774   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 775
 776   MockImageDeleter mock_image_deleter;
 777   create_image_replayer(mock_image_deleter);
 778
 779   C_SaferCond start_ctx;
 780   m_image_replayer->start(&start_ctx);
 781   ASSERT_EQ(0, start_ctx.wait());
 782
 783   // REPLAY
 784
 785   cls::journal::Tag tag =
 786     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
 787                             librbd::Journal<>::LOCAL_MIRROR_UUID,
 788                             true, 0, 0})};
 789
 790   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 791
 792   // replay_flush
 793   expect_shut_down(mock_local_replay, false, 0);
 794   EXPECT_CALL(mock_local_journal, stop_external_replay());
 795   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 796   expect_get_tag(mock_remote_journaler, tag, 0);
 797   expect_allocate_tag(mock_local_journal, 0);
 798
 799   // process
 800   EXPECT_CALL(mock_replay_entry, get_data());
 801   EXPECT_CALL(mock_local_replay, decode(_, _))
 802     .WillOnce(Return(0));
 803   expect_preprocess(mock_event_preprocessor, false, 0);
 804   expect_process(mock_local_replay, 0, 0);
 805
 806   // the next event with preprocess
 807   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 808   EXPECT_CALL(mock_replay_entry, get_data());
 809   EXPECT_CALL(mock_local_replay, decode(_, _))
 810     .WillOnce(Return(0));
 811   expect_preprocess(mock_event_preprocessor, true, 0);
 812   expect_process(mock_local_replay, 0, 0);
 813
 814   // attempt to process the next event
 815   C_SaferCond replay_ctx;
 816   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
 817
 818   // fire
 819   m_image_replayer->handle_replay_ready();
 820   ASSERT_EQ(0, replay_ctx.wait());
 821
 822   // STOP
 823
 824   MockCloseImageRequest mock_close_local_image_request;
 825   expect_shut_down(mock_local_replay, true, 0);
 826   EXPECT_CALL(mock_local_journal, remove_listener(_));
 827   EXPECT_CALL(mock_local_journal, stop_external_replay());
 828   expect_send(mock_close_local_image_request, 0);
 829
 830   expect_stop_replay(mock_remote_journaler, 0);
 831   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 832   expect_shut_down(mock_remote_journaler, 0);
 833
 834   C_SaferCond stop_ctx;
 835   m_image_replayer->stop(&stop_ctx);
 836   ASSERT_EQ(0, stop_ctx.wait());
 837 }
 838
 839 TEST_F(TestMockImageReplayer, DecodeError) {
 840   // START
 841
 842   create_local_image();
 843   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 844
 845   librbd::MockTestJournal mock_local_journal;
 846   mock_local_image_ctx.journal = &mock_local_journal;
 847
 848   journal::MockJournaler mock_remote_journaler;
 849   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 850   MockBootstrapRequest mock_bootstrap_request;
 851   MockReplay mock_local_replay;
 852   MockEventPreprocessor mock_event_preprocessor;
 853   MockReplayStatusFormatter mock_replay_status_formatter;
 854   ::journal::MockReplayEntry mock_replay_entry;
 855
 856   expect_get_or_send_update(mock_replay_status_formatter);
 857   expect_get_commit_tid_in_debug(mock_replay_entry);
 858
 859   InSequence seq;
 860   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 861               "remote mirror uuid", 0);
 862   EXPECT_CALL(mock_remote_journaler, construct());
 863   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 864
 865   EXPECT_CALL(mock_local_journal, add_listener(_));
 866
 867   expect_init(mock_remote_journaler, 0);
 868
 869   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 870   expect_get_cached_client(mock_remote_journaler, 0);
 871
 872   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 873
 874   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 875
 876   MockImageDeleter mock_image_deleter;
 877   create_image_replayer(mock_image_deleter);
 878
 879   C_SaferCond start_ctx;
 880   m_image_replayer->start(&start_ctx);
 881   ASSERT_EQ(0, start_ctx.wait());
 882
 883   // REPLAY
 884
 885   cls::journal::Tag tag =
 886     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
 887                             librbd::Journal<>::LOCAL_MIRROR_UUID,
 888                             true, 0, 0})};
 889
 890   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 891
 892   // replay_flush
 893   expect_shut_down(mock_local_replay, false, 0);
 894   EXPECT_CALL(mock_local_journal, stop_external_replay());
 895   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 896   expect_get_tag(mock_remote_journaler, tag, 0);
 897   expect_allocate_tag(mock_local_journal, 0);
 898
 899   // process
 900   EXPECT_CALL(mock_replay_entry, get_data());
 901   EXPECT_CALL(mock_local_replay, decode(_, _))
 902     .WillOnce(Return(-EINVAL));
 903
 904   // stop on error
 905   expect_shut_down(mock_local_replay, true, 0);
 906   EXPECT_CALL(mock_local_journal, remove_listener(_));
 907   EXPECT_CALL(mock_local_journal, stop_external_replay());
 908
 909   MockCloseImageRequest mock_close_local_image_request;
 910   C_SaferCond close_ctx;
 911   EXPECT_CALL(mock_close_local_image_request, send())
 912     .WillOnce(Invoke([&mock_close_local_image_request, &close_ctx]() {
 913           *mock_close_local_image_request.image_ctx = nullptr;
 914           mock_close_local_image_request.on_finish->complete(0);
 915           close_ctx.complete(0);
 916         }));
 917
 918   expect_stop_replay(mock_remote_journaler, 0);
 919   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 920   expect_shut_down(mock_remote_journaler, 0);
 921
 922   // fire
 923   m_image_replayer->handle_replay_ready();
 924   ASSERT_EQ(0, close_ctx.wait());
 925
 926   while (!m_image_replayer->is_stopped()) {
 927     usleep(1000);
 928   }
 929 }
 930
 931 TEST_F(TestMockImageReplayer, DelayedReplay) {
 932
 933   // START
 934
 935   create_local_image();
 936   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 937
 938   librbd::MockTestJournal mock_local_journal;
 939   mock_local_image_ctx.journal = &mock_local_journal;
 940
 941   journal::MockJournaler mock_remote_journaler;
 942   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 943   MockBootstrapRequest mock_bootstrap_request;
 944   MockReplay mock_local_replay;
 945   MockEventPreprocessor mock_event_preprocessor;
 946   MockReplayStatusFormatter mock_replay_status_formatter;
 947   ::journal::MockReplayEntry mock_replay_entry;
 948
 949   expect_get_or_send_update(mock_replay_status_formatter);
 950   expect_get_commit_tid_in_debug(mock_replay_entry);
 951   expect_committed(mock_remote_journaler, 1);
 952
 953   InSequence seq;
 954   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 955               "remote mirror uuid", 0);
 956   EXPECT_CALL(mock_remote_journaler, construct());
 957   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 958
 959   EXPECT_CALL(mock_local_journal, add_listener(_));
 960
 961   expect_init(mock_remote_journaler, 0);
 962
 963   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 964   expect_get_cached_client(mock_remote_journaler, 0);
 965
 966   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 967
 968   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 969
 970   MockImageDeleter mock_image_deleter;
 971   create_image_replayer(mock_image_deleter);
 972
 973   C_SaferCond start_ctx;
 974   m_image_replayer->start(&start_ctx);
 975   ASSERT_EQ(0, start_ctx.wait());
 976
 977   // REPLAY
 978
 979   cls::journal::Tag tag =
 980     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
 981                             librbd::Journal<>::LOCAL_MIRROR_UUID,
 982                             true, 0, 0})};
 983
 984   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 985
 986   // replay_flush
 987   expect_shut_down(mock_local_replay, false, 0);
 988   EXPECT_CALL(mock_local_journal, stop_external_replay());
 989   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 990   expect_get_tag(mock_remote_journaler, tag, 0);
 991   expect_allocate_tag(mock_local_journal, 0);
 992
 993   // process with delay
 994   EXPECT_CALL(mock_replay_entry, get_data());
 995   librbd::journal::EventEntry event_entry(
 996     librbd::journal::AioDiscardEvent(123, 345, false), ceph_clock_now());
 997   EXPECT_CALL(mock_local_replay, decode(_, _))
 998     .WillOnce(DoAll(SetArgPointee<1>(event_entry),
 999                     Return(0)));
1000   expect_preprocess(mock_event_preprocessor, false, 0);
1001   expect_process(mock_local_replay, 0, 0);
1002
1003   // attempt to process the next event
1004   C_SaferCond replay_ctx;
1005   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1006
1007   // fire
1008   mock_local_image_ctx.mirroring_replay_delay = 2;
1009   m_image_replayer->handle_replay_ready();
1010   ASSERT_EQ(0, replay_ctx.wait());
1011
1012   // add a pending (delayed) entry before stop
1013   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1014   EXPECT_CALL(mock_replay_entry, get_data());
1015   C_SaferCond decode_ctx;
1016   EXPECT_CALL(mock_local_replay, decode(_, _))
1017     .WillOnce(DoAll(Invoke([&decode_ctx](bufferlist::iterator* it,
1018                                          librbd::journal::EventEntry *e) {
1019                              decode_ctx.complete(0);
1020                            }),
1021                     Return(0)));
1022
1023   mock_local_image_ctx.mirroring_replay_delay = 10;
1024   m_image_replayer->handle_replay_ready();
1025   ASSERT_EQ(0, decode_ctx.wait());
1026
1027   // STOP
1028
1029   MockCloseImageRequest mock_close_local_image_request;
1030
1031   expect_shut_down(mock_local_replay, true, 0);
1032   EXPECT_CALL(mock_local_journal, remove_listener(_));
1033   EXPECT_CALL(mock_local_journal, stop_external_replay());
1034   expect_send(mock_close_local_image_request, 0);
1035
1036   expect_stop_replay(mock_remote_journaler, 0);
1037   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1038   expect_shut_down(mock_remote_journaler, 0);
1039
1040   C_SaferCond stop_ctx;
1041   m_image_replayer->stop(&stop_ctx);
1042   ASSERT_EQ(0, stop_ctx.wait());
1043 }
1044
1045 } // namespace mirror
1046 } // namespace rbd