ceph/src/test/rbd_mirror/test_mock_ImageReplayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "cls/journal/cls_journal_types.h"
   5 #include "librbd/journal/Replay.h"
   6 #include "librbd/journal/Types.h"
   7 #include "tools/rbd_mirror/ImageReplayer.h"
   8 #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
   9 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
  10 #include "tools/rbd_mirror/image_replayer/EventPreprocessor.h"
  11 #include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h"
  12 #include "tools/rbd_mirror/ImageSyncThrottler.h"
  13 #include "test/rbd_mirror/test_mock_fixture.h"
  14 #include "test/journal/mock/MockJournaler.h"
  15 #include "test/librbd/mock/MockImageCtx.h"
  16 #include "test/librbd/mock/MockJournal.h"
  17
  18 namespace librbd {
  19
  20 namespace {
  21
  22 struct MockTestJournal;
  23
  24 struct MockTestImageCtx : public MockImageCtx {
  25   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  26     : librbd::MockImageCtx(image_ctx) {
  27   }
  28   MockTestJournal *journal = nullptr;
  29 };
  30
  31 struct MockTestJournal : public MockJournal {
  32   MOCK_METHOD2(start_external_replay, void(journal::Replay<MockTestImageCtx> **,
  33                                            Context *on_start));
  34   MOCK_METHOD0(stop_external_replay, void());
  35 };
  36
  37 } // anonymous namespace
  38
  39 namespace journal {
  40
  41 template<>
  42 struct Replay<MockTestImageCtx> {
  43   MOCK_METHOD2(decode, int(bufferlist::iterator *, EventEntry *));
  44   MOCK_METHOD3(process, void(const EventEntry &, Context *, Context *));
  45   MOCK_METHOD1(flush, void(Context*));
  46   MOCK_METHOD2(shut_down, void(bool, Context*));
  47 };
  48
  49 template <>
  50 struct TypeTraits<MockTestImageCtx> {
  51   typedef ::journal::MockJournalerProxy Journaler;
  52   typedef ::journal::MockReplayEntryProxy ReplayEntry;
  53 };
  54
  55 struct MirrorPeerClientMeta;
  56
  57 } // namespace journal
  58 } // namespace librbd
  59
  60 namespace rbd {
  61 namespace mirror {
  62
  63 template<>
  64 class ImageSync<librbd::MockTestImageCtx> {
  65 public:
  66   static ImageSync* create(librbd::MockTestImageCtx *local_image_ctx,
  67                            librbd::MockTestImageCtx *remote_image_ctx,
  68                            SafeTimer *timer, Mutex *timer_lock,
  69                            const std::string &mirror_uuid,
  70                            journal::MockJournaler *journaler,
  71                            librbd::journal::MirrorPeerClientMeta *client_meta,
  72                            ContextWQ *work_queue, Context *on_finish,
  73                            ProgressContext *progress_ctx = nullptr) {
  74     assert(0 == "unexpected call");
  75     return nullptr;
  76   }
  77
  78   void send() {
  79   }
  80 };
  81
  82 namespace image_replayer {
  83
  84 using ::testing::_;
  85 using ::testing::AtLeast;
  86 using ::testing::DoAll;
  87 using ::testing::InSequence;
  88 using ::testing::Invoke;
  89 using ::testing::MatcherCast;
  90 using ::testing::Return;
  91 using ::testing::SetArgPointee;
  92 using ::testing::WithArg;
  93
  94 template<>
  95 struct PrepareLocalImageRequest<librbd::MockTestImageCtx> {
  96   static PrepareLocalImageRequest* s_instance;
  97   std::string *local_image_id = nullptr;
  98   std::string *tag_owner = nullptr;
  99   Context *on_finish = nullptr;
 100
 101   static PrepareLocalImageRequest* create(librados::IoCtx &,
 102                                           const std::string &global_image_id,
 103                                           std::string *local_image_id,
 104                                           std::string *tag_owner,
 105                                           ContextWQ *work_queue,
 106                                           Context *on_finish) {
 107     assert(s_instance != nullptr);
 108     s_instance->local_image_id = local_image_id;
 109     s_instance->tag_owner = tag_owner;
 110     s_instance->on_finish = on_finish;
 111     return s_instance;
 112   }
 113
 114   PrepareLocalImageRequest() {
 115     s_instance = this;
 116   }
 117
 118   MOCK_METHOD0(send, void());
 119 };
 120
 121 template<>
 122 struct BootstrapRequest<librbd::MockTestImageCtx> {
 123   static BootstrapRequest* s_instance;
 124   librbd::MockTestImageCtx **image_ctx = nullptr;
 125   Context *on_finish = nullptr;
 126   bool *do_resync = nullptr;
 127
 128   static BootstrapRequest* create(librados::IoCtx &local_io_ctx,
 129         librados::IoCtx &remote_io_ctx,
 130         rbd::mirror::ImageSyncThrottlerRef<librbd::MockTestImageCtx> image_sync_throttler,
 131         librbd::MockTestImageCtx **local_image_ctx,
 132         const std::string &local_image_name,
 133         const std::string &remote_image_id,
 134         const std::string &global_image_id,
 135         ContextWQ *work_queue, SafeTimer *timer,
 136         Mutex *timer_lock,
 137         const std::string &local_mirror_uuid,
 138         const std::string &remote_mirror_uuid,
 139         ::journal::MockJournalerProxy *journaler,
 140         librbd::journal::MirrorPeerClientMeta *client_meta,
 141         Context *on_finish,
 142         bool *do_resync,
 143         rbd::mirror::ProgressContext *progress_ctx = nullptr) {
 144     assert(s_instance != nullptr);
 145     s_instance->image_ctx = local_image_ctx;
 146     s_instance->on_finish = on_finish;
 147     s_instance->do_resync = do_resync;
 148     return s_instance;
 149   }
 150
 151   BootstrapRequest() {
 152     assert(s_instance == nullptr);
 153     s_instance = this;
 154   }
 155
 156   ~BootstrapRequest() {
 157     assert(s_instance == this);
 158     s_instance = nullptr;
 159   }
 160
 161   void put() {
 162   }
 163
 164   void get() {
 165   }
 166
 167   MOCK_METHOD0(send, void());
 168   MOCK_METHOD0(cancel, void());
 169 };
 170
 171 template<>
 172 struct CloseImageRequest<librbd::MockTestImageCtx> {
 173   static CloseImageRequest* s_instance;
 174   librbd::MockTestImageCtx **image_ctx = nullptr;
 175   Context *on_finish = nullptr;
 176
 177   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
 178                                    Context *on_finish) {
 179     assert(s_instance != nullptr);
 180     s_instance->image_ctx = image_ctx;
 181     s_instance->on_finish = on_finish;
 182     return s_instance;
 183   }
 184
 185   CloseImageRequest() {
 186     assert(s_instance == nullptr);
 187     s_instance = this;
 188   }
 189
 190   ~CloseImageRequest() {
 191     assert(s_instance == this);
 192     s_instance = nullptr;
 193   }
 194
 195   MOCK_METHOD0(send, void());
 196 };
 197
 198 template<>
 199 struct EventPreprocessor<librbd::MockTestImageCtx> {
 200   static EventPreprocessor *s_instance;
 201
 202   static EventPreprocessor *create(librbd::MockTestImageCtx &local_image_ctx,
 203                                    ::journal::MockJournalerProxy &remote_journaler,
 204                                    const std::string &local_mirror_uuid,
 205                                    librbd::journal::MirrorPeerClientMeta *client_meta,
 206                                    ContextWQ *work_queue) {
 207     assert(s_instance != nullptr);
 208     return s_instance;
 209   }
 210
 211   static void destroy(EventPreprocessor* processor) {
 212   }
 213
 214   EventPreprocessor() {
 215     assert(s_instance == nullptr);
 216     s_instance = this;
 217   }
 218
 219   ~EventPreprocessor() {
 220     assert(s_instance == this);
 221     s_instance = nullptr;
 222   }
 223
 224   MOCK_METHOD1(is_required, bool(const librbd::journal::EventEntry &));
 225   MOCK_METHOD2(preprocess, void(librbd::journal::EventEntry *, Context *));
 226 };
 227
 228 template<>
 229 struct ReplayStatusFormatter<librbd::MockTestImageCtx> {
 230   static ReplayStatusFormatter* s_instance;
 231
 232   static ReplayStatusFormatter* create(::journal::MockJournalerProxy *journaler,
 233                                        const std::string &mirror_uuid) {
 234     assert(s_instance != nullptr);
 235     return s_instance;
 236   }
 237
 238   static void destroy(ReplayStatusFormatter* formatter) {
 239   }
 240
 241   ReplayStatusFormatter() {
 242     assert(s_instance == nullptr);
 243     s_instance = this;
 244   }
 245
 246   ~ReplayStatusFormatter() {
 247     assert(s_instance == this);
 248     s_instance = nullptr;
 249   }
 250
 251   MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish));
 252 };
 253
 254 BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 255 CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 256 EventPreprocessor<librbd::MockTestImageCtx>* EventPreprocessor<librbd::MockTestImageCtx>::s_instance = nullptr;
 257 PrepareLocalImageRequest<librbd::MockTestImageCtx>* PrepareLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 258 ReplayStatusFormatter<librbd::MockTestImageCtx>* ReplayStatusFormatter<librbd::MockTestImageCtx>::s_instance = nullptr;
 259
 260 } // namespace image_replayer
 261 } // namespace mirror
 262 } // namespace rbd
 263
 264 // template definitions
 265 #include "tools/rbd_mirror/ImageReplayer.cc"
 266 #include "tools/rbd_mirror/ImageSyncThrottler.cc"
 267
 268 namespace rbd {
 269 namespace mirror {
 270
 271 class TestMockImageReplayer : public TestMockFixture {
 272 public:
 273   typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
 274   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
 275   typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor;
 276   typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest;
 277   typedef ReplayStatusFormatter<librbd::MockTestImageCtx> MockReplayStatusFormatter;
 278   typedef librbd::journal::Replay<librbd::MockTestImageCtx> MockReplay;
 279   typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer;
 280
 281   void SetUp() override {
 282     TestMockFixture::SetUp();
 283
 284     librbd::RBD rbd;
 285     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size));
 286     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx));
 287
 288     m_image_deleter.reset(new rbd::mirror::ImageDeleter(m_threads->work_queue,
 289                                                         m_threads->timer,
 290                                                         &m_threads->timer_lock));
 291     m_image_sync_throttler.reset(
 292       new rbd::mirror::ImageSyncThrottler<librbd::MockTestImageCtx>());
 293
 294     m_image_replayer = new MockImageReplayer(
 295       m_threads, m_image_deleter, m_image_sync_throttler,
 296       rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
 297       "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id");
 298     m_image_replayer->add_remote_image(
 299       "remote_mirror_uuid", m_remote_image_ctx->id, m_remote_io_ctx);
 300   }
 301
 302   void TearDown() override {
 303     delete m_image_replayer;
 304
 305     TestMockFixture::TearDown();
 306   }
 307
 308   void create_local_image() {
 309     librbd::RBD rbd;
 310     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 311     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 312   }
 313
 314   bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) {
 315     bufferlist bl;
 316     ::encode(tag_data, bl);
 317     return bl;
 318   }
 319
 320   void expect_get_or_send_update(
 321     MockReplayStatusFormatter &mock_replay_status_formatter) {
 322     EXPECT_CALL(mock_replay_status_formatter, get_or_send_update(_, _))
 323       .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)),
 324                             Return(true)));
 325   }
 326
 327   void expect_send(MockPrepareLocalImageRequest &mock_request,
 328                    const std::string &local_image_id,
 329                    const std::string &tag_owner,
 330                    int r) {
 331     EXPECT_CALL(mock_request, send())
 332       .WillOnce(Invoke([&mock_request, local_image_id, tag_owner, r]() {
 333           if (r == 0) {
 334             *mock_request.local_image_id = local_image_id;
 335             *mock_request.tag_owner = tag_owner;
 336           }
 337           mock_request.on_finish->complete(r);
 338         }));
 339   }
 340
 341   void expect_send(MockBootstrapRequest &mock_bootstrap_request,
 342                    librbd::MockTestImageCtx &mock_local_image_ctx,
 343                    bool do_resync, int r) {
 344     EXPECT_CALL(mock_bootstrap_request, send())
 345       .WillOnce(Invoke([&mock_bootstrap_request, &mock_local_image_ctx,
 346                         do_resync, r]() {
 347             if (r == 0) {
 348               *mock_bootstrap_request.image_ctx = &mock_local_image_ctx;
 349               *mock_bootstrap_request.do_resync = do_resync;
 350             }
 351             mock_bootstrap_request.on_finish->complete(r);
 352           }));
 353   }
 354
 355   void expect_start_external_replay(librbd::MockTestJournal &mock_journal,
 356                                     MockReplay *mock_replay, int r) {
 357     EXPECT_CALL(mock_journal, start_external_replay(_, _))
 358       .WillOnce(DoAll(SetArgPointee<0>(mock_replay),
 359                       WithArg<1>(CompleteContext(r))));
 360   }
 361
 362   void expect_init(::journal::MockJournaler &mock_journaler, int r) {
 363     EXPECT_CALL(mock_journaler, init(_))
 364       .WillOnce(CompleteContext(r));
 365   }
 366
 367   void expect_get_cached_client(::journal::MockJournaler &mock_journaler,
 368                                 int r) {
 369     librbd::journal::ImageClientMeta image_client_meta;
 370     image_client_meta.tag_class = 0;
 371
 372     librbd::journal::ClientData client_data;
 373     client_data.client_meta = image_client_meta;
 374
 375     cls::journal::Client client;
 376     ::encode(client_data, client.data);
 377
 378     EXPECT_CALL(mock_journaler, get_cached_client("local_mirror_uuid", _))
 379       .WillOnce(DoAll(SetArgPointee<1>(client),
 380                       Return(r)));
 381   }
 382
 383   void expect_stop_replay(::journal::MockJournaler &mock_journaler, int r) {
 384     EXPECT_CALL(mock_journaler, stop_replay(_))
 385       .WillOnce(CompleteContext(r));
 386   }
 387
 388   void expect_shut_down(MockReplay &mock_replay, bool cancel_ops, int r) {
 389     EXPECT_CALL(mock_replay, shut_down(cancel_ops, _))
 390       .WillOnce(WithArg<1>(CompleteContext(r)));
 391   }
 392
 393   void expect_shut_down(journal::MockJournaler &mock_journaler, int r) {
 394     EXPECT_CALL(mock_journaler, shut_down(_))
 395       .WillOnce(CompleteContext(r));
 396   }
 397
 398   void expect_send(MockCloseImageRequest &mock_close_image_request, int r) {
 399     EXPECT_CALL(mock_close_image_request, send())
 400       .WillOnce(Invoke([&mock_close_image_request, r]() {
 401             *mock_close_image_request.image_ctx = nullptr;
 402             mock_close_image_request.on_finish->complete(r);
 403           }));
 404   }
 405
 406   void expect_get_commit_tid_in_debug(
 407     ::journal::MockReplayEntry &mock_replay_entry) {
 408     // It is used in debug messages and depends on debug level
 409     EXPECT_CALL(mock_replay_entry, get_commit_tid())
 410     .Times(AtLeast(0))
 411       .WillRepeatedly(Return(0));
 412   }
 413
 414   void expect_committed(::journal::MockJournaler &mock_journaler, int times) {
 415     EXPECT_CALL(mock_journaler, committed(
 416                   MatcherCast<const ::journal::MockReplayEntryProxy&>(_)))
 417       .Times(times);
 418   }
 419
 420   void expect_try_pop_front(::journal::MockJournaler &mock_journaler,
 421                             uint64_t replay_tag_tid, bool entries_available) {
 422     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 423       .WillOnce(DoAll(SetArgPointee<0>(::journal::MockReplayEntryProxy()),
 424                       SetArgPointee<1>(replay_tag_tid),
 425                       Return(entries_available)));
 426   }
 427
 428   void expect_try_pop_front_return_no_entries(
 429     ::journal::MockJournaler &mock_journaler, Context *on_finish) {
 430     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 431       .WillOnce(DoAll(Invoke([on_finish](::journal::MockReplayEntryProxy *e,
 432                                          uint64_t *t) {
 433                                on_finish->complete(0);
 434                              }),
 435                       Return(false)));
 436   }
 437
 438   void expect_get_tag(::journal::MockJournaler &mock_journaler,
 439                       const cls::journal::Tag &tag, int r) {
 440     EXPECT_CALL(mock_journaler, get_tag(_, _, _))
 441       .WillOnce(DoAll(SetArgPointee<1>(tag),
 442                       WithArg<2>(CompleteContext(r))));
 443   }
 444
 445   void expect_allocate_tag(librbd::MockTestJournal &mock_journal, int r) {
 446     EXPECT_CALL(mock_journal, allocate_tag(_, _, _))
 447       .WillOnce(WithArg<2>(CompleteContext(r)));
 448   }
 449
 450   void expect_preprocess(MockEventPreprocessor &mock_event_preprocessor,
 451                          bool required, int r) {
 452     EXPECT_CALL(mock_event_preprocessor, is_required(_))
 453       .WillOnce(Return(required));
 454     if (required) {
 455       EXPECT_CALL(mock_event_preprocessor, preprocess(_, _))
 456         .WillOnce(WithArg<1>(CompleteContext(r)));
 457     }
 458   }
 459
 460   void expect_process(MockReplay &mock_replay,
 461                       int on_ready_r, int on_commit_r) {
 462     EXPECT_CALL(mock_replay, process(_, _, _))
 463       .WillOnce(DoAll(WithArg<1>(CompleteContext(on_ready_r)),
 464                       WithArg<2>(CompleteContext(on_commit_r))));
 465   }
 466
 467   librbd::ImageCtx *m_remote_image_ctx;
 468   librbd::ImageCtx *m_local_image_ctx = nullptr;
 469   std::shared_ptr<rbd::mirror::ImageDeleter> m_image_deleter;
 470   std::shared_ptr<rbd::mirror::ImageSyncThrottler<librbd::MockTestImageCtx>> m_image_sync_throttler;
 471   MockImageReplayer *m_image_replayer;
 472 };
 473
 474 TEST_F(TestMockImageReplayer, StartStop) {
 475   // START
 476
 477   create_local_image();
 478   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 479
 480   librbd::MockTestJournal mock_local_journal;
 481   mock_local_image_ctx.journal = &mock_local_journal;
 482
 483   journal::MockJournaler mock_remote_journaler;
 484   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 485   MockBootstrapRequest mock_bootstrap_request;
 486   MockReplay mock_local_replay;
 487   MockEventPreprocessor mock_event_preprocessor;
 488   MockReplayStatusFormatter mock_replay_status_formatter;
 489
 490   expect_get_or_send_update(mock_replay_status_formatter);
 491
 492   InSequence seq;
 493   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 494               "remote mirror uuid", 0);
 495   EXPECT_CALL(mock_remote_journaler, construct());
 496   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 497
 498   EXPECT_CALL(mock_local_journal, add_listener(_));
 499
 500   expect_init(mock_remote_journaler, 0);
 501
 502   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 503   expect_get_cached_client(mock_remote_journaler, 0);
 504
 505   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 506
 507   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 508
 509   C_SaferCond start_ctx;
 510   m_image_replayer->start(&start_ctx);
 511   ASSERT_EQ(0, start_ctx.wait());
 512
 513   // STOP
 514
 515   MockCloseImageRequest mock_close_local_image_request;
 516
 517   expect_stop_replay(mock_remote_journaler, 0);
 518   expect_shut_down(mock_local_replay, true, 0);
 519
 520   EXPECT_CALL(mock_local_journal, remove_listener(_));
 521   EXPECT_CALL(mock_local_journal, stop_external_replay());
 522
 523   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 524   expect_shut_down(mock_remote_journaler, 0);
 525
 526   expect_send(mock_close_local_image_request, 0);
 527
 528   C_SaferCond stop_ctx;
 529   m_image_replayer->stop(&stop_ctx);
 530   ASSERT_EQ(0, stop_ctx.wait());
 531 }
 532
 533 TEST_F(TestMockImageReplayer, LocalImagePrimary) {
 534   create_local_image();
 535   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 536
 537   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 538   MockReplayStatusFormatter mock_replay_status_formatter;
 539
 540   expect_get_or_send_update(mock_replay_status_formatter);
 541
 542   InSequence seq;
 543   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 544               "", 0);
 545
 546   C_SaferCond start_ctx;
 547   m_image_replayer->start(&start_ctx);
 548   ASSERT_EQ(0, start_ctx.wait());
 549 }
 550
 551 TEST_F(TestMockImageReplayer, LocalImageDNE) {
 552   create_local_image();
 553   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 554
 555   journal::MockJournaler mock_remote_journaler;
 556   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 557   MockBootstrapRequest mock_bootstrap_request;
 558   MockReplayStatusFormatter mock_replay_status_formatter;
 559
 560   expect_get_or_send_update(mock_replay_status_formatter);
 561
 562   InSequence seq;
 563   expect_send(mock_prepare_local_image_request, "", "", -ENOENT);
 564   EXPECT_CALL(mock_remote_journaler, construct());
 565   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EREMOTEIO);
 566
 567   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 568   expect_shut_down(mock_remote_journaler, 0);
 569
 570   C_SaferCond start_ctx;
 571   m_image_replayer->start(&start_ctx);
 572   ASSERT_EQ(0, start_ctx.wait());
 573 }
 574
 575 TEST_F(TestMockImageReplayer, PrepareLocalImageError) {
 576   create_local_image();
 577   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 578
 579   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 580   MockReplayStatusFormatter mock_replay_status_formatter;
 581
 582   expect_get_or_send_update(mock_replay_status_formatter);
 583
 584   InSequence seq;
 585   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 586               "remote mirror uuid", -EINVAL);
 587
 588   C_SaferCond start_ctx;
 589   m_image_replayer->start(&start_ctx);
 590   ASSERT_EQ(-EINVAL, start_ctx.wait());
 591 }
 592
 593 TEST_F(TestMockImageReplayer, BootstrapError) {
 594
 595   create_local_image();
 596   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 597
 598   journal::MockJournaler mock_remote_journaler;
 599   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 600   MockBootstrapRequest mock_bootstrap_request;
 601   MockReplayStatusFormatter mock_replay_status_formatter;
 602
 603   expect_get_or_send_update(mock_replay_status_formatter);
 604
 605   InSequence seq;
 606   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 607               "remote mirror uuid", 0);
 608   EXPECT_CALL(mock_remote_journaler, construct());
 609   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EINVAL);
 610
 611   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 612   expect_shut_down(mock_remote_journaler, 0);
 613
 614   C_SaferCond start_ctx;
 615   m_image_replayer->start(&start_ctx);
 616   ASSERT_EQ(-EINVAL, start_ctx.wait());
 617 }
 618
 619 TEST_F(TestMockImageReplayer, StartExternalReplayError) {
 620   // START
 621
 622   create_local_image();
 623   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 624
 625   librbd::MockTestJournal mock_local_journal;
 626   mock_local_image_ctx.journal = &mock_local_journal;
 627
 628   journal::MockJournaler mock_remote_journaler;
 629   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 630   MockBootstrapRequest mock_bootstrap_request;
 631   MockReplay mock_local_replay;
 632   MockEventPreprocessor mock_event_preprocessor;
 633   MockReplayStatusFormatter mock_replay_status_formatter;
 634
 635   expect_get_or_send_update(mock_replay_status_formatter);
 636
 637   InSequence seq;
 638   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 639               "remote mirror uuid", 0);
 640   EXPECT_CALL(mock_remote_journaler, construct());
 641   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 642
 643   EXPECT_CALL(mock_local_journal, add_listener(_));
 644
 645   expect_init(mock_remote_journaler, 0);
 646
 647   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 648   expect_get_cached_client(mock_remote_journaler, 0);
 649
 650   expect_start_external_replay(mock_local_journal, nullptr, -EINVAL);
 651
 652   MockCloseImageRequest mock_close_local_image_request;
 653
 654   EXPECT_CALL(mock_local_journal, remove_listener(_));
 655
 656   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 657   expect_shut_down(mock_remote_journaler, 0);
 658
 659   expect_send(mock_close_local_image_request, 0);
 660
 661   C_SaferCond start_ctx;
 662   m_image_replayer->start(&start_ctx);
 663   ASSERT_EQ(-EINVAL, start_ctx.wait());
 664 }
 665
 666 TEST_F(TestMockImageReplayer, StopError) {
 667   // START
 668
 669   create_local_image();
 670   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 671
 672   librbd::MockTestJournal mock_local_journal;
 673   mock_local_image_ctx.journal = &mock_local_journal;
 674
 675   journal::MockJournaler mock_remote_journaler;
 676   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 677   MockBootstrapRequest mock_bootstrap_request;
 678   MockReplay mock_local_replay;
 679   MockEventPreprocessor mock_event_preprocessor;
 680   MockReplayStatusFormatter mock_replay_status_formatter;
 681
 682   expect_get_or_send_update(mock_replay_status_formatter);
 683
 684   InSequence seq;
 685   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 686               "remote mirror uuid", 0);
 687   EXPECT_CALL(mock_remote_journaler, construct());
 688   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 689
 690   EXPECT_CALL(mock_local_journal, add_listener(_));
 691
 692   expect_init(mock_remote_journaler, 0);
 693
 694   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 695   expect_get_cached_client(mock_remote_journaler, 0);
 696
 697   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 698
 699   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 700
 701   C_SaferCond start_ctx;
 702   m_image_replayer->start(&start_ctx);
 703   ASSERT_EQ(0, start_ctx.wait());
 704
 705   // STOP (errors are ignored)
 706
 707   MockCloseImageRequest mock_close_local_image_request;
 708
 709   expect_stop_replay(mock_remote_journaler, -EINVAL);
 710   expect_shut_down(mock_local_replay, true, -EINVAL);
 711
 712   EXPECT_CALL(mock_local_journal, remove_listener(_));
 713   EXPECT_CALL(mock_local_journal, stop_external_replay());
 714
 715   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 716   expect_shut_down(mock_remote_journaler, -EINVAL);
 717
 718   expect_send(mock_close_local_image_request, -EINVAL);
 719
 720   C_SaferCond stop_ctx;
 721   m_image_replayer->stop(&stop_ctx);
 722   ASSERT_EQ(0, stop_ctx.wait());
 723 }
 724
 725 TEST_F(TestMockImageReplayer, Replay) {
 726   // START
 727
 728   create_local_image();
 729   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 730
 731   librbd::MockTestJournal mock_local_journal;
 732   mock_local_image_ctx.journal = &mock_local_journal;
 733
 734   journal::MockJournaler mock_remote_journaler;
 735   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 736   MockBootstrapRequest mock_bootstrap_request;
 737   MockReplay mock_local_replay;
 738   MockEventPreprocessor mock_event_preprocessor;
 739   MockReplayStatusFormatter mock_replay_status_formatter;
 740   ::journal::MockReplayEntry mock_replay_entry;
 741
 742   expect_get_or_send_update(mock_replay_status_formatter);
 743   expect_get_commit_tid_in_debug(mock_replay_entry);
 744   expect_committed(mock_remote_journaler, 2);
 745
 746   InSequence seq;
 747   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 748               "remote mirror uuid", 0);
 749   EXPECT_CALL(mock_remote_journaler, construct());
 750   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 751
 752   EXPECT_CALL(mock_local_journal, add_listener(_));
 753
 754   expect_init(mock_remote_journaler, 0);
 755
 756   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 757   expect_get_cached_client(mock_remote_journaler, 0);
 758
 759   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 760
 761   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 762
 763   C_SaferCond start_ctx;
 764   m_image_replayer->start(&start_ctx);
 765   ASSERT_EQ(0, start_ctx.wait());
 766
 767   // REPLAY
 768
 769   cls::journal::Tag tag =
 770     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
 771                             librbd::Journal<>::LOCAL_MIRROR_UUID,
 772                             true, 0, 0})};
 773
 774   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 775
 776   // replay_flush
 777   expect_shut_down(mock_local_replay, false, 0);
 778   EXPECT_CALL(mock_local_journal, stop_external_replay());
 779   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 780   expect_get_tag(mock_remote_journaler, tag, 0);
 781   expect_allocate_tag(mock_local_journal, 0);
 782
 783   // process
 784   EXPECT_CALL(mock_replay_entry, get_data());
 785   EXPECT_CALL(mock_local_replay, decode(_, _))
 786     .WillOnce(Return(0));
 787   expect_preprocess(mock_event_preprocessor, false, 0);
 788   expect_process(mock_local_replay, 0, 0);
 789
 790   // the next event with preprocess
 791   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 792   EXPECT_CALL(mock_replay_entry, get_data());
 793   EXPECT_CALL(mock_local_replay, decode(_, _))
 794     .WillOnce(Return(0));
 795   expect_preprocess(mock_event_preprocessor, true, 0);
 796   expect_process(mock_local_replay, 0, 0);
 797
 798   // attempt to process the next event
 799   C_SaferCond replay_ctx;
 800   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
 801
 802   // fire
 803   m_image_replayer->handle_replay_ready();
 804   ASSERT_EQ(0, replay_ctx.wait());
 805
 806   // STOP
 807
 808   MockCloseImageRequest mock_close_local_image_request;
 809
 810   expect_stop_replay(mock_remote_journaler, 0);
 811   expect_shut_down(mock_local_replay, true, 0);
 812
 813   EXPECT_CALL(mock_local_journal, remove_listener(_));
 814   EXPECT_CALL(mock_local_journal, stop_external_replay());
 815
 816   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 817   expect_shut_down(mock_remote_journaler, 0);
 818
 819   expect_send(mock_close_local_image_request, 0);
 820
 821   C_SaferCond stop_ctx;
 822   m_image_replayer->stop(&stop_ctx);
 823   ASSERT_EQ(0, stop_ctx.wait());
 824 }
 825
 826 TEST_F(TestMockImageReplayer, DecodeError) {
 827   // START
 828
 829   create_local_image();
 830   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 831
 832   librbd::MockTestJournal mock_local_journal;
 833   mock_local_image_ctx.journal = &mock_local_journal;
 834
 835   journal::MockJournaler mock_remote_journaler;
 836   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 837   MockBootstrapRequest mock_bootstrap_request;
 838   MockReplay mock_local_replay;
 839   MockEventPreprocessor mock_event_preprocessor;
 840   MockReplayStatusFormatter mock_replay_status_formatter;
 841   ::journal::MockReplayEntry mock_replay_entry;
 842
 843   expect_get_or_send_update(mock_replay_status_formatter);
 844   expect_get_commit_tid_in_debug(mock_replay_entry);
 845
 846   InSequence seq;
 847   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 848               "remote mirror uuid", 0);
 849   EXPECT_CALL(mock_remote_journaler, construct());
 850   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 851
 852   EXPECT_CALL(mock_local_journal, add_listener(_));
 853
 854   expect_init(mock_remote_journaler, 0);
 855
 856   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 857   expect_get_cached_client(mock_remote_journaler, 0);
 858
 859   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 860
 861   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 862
 863   C_SaferCond start_ctx;
 864   m_image_replayer->start(&start_ctx);
 865   ASSERT_EQ(0, start_ctx.wait());
 866
 867   // REPLAY
 868
 869   cls::journal::Tag tag =
 870     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
 871                             librbd::Journal<>::LOCAL_MIRROR_UUID,
 872                             true, 0, 0})};
 873
 874   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 875
 876   // replay_flush
 877   expect_shut_down(mock_local_replay, false, 0);
 878   EXPECT_CALL(mock_local_journal, stop_external_replay());
 879   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 880   expect_get_tag(mock_remote_journaler, tag, 0);
 881   expect_allocate_tag(mock_local_journal, 0);
 882
 883   // process
 884   EXPECT_CALL(mock_replay_entry, get_data());
 885   EXPECT_CALL(mock_local_replay, decode(_, _))
 886     .WillOnce(Return(-EINVAL));
 887
 888   // stop on error
 889   expect_stop_replay(mock_remote_journaler, 0);
 890   expect_shut_down(mock_local_replay, true, 0);
 891
 892   EXPECT_CALL(mock_local_journal, remove_listener(_));
 893   EXPECT_CALL(mock_local_journal, stop_external_replay());
 894
 895   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 896   expect_shut_down(mock_remote_journaler, 0);
 897
 898   MockCloseImageRequest mock_close_local_image_request;
 899   C_SaferCond close_ctx;
 900   EXPECT_CALL(mock_close_local_image_request, send())
 901     .WillOnce(Invoke([&mock_close_local_image_request, &close_ctx]() {
 902           *mock_close_local_image_request.image_ctx = nullptr;
 903           mock_close_local_image_request.on_finish->complete(0);
 904           close_ctx.complete(0);
 905         }));
 906
 907   // fire
 908   m_image_replayer->handle_replay_ready();
 909   ASSERT_EQ(0, close_ctx.wait());
 910
 911   while (!m_image_replayer->is_stopped()) {
 912     usleep(1000);
 913   }
 914 }
 915
 916 TEST_F(TestMockImageReplayer, DelayedReplay) {
 917
 918   // START
 919
 920   create_local_image();
 921   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 922
 923   librbd::MockTestJournal mock_local_journal;
 924   mock_local_image_ctx.journal = &mock_local_journal;
 925
 926   journal::MockJournaler mock_remote_journaler;
 927   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 928   MockBootstrapRequest mock_bootstrap_request;
 929   MockReplay mock_local_replay;
 930   MockEventPreprocessor mock_event_preprocessor;
 931   MockReplayStatusFormatter mock_replay_status_formatter;
 932   ::journal::MockReplayEntry mock_replay_entry;
 933
 934   expect_get_or_send_update(mock_replay_status_formatter);
 935   expect_get_commit_tid_in_debug(mock_replay_entry);
 936   expect_committed(mock_remote_journaler, 1);
 937
 938   InSequence seq;
 939   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 940               "remote mirror uuid", 0);
 941   EXPECT_CALL(mock_remote_journaler, construct());
 942   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 943
 944   EXPECT_CALL(mock_local_journal, add_listener(_));
 945
 946   expect_init(mock_remote_journaler, 0);
 947
 948   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 949   expect_get_cached_client(mock_remote_journaler, 0);
 950
 951   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 952
 953   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 954
 955   C_SaferCond start_ctx;
 956   m_image_replayer->start(&start_ctx);
 957   ASSERT_EQ(0, start_ctx.wait());
 958
 959   // REPLAY
 960
 961   cls::journal::Tag tag =
 962     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
 963                             librbd::Journal<>::LOCAL_MIRROR_UUID,
 964                             true, 0, 0})};
 965
 966   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 967
 968   // replay_flush
 969   expect_shut_down(mock_local_replay, false, 0);
 970   EXPECT_CALL(mock_local_journal, stop_external_replay());
 971   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 972   expect_get_tag(mock_remote_journaler, tag, 0);
 973   expect_allocate_tag(mock_local_journal, 0);
 974
 975   // process with delay
 976   EXPECT_CALL(mock_replay_entry, get_data());
 977   librbd::journal::EventEntry event_entry(
 978     librbd::journal::AioDiscardEvent(123, 345, false), ceph_clock_now());
 979   EXPECT_CALL(mock_local_replay, decode(_, _))
 980     .WillOnce(DoAll(SetArgPointee<1>(event_entry),
 981                     Return(0)));
 982   expect_preprocess(mock_event_preprocessor, false, 0);
 983   expect_process(mock_local_replay, 0, 0);
 984
 985   // attempt to process the next event
 986   C_SaferCond replay_ctx;
 987   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
 988
 989   // fire
 990   mock_local_image_ctx.mirroring_replay_delay = 2;
 991   m_image_replayer->handle_replay_ready();
 992   ASSERT_EQ(0, replay_ctx.wait());
 993
 994   // add a pending (delayed) entry before stop
 995   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
 996   EXPECT_CALL(mock_replay_entry, get_data());
 997   C_SaferCond decode_ctx;
 998   EXPECT_CALL(mock_local_replay, decode(_, _))
 999     .WillOnce(DoAll(Invoke([&decode_ctx](bufferlist::iterator* it,
1000                                          librbd::journal::EventEntry *e) {
1001                              decode_ctx.complete(0);
1002                            }),
1003                     Return(0)));
1004
1005   mock_local_image_ctx.mirroring_replay_delay = 10;
1006   m_image_replayer->handle_replay_ready();
1007   ASSERT_EQ(0, decode_ctx.wait());
1008
1009   // STOP
1010
1011   MockCloseImageRequest mock_close_local_image_request;
1012
1013   expect_stop_replay(mock_remote_journaler, 0);
1014   expect_shut_down(mock_local_replay, true, 0);
1015
1016   EXPECT_CALL(mock_local_journal, remove_listener(_));
1017   EXPECT_CALL(mock_local_journal, stop_external_replay());
1018
1019   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1020   expect_shut_down(mock_remote_journaler, 0);
1021
1022   expect_send(mock_close_local_image_request, 0);
1023
1024   C_SaferCond stop_ctx;
1025   m_image_replayer->stop(&stop_ctx);
1026   ASSERT_EQ(0, stop_ctx.wait());
1027 }
1028
1029 } // namespace mirror
1030 } // namespace rbd