ceph/src/test/rbd_mirror/test_mock_ImageReplayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "cls/journal/cls_journal_types.h"
   5 #include "librbd/journal/Replay.h"
   6 #include "librbd/journal/Types.h"
   7 #include "tools/rbd_mirror/ImageDeleter.h"
   8 #include "tools/rbd_mirror/ImageReplayer.h"
   9 #include "tools/rbd_mirror/InstanceWatcher.h"
  10 #include "tools/rbd_mirror/Threads.h"
  11 #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
  12 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
  13 #include "tools/rbd_mirror/image_replayer/EventPreprocessor.h"
  14 #include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h"
  15 #include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h"
  16 #include "test/rbd_mirror/test_mock_fixture.h"
  17 #include "test/journal/mock/MockJournaler.h"
  18 #include "test/librbd/mock/MockImageCtx.h"
  19 #include "test/librbd/mock/MockJournal.h"
  20 #include "test/rbd_mirror/mock/MockContextWQ.h"
  21 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  22
  23 namespace librbd {
  24
  25 namespace {
  26
  27 struct MockTestJournal;
  28
  29 struct MockTestImageCtx : public MockImageCtx {
  30   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  31     : librbd::MockImageCtx(image_ctx) {
  32   }
  33   MockTestJournal *journal = nullptr;
  34 };
  35
  36 struct MockTestJournal : public MockJournal {
  37   MOCK_METHOD2(start_external_replay, void(journal::Replay<MockTestImageCtx> **,
  38                                            Context *on_start));
  39   MOCK_METHOD0(stop_external_replay, void());
  40 };
  41
  42 } // anonymous namespace
  43
  44 namespace journal {
  45
  46 template<>
  47 struct Replay<MockTestImageCtx> {
  48   MOCK_METHOD2(decode, int(bufferlist::const_iterator *, EventEntry *));
  49   MOCK_METHOD3(process, void(const EventEntry &, Context *, Context *));
  50   MOCK_METHOD1(flush, void(Context*));
  51   MOCK_METHOD2(shut_down, void(bool, Context*));
  52 };
  53
  54 template <>
  55 struct TypeTraits<MockTestImageCtx> {
  56   typedef ::journal::MockJournalerProxy Journaler;
  57   typedef ::journal::MockReplayEntryProxy ReplayEntry;
  58 };
  59
  60 struct MirrorPeerClientMeta;
  61
  62 } // namespace journal
  63 } // namespace librbd
  64
  65 namespace rbd {
  66 namespace mirror {
  67
  68 template <>
  69 struct ImageDeleter<librbd::MockTestImageCtx> {
  70   static ImageDeleter* s_instance;
  71
  72   static void trash_move(librados::IoCtx& local_io_ctx,
  73                          const std::string& global_image_id, bool resync,
  74                          MockContextWQ* work_queue, Context* on_finish) {
  75     ceph_assert(s_instance != nullptr);
  76     s_instance->trash_move(global_image_id, resync, on_finish);
  77   }
  78
  79   MOCK_METHOD3(trash_move, void(const std::string&, bool, Context*));
  80
  81   ImageDeleter() {
  82     s_instance = this;
  83   }
  84 };
  85
  86 ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr;
  87
  88 template <>
  89 struct Threads<librbd::MockTestImageCtx> {
  90   MockSafeTimer *timer;
  91   Mutex &timer_lock;
  92
  93   MockContextWQ *work_queue;
  94
  95   Threads(Threads<librbd::ImageCtx> *threads)
  96     : timer(new MockSafeTimer()),
  97       timer_lock(threads->timer_lock),
  98       work_queue(new MockContextWQ()) {
  99   }
 100   ~Threads() {
 101     delete timer;
 102     delete work_queue;
 103   }
 104 };
 105
 106 template<>
 107 class InstanceWatcher<librbd::MockTestImageCtx> {
 108 };
 109
 110 namespace image_replayer {
 111
 112 using ::testing::_;
 113 using ::testing::AtLeast;
 114 using ::testing::DoAll;
 115 using ::testing::InSequence;
 116 using ::testing::Invoke;
 117 using ::testing::MatcherCast;
 118 using ::testing::Return;
 119 using ::testing::ReturnArg;
 120 using ::testing::SetArgPointee;
 121 using ::testing::WithArg;
 122
 123 template<>
 124 struct PrepareLocalImageRequest<librbd::MockTestImageCtx> {
 125   static PrepareLocalImageRequest* s_instance;
 126   std::string *local_image_id = nullptr;
 127   std::string *local_image_name = nullptr;
 128   std::string *tag_owner = nullptr;
 129   Context *on_finish = nullptr;
 130
 131   static PrepareLocalImageRequest* create(librados::IoCtx &,
 132                                           const std::string &global_image_id,
 133                                           std::string *local_image_id,
 134                                           std::string *local_image_name,
 135                                           std::string *tag_owner,
 136                                           MockContextWQ *work_queue,
 137                                           Context *on_finish) {
 138     ceph_assert(s_instance != nullptr);
 139     s_instance->local_image_id = local_image_id;
 140     s_instance->local_image_name = local_image_name;
 141     s_instance->tag_owner = tag_owner;
 142     s_instance->on_finish = on_finish;
 143     return s_instance;
 144   }
 145
 146   PrepareLocalImageRequest() {
 147     s_instance = this;
 148   }
 149
 150   MOCK_METHOD0(send, void());
 151 };
 152
 153 template<>
 154 struct PrepareRemoteImageRequest<librbd::MockTestImageCtx> {
 155   static PrepareRemoteImageRequest* s_instance;
 156   std::string *remote_mirror_uuid = nullptr;
 157   std::string *remote_image_id = nullptr;
 158   cls::journal::ClientState *client_state;
 159   ::journal::MockJournalerProxy **remote_journaler = nullptr;
 160   librbd::journal::MirrorPeerClientMeta *client_meta = nullptr;
 161   Context *on_finish = nullptr;
 162
 163   static PrepareRemoteImageRequest* create(Threads<librbd::MockTestImageCtx> *threads,
 164                                            librados::IoCtx &,
 165                                            const std::string &global_image_id,
 166                                            const std::string &local_mirror_uuid,
 167                                            const std::string &local_image_id,
 168                                            const journal::Settings &settings,
 169                                            std::string *remote_mirror_uuid,
 170                                            std::string *remote_image_id,
 171                                            ::journal::MockJournalerProxy **remote_journaler,
 172                                            cls::journal::ClientState *client_state,
 173                                            librbd::journal::MirrorPeerClientMeta *client_meta,
 174                                            Context *on_finish) {
 175     ceph_assert(s_instance != nullptr);
 176     s_instance->remote_mirror_uuid = remote_mirror_uuid;
 177     s_instance->remote_image_id = remote_image_id;
 178     s_instance->remote_journaler = remote_journaler;
 179     s_instance->client_state = client_state;
 180     s_instance->client_meta = client_meta;
 181     s_instance->on_finish = on_finish;
 182     return s_instance;
 183   }
 184
 185   PrepareRemoteImageRequest() {
 186     s_instance = this;
 187   }
 188
 189   MOCK_METHOD0(send, void());
 190 };
 191
 192 template<>
 193 struct BootstrapRequest<librbd::MockTestImageCtx> {
 194   static BootstrapRequest* s_instance;
 195   librbd::MockTestImageCtx **image_ctx = nullptr;
 196   Context *on_finish = nullptr;
 197   bool *do_resync = nullptr;
 198
 199   static BootstrapRequest* create(
 200       Threads<librbd::MockTestImageCtx>* threads,
 201       librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx,
 202       rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
 203       librbd::MockTestImageCtx **local_image_ctx,
 204       const std::string &local_image_name, const std::string &remote_image_id,
 205       const std::string &global_image_id,
 206       const std::string &local_mirror_uuid,
 207       const std::string &remote_mirror_uuid,
 208       ::journal::MockJournalerProxy *journaler,
 209       cls::journal::ClientState *client_state,
 210       librbd::journal::MirrorPeerClientMeta *client_meta,
 211       Context *on_finish, bool *do_resync,
 212       rbd::mirror::ProgressContext *progress_ctx = nullptr) {
 213     ceph_assert(s_instance != nullptr);
 214     s_instance->image_ctx = local_image_ctx;
 215     s_instance->on_finish = on_finish;
 216     s_instance->do_resync = do_resync;
 217     return s_instance;
 218   }
 219
 220   BootstrapRequest() {
 221     ceph_assert(s_instance == nullptr);
 222     s_instance = this;
 223   }
 224
 225   ~BootstrapRequest() {
 226     ceph_assert(s_instance == this);
 227     s_instance = nullptr;
 228   }
 229
 230   void put() {
 231   }
 232
 233   void get() {
 234   }
 235
 236   inline bool is_syncing() const {
 237     return false;
 238   }
 239
 240   MOCK_METHOD0(send, void());
 241   MOCK_METHOD0(cancel, void());
 242 };
 243
 244 template<>
 245 struct CloseImageRequest<librbd::MockTestImageCtx> {
 246   static CloseImageRequest* s_instance;
 247   librbd::MockTestImageCtx **image_ctx = nullptr;
 248   Context *on_finish = nullptr;
 249
 250   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
 251                                    Context *on_finish) {
 252     ceph_assert(s_instance != nullptr);
 253     s_instance->image_ctx = image_ctx;
 254     s_instance->on_finish = on_finish;
 255     return s_instance;
 256   }
 257
 258   CloseImageRequest() {
 259     ceph_assert(s_instance == nullptr);
 260     s_instance = this;
 261   }
 262
 263   ~CloseImageRequest() {
 264     ceph_assert(s_instance == this);
 265     s_instance = nullptr;
 266   }
 267
 268   MOCK_METHOD0(send, void());
 269 };
 270
 271 template<>
 272 struct EventPreprocessor<librbd::MockTestImageCtx> {
 273   static EventPreprocessor *s_instance;
 274
 275   static EventPreprocessor *create(librbd::MockTestImageCtx &local_image_ctx,
 276                                    ::journal::MockJournalerProxy &remote_journaler,
 277                                    const std::string &local_mirror_uuid,
 278                                    librbd::journal::MirrorPeerClientMeta *client_meta,
 279                                    MockContextWQ *work_queue) {
 280     ceph_assert(s_instance != nullptr);
 281     return s_instance;
 282   }
 283
 284   static void destroy(EventPreprocessor* processor) {
 285   }
 286
 287   EventPreprocessor() {
 288     ceph_assert(s_instance == nullptr);
 289     s_instance = this;
 290   }
 291
 292   ~EventPreprocessor() {
 293     ceph_assert(s_instance == this);
 294     s_instance = nullptr;
 295   }
 296
 297   MOCK_METHOD1(is_required, bool(const librbd::journal::EventEntry &));
 298   MOCK_METHOD2(preprocess, void(librbd::journal::EventEntry *, Context *));
 299 };
 300
 301 template<>
 302 struct ReplayStatusFormatter<librbd::MockTestImageCtx> {
 303   static ReplayStatusFormatter* s_instance;
 304
 305   static ReplayStatusFormatter* create(::journal::MockJournalerProxy *journaler,
 306                                        const std::string &mirror_uuid) {
 307     ceph_assert(s_instance != nullptr);
 308     return s_instance;
 309   }
 310
 311   static void destroy(ReplayStatusFormatter* formatter) {
 312   }
 313
 314   ReplayStatusFormatter() {
 315     ceph_assert(s_instance == nullptr);
 316     s_instance = this;
 317   }
 318
 319   ~ReplayStatusFormatter() {
 320     ceph_assert(s_instance == this);
 321     s_instance = nullptr;
 322   }
 323
 324   MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish));
 325 };
 326
 327 BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 328 CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 329 EventPreprocessor<librbd::MockTestImageCtx>* EventPreprocessor<librbd::MockTestImageCtx>::s_instance = nullptr;
 330 PrepareLocalImageRequest<librbd::MockTestImageCtx>* PrepareLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 331 PrepareRemoteImageRequest<librbd::MockTestImageCtx>* PrepareRemoteImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 332 ReplayStatusFormatter<librbd::MockTestImageCtx>* ReplayStatusFormatter<librbd::MockTestImageCtx>::s_instance = nullptr;
 333
 334 } // namespace image_replayer
 335 } // namespace mirror
 336 } // namespace rbd
 337
 338 // template definitions
 339 #include "tools/rbd_mirror/ImageReplayer.cc"
 340
 341 namespace rbd {
 342 namespace mirror {
 343
 344 class TestMockImageReplayer : public TestMockFixture {
 345 public:
 346   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 347   typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter;
 348   typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
 349   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
 350   typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor;
 351   typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest;
 352   typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest;
 353   typedef ReplayStatusFormatter<librbd::MockTestImageCtx> MockReplayStatusFormatter;
 354   typedef librbd::journal::Replay<librbd::MockTestImageCtx> MockReplay;
 355   typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer;
 356   typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
 357
 358   void SetUp() override {
 359     TestMockFixture::SetUp();
 360
 361     librbd::RBD rbd;
 362     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size));
 363     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx));
 364   }
 365
 366   void TearDown() override {
 367     delete m_image_replayer;
 368
 369     TestMockFixture::TearDown();
 370   }
 371
 372   void create_local_image() {
 373     librbd::RBD rbd;
 374     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 375     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 376   }
 377
 378   void expect_work_queue_repeatedly(MockThreads &mock_threads) {
 379     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 380       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 381           m_threads->work_queue->queue(ctx, r);
 382         }));
 383   }
 384
 385   void expect_add_event_after_repeatedly(MockThreads &mock_threads) {
 386     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 387       .WillRepeatedly(
 388         DoAll(Invoke([this](double seconds, Context *ctx) {
 389                        m_threads->timer->add_event_after(seconds, ctx);
 390                      }),
 391           ReturnArg<1>()));
 392     EXPECT_CALL(*mock_threads.timer, cancel_event(_))
 393       .WillRepeatedly(
 394         Invoke([this](Context *ctx) {
 395           return m_threads->timer->cancel_event(ctx);
 396         }));
 397   }
 398
 399   void expect_flush_repeatedly(MockReplay& mock_replay,
 400                                journal::MockJournaler& mock_journal) {
 401     EXPECT_CALL(mock_replay, flush(_))
 402       .WillRepeatedly(Invoke([this](Context* ctx) {
 403                         m_threads->work_queue->queue(ctx, 0);
 404                       }));
 405     EXPECT_CALL(mock_journal, flush_commit_position(_))
 406       .WillRepeatedly(Invoke([this](Context* ctx) {
 407                         m_threads->work_queue->queue(ctx, 0);
 408                       }));
 409   }
 410
 411   void expect_trash_move(MockImageDeleter& mock_image_deleter,
 412                          const std::string& global_image_id,
 413                          bool ignore_orphan, int r) {
 414     EXPECT_CALL(mock_image_deleter,
 415                 trash_move(global_image_id, ignore_orphan, _))
 416       .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) {
 417                              m_threads->work_queue->queue(ctx, r);
 418                            })));
 419   }
 420
 421   bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) {
 422     bufferlist bl;
 423     encode(tag_data, bl);
 424     return bl;
 425   }
 426
 427   void expect_get_or_send_update(
 428     MockReplayStatusFormatter &mock_replay_status_formatter) {
 429     EXPECT_CALL(mock_replay_status_formatter, get_or_send_update(_, _))
 430       .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)),
 431                             Return(true)));
 432   }
 433
 434   void expect_send(MockPrepareLocalImageRequest &mock_request,
 435                    const std::string &local_image_id,
 436                    const std::string &local_image_name,
 437                    const std::string &tag_owner,
 438                    int r) {
 439     EXPECT_CALL(mock_request, send())
 440       .WillOnce(Invoke([&mock_request, local_image_id, local_image_name, tag_owner, r]() {
 441           if (r == 0) {
 442             *mock_request.local_image_id = local_image_id;
 443             *mock_request.local_image_name = local_image_name;
 444             *mock_request.tag_owner = tag_owner;
 445           }
 446           mock_request.on_finish->complete(r);
 447         }));
 448   }
 449
 450   void expect_send(MockPrepareRemoteImageRequest& mock_request,
 451                    const std::string& mirror_uuid, const std::string& image_id,
 452                    int r) {
 453     EXPECT_CALL(mock_request, send())
 454       .WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() {
 455                   if (r >= 0) {
 456                     *mock_request.remote_journaler = new ::journal::MockJournalerProxy();
 457                   }
 458
 459                   *mock_request.remote_mirror_uuid = mirror_uuid;
 460                   *mock_request.remote_image_id = image_id;
 461                   mock_request.on_finish->complete(r);
 462                 }));
 463   }
 464
 465   void expect_send(MockBootstrapRequest &mock_bootstrap_request,
 466                    librbd::MockTestImageCtx &mock_local_image_ctx,
 467                    bool do_resync, int r) {
 468     EXPECT_CALL(mock_bootstrap_request, send())
 469       .WillOnce(Invoke([&mock_bootstrap_request, &mock_local_image_ctx,
 470                         do_resync, r]() {
 471             if (r == 0) {
 472               *mock_bootstrap_request.image_ctx = &mock_local_image_ctx;
 473               *mock_bootstrap_request.do_resync = do_resync;
 474             }
 475             mock_bootstrap_request.on_finish->complete(r);
 476           }));
 477   }
 478
 479   void expect_start_external_replay(librbd::MockTestJournal &mock_journal,
 480                                     MockReplay *mock_replay, int r) {
 481     EXPECT_CALL(mock_journal, start_external_replay(_, _))
 482       .WillOnce(DoAll(SetArgPointee<0>(mock_replay),
 483                       WithArg<1>(CompleteContext(r))));
 484   }
 485
 486   void expect_init(::journal::MockJournaler &mock_journaler, int r) {
 487     EXPECT_CALL(mock_journaler, init(_))
 488       .WillOnce(CompleteContext(r));
 489   }
 490
 491   void expect_get_cached_client(::journal::MockJournaler &mock_journaler,
 492                                 int r) {
 493     librbd::journal::ImageClientMeta image_client_meta;
 494     image_client_meta.tag_class = 0;
 495
 496     librbd::journal::ClientData client_data;
 497     client_data.client_meta = image_client_meta;
 498
 499     cls::journal::Client client;
 500     encode(client_data, client.data);
 501
 502     EXPECT_CALL(mock_journaler, get_cached_client("local_mirror_uuid", _))
 503       .WillOnce(DoAll(SetArgPointee<1>(client),
 504                       Return(r)));
 505   }
 506
 507   void expect_stop_replay(::journal::MockJournaler &mock_journaler, int r) {
 508     EXPECT_CALL(mock_journaler, stop_replay(_))
 509       .WillOnce(CompleteContext(r));
 510   }
 511
 512   void expect_flush(MockReplay &mock_replay, int r) {
 513     EXPECT_CALL(mock_replay, flush(_)).WillOnce(CompleteContext(r));
 514   }
 515
 516   void expect_shut_down(MockReplay &mock_replay, bool cancel_ops, int r) {
 517     EXPECT_CALL(mock_replay, shut_down(cancel_ops, _))
 518       .WillOnce(WithArg<1>(CompleteContext(r)));
 519   }
 520
 521   void expect_shut_down(journal::MockJournaler &mock_journaler, int r) {
 522     EXPECT_CALL(mock_journaler, shut_down(_))
 523       .WillOnce(CompleteContext(r));
 524   }
 525
 526   void expect_send(MockCloseImageRequest &mock_close_image_request, int r) {
 527     EXPECT_CALL(mock_close_image_request, send())
 528       .WillOnce(Invoke([&mock_close_image_request, r]() {
 529             *mock_close_image_request.image_ctx = nullptr;
 530             mock_close_image_request.on_finish->complete(r);
 531           }));
 532   }
 533
 534   void expect_get_commit_tid_in_debug(
 535     ::journal::MockReplayEntry &mock_replay_entry) {
 536     // It is used in debug messages and depends on debug level
 537     EXPECT_CALL(mock_replay_entry, get_commit_tid())
 538     .Times(AtLeast(0))
 539       .WillRepeatedly(Return(0));
 540   }
 541
 542   void expect_get_tag_tid_in_debug(librbd::MockTestJournal &mock_journal) {
 543     // It is used in debug messages and depends on debug level
 544     EXPECT_CALL(mock_journal, get_tag_tid()).Times(AtLeast(0))
 545       .WillRepeatedly(Return(0));
 546   }
 547
 548   void expect_committed(::journal::MockReplayEntry &mock_replay_entry,
 549                         ::journal::MockJournaler &mock_journaler, int times) {
 550     EXPECT_CALL(mock_replay_entry, get_data()).Times(times);
 551     EXPECT_CALL(mock_journaler, committed(
 552                   MatcherCast<const ::journal::MockReplayEntryProxy&>(_)))
 553       .Times(times);
 554   }
 555
 556   void expect_try_pop_front(::journal::MockJournaler &mock_journaler,
 557                             uint64_t replay_tag_tid, bool entries_available) {
 558     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 559       .WillOnce(DoAll(SetArgPointee<0>(::journal::MockReplayEntryProxy()),
 560                       SetArgPointee<1>(replay_tag_tid),
 561                       Return(entries_available)));
 562   }
 563
 564   void expect_try_pop_front_return_no_entries(
 565     ::journal::MockJournaler &mock_journaler, Context *on_finish) {
 566     EXPECT_CALL(mock_journaler, try_pop_front(_, _))
 567       .WillOnce(DoAll(Invoke([on_finish](::journal::MockReplayEntryProxy *e,
 568                                          uint64_t *t) {
 569                                on_finish->complete(0);
 570                              }),
 571                       Return(false)));
 572   }
 573
 574   void expect_get_tag(::journal::MockJournaler &mock_journaler,
 575                       const cls::journal::Tag &tag, int r) {
 576     EXPECT_CALL(mock_journaler, get_tag(_, _, _))
 577       .WillOnce(DoAll(SetArgPointee<1>(tag),
 578                       WithArg<2>(CompleteContext(r))));
 579   }
 580
 581   void expect_allocate_tag(librbd::MockTestJournal &mock_journal, int r) {
 582     EXPECT_CALL(mock_journal, allocate_tag(_, _, _))
 583       .WillOnce(WithArg<2>(CompleteContext(r)));
 584   }
 585
 586   void expect_preprocess(MockEventPreprocessor &mock_event_preprocessor,
 587                          bool required, int r) {
 588     EXPECT_CALL(mock_event_preprocessor, is_required(_))
 589       .WillOnce(Return(required));
 590     if (required) {
 591       EXPECT_CALL(mock_event_preprocessor, preprocess(_, _))
 592         .WillOnce(WithArg<1>(CompleteContext(r)));
 593     }
 594   }
 595
 596   void expect_process(MockReplay &mock_replay,
 597                       int on_ready_r, int on_commit_r) {
 598     EXPECT_CALL(mock_replay, process(_, _, _))
 599       .WillOnce(DoAll(WithArg<1>(CompleteContext(on_ready_r)),
 600                       WithArg<2>(CompleteContext(on_commit_r))));
 601   }
 602
 603   void create_image_replayer(MockThreads &mock_threads) {
 604     m_image_replayer = new MockImageReplayer(
 605       &mock_threads, &m_instance_watcher,
 606       rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
 607       "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id");
 608     m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx);
 609   }
 610
 611   librbd::ImageCtx *m_remote_image_ctx;
 612   librbd::ImageCtx *m_local_image_ctx = nullptr;
 613   MockInstanceWatcher m_instance_watcher;
 614   MockImageReplayer *m_image_replayer = nullptr;
 615 };
 616
 617 TEST_F(TestMockImageReplayer, StartStop) {
 618   // START
 619
 620   create_local_image();
 621   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 622
 623   librbd::MockTestJournal mock_local_journal;
 624   mock_local_image_ctx.journal = &mock_local_journal;
 625
 626   journal::MockJournaler mock_remote_journaler;
 627   MockThreads mock_threads(m_threads);
 628   expect_work_queue_repeatedly(mock_threads);
 629   expect_add_event_after_repeatedly(mock_threads);
 630
 631   MockImageDeleter mock_image_deleter;
 632   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 633   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 634   MockBootstrapRequest mock_bootstrap_request;
 635   MockReplay mock_local_replay;
 636   MockEventPreprocessor mock_event_preprocessor;
 637   MockReplayStatusFormatter mock_replay_status_formatter;
 638
 639   expect_flush_repeatedly(mock_local_replay, mock_remote_journaler);
 640   expect_get_or_send_update(mock_replay_status_formatter);
 641
 642   InSequence seq;
 643   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 644               mock_local_image_ctx.name, "remote mirror uuid", 0);
 645   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 646               m_remote_image_ctx->id, 0);
 647   EXPECT_CALL(mock_remote_journaler, construct());
 648   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 649
 650   EXPECT_CALL(mock_local_journal, add_listener(_));
 651
 652   expect_init(mock_remote_journaler, 0);
 653
 654   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 655   expect_get_cached_client(mock_remote_journaler, 0);
 656
 657   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
 658
 659   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
 660
 661   create_image_replayer(mock_threads);
 662
 663   C_SaferCond start_ctx;
 664   m_image_replayer->start(&start_ctx);
 665   ASSERT_EQ(0, start_ctx.wait());
 666   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 667             m_image_replayer->get_health_state());
 668
 669   // STOP
 670
 671   MockCloseImageRequest mock_close_local_image_request;
 672
 673   expect_shut_down(mock_local_replay, true, 0);
 674   EXPECT_CALL(mock_local_journal, remove_listener(_));
 675   EXPECT_CALL(mock_local_journal, stop_external_replay());
 676   expect_send(mock_close_local_image_request, 0);
 677
 678   expect_stop_replay(mock_remote_journaler, 0);
 679   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 680   expect_shut_down(mock_remote_journaler, 0);
 681
 682   C_SaferCond stop_ctx;
 683   m_image_replayer->stop(&stop_ctx);
 684   ASSERT_EQ(0, stop_ctx.wait());
 685   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 686             m_image_replayer->get_health_state());
 687 }
 688
 689 TEST_F(TestMockImageReplayer, LocalImagePrimary) {
 690   create_local_image();
 691   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 692
 693   journal::MockJournaler mock_remote_journaler;
 694   MockThreads mock_threads(m_threads);
 695   expect_work_queue_repeatedly(mock_threads);
 696   expect_add_event_after_repeatedly(mock_threads);
 697
 698   MockImageDeleter mock_image_deleter;
 699   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 700   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 701   MockReplayStatusFormatter mock_replay_status_formatter;
 702
 703   expect_get_or_send_update(mock_replay_status_formatter);
 704
 705   InSequence seq;
 706   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 707               mock_local_image_ctx.name, "", 0);
 708   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 709               "remote image id", 0);
 710   EXPECT_CALL(mock_remote_journaler, construct());
 711   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 712   expect_shut_down(mock_remote_journaler, 0);
 713
 714   create_image_replayer(mock_threads);
 715
 716   C_SaferCond start_ctx;
 717   m_image_replayer->start(&start_ctx);
 718   ASSERT_EQ(0, start_ctx.wait());
 719 }
 720
 721 TEST_F(TestMockImageReplayer, LocalImageDNE) {
 722   create_local_image();
 723   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 724
 725   journal::MockJournaler mock_remote_journaler;
 726   MockThreads mock_threads(m_threads);
 727   expect_work_queue_repeatedly(mock_threads);
 728   expect_add_event_after_repeatedly(mock_threads);
 729
 730   MockImageDeleter mock_image_deleter;
 731   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 732   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 733   MockBootstrapRequest mock_bootstrap_request;
 734   MockReplayStatusFormatter mock_replay_status_formatter;
 735
 736   expect_get_or_send_update(mock_replay_status_formatter);
 737
 738   InSequence seq;
 739   expect_send(mock_prepare_local_image_request, "", "", "", -ENOENT);
 740   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 741               m_remote_image_ctx->id, 0);
 742   EXPECT_CALL(mock_remote_journaler, construct());
 743   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EREMOTEIO);
 744
 745   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 746   expect_shut_down(mock_remote_journaler, 0);
 747
 748   create_image_replayer(mock_threads);
 749
 750   C_SaferCond start_ctx;
 751   m_image_replayer->start(&start_ctx);
 752   ASSERT_EQ(-EREMOTEIO, start_ctx.wait());
 753 }
 754
 755 TEST_F(TestMockImageReplayer, PrepareLocalImageError) {
 756   create_local_image();
 757   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 758
 759   MockThreads mock_threads(m_threads);
 760   expect_work_queue_repeatedly(mock_threads);
 761   expect_add_event_after_repeatedly(mock_threads);
 762
 763   MockImageDeleter mock_image_deleter;
 764   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 765   MockReplayStatusFormatter mock_replay_status_formatter;
 766
 767   expect_get_or_send_update(mock_replay_status_formatter);
 768
 769   InSequence seq;
 770   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 771               mock_local_image_ctx.name, "remote mirror uuid", -EINVAL);
 772
 773   create_image_replayer(mock_threads);
 774
 775   C_SaferCond start_ctx;
 776   m_image_replayer->start(&start_ctx);
 777   ASSERT_EQ(-EINVAL, start_ctx.wait());
 778 }
 779
 780 TEST_F(TestMockImageReplayer, GetRemoteImageIdDNE) {
 781   create_local_image();
 782   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 783
 784   MockThreads mock_threads(m_threads);
 785   expect_work_queue_repeatedly(mock_threads);
 786   expect_add_event_after_repeatedly(mock_threads);
 787
 788   MockImageDeleter mock_image_deleter;
 789   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 790   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 791   MockReplayStatusFormatter mock_replay_status_formatter;
 792
 793   expect_get_or_send_update(mock_replay_status_formatter);
 794
 795   InSequence seq;
 796   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 797               mock_local_image_ctx.name, "remote mirror uuid", 0);
 798   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 799               "", -ENOENT);
 800   expect_trash_move(mock_image_deleter, "global image id", false, 0);
 801
 802   create_image_replayer(mock_threads);
 803
 804   C_SaferCond start_ctx;
 805   m_image_replayer->start(&start_ctx);
 806   ASSERT_EQ(0, start_ctx.wait());
 807 }
 808
 809 TEST_F(TestMockImageReplayer, GetRemoteImageIdNonLinkedDNE) {
 810   create_local_image();
 811   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 812
 813   MockThreads mock_threads(m_threads);
 814   expect_work_queue_repeatedly(mock_threads);
 815   expect_add_event_after_repeatedly(mock_threads);
 816
 817   MockImageDeleter mock_image_deleter;
 818   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 819   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 820   MockReplayStatusFormatter mock_replay_status_formatter;
 821
 822   expect_get_or_send_update(mock_replay_status_formatter);
 823
 824   InSequence seq;
 825   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 826               mock_local_image_ctx.name, "some other mirror uuid", 0);
 827   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 828               "", -ENOENT);
 829
 830   create_image_replayer(mock_threads);
 831
 832   C_SaferCond start_ctx;
 833   m_image_replayer->start(&start_ctx);
 834   ASSERT_EQ(-ENOENT, start_ctx.wait());
 835 }
 836
 837 TEST_F(TestMockImageReplayer, GetRemoteImageIdError) {
 838   create_local_image();
 839   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 840
 841   MockThreads mock_threads(m_threads);
 842   expect_work_queue_repeatedly(mock_threads);
 843   expect_add_event_after_repeatedly(mock_threads);
 844
 845   MockImageDeleter mock_image_deleter;
 846   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 847   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 848   MockReplayStatusFormatter mock_replay_status_formatter;
 849
 850   expect_get_or_send_update(mock_replay_status_formatter);
 851
 852   InSequence seq;
 853   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 854               mock_local_image_ctx.name, "remote mirror uuid", 0);
 855   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 856               m_remote_image_ctx->id, -EINVAL);
 857
 858   create_image_replayer(mock_threads);
 859
 860   C_SaferCond start_ctx;
 861   m_image_replayer->start(&start_ctx);
 862   ASSERT_EQ(-EINVAL, start_ctx.wait());
 863 }
 864
 865 TEST_F(TestMockImageReplayer, BootstrapError) {
 866   create_local_image();
 867   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 868
 869   journal::MockJournaler mock_remote_journaler;
 870   MockThreads mock_threads(m_threads);
 871   expect_work_queue_repeatedly(mock_threads);
 872   expect_add_event_after_repeatedly(mock_threads);
 873
 874   MockImageDeleter mock_image_deleter;
 875   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 876   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 877   MockBootstrapRequest mock_bootstrap_request;
 878   MockReplayStatusFormatter mock_replay_status_formatter;
 879
 880   expect_get_or_send_update(mock_replay_status_formatter);
 881
 882   InSequence seq;
 883   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 884               mock_local_image_ctx.name, "remote mirror uuid", 0);
 885   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 886               m_remote_image_ctx->id, 0);
 887   EXPECT_CALL(mock_remote_journaler, construct());
 888   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EINVAL);
 889
 890   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 891   expect_shut_down(mock_remote_journaler, 0);
 892
 893   create_image_replayer(mock_threads);
 894
 895   C_SaferCond start_ctx;
 896   m_image_replayer->start(&start_ctx);
 897   ASSERT_EQ(-EINVAL, start_ctx.wait());
 898 }
 899
 900 TEST_F(TestMockImageReplayer, StopBeforeBootstrap) {
 901   create_local_image();
 902   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 903
 904   journal::MockJournaler mock_remote_journaler;
 905   MockThreads mock_threads(m_threads);
 906   expect_work_queue_repeatedly(mock_threads);
 907   expect_add_event_after_repeatedly(mock_threads);
 908
 909   MockImageDeleter mock_image_deleter;
 910   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 911   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 912   MockReplayStatusFormatter mock_replay_status_formatter;
 913
 914   expect_get_or_send_update(mock_replay_status_formatter);
 915
 916   InSequence seq;
 917   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 918               mock_local_image_ctx.name, "remote mirror uuid", 0);
 919   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 920               m_remote_image_ctx->id, 0);
 921   EXPECT_CALL(mock_remote_journaler, construct())
 922     .WillOnce(Invoke([this]() {
 923                 m_image_replayer->stop(nullptr, true);
 924               }));
 925
 926   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 927   expect_shut_down(mock_remote_journaler, 0);
 928
 929   create_image_replayer(mock_threads);
 930
 931   C_SaferCond start_ctx;
 932   m_image_replayer->start(&start_ctx);
 933   ASSERT_EQ(-ECANCELED, start_ctx.wait());
 934 }
 935
 936 TEST_F(TestMockImageReplayer, StartExternalReplayError) {
 937   // START
 938
 939   create_local_image();
 940   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 941
 942   librbd::MockTestJournal mock_local_journal;
 943   mock_local_image_ctx.journal = &mock_local_journal;
 944
 945   journal::MockJournaler mock_remote_journaler;
 946   MockThreads mock_threads(m_threads);
 947   expect_work_queue_repeatedly(mock_threads);
 948   expect_add_event_after_repeatedly(mock_threads);
 949
 950   MockImageDeleter mock_image_deleter;
 951   MockPrepareLocalImageRequest mock_prepare_local_image_request;
 952   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
 953   MockBootstrapRequest mock_bootstrap_request;
 954   MockReplay mock_local_replay;
 955   MockEventPreprocessor mock_event_preprocessor;
 956   MockReplayStatusFormatter mock_replay_status_formatter;
 957
 958   expect_get_or_send_update(mock_replay_status_formatter);
 959
 960   InSequence seq;
 961   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
 962               mock_local_image_ctx.name, "remote mirror uuid", 0);
 963   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
 964               m_remote_image_ctx->id, 0);
 965   EXPECT_CALL(mock_remote_journaler, construct());
 966   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
 967
 968   EXPECT_CALL(mock_local_journal, add_listener(_));
 969
 970   expect_init(mock_remote_journaler, 0);
 971
 972   EXPECT_CALL(mock_remote_journaler, add_listener(_));
 973   expect_get_cached_client(mock_remote_journaler, 0);
 974
 975   expect_start_external_replay(mock_local_journal, nullptr, -EINVAL);
 976
 977   MockCloseImageRequest mock_close_local_image_request;
 978   EXPECT_CALL(mock_local_journal, remove_listener(_));
 979   expect_send(mock_close_local_image_request, 0);
 980
 981   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
 982   expect_shut_down(mock_remote_journaler, 0);
 983
 984   create_image_replayer(mock_threads);
 985
 986   C_SaferCond start_ctx;
 987   m_image_replayer->start(&start_ctx);
 988   ASSERT_EQ(-EINVAL, start_ctx.wait());
 989   ASSERT_EQ(image_replayer::HEALTH_STATE_ERROR,
 990             m_image_replayer->get_health_state());
 991 }
 992
 993 TEST_F(TestMockImageReplayer, StopError) {
 994   // START
 995
 996   create_local_image();
 997   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 998
 999   librbd::MockTestJournal mock_local_journal;
1000   mock_local_image_ctx.journal = &mock_local_journal;
1001
1002   journal::MockJournaler mock_remote_journaler;
1003   MockThreads mock_threads(m_threads);
1004   expect_work_queue_repeatedly(mock_threads);
1005   expect_add_event_after_repeatedly(mock_threads);
1006
1007   MockImageDeleter mock_image_deleter;
1008   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1009   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1010   MockBootstrapRequest mock_bootstrap_request;
1011   MockReplay mock_local_replay;
1012   MockEventPreprocessor mock_event_preprocessor;
1013   MockReplayStatusFormatter mock_replay_status_formatter;
1014
1015   expect_flush_repeatedly(mock_local_replay, mock_remote_journaler);
1016   expect_get_or_send_update(mock_replay_status_formatter);
1017
1018   InSequence seq;
1019   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1020               mock_local_image_ctx.name, "remote mirror uuid", 0);
1021   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1022               m_remote_image_ctx->id, 0);
1023   EXPECT_CALL(mock_remote_journaler, construct());
1024   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1025
1026   EXPECT_CALL(mock_local_journal, add_listener(_));
1027
1028   expect_init(mock_remote_journaler, 0);
1029
1030   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1031   expect_get_cached_client(mock_remote_journaler, 0);
1032
1033   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1034
1035   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1036
1037   create_image_replayer(mock_threads);
1038
1039   C_SaferCond start_ctx;
1040   m_image_replayer->start(&start_ctx);
1041   ASSERT_EQ(0, start_ctx.wait());
1042
1043   // STOP (errors are ignored)
1044
1045   MockCloseImageRequest mock_close_local_image_request;
1046
1047   expect_shut_down(mock_local_replay, true, -EINVAL);
1048   EXPECT_CALL(mock_local_journal, remove_listener(_));
1049   EXPECT_CALL(mock_local_journal, stop_external_replay());
1050   expect_send(mock_close_local_image_request, -EINVAL);
1051
1052   expect_stop_replay(mock_remote_journaler, -EINVAL);
1053   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1054   expect_shut_down(mock_remote_journaler, -EINVAL);
1055
1056   C_SaferCond stop_ctx;
1057   m_image_replayer->stop(&stop_ctx);
1058   ASSERT_EQ(0, stop_ctx.wait());
1059 }
1060
1061 TEST_F(TestMockImageReplayer, Replay) {
1062   // START
1063
1064   create_local_image();
1065   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1066
1067   librbd::MockTestJournal mock_local_journal;
1068   mock_local_image_ctx.journal = &mock_local_journal;
1069
1070   journal::MockJournaler mock_remote_journaler;
1071   MockThreads mock_threads(m_threads);
1072   expect_work_queue_repeatedly(mock_threads);
1073   expect_add_event_after_repeatedly(mock_threads);
1074
1075   MockImageDeleter mock_image_deleter;
1076   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1077   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1078   MockBootstrapRequest mock_bootstrap_request;
1079   MockReplay mock_local_replay;
1080   MockEventPreprocessor mock_event_preprocessor;
1081   MockReplayStatusFormatter mock_replay_status_formatter;
1082   ::journal::MockReplayEntry mock_replay_entry;
1083
1084   expect_flush_repeatedly(mock_local_replay, mock_remote_journaler);
1085   expect_get_or_send_update(mock_replay_status_formatter);
1086   expect_get_commit_tid_in_debug(mock_replay_entry);
1087   expect_get_tag_tid_in_debug(mock_local_journal);
1088   expect_committed(mock_replay_entry, mock_remote_journaler, 2);
1089
1090   InSequence seq;
1091   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1092               mock_local_image_ctx.name, "remote mirror uuid", 0);
1093   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1094               m_remote_image_ctx->id, 0);
1095   EXPECT_CALL(mock_remote_journaler, construct());
1096   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1097
1098   EXPECT_CALL(mock_local_journal, add_listener(_));
1099
1100   expect_init(mock_remote_journaler, 0);
1101
1102   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1103   expect_get_cached_client(mock_remote_journaler, 0);
1104
1105   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1106
1107   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1108
1109   create_image_replayer(mock_threads);
1110
1111   C_SaferCond start_ctx;
1112   m_image_replayer->start(&start_ctx);
1113   ASSERT_EQ(0, start_ctx.wait());
1114
1115   // REPLAY
1116
1117   cls::journal::Tag tag =
1118     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1119                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1120                             true, 0, 0})};
1121
1122   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1123
1124   // replay_flush
1125   expect_shut_down(mock_local_replay, false, 0);
1126   EXPECT_CALL(mock_local_journal, stop_external_replay());
1127   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1128   expect_get_tag(mock_remote_journaler, tag, 0);
1129   expect_allocate_tag(mock_local_journal, 0);
1130
1131   // process
1132   EXPECT_CALL(mock_replay_entry, get_data());
1133   EXPECT_CALL(mock_local_replay, decode(_, _))
1134     .WillOnce(Return(0));
1135   expect_preprocess(mock_event_preprocessor, false, 0);
1136   expect_process(mock_local_replay, 0, 0);
1137
1138   // the next event with preprocess
1139   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1140   EXPECT_CALL(mock_replay_entry, get_data());
1141   EXPECT_CALL(mock_local_replay, decode(_, _))
1142     .WillOnce(Return(0));
1143   expect_preprocess(mock_event_preprocessor, true, 0);
1144   expect_process(mock_local_replay, 0, 0);
1145
1146   // attempt to process the next event
1147   C_SaferCond replay_ctx;
1148   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1149
1150   // fire
1151   m_image_replayer->handle_replay_ready();
1152   ASSERT_EQ(0, replay_ctx.wait());
1153
1154   // STOP
1155
1156   MockCloseImageRequest mock_close_local_image_request;
1157   expect_shut_down(mock_local_replay, true, 0);
1158   EXPECT_CALL(mock_local_journal, remove_listener(_));
1159   EXPECT_CALL(mock_local_journal, stop_external_replay());
1160   expect_send(mock_close_local_image_request, 0);
1161
1162   expect_stop_replay(mock_remote_journaler, 0);
1163   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1164   expect_shut_down(mock_remote_journaler, 0);
1165
1166   C_SaferCond stop_ctx;
1167   m_image_replayer->stop(&stop_ctx);
1168   ASSERT_EQ(0, stop_ctx.wait());
1169 }
1170
1171 TEST_F(TestMockImageReplayer, DecodeError) {
1172   // START
1173
1174   create_local_image();
1175   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1176
1177   librbd::MockTestJournal mock_local_journal;
1178   mock_local_image_ctx.journal = &mock_local_journal;
1179
1180   journal::MockJournaler mock_remote_journaler;
1181   MockThreads mock_threads(m_threads);
1182   expect_work_queue_repeatedly(mock_threads);
1183   expect_add_event_after_repeatedly(mock_threads);
1184
1185   MockImageDeleter mock_image_deleter;
1186   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1187   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1188   MockBootstrapRequest mock_bootstrap_request;
1189   MockReplay mock_local_replay;
1190   MockEventPreprocessor mock_event_preprocessor;
1191   MockReplayStatusFormatter mock_replay_status_formatter;
1192   ::journal::MockReplayEntry mock_replay_entry;
1193
1194   expect_flush_repeatedly(mock_local_replay, mock_remote_journaler);
1195   expect_get_or_send_update(mock_replay_status_formatter);
1196   expect_get_commit_tid_in_debug(mock_replay_entry);
1197   expect_get_tag_tid_in_debug(mock_local_journal);
1198
1199   InSequence seq;
1200   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1201               mock_local_image_ctx.name, "remote mirror uuid", 0);
1202   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1203               m_remote_image_ctx->id, 0);
1204   EXPECT_CALL(mock_remote_journaler, construct());
1205   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1206
1207   EXPECT_CALL(mock_local_journal, add_listener(_));
1208
1209   expect_init(mock_remote_journaler, 0);
1210
1211   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1212   expect_get_cached_client(mock_remote_journaler, 0);
1213
1214   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1215
1216   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1217
1218   create_image_replayer(mock_threads);
1219
1220   C_SaferCond start_ctx;
1221   m_image_replayer->start(&start_ctx);
1222   ASSERT_EQ(0, start_ctx.wait());
1223
1224   // REPLAY
1225
1226   cls::journal::Tag tag =
1227     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1228                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1229                             true, 0, 0})};
1230
1231   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1232
1233   // replay_flush
1234   expect_shut_down(mock_local_replay, false, 0);
1235   EXPECT_CALL(mock_local_journal, stop_external_replay());
1236   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1237   expect_get_tag(mock_remote_journaler, tag, 0);
1238   expect_allocate_tag(mock_local_journal, 0);
1239
1240   // process
1241   EXPECT_CALL(mock_replay_entry, get_data());
1242   EXPECT_CALL(mock_local_replay, decode(_, _))
1243     .WillOnce(Return(-EINVAL));
1244
1245   // stop on error
1246   expect_shut_down(mock_local_replay, true, 0);
1247   EXPECT_CALL(mock_local_journal, remove_listener(_));
1248   EXPECT_CALL(mock_local_journal, stop_external_replay());
1249
1250   MockCloseImageRequest mock_close_local_image_request;
1251   C_SaferCond close_ctx;
1252   EXPECT_CALL(mock_close_local_image_request, send())
1253     .WillOnce(Invoke([&mock_close_local_image_request, &close_ctx]() {
1254           *mock_close_local_image_request.image_ctx = nullptr;
1255           mock_close_local_image_request.on_finish->complete(0);
1256           close_ctx.complete(0);
1257         }));
1258
1259   expect_stop_replay(mock_remote_journaler, 0);
1260   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1261   expect_shut_down(mock_remote_journaler, 0);
1262
1263   // fire
1264   m_image_replayer->handle_replay_ready();
1265   ASSERT_EQ(0, close_ctx.wait());
1266
1267   while (!m_image_replayer->is_stopped()) {
1268     usleep(1000);
1269   }
1270 }
1271
1272 TEST_F(TestMockImageReplayer, DelayedReplay) {
1273
1274   // START
1275
1276   create_local_image();
1277   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
1278
1279   librbd::MockTestJournal mock_local_journal;
1280   mock_local_image_ctx.journal = &mock_local_journal;
1281
1282   journal::MockJournaler mock_remote_journaler;
1283   MockThreads mock_threads(m_threads);
1284   expect_work_queue_repeatedly(mock_threads);
1285   expect_add_event_after_repeatedly(mock_threads);
1286
1287   MockImageDeleter mock_image_deleter;
1288   MockPrepareLocalImageRequest mock_prepare_local_image_request;
1289   MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
1290   MockBootstrapRequest mock_bootstrap_request;
1291   MockReplay mock_local_replay;
1292   MockEventPreprocessor mock_event_preprocessor;
1293   MockReplayStatusFormatter mock_replay_status_formatter;
1294   ::journal::MockReplayEntry mock_replay_entry;
1295
1296   expect_flush_repeatedly(mock_local_replay, mock_remote_journaler);
1297   expect_get_or_send_update(mock_replay_status_formatter);
1298   expect_get_commit_tid_in_debug(mock_replay_entry);
1299   expect_get_tag_tid_in_debug(mock_local_journal);
1300   expect_committed(mock_replay_entry, mock_remote_journaler, 1);
1301
1302   InSequence seq;
1303   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
1304               mock_local_image_ctx.name, "remote mirror uuid", 0);
1305   expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
1306               m_remote_image_ctx->id, 0);
1307   EXPECT_CALL(mock_remote_journaler, construct());
1308   expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0);
1309
1310   EXPECT_CALL(mock_local_journal, add_listener(_));
1311
1312   expect_init(mock_remote_journaler, 0);
1313
1314   EXPECT_CALL(mock_remote_journaler, add_listener(_));
1315   expect_get_cached_client(mock_remote_journaler, 0);
1316
1317   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1318
1319   EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _));
1320
1321   create_image_replayer(mock_threads);
1322
1323   C_SaferCond start_ctx;
1324   m_image_replayer->start(&start_ctx);
1325   ASSERT_EQ(0, start_ctx.wait());
1326
1327   // REPLAY
1328
1329   cls::journal::Tag tag =
1330     {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID,
1331                             librbd::Journal<>::LOCAL_MIRROR_UUID,
1332                             true, 0, 0})};
1333
1334   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1335
1336   // replay_flush
1337   expect_shut_down(mock_local_replay, false, 0);
1338   EXPECT_CALL(mock_local_journal, stop_external_replay());
1339   expect_start_external_replay(mock_local_journal, &mock_local_replay, 0);
1340   expect_get_tag(mock_remote_journaler, tag, 0);
1341   expect_allocate_tag(mock_local_journal, 0);
1342
1343   // process with delay
1344   EXPECT_CALL(mock_replay_entry, get_data());
1345   librbd::journal::EventEntry event_entry(
1346     librbd::journal::AioDiscardEvent(123, 345, 0), ceph_clock_now());
1347   EXPECT_CALL(mock_local_replay, decode(_, _))
1348     .WillOnce(DoAll(SetArgPointee<1>(event_entry),
1349                     Return(0)));
1350   expect_preprocess(mock_event_preprocessor, false, 0);
1351   expect_process(mock_local_replay, 0, 0);
1352
1353   // attempt to process the next event
1354   C_SaferCond replay_ctx;
1355   expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx);
1356
1357   // fire
1358   mock_local_image_ctx.mirroring_replay_delay = 2;
1359   m_image_replayer->handle_replay_ready();
1360   ASSERT_EQ(0, replay_ctx.wait());
1361
1362   // add a pending (delayed) entry before stop
1363   expect_try_pop_front(mock_remote_journaler, tag.tid, true);
1364   EXPECT_CALL(mock_replay_entry, get_data());
1365   C_SaferCond decode_ctx;
1366   EXPECT_CALL(mock_local_replay, decode(_, _))
1367     .WillOnce(DoAll(Invoke([&decode_ctx](bufferlist::const_iterator* it,
1368                                          librbd::journal::EventEntry *e) {
1369                              decode_ctx.complete(0);
1370                            }),
1371                     Return(0)));
1372
1373   mock_local_image_ctx.mirroring_replay_delay = 10;
1374   m_image_replayer->handle_replay_ready();
1375   ASSERT_EQ(0, decode_ctx.wait());
1376
1377   // STOP
1378
1379   MockCloseImageRequest mock_close_local_image_request;
1380
1381   expect_shut_down(mock_local_replay, true, 0);
1382   EXPECT_CALL(mock_local_journal, remove_listener(_));
1383   EXPECT_CALL(mock_local_journal, stop_external_replay());
1384   expect_send(mock_close_local_image_request, 0);
1385
1386   expect_stop_replay(mock_remote_journaler, 0);
1387   EXPECT_CALL(mock_remote_journaler, remove_listener(_));
1388   expect_shut_down(mock_remote_journaler, 0);
1389
1390   C_SaferCond stop_ctx;
1391   m_image_replayer->stop(&stop_ctx);
1392   ASSERT_EQ(0, stop_ctx.wait());
1393 }
1394
1395
1396 } // namespace mirror
1397 } // namespace rbd