ceph/src/test/rbd_mirror/test_mock_ImageReplayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "cls/journal/cls_journal_types.h"
   5 #include "librbd/journal/Types.h"
   6 #include "librbd/journal/TypeTraits.h"
   7 #include "tools/rbd_mirror/ImageDeleter.h"
   8 #include "tools/rbd_mirror/ImageReplayer.h"
   9 #include "tools/rbd_mirror/InstanceWatcher.h"
  10 #include "tools/rbd_mirror/MirrorStatusUpdater.h"
  11 #include "tools/rbd_mirror/Threads.h"
  12 #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
  13 #include "tools/rbd_mirror/image_replayer/Replayer.h"
  14 #include "tools/rbd_mirror/image_replayer/ReplayerListener.h"
  15 #include "tools/rbd_mirror/image_replayer/StateBuilder.h"
  16 #include "tools/rbd_mirror/image_replayer/Utils.h"
  17 #include "test/rbd_mirror/test_mock_fixture.h"
  18 #include "test/librbd/mock/MockImageCtx.h"
  19 #include "test/rbd_mirror/mock/MockContextWQ.h"
  20 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  21
  22 namespace librbd {
  23
  24 namespace {
  25
  26 struct MockTestImageCtx : public MockImageCtx {
  27   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  28     : librbd::MockImageCtx(image_ctx) {
  29   }
  30 };
  31
  32 } // anonymous namespace
  33
  34 } // namespace librbd
  35
  36 namespace rbd {
  37 namespace mirror {
  38
  39 template <>
  40 struct ImageDeleter<librbd::MockTestImageCtx> {
  41   static ImageDeleter* s_instance;
  42
  43   static void trash_move(librados::IoCtx& local_io_ctx,
  44                          const std::string& global_image_id, bool resync,
  45                          MockContextWQ* work_queue, Context* on_finish) {
  46     ceph_assert(s_instance != nullptr);
  47     s_instance->trash_move(global_image_id, resync, on_finish);
  48   }
  49
  50   MOCK_METHOD3(trash_move, void(const std::string&, bool, Context*));
  51
  52   ImageDeleter() {
  53     s_instance = this;
  54   }
  55 };
  56
  57 ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr;
  58
  59 template <>
  60 struct MirrorStatusUpdater<librbd::MockTestImageCtx> {
  61
  62   MOCK_METHOD1(exists, bool(const std::string&));
  63   MOCK_METHOD3(set_mirror_image_status,
  64                void(const std::string&, const cls::rbd::MirrorImageSiteStatus&,
  65                     bool));
  66   MOCK_METHOD2(remove_refresh_mirror_image_status, void(const std::string&,
  67                                                         Context*));
  68   MOCK_METHOD3(remove_mirror_image_status, void(const std::string&, bool,
  69                                                 Context*));
  70 };
  71
  72 template <>
  73 struct Threads<librbd::MockTestImageCtx> {
  74   MockSafeTimer *timer;
  75   ceph::mutex &timer_lock;
  76
  77   MockContextWQ *work_queue;
  78
  79   Threads(Threads<librbd::ImageCtx> *threads)
  80     : timer(new MockSafeTimer()),
  81       timer_lock(threads->timer_lock),
  82       work_queue(new MockContextWQ()) {
  83   }
  84   ~Threads() {
  85     delete timer;
  86     delete work_queue;
  87   }
  88 };
  89
  90 template<>
  91 class InstanceWatcher<librbd::MockTestImageCtx> {
  92 };
  93
  94 namespace image_replayer {
  95
  96 template<>
  97 struct BootstrapRequest<librbd::MockTestImageCtx> {
  98   static BootstrapRequest* s_instance;
  99
 100   StateBuilder<librbd::MockTestImageCtx>** state_builder = nullptr;
 101   bool *do_resync = nullptr;
 102   Context *on_finish = nullptr;
 103
 104   static BootstrapRequest* create(
 105       Threads<librbd::MockTestImageCtx>* threads,
 106       librados::IoCtx &local_io_ctx,
 107       librados::IoCtx& remote_io_ctx,
 108       rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
 109       const std::string &global_image_id,
 110       const std::string &local_mirror_uuid,
 111       const RemotePoolMeta& remote_pool_meta,
 112       ::journal::CacheManagerHandler *cache_manager_handler,
 113       PoolMetaCache* pool_meta_cache,
 114       rbd::mirror::ProgressContext *progress_ctx,
 115       StateBuilder<librbd::MockTestImageCtx>** state_builder,
 116       bool *do_resync, Context *on_finish) {
 117     ceph_assert(s_instance != nullptr);
 118     s_instance->state_builder = state_builder;
 119     s_instance->do_resync = do_resync;
 120     s_instance->on_finish = on_finish;
 121     return s_instance;
 122   }
 123
 124   BootstrapRequest() {
 125     ceph_assert(s_instance == nullptr);
 126     s_instance = this;
 127   }
 128
 129   ~BootstrapRequest() {
 130     ceph_assert(s_instance == this);
 131     s_instance = nullptr;
 132   }
 133
 134   void put() {
 135   }
 136
 137   void get() {
 138   }
 139
 140   std::string get_local_image_name() const {
 141     return "local image name";
 142   }
 143
 144   inline bool is_syncing() const {
 145     return false;
 146   }
 147
 148   MOCK_METHOD0(send, void());
 149   MOCK_METHOD0(cancel, void());
 150 };
 151
 152 struct MockReplayer : public Replayer {
 153   image_replayer::ReplayerListener* replayer_listener;
 154
 155   MOCK_METHOD0(destroy, void());
 156
 157   MOCK_METHOD1(init, void(Context*));
 158   MOCK_METHOD1(shut_down, void(Context*));
 159   MOCK_METHOD1(flush, void(Context*));
 160
 161   MOCK_METHOD2(get_replay_status, bool(std::string*, Context*));
 162
 163   MOCK_CONST_METHOD0(is_replaying, bool());
 164   MOCK_CONST_METHOD0(is_resync_requested, bool());
 165   MOCK_CONST_METHOD0(get_error_code, int());
 166   MOCK_CONST_METHOD0(get_error_description, std::string());
 167 };
 168
 169 template <>
 170 struct StateBuilder<librbd::MockTestImageCtx> {
 171   static StateBuilder* s_instance;
 172
 173   librbd::MockTestImageCtx* local_image_ctx = nullptr;
 174   std::string local_image_id;
 175   std::string remote_image_id;
 176
 177   void destroy() {
 178   }
 179
 180   MOCK_METHOD1(close, void(Context*));
 181   MOCK_METHOD5(create_replayer, Replayer*(Threads<librbd::MockTestImageCtx>*,
 182                                           InstanceWatcher<librbd::MockTestImageCtx>*,
 183                                           const std::string&, PoolMetaCache*,
 184                                           ReplayerListener*));
 185
 186   StateBuilder() {
 187     s_instance = this;
 188   }
 189 };
 190
 191 BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 192 StateBuilder<librbd::MockTestImageCtx>* StateBuilder<librbd::MockTestImageCtx>::s_instance = nullptr;
 193
 194 } // namespace image_replayer
 195 } // namespace mirror
 196 } // namespace rbd
 197
 198 // template definitions
 199 #include "tools/rbd_mirror/ImageReplayer.cc"
 200
 201 namespace rbd {
 202 namespace mirror {
 203
 204 using ::testing::_;
 205 using ::testing::AtLeast;
 206 using ::testing::DoAll;
 207 using ::testing::InSequence;
 208 using ::testing::Invoke;
 209 using ::testing::MatcherCast;
 210 using ::testing::Return;
 211 using ::testing::ReturnArg;
 212 using ::testing::SetArgPointee;
 213 using ::testing::WithArg;
 214
 215 class TestMockImageReplayer : public TestMockFixture {
 216 public:
 217   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 218   typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter;
 219   typedef MirrorStatusUpdater<librbd::MockTestImageCtx> MockMirrorStatusUpdater;
 220   typedef image_replayer::BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
 221   typedef image_replayer::StateBuilder<librbd::MockTestImageCtx> MockStateBuilder;
 222   typedef image_replayer::MockReplayer MockReplayer;
 223   typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer;
 224   typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
 225
 226   void SetUp() override {
 227     TestMockFixture::SetUp();
 228
 229     librbd::RBD rbd;
 230     ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size));
 231     ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx));
 232   }
 233
 234   void TearDown() override {
 235     delete m_image_replayer;
 236
 237     TestMockFixture::TearDown();
 238   }
 239
 240   void create_local_image() {
 241     librbd::RBD rbd;
 242     ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size));
 243     ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx));
 244   }
 245
 246   void expect_work_queue_repeatedly(MockThreads &mock_threads) {
 247     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 248       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 249           m_threads->work_queue->queue(ctx, r);
 250         }));
 251   }
 252
 253   void expect_add_event_after_repeatedly(MockThreads &mock_threads) {
 254     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 255       .WillRepeatedly(
 256         DoAll(Invoke([this](double seconds, Context *ctx) {
 257                        m_threads->timer->add_event_after(seconds, ctx);
 258                      }),
 259           ReturnArg<1>()));
 260     EXPECT_CALL(*mock_threads.timer, cancel_event(_))
 261       .WillRepeatedly(
 262         Invoke([this](Context *ctx) {
 263           return m_threads->timer->cancel_event(ctx);
 264         }));
 265   }
 266
 267   void expect_trash_move(MockImageDeleter& mock_image_deleter,
 268                          const std::string& global_image_id,
 269                          bool ignore_orphan, int r) {
 270     EXPECT_CALL(mock_image_deleter,
 271                 trash_move(global_image_id, ignore_orphan, _))
 272       .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) {
 273                              m_threads->work_queue->queue(ctx, r);
 274                            })));
 275   }
 276
 277   bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) {
 278     bufferlist bl;
 279     encode(tag_data, bl);
 280     return bl;
 281   }
 282
 283   void expect_send(MockBootstrapRequest& mock_bootstrap_request,
 284                    MockStateBuilder& mock_state_builder,
 285                    librbd::MockTestImageCtx& mock_local_image_ctx,
 286                    bool do_resync, bool set_local_image, int r) {
 287     EXPECT_CALL(mock_bootstrap_request, send())
 288       .WillOnce(Invoke([this, &mock_bootstrap_request, &mock_state_builder,
 289                         &mock_local_image_ctx, set_local_image, do_resync,
 290                         r]() {
 291             if (r == 0 || r == -ENOLINK) {
 292               mock_state_builder.local_image_id = mock_local_image_ctx.id;
 293               mock_state_builder.remote_image_id = m_remote_image_ctx->id;
 294               *mock_bootstrap_request.state_builder = &mock_state_builder;
 295             }
 296             if (r == 0) {
 297               mock_state_builder.local_image_ctx = &mock_local_image_ctx;
 298               *mock_bootstrap_request.do_resync = do_resync;
 299             }
 300             if (r < 0 && r != -ENOENT) {
 301               mock_state_builder.remote_image_id = "";
 302             }
 303             if (r == -ENOENT) {
 304               *mock_bootstrap_request.state_builder = &mock_state_builder;
 305             }
 306             if (set_local_image) {
 307               mock_state_builder.local_image_id = mock_local_image_ctx.id;
 308             }
 309             mock_bootstrap_request.on_finish->complete(r);
 310           }));
 311   }
 312
 313   void expect_create_replayer(MockStateBuilder& mock_state_builder,
 314                               MockReplayer& mock_replayer) {
 315     EXPECT_CALL(mock_state_builder, create_replayer(_, _, _, _, _))
 316       .WillOnce(WithArg<4>(
 317         Invoke([&mock_replayer]
 318                (image_replayer::ReplayerListener* replayer_listener) {
 319           mock_replayer.replayer_listener = replayer_listener;
 320           return &mock_replayer;
 321         })));
 322   }
 323
 324   void expect_close(MockStateBuilder& mock_state_builder, int r) {
 325     EXPECT_CALL(mock_state_builder, close(_))
 326       .WillOnce(Invoke([this, r](Context* ctx) {
 327                   m_threads->work_queue->queue(ctx, r);
 328                 }));
 329   }
 330
 331   void expect_init(MockReplayer& mock_replayer, int r) {
 332     EXPECT_CALL(mock_replayer, init(_))
 333       .WillOnce(Invoke([this, r](Context* ctx) {
 334                   m_threads->work_queue->queue(ctx, r);
 335                 }));
 336   }
 337
 338   void expect_shut_down(MockReplayer& mock_replayer, int r) {
 339     EXPECT_CALL(mock_replayer, shut_down(_))
 340       .WillOnce(Invoke([this, r](Context* ctx) {
 341                   m_threads->work_queue->queue(ctx, r);
 342                 }));
 343     EXPECT_CALL(mock_replayer, destroy());
 344   }
 345
 346   void expect_get_replay_status(MockReplayer& mock_replayer) {
 347     EXPECT_CALL(mock_replayer, get_replay_status(_, _))
 348       .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)),
 349                             Return(true)));
 350   }
 351
 352   void expect_set_mirror_image_status_repeatedly() {
 353     EXPECT_CALL(m_local_status_updater, set_mirror_image_status(_, _, _))
 354       .WillRepeatedly(Invoke([](auto, auto, auto){}));
 355     EXPECT_CALL(m_remote_status_updater, set_mirror_image_status(_, _, _))
 356       .WillRepeatedly(Invoke([](auto, auto, auto){}));
 357   }
 358
 359   void expect_mirror_image_status_exists(bool exists) {
 360     EXPECT_CALL(m_local_status_updater, exists(_))
 361       .WillOnce(Return(exists));
 362     EXPECT_CALL(m_remote_status_updater, exists(_))
 363       .WillOnce(Return(exists));
 364   }
 365
 366   void create_image_replayer(MockThreads &mock_threads) {
 367     m_image_replayer = new MockImageReplayer(
 368         m_local_io_ctx, "local_mirror_uuid", "global image id",
 369         &mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr,
 370         nullptr);
 371     m_image_replayer->add_peer({"peer_uuid", m_remote_io_ctx,
 372                                 {"remote mirror uuid",
 373                                  "remote mirror peer uuid"},
 374                                 &m_remote_status_updater});
 375   }
 376
 377   void wait_for_stopped() {
 378     for (int i = 0; i < 10000; i++) {
 379       if (m_image_replayer->is_stopped()) {
 380         break;
 381       }
 382       usleep(1000);
 383     }
 384     ASSERT_TRUE(m_image_replayer->is_stopped());
 385   }
 386
 387   librbd::ImageCtx *m_remote_image_ctx;
 388   librbd::ImageCtx *m_local_image_ctx = nullptr;
 389   MockInstanceWatcher m_instance_watcher;
 390   MockMirrorStatusUpdater m_local_status_updater;
 391   MockMirrorStatusUpdater m_remote_status_updater;
 392   MockImageReplayer *m_image_replayer = nullptr;
 393 };
 394
 395 TEST_F(TestMockImageReplayer, StartStop) {
 396   // START
 397
 398   create_local_image();
 399   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 400
 401   MockThreads mock_threads(m_threads);
 402   expect_work_queue_repeatedly(mock_threads);
 403   expect_add_event_after_repeatedly(mock_threads);
 404
 405   MockImageDeleter mock_image_deleter;
 406   MockReplayer mock_replayer;
 407
 408   expect_get_replay_status(mock_replayer);
 409   expect_set_mirror_image_status_repeatedly();
 410
 411   InSequence seq;
 412   MockBootstrapRequest mock_bootstrap_request;
 413   MockStateBuilder mock_state_builder;
 414   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 415               false, false, 0);
 416
 417   expect_create_replayer(mock_state_builder, mock_replayer);
 418   expect_init(mock_replayer, 0);
 419
 420   create_image_replayer(mock_threads);
 421
 422   C_SaferCond start_ctx;
 423   m_image_replayer->start(&start_ctx);
 424   ASSERT_EQ(0, start_ctx.wait());
 425   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 426             m_image_replayer->get_health_state());
 427
 428   // STOP
 429   expect_shut_down(mock_replayer, 0);
 430   expect_close(mock_state_builder, 0);
 431   expect_mirror_image_status_exists(false);
 432
 433   C_SaferCond stop_ctx;
 434   m_image_replayer->stop(&stop_ctx);
 435   ASSERT_EQ(0, stop_ctx.wait());
 436   ASSERT_EQ(image_replayer::HEALTH_STATE_OK,
 437             m_image_replayer->get_health_state());
 438 }
 439
 440 TEST_F(TestMockImageReplayer, LocalImagePrimary) {
 441   create_local_image();
 442   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 443
 444   MockThreads mock_threads(m_threads);
 445   expect_work_queue_repeatedly(mock_threads);
 446   expect_add_event_after_repeatedly(mock_threads);
 447
 448   MockImageDeleter mock_image_deleter;
 449   MockBootstrapRequest mock_bootstrap_request;
 450
 451   expect_set_mirror_image_status_repeatedly();
 452
 453   InSequence seq;
 454
 455   MockStateBuilder mock_state_builder;
 456   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 457               false, false, -ENOMSG);
 458
 459   expect_mirror_image_status_exists(false);
 460
 461   create_image_replayer(mock_threads);
 462
 463   C_SaferCond start_ctx;
 464   m_image_replayer->start(&start_ctx);
 465   ASSERT_EQ(0, start_ctx.wait());
 466 }
 467
 468 TEST_F(TestMockImageReplayer, MetadataCleanup) {
 469   // START
 470
 471   create_local_image();
 472   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 473
 474   MockThreads mock_threads(m_threads);
 475   expect_work_queue_repeatedly(mock_threads);
 476   expect_add_event_after_repeatedly(mock_threads);
 477
 478   MockImageDeleter mock_image_deleter;
 479   MockBootstrapRequest mock_bootstrap_request;
 480   MockReplayer mock_replayer;
 481
 482   expect_get_replay_status(mock_replayer);
 483   expect_set_mirror_image_status_repeatedly();
 484
 485   InSequence seq;
 486
 487   MockStateBuilder mock_state_builder;
 488   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 489               false, true, -ENOLINK);
 490
 491   expect_close(mock_state_builder, 0);
 492   expect_trash_move(mock_image_deleter, "global image id", false, 0);
 493   expect_mirror_image_status_exists(false);
 494
 495   create_image_replayer(mock_threads);
 496
 497   C_SaferCond start_ctx;
 498   m_image_replayer->start(&start_ctx);
 499   ASSERT_EQ(0, start_ctx.wait());
 500 }
 501
 502 TEST_F(TestMockImageReplayer, BootstrapRemoteDeleted) {
 503   create_local_image();
 504   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 505
 506   MockThreads mock_threads(m_threads);
 507   expect_work_queue_repeatedly(mock_threads);
 508   expect_add_event_after_repeatedly(mock_threads);
 509
 510   MockImageDeleter mock_image_deleter;
 511
 512   expect_set_mirror_image_status_repeatedly();
 513
 514   InSequence seq;
 515
 516   MockBootstrapRequest mock_bootstrap_request;
 517   MockStateBuilder mock_state_builder;
 518   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 519               false, false, -ENOLINK);
 520
 521   expect_close(mock_state_builder, 0);
 522
 523   expect_trash_move(mock_image_deleter, "global image id", false, 0);
 524   expect_mirror_image_status_exists(false);
 525
 526   create_image_replayer(mock_threads);
 527
 528   C_SaferCond start_ctx;
 529   m_image_replayer->start(&start_ctx);
 530   ASSERT_EQ(0, start_ctx.wait());
 531 }
 532
 533 TEST_F(TestMockImageReplayer, BootstrapResyncRequested) {
 534   create_local_image();
 535   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 536
 537   MockThreads mock_threads(m_threads);
 538   expect_work_queue_repeatedly(mock_threads);
 539   expect_add_event_after_repeatedly(mock_threads);
 540
 541   MockImageDeleter mock_image_deleter;
 542
 543   expect_set_mirror_image_status_repeatedly();
 544
 545   InSequence seq;
 546
 547   MockBootstrapRequest mock_bootstrap_request;
 548   MockStateBuilder mock_state_builder;
 549   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 550               true, false, 0);
 551
 552   expect_close(mock_state_builder, 0);
 553
 554   expect_trash_move(mock_image_deleter, "global image id", true, 0);
 555   expect_mirror_image_status_exists(false);
 556
 557   create_image_replayer(mock_threads);
 558
 559   C_SaferCond start_ctx;
 560   m_image_replayer->start(&start_ctx);
 561   ASSERT_EQ(0, start_ctx.wait());
 562 }
 563
 564 TEST_F(TestMockImageReplayer, BootstrapError) {
 565   create_local_image();
 566   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 567
 568   MockThreads mock_threads(m_threads);
 569   expect_work_queue_repeatedly(mock_threads);
 570   expect_add_event_after_repeatedly(mock_threads);
 571
 572   MockImageDeleter mock_image_deleter;
 573   MockBootstrapRequest mock_bootstrap_request;
 574
 575   expect_set_mirror_image_status_repeatedly();
 576
 577   InSequence seq;
 578   MockStateBuilder mock_state_builder;
 579   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 580               false, false, -EINVAL);
 581
 582   expect_mirror_image_status_exists(false);
 583
 584   create_image_replayer(mock_threads);
 585
 586   C_SaferCond start_ctx;
 587   m_image_replayer->start(&start_ctx);
 588   ASSERT_EQ(-EINVAL, start_ctx.wait());
 589 }
 590
 591 TEST_F(TestMockImageReplayer, BootstrapCancel) {
 592   create_local_image();
 593   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 594
 595   MockThreads mock_threads(m_threads);
 596   expect_work_queue_repeatedly(mock_threads);
 597   expect_add_event_after_repeatedly(mock_threads);
 598
 599   MockImageDeleter mock_image_deleter;
 600
 601   expect_set_mirror_image_status_repeatedly();
 602
 603   InSequence seq;
 604
 605   create_image_replayer(mock_threads);
 606
 607   MockBootstrapRequest mock_bootstrap_request;
 608   MockStateBuilder mock_state_builder;
 609   EXPECT_CALL(mock_bootstrap_request, send())
 610     .WillOnce(Invoke([this, &mock_bootstrap_request]() {
 611         m_image_replayer->stop(nullptr);
 612         mock_bootstrap_request.on_finish->complete(-ECANCELED);
 613       }));
 614   EXPECT_CALL(mock_bootstrap_request, cancel());
 615
 616   expect_mirror_image_status_exists(false);
 617
 618   C_SaferCond start_ctx;
 619   m_image_replayer->start(&start_ctx);
 620   ASSERT_EQ(-ECANCELED, start_ctx.wait());
 621 }
 622
 623 TEST_F(TestMockImageReplayer, BootstrapRemoteDeletedCancel) {
 624   create_local_image();
 625   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 626
 627   MockThreads mock_threads(m_threads);
 628   expect_work_queue_repeatedly(mock_threads);
 629   expect_add_event_after_repeatedly(mock_threads);
 630
 631   MockImageDeleter mock_image_deleter;
 632
 633   expect_set_mirror_image_status_repeatedly();
 634
 635   InSequence seq;
 636
 637   MockBootstrapRequest mock_bootstrap_request;
 638   MockStateBuilder mock_state_builder;
 639   EXPECT_CALL(mock_bootstrap_request, send())
 640     .WillOnce(Invoke([this, &mock_bootstrap_request, &mock_state_builder,
 641                       &mock_local_image_ctx]() {
 642         mock_state_builder.local_image_id = mock_local_image_ctx.id;
 643         mock_state_builder.remote_image_id = "";
 644         *mock_bootstrap_request.state_builder = &mock_state_builder;
 645         m_image_replayer->stop(nullptr);
 646         mock_bootstrap_request.on_finish->complete(-ENOLINK);
 647       }));
 648   EXPECT_CALL(mock_bootstrap_request, cancel());
 649
 650   expect_close(mock_state_builder, 0);
 651
 652   expect_trash_move(mock_image_deleter, "global image id", false, 0);
 653   expect_mirror_image_status_exists(false);
 654
 655   create_image_replayer(mock_threads);
 656
 657   C_SaferCond start_ctx;
 658   m_image_replayer->start(&start_ctx);
 659   ASSERT_EQ(-ECANCELED, start_ctx.wait());
 660 }
 661
 662 TEST_F(TestMockImageReplayer, StopError) {
 663   // START
 664
 665   create_local_image();
 666   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 667
 668   MockThreads mock_threads(m_threads);
 669   expect_work_queue_repeatedly(mock_threads);
 670   expect_add_event_after_repeatedly(mock_threads);
 671
 672   MockImageDeleter mock_image_deleter;
 673   MockBootstrapRequest mock_bootstrap_request;
 674   MockReplayer mock_replayer;
 675
 676   expect_get_replay_status(mock_replayer);
 677   expect_set_mirror_image_status_repeatedly();
 678
 679   InSequence seq;
 680   MockStateBuilder mock_state_builder;
 681   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 682               false, false, 0);
 683
 684   expect_create_replayer(mock_state_builder, mock_replayer);
 685   expect_init(mock_replayer, 0);
 686
 687   create_image_replayer(mock_threads);
 688
 689   C_SaferCond start_ctx;
 690   m_image_replayer->start(&start_ctx);
 691   ASSERT_EQ(0, start_ctx.wait());
 692
 693   // STOP (errors are ignored)
 694
 695   expect_shut_down(mock_replayer, -EINVAL);
 696   expect_close(mock_state_builder, -EINVAL);
 697   expect_mirror_image_status_exists(false);
 698
 699   C_SaferCond stop_ctx;
 700   m_image_replayer->stop(&stop_ctx);
 701   ASSERT_EQ(0, stop_ctx.wait());
 702 }
 703
 704 TEST_F(TestMockImageReplayer, ReplayerError) {
 705   create_local_image();
 706   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 707
 708   MockThreads mock_threads(m_threads);
 709   expect_work_queue_repeatedly(mock_threads);
 710   expect_add_event_after_repeatedly(mock_threads);
 711
 712   MockImageDeleter mock_image_deleter;
 713   MockBootstrapRequest mock_bootstrap_request;
 714   MockReplayer mock_replayer;
 715
 716   expect_set_mirror_image_status_repeatedly();
 717
 718   InSequence seq;
 719   MockStateBuilder mock_state_builder;
 720   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 721               false, false, 0);
 722
 723   expect_create_replayer(mock_state_builder, mock_replayer);
 724   expect_init(mock_replayer, -EINVAL);
 725   EXPECT_CALL(mock_replayer, get_error_description())
 726     .WillOnce(Return("FAIL"));
 727
 728   EXPECT_CALL(mock_replayer, destroy());
 729   expect_close(mock_state_builder, -EINVAL);
 730
 731   expect_mirror_image_status_exists(false);
 732   create_image_replayer(mock_threads);
 733
 734   C_SaferCond start_ctx;
 735   m_image_replayer->start(&start_ctx);
 736   ASSERT_EQ(-EINVAL, start_ctx.wait());
 737 }
 738
 739 TEST_F(TestMockImageReplayer, ReplayerResync) {
 740   // START
 741   create_local_image();
 742   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 743
 744   MockThreads mock_threads(m_threads);
 745   expect_work_queue_repeatedly(mock_threads);
 746   expect_add_event_after_repeatedly(mock_threads);
 747
 748   MockImageDeleter mock_image_deleter;
 749   MockBootstrapRequest mock_bootstrap_request;
 750   MockReplayer mock_replayer;
 751
 752   expect_get_replay_status(mock_replayer);
 753   expect_set_mirror_image_status_repeatedly();
 754
 755   InSequence seq;
 756   MockStateBuilder mock_state_builder;
 757   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 758               false, false, 0);
 759
 760   expect_create_replayer(mock_state_builder, mock_replayer);
 761   expect_init(mock_replayer, 0);
 762
 763   create_image_replayer(mock_threads);
 764
 765   C_SaferCond start_ctx;
 766   m_image_replayer->start(&start_ctx);
 767   ASSERT_EQ(0, start_ctx.wait());
 768
 769   // NOTIFY
 770   EXPECT_CALL(mock_replayer, is_resync_requested())
 771     .WillOnce(Return(true));
 772   expect_shut_down(mock_replayer, 0);
 773   expect_close(mock_state_builder, 0);
 774   expect_trash_move(mock_image_deleter, "global image id", true, 0);
 775   expect_mirror_image_status_exists(false);
 776   mock_replayer.replayer_listener->handle_notification();
 777   ASSERT_FALSE(m_image_replayer->is_running());
 778
 779   wait_for_stopped();
 780 }
 781
 782 TEST_F(TestMockImageReplayer, ReplayerInterrupted) {
 783   // START
 784   create_local_image();
 785   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 786
 787   MockThreads mock_threads(m_threads);
 788   expect_work_queue_repeatedly(mock_threads);
 789   expect_add_event_after_repeatedly(mock_threads);
 790
 791   MockImageDeleter mock_image_deleter;
 792   MockBootstrapRequest mock_bootstrap_request;
 793   MockReplayer mock_replayer;
 794
 795   expect_get_replay_status(mock_replayer);
 796   expect_set_mirror_image_status_repeatedly();
 797
 798   InSequence seq;
 799   MockStateBuilder mock_state_builder;
 800   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 801               false, false, 0);
 802
 803   expect_create_replayer(mock_state_builder, mock_replayer);
 804   expect_init(mock_replayer, 0);
 805
 806   create_image_replayer(mock_threads);
 807
 808   C_SaferCond start_ctx;
 809   m_image_replayer->start(&start_ctx);
 810   ASSERT_EQ(0, start_ctx.wait());
 811
 812   // NOTIFY
 813   EXPECT_CALL(mock_replayer, is_resync_requested())
 814     .WillOnce(Return(false));
 815   EXPECT_CALL(mock_replayer, is_replaying())
 816     .WillOnce(Return(false));
 817   EXPECT_CALL(mock_replayer, get_error_code())
 818     .WillOnce(Return(-EINVAL));
 819   EXPECT_CALL(mock_replayer, get_error_description())
 820     .WillOnce(Return("INVALID"));
 821   expect_shut_down(mock_replayer, 0);
 822   expect_close(mock_state_builder, 0);
 823   expect_mirror_image_status_exists(false);
 824   mock_replayer.replayer_listener->handle_notification();
 825   ASSERT_FALSE(m_image_replayer->is_running());
 826
 827   wait_for_stopped();
 828 }
 829
 830 TEST_F(TestMockImageReplayer, ReplayerRenamed) {
 831   // START
 832   create_local_image();
 833   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 834
 835   MockThreads mock_threads(m_threads);
 836   expect_work_queue_repeatedly(mock_threads);
 837   expect_add_event_after_repeatedly(mock_threads);
 838
 839   MockImageDeleter mock_image_deleter;
 840   MockBootstrapRequest mock_bootstrap_request;
 841   MockReplayer mock_replayer;
 842
 843   expect_get_replay_status(mock_replayer);
 844   expect_set_mirror_image_status_repeatedly();
 845
 846   InSequence seq;
 847   MockStateBuilder mock_state_builder;
 848   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 849               false, false, 0);
 850
 851   expect_create_replayer(mock_state_builder, mock_replayer);
 852   expect_init(mock_replayer, 0);
 853
 854   create_image_replayer(mock_threads);
 855
 856   C_SaferCond start_ctx;
 857   m_image_replayer->start(&start_ctx);
 858   ASSERT_EQ(0, start_ctx.wait());
 859
 860   // NOTIFY
 861   EXPECT_CALL(mock_replayer, is_resync_requested())
 862     .WillOnce(Return(false));
 863   EXPECT_CALL(mock_replayer, is_replaying())
 864     .WillOnce(Return(true));
 865   mock_local_image_ctx.name = "NEW NAME";
 866   mock_replayer.replayer_listener->handle_notification();
 867
 868   // STOP
 869   expect_shut_down(mock_replayer, 0);
 870   expect_close(mock_state_builder, 0);
 871   expect_mirror_image_status_exists(false);
 872
 873   C_SaferCond stop_ctx;
 874   m_image_replayer->stop(&stop_ctx);
 875   ASSERT_EQ(0, stop_ctx.wait());
 876
 877   auto image_spec = image_replayer::util::compute_image_spec(
 878     m_local_io_ctx, "NEW NAME");
 879   ASSERT_EQ(image_spec, m_image_replayer->get_name());
 880 }
 881
 882 TEST_F(TestMockImageReplayer, StopJoinInterruptedReplayer) {
 883   // START
 884   create_local_image();
 885   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 886
 887   MockThreads mock_threads(m_threads);
 888   expect_work_queue_repeatedly(mock_threads);
 889   expect_add_event_after_repeatedly(mock_threads);
 890
 891   MockReplayer mock_replayer;
 892   expect_get_replay_status(mock_replayer);
 893   expect_set_mirror_image_status_repeatedly();
 894
 895   InSequence seq;
 896   MockBootstrapRequest mock_bootstrap_request;
 897   MockStateBuilder mock_state_builder;
 898   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 899               false, false, 0);
 900
 901   expect_create_replayer(mock_state_builder, mock_replayer);
 902   expect_init(mock_replayer, 0);
 903
 904   create_image_replayer(mock_threads);
 905
 906   C_SaferCond start_ctx;
 907   m_image_replayer->start(&start_ctx);
 908   ASSERT_EQ(0, start_ctx.wait());
 909
 910   // NOTIFY
 911   EXPECT_CALL(mock_replayer, is_resync_requested())
 912     .WillOnce(Return(false));
 913   EXPECT_CALL(mock_replayer, is_replaying())
 914     .WillOnce(Return(false));
 915   EXPECT_CALL(mock_replayer, get_error_code())
 916     .WillOnce(Return(-EINVAL));
 917   EXPECT_CALL(mock_replayer, get_error_description())
 918     .WillOnce(Return("INVALID"));
 919   const double DELAY = 10;
 920   EXPECT_CALL(mock_replayer, shut_down(_))
 921     .WillOnce(Invoke([this, DELAY](Context* ctx) {
 922                 std::lock_guard l(m_threads->timer_lock);
 923                 m_threads->timer->add_event_after(DELAY, ctx);
 924               }));
 925   EXPECT_CALL(mock_replayer, destroy());
 926   expect_close(mock_state_builder, 0);
 927   expect_mirror_image_status_exists(false);
 928
 929   mock_replayer.replayer_listener->handle_notification();
 930   ASSERT_FALSE(m_image_replayer->is_running());
 931
 932   C_SaferCond stop_ctx;
 933   m_image_replayer->stop(&stop_ctx);
 934   ASSERT_EQ(ETIMEDOUT, stop_ctx.wait_for(DELAY * 3 / 4));
 935   ASSERT_EQ(0, stop_ctx.wait_for(DELAY));
 936 }
 937
 938 TEST_F(TestMockImageReplayer, StopJoinRequestedStop) {
 939   // START
 940   create_local_image();
 941   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 942
 943   MockThreads mock_threads(m_threads);
 944   expect_work_queue_repeatedly(mock_threads);
 945   expect_add_event_after_repeatedly(mock_threads);
 946
 947   MockReplayer mock_replayer;
 948   expect_get_replay_status(mock_replayer);
 949   expect_set_mirror_image_status_repeatedly();
 950
 951   InSequence seq;
 952   MockBootstrapRequest mock_bootstrap_request;
 953   MockStateBuilder mock_state_builder;
 954   expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx,
 955               false, false, 0);
 956
 957   expect_create_replayer(mock_state_builder, mock_replayer);
 958   expect_init(mock_replayer, 0);
 959
 960   create_image_replayer(mock_threads);
 961
 962   C_SaferCond start_ctx;
 963   m_image_replayer->start(&start_ctx);
 964   ASSERT_EQ(0, start_ctx.wait());
 965
 966   // STOP
 967   const double DELAY = 10;
 968   EXPECT_CALL(mock_replayer, shut_down(_))
 969     .WillOnce(Invoke([this, DELAY](Context* ctx) {
 970                 std::lock_guard l(m_threads->timer_lock);
 971                 m_threads->timer->add_event_after(DELAY, ctx);
 972               }));
 973   EXPECT_CALL(mock_replayer, destroy());
 974   expect_close(mock_state_builder, 0);
 975   expect_mirror_image_status_exists(false);
 976
 977   C_SaferCond stop_ctx1;
 978   m_image_replayer->stop(&stop_ctx1);
 979
 980   C_SaferCond stop_ctx2;
 981   m_image_replayer->stop(&stop_ctx2);
 982   ASSERT_EQ(ETIMEDOUT, stop_ctx2.wait_for(DELAY * 3 / 4));
 983   ASSERT_EQ(0, stop_ctx2.wait_for(DELAY));
 984
 985   ASSERT_EQ(0, stop_ctx1.wait_for(0));
 986 }
 987
 988 } // namespace mirror
 989 } // namespace rbd