ceph/src/test/rbd_mirror/test_mock_PoolWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "test/rbd_mirror/test_mock_fixture.h"
   5 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
   6 #include "test/librados_test_stub/MockTestMemRadosClient.h"
   7 #include "test/librbd/mock/MockImageCtx.h"
   8 #include "test/rbd_mirror/mock/MockContextWQ.h"
   9 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  10 #include "librbd/MirroringWatcher.h"
  11 #include "tools/rbd_mirror/Threads.h"
  12 #include "tools/rbd_mirror/PoolWatcher.h"
  13 #include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h"
  14 #include "include/stringify.h"
  15
  16 namespace librbd {
  17 namespace {
  18
  19 struct MockTestImageCtx : public librbd::MockImageCtx {
  20   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  21     : librbd::MockImageCtx(image_ctx) {
  22   }
  23 };
  24
  25 } // anonymous namespace
  26
  27 struct MockMirroringWatcher {
  28   static MockMirroringWatcher *s_instance;
  29   static MockMirroringWatcher &get_instance() {
  30     assert(s_instance != nullptr);
  31     return *s_instance;
  32   }
  33
  34   MockMirroringWatcher() {
  35     s_instance = this;
  36   }
  37
  38   MOCK_CONST_METHOD0(is_unregistered, bool());
  39   MOCK_METHOD1(register_watch, void(Context*));
  40   MOCK_METHOD1(unregister_watch, void(Context*));
  41
  42   MOCK_CONST_METHOD0(get_oid, std::string());
  43 };
  44
  45 template <>
  46 struct MirroringWatcher<MockTestImageCtx> {
  47   static MirroringWatcher *s_instance;
  48
  49   MirroringWatcher(librados::IoCtx &io_ctx, ::MockContextWQ *work_queue) {
  50     s_instance = this;
  51   }
  52   virtual ~MirroringWatcher() {
  53   }
  54
  55   static MirroringWatcher<MockTestImageCtx> &get_instance() {
  56     assert(s_instance != nullptr);
  57     return *s_instance;
  58   }
  59
  60   virtual void handle_rewatch_complete(int r) = 0;
  61
  62   virtual void handle_mode_updated(cls::rbd::MirrorMode mirror_mode) = 0;
  63   virtual void handle_image_updated(cls::rbd::MirrorImageState state,
  64                                     const std::string &remote_image_id,
  65                                     const std::string &global_image_id) = 0;
  66
  67   bool is_unregistered() const {
  68     return MockMirroringWatcher::get_instance().is_unregistered();
  69   }
  70   void register_watch(Context *ctx) {
  71     MockMirroringWatcher::get_instance().register_watch(ctx);
  72   }
  73   void unregister_watch(Context *ctx) {
  74     MockMirroringWatcher::get_instance().unregister_watch(ctx);
  75   }
  76   std::string get_oid() const {
  77     return MockMirroringWatcher::get_instance().get_oid();
  78   }
  79 };
  80
  81 MockMirroringWatcher *MockMirroringWatcher::s_instance = nullptr;
  82 MirroringWatcher<MockTestImageCtx> *MirroringWatcher<MockTestImageCtx>::s_instance = nullptr;
  83
  84 } // namespace librbd
  85
  86 namespace rbd {
  87 namespace mirror {
  88
  89 template <>
  90 struct Threads<librbd::MockTestImageCtx> {
  91   MockSafeTimer *timer;
  92   Mutex &timer_lock;
  93
  94   MockContextWQ *work_queue;
  95
  96   Threads(Threads<librbd::ImageCtx> *threads)
  97     : timer(new MockSafeTimer()),
  98       timer_lock(threads->timer_lock),
  99       work_queue(new MockContextWQ()) {
 100   }
 101   ~Threads() {
 102     delete timer;
 103     delete work_queue;
 104   }
 105 };
 106
 107 namespace pool_watcher {
 108
 109 template <>
 110 struct RefreshImagesRequest<librbd::MockTestImageCtx> {
 111   ImageIds *image_ids = nullptr;
 112   Context *on_finish = nullptr;
 113   static RefreshImagesRequest *s_instance;
 114   static RefreshImagesRequest *create(librados::IoCtx &io_ctx,
 115                                       ImageIds *image_ids,
 116                                       Context *on_finish) {
 117     assert(s_instance != nullptr);
 118     s_instance->image_ids = image_ids;
 119     s_instance->on_finish = on_finish;
 120     return s_instance;
 121   }
 122
 123   MOCK_METHOD0(send, void());
 124
 125   RefreshImagesRequest() {
 126     s_instance = this;
 127   }
 128 };
 129
 130 RefreshImagesRequest<librbd::MockTestImageCtx> *RefreshImagesRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 131
 132 } // namespace pool_watcher
 133
 134 } // namespace mirror
 135 } // namespace rbd
 136
 137 // template definitions
 138 #include "tools/rbd_mirror/PoolWatcher.cc"
 139
 140 namespace rbd {
 141 namespace mirror {
 142
 143 using ::testing::_;
 144 using ::testing::DoAll;
 145 using ::testing::InSequence;
 146 using ::testing::Invoke;
 147 using ::testing::Return;
 148 using ::testing::StrEq;
 149 using ::testing::WithArg;
 150 using ::testing::WithoutArgs;
 151
 152 class TestMockPoolWatcher : public TestMockFixture {
 153 public:
 154   typedef PoolWatcher<librbd::MockTestImageCtx> MockPoolWatcher;
 155   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 156   typedef pool_watcher::RefreshImagesRequest<librbd::MockTestImageCtx> MockRefreshImagesRequest;
 157   typedef librbd::MockMirroringWatcher MockMirroringWatcher;
 158   typedef librbd::MirroringWatcher<librbd::MockTestImageCtx> MirroringWatcher;
 159
 160   struct MockListener : MockPoolWatcher::Listener {
 161     TestMockPoolWatcher *test;
 162
 163     MockListener(TestMockPoolWatcher *test) : test(test) {
 164     }
 165
 166     MOCK_METHOD3(mock_handle_update, void(const std::string &, const ImageIds &,
 167                                           const ImageIds &));
 168     void handle_update(const std::string &mirror_uuid,
 169                        ImageIds &&added_image_ids,
 170                        ImageIds &&removed_image_ids) override {
 171       mock_handle_update(mirror_uuid, added_image_ids, removed_image_ids);
 172     }
 173   };
 174
 175   TestMockPoolWatcher() : m_lock("TestMockPoolWatcher::m_lock") {
 176   }
 177
 178   void expect_work_queue(MockThreads &mock_threads) {
 179     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 180       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 181           m_threads->work_queue->queue(ctx, r);
 182         }));
 183   }
 184
 185   void expect_mirroring_watcher_is_unregistered(MockMirroringWatcher &mock_mirroring_watcher,
 186                                                 bool unregistered) {
 187     EXPECT_CALL(mock_mirroring_watcher, is_unregistered())
 188       .WillOnce(Return(unregistered));
 189   }
 190
 191   void expect_mirroring_watcher_register(MockMirroringWatcher &mock_mirroring_watcher,
 192                                          int r) {
 193     EXPECT_CALL(mock_mirroring_watcher, register_watch(_))
 194       .WillOnce(CompleteContext(r));
 195   }
 196
 197   void expect_mirroring_watcher_unregister(MockMirroringWatcher &mock_mirroring_watcher,
 198                                          int r) {
 199     EXPECT_CALL(mock_mirroring_watcher, unregister_watch(_))
 200       .WillOnce(CompleteContext(r));
 201   }
 202
 203   void expect_refresh_images(MockRefreshImagesRequest &request,
 204                              const ImageIds &image_ids, int r) {
 205     EXPECT_CALL(request, send())
 206       .WillOnce(Invoke([&request, image_ids, r]() {
 207           *request.image_ids = image_ids;
 208           request.on_finish->complete(r);
 209         }));
 210   }
 211
 212   void expect_listener_handle_update(MockListener &mock_listener,
 213                                      const std::string &mirror_uuid,
 214                                      const ImageIds &added_image_ids,
 215                                      const ImageIds &removed_image_ids) {
 216     EXPECT_CALL(mock_listener, mock_handle_update(mirror_uuid, added_image_ids,
 217                                                   removed_image_ids))
 218       .WillOnce(WithoutArgs(Invoke([this]() {
 219           Mutex::Locker locker(m_lock);
 220           ++m_update_count;
 221           m_cond.Signal();
 222         })));
 223   }
 224
 225   void expect_mirror_uuid_get(librados::IoCtx &io_ctx,
 226                               const std::string &uuid, int r) {
 227     bufferlist out_bl;
 228     ::encode(uuid, out_bl);
 229
 230     EXPECT_CALL(get_mock_io_ctx(io_ctx),
 231                 exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"),
 232                      _, _, _))
 233       .WillOnce(DoAll(WithArg<5>(Invoke([this, out_bl](bufferlist *bl) {
 234                           *bl = out_bl;
 235                         })),
 236                       Return(r)));
 237   }
 238
 239   void expect_timer_add_event(MockThreads &mock_threads) {
 240     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 241       .WillOnce(WithArg<1>(Invoke([this](Context *ctx) {
 242           auto wrapped_ctx = new FunctionContext([this, ctx](int r) {
 243               Mutex::Locker timer_locker(m_threads->timer_lock);
 244               ctx->complete(r);
 245             });
 246           m_threads->work_queue->queue(wrapped_ctx, 0);
 247         })));
 248   }
 249
 250   int when_shut_down(MockPoolWatcher &mock_pool_watcher) {
 251     C_SaferCond ctx;
 252     mock_pool_watcher.shut_down(&ctx);
 253     return ctx.wait();
 254   }
 255
 256   bool wait_for_update(uint32_t count) {
 257     Mutex::Locker locker(m_lock);
 258     while (m_update_count < count) {
 259       if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) {
 260         break;
 261       }
 262     }
 263     if (m_update_count < count) {
 264       return false;
 265     }
 266
 267     m_update_count -= count;
 268     return true;
 269   }
 270
 271   Mutex m_lock;
 272   Cond m_cond;
 273   uint32_t m_update_count = 0;
 274 };
 275
 276 TEST_F(TestMockPoolWatcher, EmptyPool) {
 277   MockThreads mock_threads(m_threads);
 278   expect_work_queue(mock_threads);
 279
 280   InSequence seq;
 281   MockMirroringWatcher mock_mirroring_watcher;
 282   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 283   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 284
 285   MockRefreshImagesRequest mock_refresh_images_request;
 286   expect_refresh_images(mock_refresh_images_request, {}, 0);
 287   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 288
 289   MockListener mock_listener(this);
 290   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 291
 292   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 293                                     mock_listener);
 294   C_SaferCond ctx;
 295   mock_pool_watcher.init(&ctx);
 296   ASSERT_EQ(0, ctx.wait());
 297
 298   ASSERT_TRUE(wait_for_update(1));
 299   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 300   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 301 }
 302
 303 TEST_F(TestMockPoolWatcher, NonEmptyPool) {
 304   MockThreads mock_threads(m_threads);
 305   expect_work_queue(mock_threads);
 306
 307   InSequence seq;
 308   MockMirroringWatcher mock_mirroring_watcher;
 309   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 310   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 311
 312   ImageIds image_ids{
 313     {"global id 1", "remote id 1"},
 314     {"global id 2", "remote id 2"}};
 315   MockRefreshImagesRequest mock_refresh_images_request;
 316   expect_refresh_images(mock_refresh_images_request, image_ids, 0);
 317   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 318
 319   MockListener mock_listener(this);
 320   expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 321
 322   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 323                                     mock_listener);
 324   C_SaferCond ctx;
 325   mock_pool_watcher.init(&ctx);
 326   ASSERT_EQ(0, ctx.wait());
 327
 328   ASSERT_TRUE(wait_for_update(1));
 329   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 330   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 331 }
 332
 333 TEST_F(TestMockPoolWatcher, NotifyDuringRefresh) {
 334   MockThreads mock_threads(m_threads);
 335   expect_work_queue(mock_threads);
 336
 337   InSequence seq;
 338   MockMirroringWatcher mock_mirroring_watcher;
 339   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 340   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 341
 342   ImageIds image_ids{
 343     {"global id 1", "remote id 1"},
 344     {"global id 2", "remote id 2"}};
 345   MockRefreshImagesRequest mock_refresh_images_request;
 346   bool refresh_sent = false;
 347   EXPECT_CALL(mock_refresh_images_request, send())
 348     .WillOnce(Invoke([this, &mock_refresh_images_request, &image_ids,
 349                       &refresh_sent]() {
 350        *mock_refresh_images_request.image_ids = image_ids;
 351
 352         Mutex::Locker locker(m_lock);
 353         refresh_sent = true;
 354         m_cond.Signal();
 355       }));
 356
 357   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 358
 359   MockListener mock_listener(this);
 360   image_ids = {
 361     {"global id 1", "remote id 1a"},
 362     {"global id 3", "remote id 3"}};
 363   expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 364
 365   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 366                                     mock_listener);
 367   mock_pool_watcher.init(nullptr);
 368
 369   {
 370     Mutex::Locker locker(m_lock);
 371     while (!refresh_sent) {
 372       m_cond.Wait(m_lock);
 373     }
 374   }
 375
 376   MirroringWatcher::get_instance().handle_image_updated(
 377     cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "remote id 2", "global id 2");
 378   MirroringWatcher::get_instance().handle_image_updated(
 379     cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 1a", "global id 1");
 380   MirroringWatcher::get_instance().handle_image_updated(
 381     cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 3", "global id 3");
 382
 383   mock_refresh_images_request.on_finish->complete(0);
 384   ASSERT_TRUE(wait_for_update(1));
 385
 386   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 387   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 388 }
 389
 390 TEST_F(TestMockPoolWatcher, Notify) {
 391   MockThreads mock_threads(m_threads);
 392
 393   InSequence seq;
 394   MockMirroringWatcher mock_mirroring_watcher;
 395   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 396   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 397
 398   ImageIds image_ids{
 399     {"global id 1", "remote id 1"},
 400     {"global id 2", "remote id 2"}};
 401   MockRefreshImagesRequest mock_refresh_images_request;
 402   expect_refresh_images(mock_refresh_images_request, image_ids, 0);
 403   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 404
 405   EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 406     .WillOnce(Invoke([this](Context *ctx, int r) {
 407         m_threads->work_queue->queue(ctx, r);
 408       }));
 409
 410   MockListener mock_listener(this);
 411   expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 412
 413   Context *notify_ctx = nullptr;
 414   EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 415     .WillOnce(Invoke([this, &notify_ctx](Context *ctx, int r) {
 416         Mutex::Locker locker(m_lock);
 417         ASSERT_EQ(nullptr, notify_ctx);
 418         notify_ctx = ctx;
 419         m_cond.Signal();
 420       }));
 421   expect_listener_handle_update(
 422     mock_listener, "remote uuid",
 423     {{"global id 1", "remote id 1a"}, {"global id 3", "remote id 3"}},
 424     {{"global id 1", "remote id 1"}, {"global id 2", "remote id 2"}});
 425
 426   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 427                                     mock_listener);
 428   C_SaferCond ctx;
 429   mock_pool_watcher.init(&ctx);
 430   ASSERT_EQ(0, ctx.wait());
 431   ASSERT_TRUE(wait_for_update(1));
 432
 433   C_SaferCond flush_ctx;
 434   m_threads->work_queue->queue(&flush_ctx, 0);
 435   ASSERT_EQ(0, flush_ctx.wait());
 436
 437   MirroringWatcher::get_instance().handle_image_updated(
 438     cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "remote id 2", "global id 2");
 439   MirroringWatcher::get_instance().handle_image_updated(
 440     cls::rbd::MIRROR_IMAGE_STATE_DISABLED, "remote id 2", "global id 2");
 441   MirroringWatcher::get_instance().handle_image_updated(
 442     cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 1a", "global id 1");
 443   MirroringWatcher::get_instance().handle_image_updated(
 444     cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 3", "global id 3");
 445   notify_ctx->complete(0);
 446
 447   ASSERT_TRUE(wait_for_update(1));
 448
 449   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 450   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 451 }
 452
 453 TEST_F(TestMockPoolWatcher, RegisterWatcherBlacklist) {
 454   MockThreads mock_threads(m_threads);
 455   expect_work_queue(mock_threads);
 456
 457   InSequence seq;
 458   MockMirroringWatcher mock_mirroring_watcher;
 459   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 460   expect_mirroring_watcher_register(mock_mirroring_watcher, -EBLACKLISTED);
 461
 462   MockListener mock_listener(this);
 463   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 464                                     mock_listener);
 465   C_SaferCond ctx;
 466   mock_pool_watcher.init(&ctx);
 467   ASSERT_EQ(-EBLACKLISTED, ctx.wait());
 468   ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
 469
 470   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 471   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 472 }
 473
 474 TEST_F(TestMockPoolWatcher, RegisterWatcherMissing) {
 475   MockThreads mock_threads(m_threads);
 476   expect_work_queue(mock_threads);
 477
 478   InSequence seq;
 479   MockMirroringWatcher mock_mirroring_watcher;
 480   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 481   expect_mirroring_watcher_register(mock_mirroring_watcher, -ENOENT);
 482   expect_timer_add_event(mock_threads);
 483
 484   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 485   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 486
 487   MockRefreshImagesRequest mock_refresh_images_request;
 488   expect_refresh_images(mock_refresh_images_request, {}, 0);
 489   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 490
 491   MockListener mock_listener(this);
 492   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 493
 494   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 495                                     mock_listener);
 496   C_SaferCond ctx;
 497   mock_pool_watcher.init(&ctx);
 498   ASSERT_EQ(0, ctx.wait());
 499
 500   ASSERT_TRUE(wait_for_update(1));
 501   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 502   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 503 }
 504
 505 TEST_F(TestMockPoolWatcher, RegisterWatcherError) {
 506   MockThreads mock_threads(m_threads);
 507   expect_work_queue(mock_threads);
 508
 509   InSequence seq;
 510   MockMirroringWatcher mock_mirroring_watcher;
 511   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 512   expect_mirroring_watcher_register(mock_mirroring_watcher, -EINVAL);
 513   expect_timer_add_event(mock_threads);
 514
 515   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 516   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 517
 518   MockRefreshImagesRequest mock_refresh_images_request;
 519   expect_refresh_images(mock_refresh_images_request, {}, 0);
 520   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 521
 522   MockListener mock_listener(this);
 523   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 524
 525   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 526                                     mock_listener);
 527   C_SaferCond ctx;
 528   mock_pool_watcher.init(&ctx);
 529   ASSERT_EQ(0, ctx.wait());
 530
 531   ASSERT_TRUE(wait_for_update(1));
 532   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 533   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 534 }
 535
 536 TEST_F(TestMockPoolWatcher, RefreshBlacklist) {
 537   MockThreads mock_threads(m_threads);
 538   expect_work_queue(mock_threads);
 539
 540   InSequence seq;
 541   MockMirroringWatcher mock_mirroring_watcher;
 542   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 543   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 544
 545   MockRefreshImagesRequest mock_refresh_images_request;
 546   expect_refresh_images(mock_refresh_images_request, {}, -EBLACKLISTED);
 547
 548   MockListener mock_listener(this);
 549   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 550                                     mock_listener);
 551   C_SaferCond ctx;
 552   mock_pool_watcher.init(&ctx);
 553   ASSERT_EQ(-EBLACKLISTED, ctx.wait());
 554   ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
 555
 556   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 557   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 558 }
 559
 560 TEST_F(TestMockPoolWatcher, RefreshMissing) {
 561   MockThreads mock_threads(m_threads);
 562   expect_work_queue(mock_threads);
 563
 564   InSequence seq;
 565   MockMirroringWatcher mock_mirroring_watcher;
 566   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 567   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 568
 569   MockRefreshImagesRequest mock_refresh_images_request;
 570   expect_refresh_images(mock_refresh_images_request, {}, -ENOENT);
 571   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 572
 573   MockListener mock_listener(this);
 574   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 575
 576   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 577                                     mock_listener);
 578   C_SaferCond ctx;
 579   mock_pool_watcher.init(&ctx);
 580   ASSERT_EQ(0, ctx.wait());
 581
 582   ASSERT_TRUE(wait_for_update(1));
 583   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 584   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 585 }
 586
 587 TEST_F(TestMockPoolWatcher, RefreshError) {
 588   MockThreads mock_threads(m_threads);
 589   expect_work_queue(mock_threads);
 590
 591   InSequence seq;
 592   MockMirroringWatcher mock_mirroring_watcher;
 593   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 594   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 595
 596   MockRefreshImagesRequest mock_refresh_images_request;
 597   expect_refresh_images(mock_refresh_images_request, {}, -EINVAL);
 598   expect_timer_add_event(mock_threads);
 599
 600   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 601   expect_refresh_images(mock_refresh_images_request, {}, 0);
 602   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 603
 604   MockListener mock_listener(this);
 605   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 606
 607   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 608                                     mock_listener);
 609   C_SaferCond ctx;
 610   mock_pool_watcher.init(&ctx);
 611   ASSERT_EQ(0, ctx.wait());
 612
 613   ASSERT_TRUE(wait_for_update(1));
 614   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 615   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 616 }
 617
 618 TEST_F(TestMockPoolWatcher, GetMirrorUuidBlacklist) {
 619   MockThreads mock_threads(m_threads);
 620   expect_work_queue(mock_threads);
 621
 622   InSequence seq;
 623   MockMirroringWatcher mock_mirroring_watcher;
 624   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 625   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 626
 627   MockRefreshImagesRequest mock_refresh_images_request;
 628   expect_refresh_images(mock_refresh_images_request, {}, 0);
 629   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EBLACKLISTED);
 630
 631   MockListener mock_listener(this);
 632   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 633                                     mock_listener);
 634   C_SaferCond ctx;
 635   mock_pool_watcher.init(&ctx);
 636   ASSERT_EQ(-EBLACKLISTED, ctx.wait());
 637   ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
 638
 639   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 640   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 641 }
 642
 643 TEST_F(TestMockPoolWatcher, GetMirrorUuidMissing) {
 644   MockThreads mock_threads(m_threads);
 645   expect_work_queue(mock_threads);
 646
 647   InSequence seq;
 648   MockMirroringWatcher mock_mirroring_watcher;
 649   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 650   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 651
 652   MockRefreshImagesRequest mock_refresh_images_request;
 653   expect_refresh_images(mock_refresh_images_request, {}, 0);
 654   expect_mirror_uuid_get(m_remote_io_ctx, "", -ENOENT);
 655   expect_timer_add_event(mock_threads);
 656
 657   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 658   expect_refresh_images(mock_refresh_images_request, {}, 0);
 659   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 660
 661   MockListener mock_listener(this);
 662   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 663
 664   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 665                                     mock_listener);
 666   C_SaferCond ctx;
 667   mock_pool_watcher.init(&ctx);
 668   ASSERT_EQ(-ENOENT, ctx.wait());
 669
 670   ASSERT_TRUE(wait_for_update(1));
 671   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 672   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 673 }
 674
 675 TEST_F(TestMockPoolWatcher, GetMirrorUuidError) {
 676   MockThreads mock_threads(m_threads);
 677   expect_work_queue(mock_threads);
 678
 679   InSequence seq;
 680   MockMirroringWatcher mock_mirroring_watcher;
 681   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 682   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 683
 684   MockRefreshImagesRequest mock_refresh_images_request;
 685   expect_refresh_images(mock_refresh_images_request, {}, 0);
 686   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EINVAL);
 687   expect_timer_add_event(mock_threads);
 688
 689   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 690   expect_refresh_images(mock_refresh_images_request, {}, 0);
 691   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 692
 693   MockListener mock_listener(this);
 694   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 695
 696   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 697                                     mock_listener);
 698   C_SaferCond ctx;
 699   mock_pool_watcher.init(&ctx);
 700   ASSERT_EQ(0, ctx.wait());
 701
 702   ASSERT_TRUE(wait_for_update(1));
 703   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 704   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 705 }
 706
 707 TEST_F(TestMockPoolWatcher, Rewatch) {
 708   MockThreads mock_threads(m_threads);
 709   expect_work_queue(mock_threads);
 710
 711   InSequence seq;
 712   MockMirroringWatcher mock_mirroring_watcher;
 713   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 714   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 715
 716   MockRefreshImagesRequest mock_refresh_images_request;
 717   expect_refresh_images(mock_refresh_images_request, {}, 0);
 718   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 719
 720   MockListener mock_listener(this);
 721   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 722
 723   expect_timer_add_event(mock_threads);
 724   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 725   expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0);
 726   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 727   expect_listener_handle_update(mock_listener, "remote uuid",
 728                                 {{"global id", "image id"}}, {});
 729
 730   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 731                                     mock_listener);
 732   C_SaferCond ctx;
 733   mock_pool_watcher.init(&ctx);
 734   ASSERT_EQ(0, ctx.wait());
 735   ASSERT_TRUE(wait_for_update(1));
 736
 737   MirroringWatcher::get_instance().handle_rewatch_complete(0);
 738   ASSERT_TRUE(wait_for_update(1));
 739
 740   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 741   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 742 }
 743
 744 TEST_F(TestMockPoolWatcher, RewatchBlacklist) {
 745   MockThreads mock_threads(m_threads);
 746   expect_work_queue(mock_threads);
 747
 748   InSequence seq;
 749   MockMirroringWatcher mock_mirroring_watcher;
 750   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 751   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 752
 753   MockRefreshImagesRequest mock_refresh_images_request;
 754   expect_refresh_images(mock_refresh_images_request, {}, 0);
 755   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 756
 757   MockListener mock_listener(this);
 758   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 759
 760   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 761                                     mock_listener);
 762   C_SaferCond ctx;
 763   mock_pool_watcher.init(&ctx);
 764   ASSERT_EQ(0, ctx.wait());
 765   ASSERT_TRUE(wait_for_update(1));
 766
 767   MirroringWatcher::get_instance().handle_rewatch_complete(-EBLACKLISTED);
 768   ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
 769
 770   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 771   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 772 }
 773
 774 TEST_F(TestMockPoolWatcher, RewatchError) {
 775   MockThreads mock_threads(m_threads);
 776   expect_work_queue(mock_threads);
 777
 778   InSequence seq;
 779   MockMirroringWatcher mock_mirroring_watcher;
 780   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 781   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 782
 783   MockRefreshImagesRequest mock_refresh_images_request;
 784   expect_refresh_images(mock_refresh_images_request, {}, 0);
 785   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 786
 787   MockListener mock_listener(this);
 788   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 789
 790   expect_timer_add_event(mock_threads);
 791   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 792   expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0);
 793   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 794   expect_listener_handle_update(mock_listener, "remote uuid",
 795                                 {{"global id", "image id"}}, {});
 796
 797   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 798                                     mock_listener);
 799   C_SaferCond ctx;
 800   mock_pool_watcher.init(&ctx);
 801   ASSERT_EQ(0, ctx.wait());
 802   ASSERT_TRUE(wait_for_update(1));
 803
 804   MirroringWatcher::get_instance().handle_rewatch_complete(-EINVAL);
 805   ASSERT_TRUE(wait_for_update(1));
 806
 807   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 808   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 809 }
 810
 811 TEST_F(TestMockPoolWatcher, DeferredRefresh) {
 812   MockThreads mock_threads(m_threads);
 813   expect_work_queue(mock_threads);
 814
 815   InSequence seq;
 816   MockMirroringWatcher mock_mirroring_watcher;
 817   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 818   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 819
 820   MockRefreshImagesRequest mock_refresh_images_request;
 821
 822   EXPECT_CALL(mock_refresh_images_request, send())
 823     .WillOnce(Invoke([&mock_refresh_images_request]() {
 824         *mock_refresh_images_request.image_ids = {};
 825         MirroringWatcher::get_instance().handle_rewatch_complete(0);
 826         mock_refresh_images_request.on_finish->complete(0);
 827         }));
 828   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 829   expect_timer_add_event(mock_threads);
 830
 831   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 832   expect_refresh_images(mock_refresh_images_request, {}, 0);
 833   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 834
 835   MockListener mock_listener(this);
 836   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 837
 838   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 839                                     mock_listener);
 840   C_SaferCond ctx;
 841   mock_pool_watcher.init(&ctx);
 842   ASSERT_EQ(0, ctx.wait());
 843   ASSERT_TRUE(wait_for_update(1));
 844
 845   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 846   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 847 }
 848
 849 TEST_F(TestMockPoolWatcher, MirrorUuidUpdated) {
 850   MockThreads mock_threads(m_threads);
 851   expect_work_queue(mock_threads);
 852
 853   InSequence seq;
 854   MockMirroringWatcher mock_mirroring_watcher;
 855   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 856   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 857
 858   ImageIds image_ids{
 859     {"global id 1", "remote id 1"},
 860     {"global id 2", "remote id 2"}};
 861   MockRefreshImagesRequest mock_refresh_images_request;
 862   expect_refresh_images(mock_refresh_images_request, image_ids, 0);
 863   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 864
 865   MockListener mock_listener(this);
 866   expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 867
 868   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 869                                     mock_listener);
 870   C_SaferCond ctx;
 871   mock_pool_watcher.init(&ctx);
 872   ASSERT_EQ(0, ctx.wait());
 873
 874   ASSERT_TRUE(wait_for_update(1));
 875
 876   expect_timer_add_event(mock_threads);
 877   ImageIds new_image_ids{
 878     {"global id 1", "remote id 1"}};
 879   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 880   expect_refresh_images(mock_refresh_images_request, new_image_ids, 0);
 881   expect_mirror_uuid_get(m_remote_io_ctx, "updated uuid", 0);
 882   expect_listener_handle_update(mock_listener, "remote uuid", {}, image_ids);
 883   expect_listener_handle_update(mock_listener, "updated uuid", new_image_ids,
 884                                 {});
 885
 886   MirroringWatcher::get_instance().handle_rewatch_complete(0);
 887   ASSERT_TRUE(wait_for_update(2));
 888
 889   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 890   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 891 }
 892
 893 } // namespace mirror
 894 } // namespace rbd