ceph/src/test/rbd_mirror/test_mock_ImageMap.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "test/rbd_mirror/test_mock_fixture.h"
   5 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
   6 #include "test/librados_test_stub/MockTestMemRadosClient.h"
   7 #include "test/librbd/mock/MockImageCtx.h"
   8 #include "test/rbd_mirror/mock/MockContextWQ.h"
   9 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  10 #include "librbd/MirroringWatcher.h"
  11 #include "tools/rbd_mirror/Threads.h"
  12 #include "tools/rbd_mirror/ImageMap.h"
  13 #include "tools/rbd_mirror/image_map/LoadRequest.h"
  14 #include "tools/rbd_mirror/image_map/UpdateRequest.h"
  15 #include "tools/rbd_mirror/image_map/Types.h"
  16 #include "include/stringify.h"
  17
  18 namespace librbd {
  19 namespace {
  20
  21 struct MockTestImageCtx : public librbd::MockImageCtx {
  22   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  23     : librbd::MockImageCtx(image_ctx) {
  24   }
  25 };
  26
  27 } // anonymous namespace
  28
  29 } // namespace librbd
  30
  31 namespace rbd {
  32 namespace mirror {
  33
  34 template <>
  35 struct Threads<librbd::MockTestImageCtx> {
  36   MockSafeTimer *timer;
  37   Mutex &timer_lock;
  38
  39   MockContextWQ *work_queue;
  40
  41   Threads(Threads<librbd::ImageCtx> *threads)
  42     : timer(new MockSafeTimer()),
  43       timer_lock(threads->timer_lock),
  44       work_queue(new MockContextWQ()) {
  45   }
  46   ~Threads() {
  47     delete timer;
  48     delete work_queue;
  49   }
  50 };
  51
  52 namespace image_map {
  53
  54 template <>
  55 struct LoadRequest<librbd::MockTestImageCtx> {
  56   std::map<std::string, cls::rbd::MirrorImageMap> *image_map;
  57   Context *on_finish = nullptr;
  58
  59   static LoadRequest *s_instance;
  60   static LoadRequest *create(librados::IoCtx &ioctx,
  61                              std::map<std::string, cls::rbd::MirrorImageMap> *image_map,
  62                              Context *on_finish) {
  63     ceph_assert(s_instance != nullptr);
  64     s_instance->image_map = image_map;
  65     s_instance->on_finish = on_finish;
  66     return s_instance;
  67   }
  68
  69   MOCK_METHOD0(send, void());
  70
  71   LoadRequest() {
  72     s_instance = this;
  73   }
  74 };
  75
  76 template <>
  77 struct UpdateRequest<librbd::MockTestImageCtx> {
  78   Context *on_finish = nullptr;
  79   static UpdateRequest *s_instance;
  80   static UpdateRequest *create(librados::IoCtx &ioctx,
  81                                std::map<std::string, cls::rbd::MirrorImageMap> &&update_mapping,
  82                                std::set<std::string> &&global_image_ids,
  83                                Context *on_finish) {
  84     ceph_assert(s_instance != nullptr);
  85     s_instance->on_finish = on_finish;
  86     return s_instance;
  87   }
  88
  89   MOCK_METHOD0(send, void());
  90
  91   UpdateRequest() {
  92     s_instance = this;
  93   }
  94 };
  95
  96 LoadRequest<librbd::MockTestImageCtx> *
  97 LoadRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
  98 UpdateRequest<librbd::MockTestImageCtx> *
  99 UpdateRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 100
 101 } // namespace image_map
 102
 103 } // namespace mirror
 104 } // namespace rbd
 105
 106 // template definitions
 107 #include "tools/rbd_mirror/ImageMap.cc"
 108
 109 namespace rbd {
 110 namespace mirror {
 111
 112 using ::testing::_;
 113 using ::testing::DoAll;
 114 using ::testing::WithArg;
 115 using ::testing::AtLeast;
 116 using ::testing::InSequence;
 117 using ::testing::Invoke;
 118 using ::testing::ReturnArg;
 119 using ::testing::StrEq;
 120
 121 using image_map::Listener;
 122 using image_map::LoadRequest;
 123 using image_map::UpdateRequest;
 124
 125 using ::rbd::mirror::Threads;
 126
 127 class TestMockImageMap : public TestMockFixture {
 128 public:
 129   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 130   typedef ImageMap<librbd::MockTestImageCtx> MockImageMap;
 131   typedef LoadRequest<librbd::MockTestImageCtx> MockLoadRequest;
 132   typedef UpdateRequest<librbd::MockTestImageCtx> MockUpdateRequest;
 133
 134   struct MockListener : Listener {
 135     TestMockImageMap *test_mock_image_map;
 136
 137     MockListener(TestMockImageMap *test_mock_image_map)
 138       : test_mock_image_map(test_mock_image_map) {
 139     }
 140
 141     MOCK_METHOD2(mock_acquire_image, void(const std::string &, Context*));
 142     MOCK_METHOD2(mock_release_image, void(const std::string &, Context*));
 143     MOCK_METHOD3(mock_remove_image, void(const std::string &,
 144                                          const std::string &, Context*));
 145
 146     void acquire_image(const std::string &global_image_id,
 147                        const std::string &instance_id, Context* on_finish) {
 148       mock_acquire_image(global_image_id, on_finish);
 149     }
 150
 151     void release_image(const std::string &global_image_id,
 152                        const std::string &instance_id, Context* on_finish) {
 153       mock_release_image(global_image_id, on_finish);
 154     }
 155
 156     void remove_image(const std::string &mirror_uuid,
 157                       const std::string &global_image_id,
 158                       const std::string &instance_id, Context* on_finish) {
 159       mock_remove_image(mirror_uuid, global_image_id, on_finish);
 160     }
 161   };
 162
 163   TestMockImageMap()
 164     : m_lock("TestMockImageMap::m_lock"),
 165       m_notify_update_count(0),
 166       m_map_update_count(0) {
 167   }
 168
 169   void SetUp() override {
 170     TestFixture::SetUp();
 171
 172     m_local_instance_id = stringify(m_local_io_ctx.get_instance_id());
 173
 174     EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle",
 175                                   "0"));
 176     EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_type", "simple"));
 177   }
 178
 179   void TearDown() override {
 180     EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_type", "none"));
 181
 182     TestFixture::TearDown();
 183   }
 184
 185   void expect_work_queue(MockThreads &mock_threads) {
 186     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 187       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 188             m_threads->work_queue->queue(ctx, r);
 189           }));
 190   }
 191
 192   void expect_add_event(MockThreads &mock_threads) {
 193     EXPECT_CALL(*mock_threads.timer, add_event_after(_,_))
 194       .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) {
 195              auto wrapped_ctx = new FunctionContext([this, ctx](int r) {
 196                     Mutex::Locker timer_locker(m_threads->timer_lock);
 197                     ctx->complete(r);
 198                   });
 199                 m_threads->work_queue->queue(wrapped_ctx, 0);
 200               })), ReturnArg<1>()));
 201   }
 202
 203   void expect_rebalance_event(MockThreads &mock_threads) {
 204     EXPECT_CALL(*mock_threads.timer, add_event_after(_,_))
 205       .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) {
 206                 // disable rebalance so as to not reschedule it again
 207                 CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
 208                 cct->_conf.set_val("rbd_mirror_image_policy_rebalance_timeout", "0");
 209
 210                 auto wrapped_ctx = new FunctionContext([this, ctx](int r) {
 211                     Mutex::Locker timer_locker(m_threads->timer_lock);
 212                     ctx->complete(r);
 213                   });
 214                 m_threads->work_queue->queue(wrapped_ctx, 0);
 215               })), ReturnArg<1>()));
 216   }
 217
 218   void expect_load_request(MockLoadRequest &request, int r) {
 219     EXPECT_CALL(request, send())
 220       .WillOnce(Invoke([&request, r]() {
 221             request.on_finish->complete(r);
 222           }));
 223   }
 224
 225   void expect_update_request(MockUpdateRequest &request, int r) {
 226     EXPECT_CALL(request, send())
 227       .WillOnce(Invoke([this, &request, r]() {
 228             request.on_finish->complete(r);
 229             if (r == 0) {
 230               Mutex::Locker locker(m_lock);
 231               ++m_map_update_count;
 232               m_cond.Signal();
 233             }
 234           }));
 235   }
 236
 237   void expect_listener_acquire_image(MockListener &mock_listener,
 238                                      const std::string &global_image_id,
 239                                      std::map<std::string, Context*> *peer_ack_ctxs) {
 240     EXPECT_CALL(mock_listener, mock_acquire_image(global_image_id, _))
 241       .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
 242               Mutex::Locker locker(m_lock);
 243               peer_ack_ctxs->insert({global_image_id, ctx});
 244               ++m_notify_update_count;
 245               m_cond.Signal();
 246             })));
 247   }
 248
 249   void expect_listener_release_image(MockListener &mock_listener,
 250                                      const std::string &global_image_id,
 251                                      std::map<std::string, Context*> *peer_ack_ctxs) {
 252     EXPECT_CALL(mock_listener, mock_release_image(global_image_id, _))
 253       .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
 254               Mutex::Locker locker(m_lock);
 255               peer_ack_ctxs->insert({global_image_id, ctx});
 256               ++m_notify_update_count;
 257               m_cond.Signal();
 258             })));
 259   }
 260
 261   void expect_listener_remove_image(MockListener &mock_listener,
 262                                     const std::string &mirror_uuid,
 263                                     const std::string &global_image_id,
 264                                     std::map<std::string, Context*> *peer_ack_ctxs) {
 265     EXPECT_CALL(mock_listener,
 266                 mock_remove_image(mirror_uuid, global_image_id, _))
 267       .WillOnce(WithArg<2>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
 268               Mutex::Locker locker(m_lock);
 269               peer_ack_ctxs->insert({global_image_id, ctx});
 270               ++m_notify_update_count;
 271               m_cond.Signal();
 272             })));
 273   }
 274
 275   void expect_listener_images_unmapped(MockListener &mock_listener, size_t count,
 276                                        std::set<std::string> *global_image_ids,
 277                                        std::map<std::string, Context*> *peer_ack_ctxs) {
 278     EXPECT_CALL(mock_listener, mock_release_image(_, _))
 279       .Times(count)
 280       .WillRepeatedly(Invoke([this, global_image_ids, peer_ack_ctxs](std::string global_image_id, Context* ctx) {
 281               Mutex::Locker locker(m_lock);
 282               global_image_ids->emplace(global_image_id);
 283               peer_ack_ctxs->insert({global_image_id, ctx});
 284               ++m_notify_update_count;
 285               m_cond.Signal();
 286             }));
 287   }
 288
 289   void remote_peer_ack_nowait(MockImageMap *image_map,
 290                               const std::set<std::string> &global_image_ids,
 291                               int ret,
 292                               std::map<std::string, Context*> *peer_ack_ctxs) {
 293     for (auto& global_image_id : global_image_ids) {
 294       auto it = peer_ack_ctxs->find(global_image_id);
 295       ASSERT_TRUE(it != peer_ack_ctxs->end());
 296       auto ack_ctx = it->second;
 297       peer_ack_ctxs->erase(it);
 298       ack_ctx->complete(ret);
 299       wait_for_scheduled_task();
 300     }
 301   }
 302
 303   void remote_peer_ack_wait(MockImageMap *image_map,
 304                             const std::set<std::string> &global_image_ids,
 305                             int ret,
 306                             std::map<std::string, Context*> *peer_ack_ctxs) {
 307     for (auto& global_image_id : global_image_ids) {
 308       auto it = peer_ack_ctxs->find(global_image_id);
 309       ASSERT_TRUE(it != peer_ack_ctxs->end());
 310       auto ack_ctx = it->second;
 311       peer_ack_ctxs->erase(it);
 312       ack_ctx->complete(ret);
 313       wait_for_scheduled_task();
 314       ASSERT_TRUE(wait_for_map_update(1));
 315     }
 316   }
 317
 318   void remote_peer_ack_listener_wait(MockImageMap *image_map,
 319                                      const std::set<std::string> &global_image_ids,
 320                                      int ret,
 321                                      std::map<std::string, Context*> *peer_ack_ctxs) {
 322     for (auto& global_image_id : global_image_ids) {
 323       auto it = peer_ack_ctxs->find(global_image_id);
 324       ASSERT_TRUE(it != peer_ack_ctxs->end());
 325       auto ack_ctx = it->second;
 326       peer_ack_ctxs->erase(it);
 327       ack_ctx->complete(ret);
 328       ASSERT_TRUE(wait_for_map_update(1));
 329       ASSERT_TRUE(wait_for_listener_notify(1));
 330     }
 331   }
 332
 333   void update_map_and_acquire(MockThreads &mock_threads,
 334                               MockUpdateRequest &mock_update_request,
 335                               MockListener &mock_listener,
 336                               const std::set<std::string> &global_image_ids,
 337                               int ret,
 338                               std::map<std::string, Context*> *peer_ack_ctxs) {
 339     for (auto const &global_image_id : global_image_ids) {
 340       expect_add_event(mock_threads);
 341       expect_update_request(mock_update_request, ret);
 342       expect_add_event(mock_threads);
 343       expect_listener_acquire_image(mock_listener, global_image_id,
 344                                     peer_ack_ctxs);
 345     }
 346   }
 347
 348   void update_map_request(MockThreads &mock_threads,
 349                           MockUpdateRequest &mock_update_request,
 350                           const std::set<std::string> &global_image_ids, int ret) {
 351     for (uint32_t i = 0; i < global_image_ids.size(); ++i) {
 352       expect_add_event(mock_threads);
 353       expect_update_request(mock_update_request, ret);
 354     }
 355   }
 356
 357   void wait_for_scheduled_task() {
 358     m_threads->work_queue->drain();
 359   }
 360
 361   bool wait_for_listener_notify(uint32_t count) {
 362     Mutex::Locker locker(m_lock);
 363     while (m_notify_update_count < count) {
 364       if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) {
 365         break;
 366       }
 367     }
 368
 369     if (m_notify_update_count < count) {
 370       return false;
 371     }
 372
 373     m_notify_update_count -= count;
 374     return true;
 375   }
 376
 377   bool wait_for_map_update(uint32_t count) {
 378     Mutex::Locker locker(m_lock);
 379     while (m_map_update_count < count) {
 380       if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) {
 381         break;
 382       }
 383     }
 384
 385     if (m_map_update_count < count) {
 386       return false;
 387     }
 388
 389     m_map_update_count -= count;
 390     return true;
 391   }
 392
 393   int when_shut_down(MockImageMap *image_map) {
 394     C_SaferCond ctx;
 395     image_map->shut_down(&ctx);
 396     return ctx.wait();
 397   }
 398
 399   void listener_acquire_images(MockListener &mock_listener,
 400                                const std::set<std::string> &global_image_ids,
 401                                std::map<std::string, Context*> *peer_ack_ctxs) {
 402     for (auto const &global_image_id : global_image_ids) {
 403       expect_listener_acquire_image(mock_listener, global_image_id,
 404                                     peer_ack_ctxs);
 405     }
 406   }
 407
 408   void listener_release_images(MockListener &mock_listener,
 409                                const std::set<std::string> &global_image_ids,
 410                                std::map<std::string, Context*> *peer_ack_ctxs) {
 411     for (auto const &global_image_id : global_image_ids) {
 412       expect_listener_release_image(mock_listener, global_image_id,
 413                                     peer_ack_ctxs);
 414     }
 415   }
 416
 417   void listener_remove_images(MockListener &mock_listener,
 418                               const std::string &mirror_uuid,
 419                               std::set<std::string> &global_image_ids,
 420                               std::map<std::string, Context*> *peer_ack_ctxs) {
 421     for (auto const &global_image_id : global_image_ids) {
 422       expect_listener_remove_image(mock_listener, mirror_uuid, global_image_id,
 423                                    peer_ack_ctxs);
 424     }
 425   }
 426
 427   Mutex m_lock;
 428   Cond m_cond;
 429   uint32_t m_notify_update_count;
 430   uint32_t m_map_update_count;
 431   std::string m_local_instance_id;
 432 };
 433
 434 TEST_F(TestMockImageMap, SetLocalImages) {
 435   MockThreads mock_threads(m_threads);
 436   expect_work_queue(mock_threads);
 437
 438   InSequence seq;
 439
 440   MockLoadRequest mock_load_request;
 441   expect_load_request(mock_load_request, 0);
 442
 443   MockListener mock_listener(this);
 444
 445   std::unique_ptr<MockImageMap> mock_image_map{
 446     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 447                          mock_listener)};
 448
 449   C_SaferCond cond;
 450   mock_image_map->init(&cond);
 451   ASSERT_EQ(0, cond.wait());
 452
 453   std::set<std::string> global_image_ids{
 454     "global id 1", "global id 2"
 455   };
 456   std::set<std::string> global_image_ids_ack(global_image_ids);
 457
 458   // UPDATE_MAPPING+ACQUIRE
 459   expect_add_event(mock_threads);
 460   MockUpdateRequest mock_update_request;
 461   expect_update_request(mock_update_request, 0);
 462   expect_add_event(mock_threads);
 463   std::map<std::string, Context*> peer_ack_ctxs;
 464   listener_acquire_images(mock_listener, global_image_ids, &peer_ack_ctxs);
 465
 466   // initial image list
 467   mock_image_map->update_images("", std::move(global_image_ids), {});
 468
 469   ASSERT_TRUE(wait_for_map_update(1));
 470   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 471
 472   // remote peer ACKs image acquire request
 473   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
 474                          &peer_ack_ctxs);
 475
 476   wait_for_scheduled_task();
 477   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 478 }
 479
 480 TEST_F(TestMockImageMap, AddRemoveLocalImage) {
 481   MockThreads mock_threads(m_threads);
 482   expect_work_queue(mock_threads);
 483
 484   InSequence seq;
 485
 486   MockLoadRequest mock_load_request;
 487   expect_load_request(mock_load_request, 0);
 488
 489   MockListener mock_listener(this);
 490
 491   std::unique_ptr<MockImageMap> mock_image_map{
 492     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 493                          mock_listener)};
 494
 495   C_SaferCond cond;
 496   mock_image_map->init(&cond);
 497   ASSERT_EQ(0, cond.wait());
 498
 499   std::set<std::string> initial_global_image_ids{
 500     "global id 1", "global id 2"
 501   };
 502   std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids);
 503
 504   std::set<std::string> remove_global_image_ids{
 505     "global id 1", "global id 2"
 506   };
 507   std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids);
 508
 509   // UPDATE_MAPPING+ACQUIRE
 510   expect_add_event(mock_threads);
 511   MockUpdateRequest mock_update_request;
 512   expect_update_request(mock_update_request, 0);
 513   expect_add_event(mock_threads);
 514   std::map<std::string, Context*> peer_ack_ctxs;
 515   listener_acquire_images(mock_listener, initial_global_image_ids,
 516                           &peer_ack_ctxs);
 517
 518   // initial image list
 519   mock_image_map->update_images("", std::move(initial_global_image_ids), {});
 520
 521   ASSERT_TRUE(wait_for_map_update(1));
 522   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 523
 524   // remote peer ACKs image acquire request
 525   remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
 526                          &peer_ack_ctxs);
 527
 528   // RELEASE+REMOVE_MAPPING
 529   expect_add_event(mock_threads);
 530   listener_release_images(mock_listener, remove_global_image_ids,
 531                           &peer_ack_ctxs);
 532   update_map_request(mock_threads, mock_update_request, remove_global_image_ids,
 533                      0);
 534
 535   // remove images
 536   mock_image_map->update_images("", {}, std::move(remove_global_image_ids));
 537   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size()));
 538
 539   remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 540                        &peer_ack_ctxs);
 541
 542   wait_for_scheduled_task();
 543   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 544 }
 545
 546 TEST_F(TestMockImageMap, AddRemoveRemoteImage) {
 547   MockThreads mock_threads(m_threads);
 548   expect_work_queue(mock_threads);
 549
 550   InSequence seq;
 551
 552   MockLoadRequest mock_load_request;
 553   expect_load_request(mock_load_request, 0);
 554
 555   MockListener mock_listener(this);
 556
 557   std::unique_ptr<MockImageMap> mock_image_map{
 558     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 559                          mock_listener)};
 560
 561   C_SaferCond cond;
 562   mock_image_map->init(&cond);
 563   ASSERT_EQ(0, cond.wait());
 564
 565   std::set<std::string> initial_global_image_ids{
 566     "global id 1", "global id 2"
 567   };
 568   std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids);
 569
 570   std::set<std::string> remove_global_image_ids{
 571     "global id 1", "global id 2"
 572   };
 573   std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids);
 574
 575   // UPDATE_MAPPING+ACQUIRE
 576   expect_add_event(mock_threads);
 577   MockUpdateRequest mock_update_request;
 578   expect_update_request(mock_update_request, 0);
 579   expect_add_event(mock_threads);
 580   std::map<std::string, Context*> peer_ack_ctxs;
 581   listener_acquire_images(mock_listener, initial_global_image_ids,
 582                           &peer_ack_ctxs);
 583
 584   // initial image list
 585   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids),
 586                                 {});
 587
 588   ASSERT_TRUE(wait_for_map_update(1));
 589   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 590
 591   // remote peer ACKs image acquire request
 592   remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
 593                          &peer_ack_ctxs);
 594
 595   // RELEASE+REMOVE_MAPPING
 596   std::map<std::string, Context*> peer_remove_ack_ctxs;
 597   listener_remove_images(mock_listener, "uuid1", remove_global_image_ids,
 598                          &peer_remove_ack_ctxs);
 599   expect_add_event(mock_threads);
 600   listener_release_images(mock_listener, remove_global_image_ids,
 601                           &peer_ack_ctxs);
 602   update_map_request(mock_threads, mock_update_request, remove_global_image_ids,
 603                      0);
 604
 605   // remove images
 606   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids));
 607   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2));
 608
 609   remote_peer_ack_nowait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 610                          &peer_remove_ack_ctxs);
 611   remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 612                        &peer_ack_ctxs);
 613
 614   wait_for_scheduled_task();
 615   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 616 }
 617
 618 TEST_F(TestMockImageMap, AddRemoveRemoteImageDuplicateNotification) {
 619   MockThreads mock_threads(m_threads);
 620   expect_work_queue(mock_threads);
 621
 622   InSequence seq;
 623
 624   MockLoadRequest mock_load_request;
 625   expect_load_request(mock_load_request, 0);
 626
 627   MockListener mock_listener(this);
 628
 629   std::unique_ptr<MockImageMap> mock_image_map{
 630     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 631                          mock_listener)};
 632
 633   C_SaferCond cond;
 634   mock_image_map->init(&cond);
 635   ASSERT_EQ(0, cond.wait());
 636
 637   std::set<std::string> initial_global_image_ids{
 638     "global id 1", "global id 2"
 639   };
 640   std::set<std::string> initial_global_image_ids_dup(initial_global_image_ids);
 641   std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids);
 642
 643   std::set<std::string> remove_global_image_ids{
 644     "global id 1", "global id 2"
 645   };
 646   std::set<std::string> remove_global_image_ids_dup(remove_global_image_ids);
 647   std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids);
 648
 649   // UPDATE_MAPPING+ACQUIRE
 650   expect_add_event(mock_threads);
 651   MockUpdateRequest mock_update_request;
 652   expect_update_request(mock_update_request, 0);
 653   expect_add_event(mock_threads);
 654   std::map<std::string, Context*> peer_ack_ctxs;
 655   listener_acquire_images(mock_listener, initial_global_image_ids,
 656                           &peer_ack_ctxs);
 657
 658   // initial image list
 659   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {});
 660
 661   ASSERT_TRUE(wait_for_map_update(1));
 662   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 663
 664   // trigger duplicate "add" event
 665   wait_for_scheduled_task();
 666   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids_dup), {});
 667
 668   // remote peer ACKs image acquire request
 669   remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
 670                          &peer_ack_ctxs);
 671
 672   // RELEASE+REMOVE_MAPPING
 673   std::map<std::string, Context*> peer_remove_ack_ctxs;
 674   listener_remove_images(mock_listener, "uuid1", remove_global_image_ids,
 675                          &peer_remove_ack_ctxs);
 676   expect_add_event(mock_threads);
 677   listener_release_images(mock_listener, remove_global_image_ids,
 678                           &peer_ack_ctxs);
 679   update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0);
 680
 681   // remove images
 682   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids));
 683   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2));
 684
 685   remote_peer_ack_nowait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 686                          &peer_remove_ack_ctxs);
 687   remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 688                        &peer_ack_ctxs);
 689
 690   // trigger duplicate "remove" notification
 691   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids_dup));
 692
 693   wait_for_scheduled_task();
 694   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 695 }
 696
 697 TEST_F(TestMockImageMap, AcquireImageErrorRetry) {
 698   MockThreads mock_threads(m_threads);
 699   expect_work_queue(mock_threads);
 700
 701   InSequence seq;
 702
 703   MockLoadRequest mock_load_request;
 704   expect_load_request(mock_load_request, 0);
 705
 706   MockListener mock_listener(this);
 707
 708   std::unique_ptr<MockImageMap> mock_image_map{
 709     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 710                          mock_listener)};
 711
 712   C_SaferCond cond;
 713   mock_image_map->init(&cond);
 714   ASSERT_EQ(0, cond.wait());
 715
 716   std::set<std::string> initial_global_image_ids{
 717     "global id 1", "global id 2"
 718   };
 719   std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids);
 720
 721   // UPDATE_MAPPING failure
 722   expect_add_event(mock_threads);
 723   MockUpdateRequest mock_update_request;
 724   expect_update_request(mock_update_request, -EIO);
 725
 726   // UPDATE_MAPPING+ACQUIRE
 727   expect_add_event(mock_threads);
 728   expect_update_request(mock_update_request, 0);
 729   expect_add_event(mock_threads);
 730   std::map<std::string, Context*> peer_ack_ctxs;
 731   listener_acquire_images(mock_listener, initial_global_image_ids,
 732                           &peer_ack_ctxs);
 733
 734   // initial image list
 735   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {});
 736
 737   ASSERT_TRUE(wait_for_map_update(1));
 738   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 739
 740   // remote peer ACKs image acquire request
 741   remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
 742                          &peer_ack_ctxs);
 743
 744   wait_for_scheduled_task();
 745   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 746 }
 747
 748 TEST_F(TestMockImageMap, RemoveRemoteAndLocalImage) {
 749   MockThreads mock_threads(m_threads);
 750   expect_work_queue(mock_threads);
 751
 752   InSequence seq;
 753
 754   MockLoadRequest mock_load_request;
 755   expect_load_request(mock_load_request, 0);
 756
 757   MockListener mock_listener(this);
 758
 759   std::unique_ptr<MockImageMap> mock_image_map{
 760     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 761                          mock_listener)};
 762
 763   C_SaferCond cond;
 764   mock_image_map->init(&cond);
 765   ASSERT_EQ(0, cond.wait());
 766
 767   // remote image set
 768   std::set<std::string> initial_remote_global_image_ids{
 769     "global id 1"
 770   };
 771   std::set<std::string> initial_remote_global_image_ids_ack(initial_remote_global_image_ids);
 772
 773   // local image set
 774   std::set<std::string> initial_local_global_image_ids{
 775     "global id 1"
 776   };
 777
 778   // remote/local images to remove
 779   std::set<std::string> remote_remove_global_image_ids{
 780     "global id 1"
 781   };
 782   std::set<std::string> remote_remove_global_image_ids_ack(remote_remove_global_image_ids);
 783
 784   std::set<std::string> local_remove_global_image_ids{
 785     "global id 1"
 786   };
 787   std::set<std::string> local_remove_global_image_ids_ack(local_remove_global_image_ids);
 788
 789   // UPDATE_MAPPING+ACQUIRE
 790   expect_add_event(mock_threads);
 791   MockUpdateRequest mock_update_request;
 792   expect_update_request(mock_update_request, 0);
 793   expect_add_event(mock_threads);
 794   std::map<std::string, Context*> peer_ack_ctxs;
 795   listener_acquire_images(mock_listener, initial_remote_global_image_ids,
 796                           &peer_ack_ctxs);
 797
 798   // initial remote image list
 799   mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {});
 800
 801   ASSERT_TRUE(wait_for_map_update(1));
 802   ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size()));
 803
 804   // remote peer ACKs image acquire request
 805   remote_peer_ack_nowait(mock_image_map.get(),
 806                          initial_remote_global_image_ids_ack, 0,
 807                          &peer_ack_ctxs);
 808
 809   // set initial local image list -- this is a no-op from policy pov
 810   mock_image_map->update_images("", std::move(initial_local_global_image_ids), {});
 811
 812   // remove remote images -- this should be a no-op from policy pov
 813   // except the listener notification
 814   std::map<std::string, Context*> peer_ack_remove_ctxs;
 815   listener_remove_images(mock_listener, "uuid1", remote_remove_global_image_ids,
 816                          &peer_ack_remove_ctxs);
 817
 818   mock_image_map->update_images("uuid1", {}, std::move(remote_remove_global_image_ids));
 819   ASSERT_TRUE(wait_for_listener_notify(remote_remove_global_image_ids_ack.size()));
 820
 821   // RELEASE+REMOVE_MAPPING
 822   expect_add_event(mock_threads);
 823   listener_release_images(mock_listener, local_remove_global_image_ids,
 824                           &peer_ack_ctxs);
 825   update_map_request(mock_threads, mock_update_request, local_remove_global_image_ids, 0);
 826
 827   // remove local images
 828   mock_image_map->update_images("", {}, std::move(local_remove_global_image_ids));
 829   ASSERT_TRUE(wait_for_listener_notify(local_remove_global_image_ids_ack.size()));
 830
 831   remote_peer_ack_nowait(mock_image_map.get(), local_remove_global_image_ids_ack,
 832                          0,  &peer_ack_remove_ctxs);
 833   remote_peer_ack_wait(mock_image_map.get(), local_remove_global_image_ids_ack,
 834                        0, &peer_ack_ctxs);
 835
 836   wait_for_scheduled_task();
 837   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 838 }
 839
 840 TEST_F(TestMockImageMap, AddInstance) {
 841   MockThreads mock_threads(m_threads);
 842   expect_work_queue(mock_threads);
 843
 844   InSequence seq;
 845
 846   MockLoadRequest mock_load_request;
 847   expect_load_request(mock_load_request, 0);
 848
 849   MockListener mock_listener(this);
 850
 851   std::unique_ptr<MockImageMap> mock_image_map{
 852     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 853                          mock_listener)};
 854
 855   C_SaferCond cond;
 856   mock_image_map->init(&cond);
 857   ASSERT_EQ(0, cond.wait());
 858
 859   std::set<std::string> global_image_ids{
 860     "global id 1", "global id 2", "global id 3", "global id 4", "global id 5"
 861   };
 862   std::set<std::string> global_image_ids_ack(global_image_ids);
 863
 864   // UPDATE_MAPPING+ACQUIRE
 865   expect_add_event(mock_threads);
 866   MockUpdateRequest mock_update_request;
 867   expect_update_request(mock_update_request, 0);
 868   expect_add_event(mock_threads);
 869   std::map<std::string, Context*> peer_ack_ctxs;
 870   listener_acquire_images(mock_listener, global_image_ids,
 871                           &peer_ack_ctxs);
 872
 873   // initial image list
 874   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
 875
 876   ASSERT_TRUE(wait_for_map_update(1));
 877   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 878
 879   // remote peer ACKs image acquire request
 880   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
 881                          &peer_ack_ctxs);
 882   wait_for_scheduled_task();
 883
 884   mock_image_map->update_instances_added({m_local_instance_id});
 885
 886   std::set<std::string> shuffled_global_image_ids;
 887
 888   // RELEASE+UPDATE_MAPPING+ACQUIRE
 889   expect_add_event(mock_threads);
 890   expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids,
 891                                   &peer_ack_ctxs);
 892
 893   mock_image_map->update_instances_added({"9876"});
 894
 895   wait_for_scheduled_task();
 896   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 897
 898   update_map_and_acquire(mock_threads, mock_update_request,
 899                          mock_listener, shuffled_global_image_ids, 0,
 900                          &peer_ack_ctxs);
 901   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
 902                                 0, &peer_ack_ctxs);
 903
 904   // completion shuffle action for now (re)mapped images
 905   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
 906                          &peer_ack_ctxs);
 907
 908   wait_for_scheduled_task();
 909   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 910 }
 911
 912 TEST_F(TestMockImageMap, RemoveInstance) {
 913   MockThreads mock_threads(m_threads);
 914   expect_work_queue(mock_threads);
 915
 916   InSequence seq;
 917
 918   MockLoadRequest mock_load_request;
 919   expect_load_request(mock_load_request, 0);
 920
 921   MockListener mock_listener(this);
 922
 923   std::unique_ptr<MockImageMap> mock_image_map{
 924     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 925                          mock_listener)};
 926
 927   C_SaferCond cond;
 928   mock_image_map->init(&cond);
 929   ASSERT_EQ(0, cond.wait());
 930
 931   std::set<std::string> global_image_ids{
 932     "global id 1", "global id 2", "global id 3", "global id 4", "global id 5"
 933   };
 934   std::set<std::string> global_image_ids_ack(global_image_ids);
 935
 936   expect_add_event(mock_threads);
 937
 938   // UPDATE_MAPPING+ACQUIRE
 939   MockUpdateRequest mock_update_request;
 940   expect_update_request(mock_update_request, 0);
 941   expect_add_event(mock_threads);
 942   std::map<std::string, Context*> peer_ack_ctxs;
 943   listener_acquire_images(mock_listener, global_image_ids,
 944                           &peer_ack_ctxs);
 945
 946   // set initial image list
 947   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
 948
 949   ASSERT_TRUE(wait_for_map_update(1));
 950   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 951
 952   // remote peer ACKs image acquire request -- completing action
 953   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
 954                          &peer_ack_ctxs);
 955   wait_for_scheduled_task();
 956
 957   mock_image_map->update_instances_added({m_local_instance_id});
 958
 959   std::set<std::string> shuffled_global_image_ids;
 960
 961   // RELEASE+UPDATE_MAPPING+ACQUIRE
 962   expect_add_event(mock_threads);
 963   expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids,
 964                                   &peer_ack_ctxs);
 965
 966   mock_image_map->update_instances_added({"9876"});
 967
 968   wait_for_scheduled_task();
 969   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 970
 971   update_map_and_acquire(mock_threads, mock_update_request,
 972                          mock_listener, shuffled_global_image_ids, 0,
 973                          &peer_ack_ctxs);
 974   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
 975                                 0, &peer_ack_ctxs);
 976
 977   // completion shuffle action for now (re)mapped images
 978   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
 979                          &peer_ack_ctxs);
 980   wait_for_scheduled_task();
 981
 982   shuffled_global_image_ids.clear();
 983
 984   // remove added instance
 985   expect_add_event(mock_threads);
 986   expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids,
 987                                   &peer_ack_ctxs);
 988
 989   mock_image_map->update_instances_removed({"9876"});
 990
 991   wait_for_scheduled_task();
 992   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 993
 994   update_map_and_acquire(mock_threads, mock_update_request,
 995                          mock_listener, shuffled_global_image_ids, 0,
 996                          &peer_ack_ctxs);
 997   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
 998                                 0, &peer_ack_ctxs);
 999
1000   // completion shuffle action for now (re)mapped images
1001   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1002                          &peer_ack_ctxs);
1003
1004   wait_for_scheduled_task();
1005   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1006 }
1007
1008 TEST_F(TestMockImageMap, AddInstancePingPongImageTest) {
1009   EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle", "600"));
1010
1011   MockThreads mock_threads(m_threads);
1012   expect_work_queue(mock_threads);
1013
1014   InSequence seq;
1015
1016   std::set<std::string> global_image_ids{
1017     "global id 1", "global id 2", "global id 3", "global id 4", "global id 5",
1018     "global id 6", "global id 7", "global id 8", "global id 9", "global id 10",
1019     "global id 11", "global id 12", "global id 13", "global id 14"
1020   };
1021
1022   std::map<std::string, cls::rbd::MirrorImageMap> image_mapping;
1023   for (auto& global_image_id : global_image_ids) {
1024     image_mapping[global_image_id] = {m_local_instance_id, {}, {}};
1025   }
1026
1027   // ACQUIRE
1028   MockLoadRequest mock_load_request;
1029   EXPECT_CALL(mock_load_request, send()).WillOnce(
1030     Invoke([&mock_load_request, &image_mapping]() {
1031       *mock_load_request.image_map = image_mapping;
1032       mock_load_request.on_finish->complete(0);
1033     }));
1034
1035   expect_add_event(mock_threads);
1036   MockListener mock_listener(this);
1037   std::map<std::string, Context*> peer_ack_ctxs;
1038   listener_acquire_images(mock_listener, global_image_ids,
1039                           &peer_ack_ctxs);
1040
1041   std::unique_ptr<MockImageMap> mock_image_map{
1042     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1043                          mock_listener)};
1044
1045   C_SaferCond cond;
1046   mock_image_map->init(&cond);
1047   ASSERT_EQ(0, cond.wait());
1048
1049   mock_image_map->update_instances_added({m_local_instance_id});
1050
1051   std::set<std::string> global_image_ids_ack(global_image_ids);
1052
1053   // remote peer ACKs image acquire request -- completing action
1054   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1055   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1056                          &peer_ack_ctxs);
1057   wait_for_scheduled_task();
1058
1059   // RELEASE+UPDATE_MAPPING+ACQUIRE
1060   expect_add_event(mock_threads);
1061   MockUpdateRequest mock_update_request;
1062   expect_update_request(mock_update_request, 0);
1063   expect_add_event(mock_threads);
1064   listener_acquire_images(mock_listener, global_image_ids,
1065                           &peer_ack_ctxs);
1066
1067   // set initial image list
1068   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
1069
1070   ASSERT_TRUE(wait_for_map_update(1));
1071   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1072
1073   // remote peer ACKs image acquire request -- completing action
1074   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1075                          &peer_ack_ctxs);
1076   wait_for_scheduled_task();
1077
1078   std::set<std::string> shuffled_global_image_ids;
1079
1080   // RELEASE+UPDATE_MAPPING+ACQUIRE
1081   expect_add_event(mock_threads);
1082   expect_listener_images_unmapped(mock_listener, 7, &shuffled_global_image_ids,
1083                                   &peer_ack_ctxs);
1084
1085   mock_image_map->update_instances_added({"9876"});
1086
1087   wait_for_scheduled_task();
1088   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1089
1090   update_map_and_acquire(mock_threads, mock_update_request,
1091                          mock_listener, shuffled_global_image_ids, 0,
1092                          &peer_ack_ctxs);
1093   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1094                                 0, &peer_ack_ctxs);
1095
1096   // completion shuffle action for now (re)mapped images
1097   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1098                          &peer_ack_ctxs);
1099   wait_for_scheduled_task();
1100
1101   std::set<std::string> migrated_global_image_ids(shuffled_global_image_ids);
1102   shuffled_global_image_ids.clear();
1103
1104   // RELEASE+UPDATE_MAPPING+ACQUIRE
1105   expect_add_event(mock_threads);
1106   expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids,
1107                                   &peer_ack_ctxs);
1108
1109   // add another instance
1110   mock_image_map->update_instances_added({"5432"});
1111
1112   wait_for_scheduled_task();
1113   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1114
1115   update_map_and_acquire(mock_threads, mock_update_request,
1116                          mock_listener, shuffled_global_image_ids, 0,
1117                          &peer_ack_ctxs);
1118   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1119                                 0, &peer_ack_ctxs);
1120
1121   // completion shuffle action for now (re)mapped images
1122   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1123                          &peer_ack_ctxs);
1124
1125   // shuffle set should be distinct
1126   std::set<std::string> reshuffled;
1127   std::set_intersection(migrated_global_image_ids.begin(), migrated_global_image_ids.end(),
1128                         shuffled_global_image_ids.begin(), shuffled_global_image_ids.end(),
1129                         std::inserter(reshuffled, reshuffled.begin()));
1130   ASSERT_TRUE(reshuffled.empty());
1131
1132   wait_for_scheduled_task();
1133   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1134 }
1135
1136 TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) {
1137   MockThreads mock_threads(m_threads);
1138   expect_work_queue(mock_threads);
1139
1140   InSequence seq;
1141
1142   MockLoadRequest mock_load_request;
1143   expect_load_request(mock_load_request, 0);
1144
1145   MockListener mock_listener(this);
1146
1147   std::unique_ptr<MockImageMap> mock_image_map{
1148     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1149                          mock_listener)};
1150
1151   C_SaferCond cond;
1152   mock_image_map->init(&cond);
1153   ASSERT_EQ(0, cond.wait());
1154
1155   std::set<std::string> global_image_ids{
1156     "global id 1", "global id 2", "global id 3", "remote id 4",
1157   };
1158   std::set<std::string> global_image_ids_ack(global_image_ids);
1159
1160   std::set<std::string> remove_global_image_ids{
1161     "global id 1"
1162   };
1163   std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids);
1164
1165   expect_add_event(mock_threads);
1166   // UPDATE_MAPPING+ACQUIRE
1167   MockUpdateRequest mock_update_request;
1168   expect_update_request(mock_update_request, 0);
1169   expect_add_event(mock_threads);
1170   std::map<std::string, Context*> peer_ack_ctxs;
1171   listener_acquire_images(mock_listener, global_image_ids,
1172                           &peer_ack_ctxs);
1173
1174   // initial image list
1175   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
1176
1177   ASSERT_TRUE(wait_for_map_update(1));
1178   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1179
1180   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1181                          &peer_ack_ctxs);
1182   wait_for_scheduled_task();
1183
1184   mock_image_map->update_instances_added({m_local_instance_id});
1185
1186   std::set<std::string> shuffled_global_image_ids;
1187
1188   // RELEASE+UPDATE_MAPPING+ACQUIRE
1189   expect_add_event(mock_threads);
1190   expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids,
1191                                   &peer_ack_ctxs);
1192
1193   mock_image_map->update_instances_added({"9876"});
1194
1195   wait_for_scheduled_task();
1196   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1197
1198   update_map_and_acquire(mock_threads, mock_update_request,
1199                          mock_listener, shuffled_global_image_ids, 0,
1200                          &peer_ack_ctxs);
1201   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1202                                 0, &peer_ack_ctxs);
1203
1204   // completion shuffle action for now (re)mapped images
1205   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1206                          &peer_ack_ctxs);
1207   wait_for_scheduled_task();
1208
1209   std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids);
1210
1211   // RELEASE
1212
1213   std::map<std::string, Context*> peer_ack_remove_ctxs;
1214   listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids,
1215                          &peer_ack_remove_ctxs);
1216   expect_add_event(mock_threads);
1217   listener_release_images(mock_listener, shuffled_global_image_ids,
1218                           &peer_ack_ctxs);
1219   expect_add_event(mock_threads);
1220   expect_update_request(mock_update_request, 0);
1221   expect_add_event(mock_threads);
1222   expect_update_request(mock_update_request, 0);
1223
1224   mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids));
1225   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2));
1226
1227   // instance failed -- update policy for instance removal
1228   mock_image_map->update_instances_removed({"9876"});
1229
1230   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids,
1231                          -ENOENT, &peer_ack_remove_ctxs);
1232   remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids,
1233                        -EBLACKLISTED, &peer_ack_ctxs);
1234
1235   wait_for_scheduled_task();
1236   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1237 }
1238
1239 TEST_F(TestMockImageMap, AddErrorAndRemoveImage) {
1240   MockThreads mock_threads(m_threads);
1241   expect_work_queue(mock_threads);
1242
1243   InSequence seq;
1244
1245   MockLoadRequest mock_load_request;
1246   expect_load_request(mock_load_request, 0);
1247
1248   MockListener mock_listener(this);
1249
1250   std::unique_ptr<MockImageMap> mock_image_map{
1251     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1252                          mock_listener)};
1253
1254   C_SaferCond cond;
1255   mock_image_map->init(&cond);
1256   ASSERT_EQ(0, cond.wait());
1257
1258   mock_image_map->update_instances_added({m_local_instance_id});
1259
1260   std::set<std::string> global_image_ids{
1261     "global id 1", "global id 2", "global id 3", "remote id 4",
1262   };
1263   std::set<std::string> global_image_ids_ack(global_image_ids);
1264
1265   // UPDATE_MAPPING+ACQUIRE
1266   expect_add_event(mock_threads);
1267   MockUpdateRequest mock_update_request;
1268   expect_update_request(mock_update_request, 0);
1269   expect_add_event(mock_threads);
1270   std::map<std::string, Context*> peer_ack_ctxs;
1271   listener_acquire_images(mock_listener, global_image_ids,
1272                           &peer_ack_ctxs);
1273
1274   // initial image list
1275   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
1276
1277   ASSERT_TRUE(wait_for_map_update(1));
1278   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1279
1280   // remote peer ACKs image acquire request
1281   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1282                          &peer_ack_ctxs);
1283   wait_for_scheduled_task();
1284
1285   std::set<std::string> shuffled_global_image_ids;
1286
1287   // RELEASE+UPDATE_MAPPING+ACQUIRE
1288   expect_add_event(mock_threads);
1289   expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids,
1290                                   &peer_ack_ctxs);
1291
1292   mock_image_map->update_instances_added({"9876"});
1293
1294   wait_for_scheduled_task();
1295   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1296
1297   update_map_and_acquire(mock_threads, mock_update_request,
1298                          mock_listener, shuffled_global_image_ids, 0,
1299                          &peer_ack_ctxs);
1300   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1301                                 0, &peer_ack_ctxs);
1302   wait_for_scheduled_task();
1303
1304   mock_image_map->update_instances_removed({"9876"});
1305
1306   std::set<std::string> released_global_image_ids;
1307   std::map<std::string, Context*> release_peer_ack_ctxs;
1308   expect_add_event(mock_threads);
1309   expect_listener_images_unmapped(mock_listener, 1, &released_global_image_ids,
1310                                   &release_peer_ack_ctxs);
1311   expect_add_event(mock_threads);
1312   expect_listener_images_unmapped(mock_listener, 1, &released_global_image_ids,
1313                                   &release_peer_ack_ctxs);
1314
1315   // instance blacklisted -- ACQUIRE request fails
1316   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids,
1317                          -EBLACKLISTED, &peer_ack_ctxs);
1318   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1319
1320   std::map<std::string, Context*> remap_peer_ack_ctxs;
1321   update_map_and_acquire(mock_threads, mock_update_request,
1322                          mock_listener, shuffled_global_image_ids, 0,
1323                          &remap_peer_ack_ctxs);
1324
1325   // instance blacklisted -- RELEASE request fails
1326   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1327                                 -ENOENT, &release_peer_ack_ctxs);
1328   wait_for_scheduled_task();
1329
1330   // new peer acks acquire request
1331   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1332                          &remap_peer_ack_ctxs);
1333   wait_for_scheduled_task();
1334
1335   std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids);
1336
1337   // remove image
1338   std::map<std::string, Context*> peer_ack_remove_ctxs;
1339   listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids,
1340                          &peer_ack_remove_ctxs);
1341   expect_add_event(mock_threads);
1342   listener_release_images(mock_listener, shuffled_global_image_ids,
1343                           &peer_ack_ctxs);
1344   update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids, 0);
1345
1346   mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids));
1347   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2));
1348
1349   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids_ack, 0,
1350                          &peer_ack_remove_ctxs);
1351   remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0,
1352                        &peer_ack_ctxs);
1353
1354   wait_for_scheduled_task();
1355   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1356 }
1357
1358 TEST_F(TestMockImageMap, MirrorUUIDUpdated) {
1359   MockThreads mock_threads(m_threads);
1360   expect_work_queue(mock_threads);
1361
1362   InSequence seq;
1363
1364   MockLoadRequest mock_load_request;
1365   expect_load_request(mock_load_request, 0);
1366
1367   MockListener mock_listener(this);
1368
1369   std::unique_ptr<MockImageMap> mock_image_map{
1370     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1371                          mock_listener)};
1372
1373   C_SaferCond cond;
1374   mock_image_map->init(&cond);
1375   ASSERT_EQ(0, cond.wait());
1376
1377   // remote image set
1378   std::set<std::string> initial_remote_global_image_ids{
1379     "global id 1", "global id 2", "global id 3"
1380   };
1381   std::set<std::string> initial_remote_global_image_ids_ack(initial_remote_global_image_ids);
1382
1383   // remote/local images to remove
1384   std::set<std::string> remote_removed_global_image_ids{
1385     "global id 1", "global id 2", "global id 3"
1386   };
1387   std::set<std::string> remote_removed_global_image_ids_ack(remote_removed_global_image_ids);
1388
1389   std::set<std::string> remote_added_global_image_ids{
1390     "global id 1", "global id 2", "global id 3"
1391   };
1392   std::set<std::string> remote_added_global_image_ids_ack(remote_added_global_image_ids);
1393
1394   // UPDATE_MAPPING+ACQUIRE
1395   expect_add_event(mock_threads);
1396   MockUpdateRequest mock_update_request;
1397   expect_update_request(mock_update_request, 0);
1398   expect_add_event(mock_threads);
1399   std::map<std::string, Context*> peer_ack_ctxs;
1400   listener_acquire_images(mock_listener, initial_remote_global_image_ids,
1401                           &peer_ack_ctxs);
1402
1403   // initial remote image list
1404   mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {});
1405
1406   ASSERT_TRUE(wait_for_map_update(1));
1407   ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size()));
1408
1409   // remote peer ACKs image acquire request
1410   remote_peer_ack_nowait(mock_image_map.get(),
1411                          initial_remote_global_image_ids_ack, 0,
1412                          &peer_ack_ctxs);
1413   wait_for_scheduled_task();
1414
1415   // RELEASE+REMOVE_MAPPING
1416   std::map<std::string, Context*> peer_remove_ack_ctxs;
1417   listener_remove_images(mock_listener, "uuid1", remote_removed_global_image_ids,
1418                          &peer_remove_ack_ctxs);
1419   expect_add_event(mock_threads);
1420   listener_release_images(mock_listener, remote_removed_global_image_ids,
1421                           &peer_ack_ctxs);
1422   update_map_request(mock_threads, mock_update_request, remote_removed_global_image_ids, 0);
1423
1424   mock_image_map->update_images("uuid1", {}, std::move(remote_removed_global_image_ids));
1425   ASSERT_TRUE(wait_for_listener_notify(remote_removed_global_image_ids_ack.size() * 2));
1426
1427   remote_peer_ack_nowait(mock_image_map.get(),
1428                          remote_removed_global_image_ids_ack, 0,
1429                          &peer_remove_ack_ctxs);
1430   remote_peer_ack_wait(mock_image_map.get(),
1431                        remote_removed_global_image_ids_ack, 0,
1432                        &peer_ack_ctxs);
1433
1434   // UPDATE_MAPPING+ACQUIRE
1435   expect_add_event(mock_threads);
1436   expect_update_request(mock_update_request, 0);
1437   expect_add_event(mock_threads);
1438   listener_acquire_images(mock_listener, remote_added_global_image_ids,
1439                           &peer_ack_ctxs);
1440
1441   mock_image_map->update_images("uuid2", std::move(remote_added_global_image_ids), {});
1442
1443   ASSERT_TRUE(wait_for_map_update(1));
1444   ASSERT_TRUE(wait_for_listener_notify(remote_added_global_image_ids_ack.size()));
1445
1446   // remote peer ACKs image acquire request
1447   remote_peer_ack_nowait(mock_image_map.get(),
1448                          remote_added_global_image_ids_ack, 0,
1449                          &peer_ack_ctxs);
1450
1451   wait_for_scheduled_task();
1452   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1453 }
1454
1455 TEST_F(TestMockImageMap, RebalanceImageMap) {
1456   MockThreads mock_threads(m_threads);
1457   expect_work_queue(mock_threads);
1458
1459   InSequence seq;
1460
1461   MockLoadRequest mock_load_request;
1462   expect_load_request(mock_load_request, 0);
1463
1464   MockListener mock_listener(this);
1465
1466   std::unique_ptr<MockImageMap> mock_image_map{
1467     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1468                          mock_listener)};
1469
1470   C_SaferCond cond;
1471   mock_image_map->init(&cond);
1472   ASSERT_EQ(0, cond.wait());
1473
1474   std::set<std::string> global_image_ids{
1475     "global id 1", "global id 2", "global id 3", "global id 4", "global id 5",
1476     "global id 6", "global id 7", "global id 8", "global id 9", "global id 10",
1477   };
1478   std::set<std::string> global_image_ids_ack(global_image_ids);
1479
1480   // UPDATE_MAPPING+ACQUIRE
1481   expect_add_event(mock_threads);
1482   MockUpdateRequest mock_update_request;
1483   expect_update_request(mock_update_request, 0);
1484   expect_add_event(mock_threads);
1485   std::map<std::string, Context*> peer_ack_ctxs;
1486   listener_acquire_images(mock_listener, global_image_ids,
1487                           &peer_ack_ctxs);
1488
1489   // initial image list
1490   mock_image_map->update_images("", std::move(global_image_ids), {});
1491
1492   ASSERT_TRUE(wait_for_map_update(1));
1493   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1494
1495   // remote peer ACKs image acquire request
1496   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1497                          &peer_ack_ctxs);
1498   wait_for_scheduled_task();
1499
1500   mock_image_map->update_instances_added({m_local_instance_id});
1501
1502   std::set<std::string> shuffled_global_image_ids;
1503
1504   // RELEASE+UPDATE_MAPPING+ACQUIRE
1505   expect_add_event(mock_threads);
1506   expect_listener_images_unmapped(mock_listener, 5, &shuffled_global_image_ids,
1507                                   &peer_ack_ctxs);
1508
1509   mock_image_map->update_instances_added({"9876"});
1510
1511   wait_for_scheduled_task();
1512   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1513
1514   update_map_and_acquire(mock_threads, mock_update_request,
1515                          mock_listener, shuffled_global_image_ids, 0,
1516                          &peer_ack_ctxs);
1517   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1518                                 0, &peer_ack_ctxs);
1519
1520   // completion shuffle action for now (re)mapped images
1521   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1522                          &peer_ack_ctxs);
1523
1524   wait_for_scheduled_task();
1525
1526   // remove all shuffled images -- make way for rebalance
1527   std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids);
1528
1529   // RELEASE+REMOVE_MAPPING
1530   expect_add_event(mock_threads);
1531   listener_release_images(mock_listener, shuffled_global_image_ids,
1532                           &peer_ack_ctxs);
1533   update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids,
1534                      0);
1535
1536   mock_image_map->update_images("", {}, std::move(shuffled_global_image_ids));
1537   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size()));
1538
1539   remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0,
1540                        &peer_ack_ctxs);
1541   wait_for_scheduled_task();
1542
1543   shuffled_global_image_ids.clear();
1544   shuffled_global_image_ids_ack.clear();
1545
1546   std::set<std::string> new_global_image_ids = {
1547     "global id 11"
1548   };
1549   std::set<std::string> new_global_image_ids_ack(new_global_image_ids);
1550
1551   expect_add_event(mock_threads);
1552   expect_update_request(mock_update_request, 0);
1553   expect_add_event(mock_threads);
1554   listener_acquire_images(mock_listener, new_global_image_ids, &peer_ack_ctxs);
1555
1556   expect_rebalance_event(mock_threads); // rebalance task
1557   expect_add_event(mock_threads);       // update task scheduled by
1558                                         // rebalance task
1559   expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids,
1560                                   &peer_ack_ctxs);
1561
1562   mock_image_map->update_images("", std::move(new_global_image_ids), {});
1563
1564   ASSERT_TRUE(wait_for_map_update(1));
1565   ASSERT_TRUE(wait_for_listener_notify(new_global_image_ids_ack.size()));
1566
1567   // set rebalance interval
1568   CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
1569   cct->_conf.set_val("rbd_mirror_image_policy_rebalance_timeout", "5");
1570   remote_peer_ack_nowait(mock_image_map.get(), new_global_image_ids_ack, 0,
1571                          &peer_ack_ctxs);
1572
1573   wait_for_scheduled_task();
1574   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1575
1576   update_map_and_acquire(mock_threads, mock_update_request,
1577                          mock_listener, shuffled_global_image_ids, 0,
1578                          &peer_ack_ctxs);
1579   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1580                                 0, &peer_ack_ctxs);
1581
1582   // completion shuffle action for now (re)mapped images
1583   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1584                          &peer_ack_ctxs);
1585
1586   wait_for_scheduled_task();
1587   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1588 }
1589
1590 } // namespace mirror
1591 } // namespace rbd