ceph/src/test/rbd_mirror/test_mock_ImageMap.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "test/rbd_mirror/test_mock_fixture.h"
   5 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
   6 #include "test/librados_test_stub/MockTestMemRadosClient.h"
   7 #include "test/librbd/mock/MockImageCtx.h"
   8 #include "test/rbd_mirror/mock/MockContextWQ.h"
   9 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  10 #include "librbd/MirroringWatcher.h"
  11 #include "tools/rbd_mirror/Threads.h"
  12 #include "tools/rbd_mirror/ImageMap.h"
  13 #include "tools/rbd_mirror/image_map/LoadRequest.h"
  14 #include "tools/rbd_mirror/image_map/UpdateRequest.h"
  15 #include "tools/rbd_mirror/image_map/Types.h"
  16 #include "include/stringify.h"
  17
  18 namespace librbd {
  19 namespace {
  20
  21 struct MockTestImageCtx : public librbd::MockImageCtx {
  22   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  23     : librbd::MockImageCtx(image_ctx) {
  24   }
  25 };
  26
  27 } // anonymous namespace
  28
  29 } // namespace librbd
  30
  31 namespace rbd {
  32 namespace mirror {
  33
  34 template <>
  35 struct Threads<librbd::MockTestImageCtx> {
  36   MockSafeTimer *timer;
  37   ceph::mutex &timer_lock;
  38
  39   MockContextWQ *work_queue;
  40
  41   Threads(Threads<librbd::ImageCtx> *threads)
  42     : timer(new MockSafeTimer()),
  43       timer_lock(threads->timer_lock),
  44       work_queue(new MockContextWQ()) {
  45   }
  46   ~Threads() {
  47     delete timer;
  48     delete work_queue;
  49   }
  50 };
  51
  52 namespace image_map {
  53
  54 template <>
  55 struct LoadRequest<librbd::MockTestImageCtx> {
  56   std::map<std::string, cls::rbd::MirrorImageMap> *image_map;
  57   Context *on_finish = nullptr;
  58
  59   static LoadRequest *s_instance;
  60   static LoadRequest *create(librados::IoCtx &ioctx,
  61                              std::map<std::string, cls::rbd::MirrorImageMap> *image_map,
  62                              Context *on_finish) {
  63     ceph_assert(s_instance != nullptr);
  64     s_instance->image_map = image_map;
  65     s_instance->on_finish = on_finish;
  66     return s_instance;
  67   }
  68
  69   MOCK_METHOD0(send, void());
  70
  71   LoadRequest() {
  72     s_instance = this;
  73   }
  74 };
  75
  76 template <>
  77 struct UpdateRequest<librbd::MockTestImageCtx> {
  78   Context *on_finish = nullptr;
  79   static UpdateRequest *s_instance;
  80   static UpdateRequest *create(librados::IoCtx &ioctx,
  81                                std::map<std::string, cls::rbd::MirrorImageMap> &&update_mapping,
  82                                std::set<std::string> &&global_image_ids,
  83                                Context *on_finish) {
  84     ceph_assert(s_instance != nullptr);
  85     s_instance->on_finish = on_finish;
  86     return s_instance;
  87   }
  88
  89   MOCK_METHOD0(send, void());
  90
  91   UpdateRequest() {
  92     s_instance = this;
  93   }
  94 };
  95
  96 LoadRequest<librbd::MockTestImageCtx> *
  97 LoadRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
  98 UpdateRequest<librbd::MockTestImageCtx> *
  99 UpdateRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 100
 101 } // namespace image_map
 102
 103 } // namespace mirror
 104 } // namespace rbd
 105
 106 // template definitions
 107 #include "tools/rbd_mirror/ImageMap.cc"
 108
 109 namespace rbd {
 110 namespace mirror {
 111
 112 using ::testing::_;
 113 using ::testing::DoAll;
 114 using ::testing::WithArg;
 115 using ::testing::AtLeast;
 116 using ::testing::InSequence;
 117 using ::testing::Invoke;
 118 using ::testing::ReturnArg;
 119 using ::testing::StrEq;
 120
 121 using image_map::Listener;
 122 using image_map::LoadRequest;
 123 using image_map::UpdateRequest;
 124
 125 using ::rbd::mirror::Threads;
 126
 127 class TestMockImageMap : public TestMockFixture {
 128 public:
 129   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 130   typedef ImageMap<librbd::MockTestImageCtx> MockImageMap;
 131   typedef LoadRequest<librbd::MockTestImageCtx> MockLoadRequest;
 132   typedef UpdateRequest<librbd::MockTestImageCtx> MockUpdateRequest;
 133
 134   struct MockListener : Listener {
 135     TestMockImageMap *test_mock_image_map;
 136
 137     MockListener(TestMockImageMap *test_mock_image_map)
 138       : test_mock_image_map(test_mock_image_map) {
 139     }
 140
 141     MOCK_METHOD2(mock_acquire_image, void(const std::string &, Context*));
 142     MOCK_METHOD2(mock_release_image, void(const std::string &, Context*));
 143     MOCK_METHOD3(mock_remove_image, void(const std::string &,
 144                                          const std::string &, Context*));
 145
 146     void acquire_image(const std::string &global_image_id,
 147                        const std::string &instance_id, Context* on_finish) {
 148       mock_acquire_image(global_image_id, on_finish);
 149     }
 150
 151     void release_image(const std::string &global_image_id,
 152                        const std::string &instance_id, Context* on_finish) {
 153       mock_release_image(global_image_id, on_finish);
 154     }
 155
 156     void remove_image(const std::string &mirror_uuid,
 157                       const std::string &global_image_id,
 158                       const std::string &instance_id, Context* on_finish) {
 159       mock_remove_image(mirror_uuid, global_image_id, on_finish);
 160     }
 161   };
 162
 163   TestMockImageMap() = default;
 164
 165   void SetUp() override {
 166     TestFixture::SetUp();
 167
 168     m_local_instance_id = stringify(m_local_io_ctx.get_instance_id());
 169
 170     EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle",
 171                                   "0"));
 172     EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_type", "simple"));
 173   }
 174
 175   void TearDown() override {
 176     EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_type", "none"));
 177
 178     TestFixture::TearDown();
 179   }
 180
 181   void expect_work_queue(MockThreads &mock_threads) {
 182     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 183       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 184             m_threads->work_queue->queue(ctx, r);
 185           }));
 186   }
 187
 188   void expect_add_event(MockThreads &mock_threads) {
 189     EXPECT_CALL(*mock_threads.timer, add_event_after(_,_))
 190       .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) {
 191              auto wrapped_ctx = new LambdaContext([this, ctx](int r) {
 192                     std::lock_guard timer_locker{m_threads->timer_lock};
 193                     ctx->complete(r);
 194                   });
 195                 m_threads->work_queue->queue(wrapped_ctx, 0);
 196               })), ReturnArg<1>()));
 197   }
 198
 199   void expect_rebalance_event(MockThreads &mock_threads) {
 200     EXPECT_CALL(*mock_threads.timer, add_event_after(_,_))
 201       .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) {
 202                 // disable rebalance so as to not reschedule it again
 203                 CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
 204                 cct->_conf.set_val("rbd_mirror_image_policy_rebalance_timeout", "0");
 205
 206                 auto wrapped_ctx = new LambdaContext([this, ctx](int r) {
 207                     std::lock_guard timer_locker{m_threads->timer_lock};
 208                     ctx->complete(r);
 209                   });
 210                 m_threads->work_queue->queue(wrapped_ctx, 0);
 211               })), ReturnArg<1>()));
 212   }
 213
 214   void expect_load_request(MockLoadRequest &request, int r) {
 215     EXPECT_CALL(request, send())
 216       .WillOnce(Invoke([&request, r]() {
 217             request.on_finish->complete(r);
 218           }));
 219   }
 220
 221   void expect_update_request(MockUpdateRequest &request, int r) {
 222     EXPECT_CALL(request, send())
 223       .WillOnce(Invoke([this, &request, r]() {
 224             request.on_finish->complete(r);
 225             if (r == 0) {
 226               std::lock_guard locker{m_lock};
 227               ++m_map_update_count;
 228               m_cond.notify_all();
 229             }
 230           }));
 231   }
 232
 233   void expect_listener_acquire_image(MockListener &mock_listener,
 234                                      const std::string &global_image_id,
 235                                      std::map<std::string, Context*> *peer_ack_ctxs) {
 236     EXPECT_CALL(mock_listener, mock_acquire_image(global_image_id, _))
 237       .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
 238               std::lock_guard locker{m_lock};
 239               peer_ack_ctxs->insert({global_image_id, ctx});
 240               ++m_notify_update_count;
 241               m_cond.notify_all();
 242             })));
 243   }
 244
 245   void expect_listener_release_image(MockListener &mock_listener,
 246                                      const std::string &global_image_id,
 247                                      std::map<std::string, Context*> *peer_ack_ctxs) {
 248     EXPECT_CALL(mock_listener, mock_release_image(global_image_id, _))
 249       .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
 250               std::lock_guard locker{m_lock};
 251               peer_ack_ctxs->insert({global_image_id, ctx});
 252               ++m_notify_update_count;
 253               m_cond.notify_all();
 254             })));
 255   }
 256
 257   void expect_listener_remove_image(MockListener &mock_listener,
 258                                     const std::string &mirror_uuid,
 259                                     const std::string &global_image_id,
 260                                     std::map<std::string, Context*> *peer_ack_ctxs) {
 261     EXPECT_CALL(mock_listener,
 262                 mock_remove_image(mirror_uuid, global_image_id, _))
 263       .WillOnce(WithArg<2>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
 264               std::lock_guard locker{m_lock};
 265               peer_ack_ctxs->insert({global_image_id, ctx});
 266               ++m_notify_update_count;
 267               m_cond.notify_all();
 268             })));
 269   }
 270
 271   void expect_listener_images_unmapped(MockListener &mock_listener, size_t count,
 272                                        std::set<std::string> *global_image_ids,
 273                                        std::map<std::string, Context*> *peer_ack_ctxs) {
 274     EXPECT_CALL(mock_listener, mock_release_image(_, _))
 275       .Times(count)
 276       .WillRepeatedly(Invoke([this, global_image_ids, peer_ack_ctxs](std::string global_image_id, Context* ctx) {
 277               std::lock_guard locker{m_lock};
 278               global_image_ids->emplace(global_image_id);
 279               peer_ack_ctxs->insert({global_image_id, ctx});
 280               ++m_notify_update_count;
 281               m_cond.notify_all();
 282             }));
 283   }
 284
 285   void remote_peer_ack_nowait(MockImageMap *image_map,
 286                               const std::set<std::string> &global_image_ids,
 287                               int ret,
 288                               std::map<std::string, Context*> *peer_ack_ctxs) {
 289     for (auto& global_image_id : global_image_ids) {
 290       auto it = peer_ack_ctxs->find(global_image_id);
 291       ASSERT_TRUE(it != peer_ack_ctxs->end());
 292       auto ack_ctx = it->second;
 293       peer_ack_ctxs->erase(it);
 294       ack_ctx->complete(ret);
 295       wait_for_scheduled_task();
 296     }
 297   }
 298
 299   void remote_peer_ack_wait(MockImageMap *image_map,
 300                             const std::set<std::string> &global_image_ids,
 301                             int ret,
 302                             std::map<std::string, Context*> *peer_ack_ctxs) {
 303     for (auto& global_image_id : global_image_ids) {
 304       auto it = peer_ack_ctxs->find(global_image_id);
 305       ASSERT_TRUE(it != peer_ack_ctxs->end());
 306       auto ack_ctx = it->second;
 307       peer_ack_ctxs->erase(it);
 308       ack_ctx->complete(ret);
 309       wait_for_scheduled_task();
 310       ASSERT_TRUE(wait_for_map_update(1));
 311     }
 312   }
 313
 314   void remote_peer_ack_listener_wait(MockImageMap *image_map,
 315                                      const std::set<std::string> &global_image_ids,
 316                                      int ret,
 317                                      std::map<std::string, Context*> *peer_ack_ctxs) {
 318     for (auto& global_image_id : global_image_ids) {
 319       auto it = peer_ack_ctxs->find(global_image_id);
 320       ASSERT_TRUE(it != peer_ack_ctxs->end());
 321       auto ack_ctx = it->second;
 322       peer_ack_ctxs->erase(it);
 323       ack_ctx->complete(ret);
 324       ASSERT_TRUE(wait_for_map_update(1));
 325       ASSERT_TRUE(wait_for_listener_notify(1));
 326     }
 327   }
 328
 329   void update_map_and_acquire(MockThreads &mock_threads,
 330                               MockUpdateRequest &mock_update_request,
 331                               MockListener &mock_listener,
 332                               const std::set<std::string> &global_image_ids,
 333                               int ret,
 334                               std::map<std::string, Context*> *peer_ack_ctxs) {
 335     for (auto const &global_image_id : global_image_ids) {
 336       expect_add_event(mock_threads);
 337       expect_update_request(mock_update_request, ret);
 338       expect_add_event(mock_threads);
 339       expect_listener_acquire_image(mock_listener, global_image_id,
 340                                     peer_ack_ctxs);
 341     }
 342   }
 343
 344   void update_map_request(MockThreads &mock_threads,
 345                           MockUpdateRequest &mock_update_request,
 346                           const std::set<std::string> &global_image_ids, int ret) {
 347     for (uint32_t i = 0; i < global_image_ids.size(); ++i) {
 348       expect_add_event(mock_threads);
 349       expect_update_request(mock_update_request, ret);
 350     }
 351   }
 352
 353   void wait_for_scheduled_task() {
 354     m_threads->work_queue->drain();
 355   }
 356
 357   bool wait_for_listener_notify(uint32_t count) {
 358     std::unique_lock locker{m_lock};
 359     while (m_notify_update_count < count) {
 360       if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) {
 361         break;
 362       }
 363     }
 364
 365     if (m_notify_update_count < count) {
 366       return false;
 367     }
 368
 369     m_notify_update_count -= count;
 370     return true;
 371   }
 372
 373   bool wait_for_map_update(uint32_t count) {
 374     std::unique_lock locker{m_lock};
 375     while (m_map_update_count < count) {
 376       if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) {
 377         break;
 378       }
 379     }
 380
 381     if (m_map_update_count < count) {
 382       return false;
 383     }
 384
 385     m_map_update_count -= count;
 386     return true;
 387   }
 388
 389   int when_shut_down(MockImageMap *image_map) {
 390     C_SaferCond ctx;
 391     image_map->shut_down(&ctx);
 392     return ctx.wait();
 393   }
 394
 395   void listener_acquire_images(MockListener &mock_listener,
 396                                const std::set<std::string> &global_image_ids,
 397                                std::map<std::string, Context*> *peer_ack_ctxs) {
 398     for (auto const &global_image_id : global_image_ids) {
 399       expect_listener_acquire_image(mock_listener, global_image_id,
 400                                     peer_ack_ctxs);
 401     }
 402   }
 403
 404   void listener_release_images(MockListener &mock_listener,
 405                                const std::set<std::string> &global_image_ids,
 406                                std::map<std::string, Context*> *peer_ack_ctxs) {
 407     for (auto const &global_image_id : global_image_ids) {
 408       expect_listener_release_image(mock_listener, global_image_id,
 409                                     peer_ack_ctxs);
 410     }
 411   }
 412
 413   void listener_remove_images(MockListener &mock_listener,
 414                               const std::string &mirror_uuid,
 415                               std::set<std::string> &global_image_ids,
 416                               std::map<std::string, Context*> *peer_ack_ctxs) {
 417     for (auto const &global_image_id : global_image_ids) {
 418       expect_listener_remove_image(mock_listener, mirror_uuid, global_image_id,
 419                                    peer_ack_ctxs);
 420     }
 421   }
 422
 423   ceph::mutex m_lock = ceph::make_mutex("TestMockImageMap::m_lock");
 424   ceph::condition_variable m_cond;
 425   uint32_t m_notify_update_count = 0;
 426   uint32_t m_map_update_count = 0;
 427   std::string m_local_instance_id;
 428 };
 429
 430 TEST_F(TestMockImageMap, SetLocalImages) {
 431   MockThreads mock_threads(m_threads);
 432   expect_work_queue(mock_threads);
 433
 434   InSequence seq;
 435
 436   MockLoadRequest mock_load_request;
 437   expect_load_request(mock_load_request, 0);
 438
 439   MockListener mock_listener(this);
 440
 441   std::unique_ptr<MockImageMap> mock_image_map{
 442     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 443                          mock_listener)};
 444
 445   C_SaferCond cond;
 446   mock_image_map->init(&cond);
 447   ASSERT_EQ(0, cond.wait());
 448
 449   std::set<std::string> global_image_ids{
 450     "global id 1", "global id 2"
 451   };
 452   std::set<std::string> global_image_ids_ack(global_image_ids);
 453
 454   // UPDATE_MAPPING+ACQUIRE
 455   expect_add_event(mock_threads);
 456   MockUpdateRequest mock_update_request;
 457   expect_update_request(mock_update_request, 0);
 458   expect_add_event(mock_threads);
 459   std::map<std::string, Context*> peer_ack_ctxs;
 460   listener_acquire_images(mock_listener, global_image_ids, &peer_ack_ctxs);
 461
 462   // initial image list
 463   mock_image_map->update_images("", std::move(global_image_ids), {});
 464
 465   ASSERT_TRUE(wait_for_map_update(1));
 466   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 467
 468   // remote peer ACKs image acquire request
 469   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
 470                          &peer_ack_ctxs);
 471
 472   wait_for_scheduled_task();
 473   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 474 }
 475
 476 TEST_F(TestMockImageMap, AddRemoveLocalImage) {
 477   MockThreads mock_threads(m_threads);
 478   expect_work_queue(mock_threads);
 479
 480   InSequence seq;
 481
 482   MockLoadRequest mock_load_request;
 483   expect_load_request(mock_load_request, 0);
 484
 485   MockListener mock_listener(this);
 486
 487   std::unique_ptr<MockImageMap> mock_image_map{
 488     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 489                          mock_listener)};
 490
 491   C_SaferCond cond;
 492   mock_image_map->init(&cond);
 493   ASSERT_EQ(0, cond.wait());
 494
 495   std::set<std::string> initial_global_image_ids{
 496     "global id 1", "global id 2"
 497   };
 498   std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids);
 499
 500   std::set<std::string> remove_global_image_ids{
 501     "global id 1", "global id 2"
 502   };
 503   std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids);
 504
 505   // UPDATE_MAPPING+ACQUIRE
 506   expect_add_event(mock_threads);
 507   MockUpdateRequest mock_update_request;
 508   expect_update_request(mock_update_request, 0);
 509   expect_add_event(mock_threads);
 510   std::map<std::string, Context*> peer_ack_ctxs;
 511   listener_acquire_images(mock_listener, initial_global_image_ids,
 512                           &peer_ack_ctxs);
 513
 514   // initial image list
 515   mock_image_map->update_images("", std::move(initial_global_image_ids), {});
 516
 517   ASSERT_TRUE(wait_for_map_update(1));
 518   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 519
 520   // remote peer ACKs image acquire request
 521   remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
 522                          &peer_ack_ctxs);
 523
 524   // RELEASE+REMOVE_MAPPING
 525   expect_add_event(mock_threads);
 526   listener_release_images(mock_listener, remove_global_image_ids,
 527                           &peer_ack_ctxs);
 528   update_map_request(mock_threads, mock_update_request, remove_global_image_ids,
 529                      0);
 530
 531   // remove images
 532   mock_image_map->update_images("", {}, std::move(remove_global_image_ids));
 533   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size()));
 534
 535   remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 536                        &peer_ack_ctxs);
 537
 538   wait_for_scheduled_task();
 539   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 540 }
 541
 542 TEST_F(TestMockImageMap, AddRemoveRemoteImage) {
 543   MockThreads mock_threads(m_threads);
 544   expect_work_queue(mock_threads);
 545
 546   InSequence seq;
 547
 548   MockLoadRequest mock_load_request;
 549   expect_load_request(mock_load_request, 0);
 550
 551   MockListener mock_listener(this);
 552
 553   std::unique_ptr<MockImageMap> mock_image_map{
 554     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 555                          mock_listener)};
 556
 557   C_SaferCond cond;
 558   mock_image_map->init(&cond);
 559   ASSERT_EQ(0, cond.wait());
 560
 561   std::set<std::string> initial_global_image_ids{
 562     "global id 1", "global id 2"
 563   };
 564   std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids);
 565
 566   std::set<std::string> remove_global_image_ids{
 567     "global id 1", "global id 2"
 568   };
 569   std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids);
 570
 571   // UPDATE_MAPPING+ACQUIRE
 572   expect_add_event(mock_threads);
 573   MockUpdateRequest mock_update_request;
 574   expect_update_request(mock_update_request, 0);
 575   expect_add_event(mock_threads);
 576   std::map<std::string, Context*> peer_ack_ctxs;
 577   listener_acquire_images(mock_listener, initial_global_image_ids,
 578                           &peer_ack_ctxs);
 579
 580   // initial image list
 581   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids),
 582                                 {});
 583
 584   ASSERT_TRUE(wait_for_map_update(1));
 585   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 586
 587   // remote peer ACKs image acquire request
 588   remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
 589                          &peer_ack_ctxs);
 590
 591   // RELEASE+REMOVE_MAPPING
 592   std::map<std::string, Context*> peer_remove_ack_ctxs;
 593   listener_remove_images(mock_listener, "uuid1", remove_global_image_ids,
 594                          &peer_remove_ack_ctxs);
 595   expect_add_event(mock_threads);
 596   listener_release_images(mock_listener, remove_global_image_ids,
 597                           &peer_ack_ctxs);
 598   update_map_request(mock_threads, mock_update_request, remove_global_image_ids,
 599                      0);
 600
 601   // remove images
 602   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids));
 603   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2));
 604
 605   remote_peer_ack_nowait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 606                          &peer_remove_ack_ctxs);
 607   remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 608                        &peer_ack_ctxs);
 609
 610   wait_for_scheduled_task();
 611   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 612 }
 613
 614 TEST_F(TestMockImageMap, AddRemoveRemoteImageDuplicateNotification) {
 615   MockThreads mock_threads(m_threads);
 616   expect_work_queue(mock_threads);
 617
 618   InSequence seq;
 619
 620   MockLoadRequest mock_load_request;
 621   expect_load_request(mock_load_request, 0);
 622
 623   MockListener mock_listener(this);
 624
 625   std::unique_ptr<MockImageMap> mock_image_map{
 626     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 627                          mock_listener)};
 628
 629   C_SaferCond cond;
 630   mock_image_map->init(&cond);
 631   ASSERT_EQ(0, cond.wait());
 632
 633   std::set<std::string> initial_global_image_ids{
 634     "global id 1", "global id 2"
 635   };
 636   std::set<std::string> initial_global_image_ids_dup(initial_global_image_ids);
 637   std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids);
 638
 639   std::set<std::string> remove_global_image_ids{
 640     "global id 1", "global id 2"
 641   };
 642   std::set<std::string> remove_global_image_ids_dup(remove_global_image_ids);
 643   std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids);
 644
 645   // UPDATE_MAPPING+ACQUIRE
 646   expect_add_event(mock_threads);
 647   MockUpdateRequest mock_update_request;
 648   expect_update_request(mock_update_request, 0);
 649   expect_add_event(mock_threads);
 650   std::map<std::string, Context*> peer_ack_ctxs;
 651   listener_acquire_images(mock_listener, initial_global_image_ids,
 652                           &peer_ack_ctxs);
 653
 654   // initial image list
 655   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {});
 656
 657   ASSERT_TRUE(wait_for_map_update(1));
 658   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 659
 660   // trigger duplicate "add" event
 661   wait_for_scheduled_task();
 662   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids_dup), {});
 663
 664   // remote peer ACKs image acquire request
 665   remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
 666                          &peer_ack_ctxs);
 667
 668   // RELEASE+REMOVE_MAPPING
 669   std::map<std::string, Context*> peer_remove_ack_ctxs;
 670   listener_remove_images(mock_listener, "uuid1", remove_global_image_ids,
 671                          &peer_remove_ack_ctxs);
 672   expect_add_event(mock_threads);
 673   listener_release_images(mock_listener, remove_global_image_ids,
 674                           &peer_ack_ctxs);
 675   update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0);
 676
 677   // remove images
 678   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids));
 679   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2));
 680
 681   remote_peer_ack_nowait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 682                          &peer_remove_ack_ctxs);
 683   remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
 684                        &peer_ack_ctxs);
 685
 686   // trigger duplicate "remove" notification
 687   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids_dup));
 688
 689   wait_for_scheduled_task();
 690   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 691 }
 692
 693 TEST_F(TestMockImageMap, AcquireImageErrorRetry) {
 694   MockThreads mock_threads(m_threads);
 695   expect_work_queue(mock_threads);
 696
 697   InSequence seq;
 698
 699   MockLoadRequest mock_load_request;
 700   expect_load_request(mock_load_request, 0);
 701
 702   MockListener mock_listener(this);
 703
 704   std::unique_ptr<MockImageMap> mock_image_map{
 705     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 706                          mock_listener)};
 707
 708   C_SaferCond cond;
 709   mock_image_map->init(&cond);
 710   ASSERT_EQ(0, cond.wait());
 711
 712   std::set<std::string> initial_global_image_ids{
 713     "global id 1", "global id 2"
 714   };
 715   std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids);
 716
 717   // UPDATE_MAPPING failure
 718   expect_add_event(mock_threads);
 719   MockUpdateRequest mock_update_request;
 720   expect_update_request(mock_update_request, -EIO);
 721
 722   // UPDATE_MAPPING+ACQUIRE
 723   expect_add_event(mock_threads);
 724   expect_update_request(mock_update_request, 0);
 725   expect_add_event(mock_threads);
 726   std::map<std::string, Context*> peer_ack_ctxs;
 727   listener_acquire_images(mock_listener, initial_global_image_ids,
 728                           &peer_ack_ctxs);
 729
 730   // initial image list
 731   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {});
 732
 733   ASSERT_TRUE(wait_for_map_update(1));
 734   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 735
 736   // remote peer ACKs image acquire request
 737   remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
 738                          &peer_ack_ctxs);
 739
 740   wait_for_scheduled_task();
 741   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 742 }
 743
 744 TEST_F(TestMockImageMap, RemoveRemoteAndLocalImage) {
 745   MockThreads mock_threads(m_threads);
 746   expect_work_queue(mock_threads);
 747
 748   InSequence seq;
 749
 750   MockLoadRequest mock_load_request;
 751   expect_load_request(mock_load_request, 0);
 752
 753   MockListener mock_listener(this);
 754
 755   std::unique_ptr<MockImageMap> mock_image_map{
 756     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 757                          mock_listener)};
 758
 759   C_SaferCond cond;
 760   mock_image_map->init(&cond);
 761   ASSERT_EQ(0, cond.wait());
 762
 763   // remote image set
 764   std::set<std::string> initial_remote_global_image_ids{
 765     "global id 1"
 766   };
 767   std::set<std::string> initial_remote_global_image_ids_ack(initial_remote_global_image_ids);
 768
 769   // local image set
 770   std::set<std::string> initial_local_global_image_ids{
 771     "global id 1"
 772   };
 773
 774   // remote/local images to remove
 775   std::set<std::string> remote_remove_global_image_ids{
 776     "global id 1"
 777   };
 778   std::set<std::string> remote_remove_global_image_ids_ack(remote_remove_global_image_ids);
 779
 780   std::set<std::string> local_remove_global_image_ids{
 781     "global id 1"
 782   };
 783   std::set<std::string> local_remove_global_image_ids_ack(local_remove_global_image_ids);
 784
 785   // UPDATE_MAPPING+ACQUIRE
 786   expect_add_event(mock_threads);
 787   MockUpdateRequest mock_update_request;
 788   expect_update_request(mock_update_request, 0);
 789   expect_add_event(mock_threads);
 790   std::map<std::string, Context*> peer_ack_ctxs;
 791   listener_acquire_images(mock_listener, initial_remote_global_image_ids,
 792                           &peer_ack_ctxs);
 793
 794   // initial remote image list
 795   mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {});
 796
 797   ASSERT_TRUE(wait_for_map_update(1));
 798   ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size()));
 799
 800   // remote peer ACKs image acquire request
 801   remote_peer_ack_nowait(mock_image_map.get(),
 802                          initial_remote_global_image_ids_ack, 0,
 803                          &peer_ack_ctxs);
 804
 805   // set initial local image list -- this is a no-op from policy pov
 806   mock_image_map->update_images("", std::move(initial_local_global_image_ids), {});
 807
 808   // remove remote images -- this should be a no-op from policy pov
 809   // except the listener notification
 810   std::map<std::string, Context*> peer_ack_remove_ctxs;
 811   listener_remove_images(mock_listener, "uuid1", remote_remove_global_image_ids,
 812                          &peer_ack_remove_ctxs);
 813
 814   mock_image_map->update_images("uuid1", {}, std::move(remote_remove_global_image_ids));
 815   ASSERT_TRUE(wait_for_listener_notify(remote_remove_global_image_ids_ack.size()));
 816
 817   // RELEASE+REMOVE_MAPPING
 818   expect_add_event(mock_threads);
 819   listener_release_images(mock_listener, local_remove_global_image_ids,
 820                           &peer_ack_ctxs);
 821   update_map_request(mock_threads, mock_update_request, local_remove_global_image_ids, 0);
 822
 823   // remove local images
 824   mock_image_map->update_images("", {}, std::move(local_remove_global_image_ids));
 825   ASSERT_TRUE(wait_for_listener_notify(local_remove_global_image_ids_ack.size()));
 826
 827   remote_peer_ack_nowait(mock_image_map.get(), local_remove_global_image_ids_ack,
 828                          0,  &peer_ack_remove_ctxs);
 829   remote_peer_ack_wait(mock_image_map.get(), local_remove_global_image_ids_ack,
 830                        0, &peer_ack_ctxs);
 831
 832   wait_for_scheduled_task();
 833   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 834 }
 835
 836 TEST_F(TestMockImageMap, AddInstance) {
 837   MockThreads mock_threads(m_threads);
 838   expect_work_queue(mock_threads);
 839
 840   InSequence seq;
 841
 842   MockLoadRequest mock_load_request;
 843   expect_load_request(mock_load_request, 0);
 844
 845   MockListener mock_listener(this);
 846
 847   std::unique_ptr<MockImageMap> mock_image_map{
 848     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 849                          mock_listener)};
 850
 851   C_SaferCond cond;
 852   mock_image_map->init(&cond);
 853   ASSERT_EQ(0, cond.wait());
 854
 855   std::set<std::string> global_image_ids{
 856     "global id 1", "global id 2", "global id 3", "global id 4", "global id 5"
 857   };
 858   std::set<std::string> global_image_ids_ack(global_image_ids);
 859
 860   // UPDATE_MAPPING+ACQUIRE
 861   expect_add_event(mock_threads);
 862   MockUpdateRequest mock_update_request;
 863   expect_update_request(mock_update_request, 0);
 864   expect_add_event(mock_threads);
 865   std::map<std::string, Context*> peer_ack_ctxs;
 866   listener_acquire_images(mock_listener, global_image_ids,
 867                           &peer_ack_ctxs);
 868
 869   // initial image list
 870   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
 871
 872   ASSERT_TRUE(wait_for_map_update(1));
 873   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 874
 875   // remote peer ACKs image acquire request
 876   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
 877                          &peer_ack_ctxs);
 878   wait_for_scheduled_task();
 879
 880   mock_image_map->update_instances_added({m_local_instance_id});
 881
 882   std::set<std::string> shuffled_global_image_ids;
 883
 884   // RELEASE+UPDATE_MAPPING+ACQUIRE
 885   expect_add_event(mock_threads);
 886   expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids,
 887                                   &peer_ack_ctxs);
 888
 889   mock_image_map->update_instances_added({"9876"});
 890
 891   wait_for_scheduled_task();
 892   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 893
 894   update_map_and_acquire(mock_threads, mock_update_request,
 895                          mock_listener, shuffled_global_image_ids, 0,
 896                          &peer_ack_ctxs);
 897   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
 898                                 0, &peer_ack_ctxs);
 899
 900   // completion shuffle action for now (re)mapped images
 901   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
 902                          &peer_ack_ctxs);
 903
 904   wait_for_scheduled_task();
 905   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
 906 }
 907
 908 TEST_F(TestMockImageMap, RemoveInstance) {
 909   MockThreads mock_threads(m_threads);
 910   expect_work_queue(mock_threads);
 911
 912   InSequence seq;
 913
 914   MockLoadRequest mock_load_request;
 915   expect_load_request(mock_load_request, 0);
 916
 917   MockListener mock_listener(this);
 918
 919   std::unique_ptr<MockImageMap> mock_image_map{
 920     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
 921                          mock_listener)};
 922
 923   C_SaferCond cond;
 924   mock_image_map->init(&cond);
 925   ASSERT_EQ(0, cond.wait());
 926
 927   std::set<std::string> global_image_ids{
 928     "global id 1", "global id 2", "global id 3", "global id 4", "global id 5"
 929   };
 930   std::set<std::string> global_image_ids_ack(global_image_ids);
 931
 932   expect_add_event(mock_threads);
 933
 934   // UPDATE_MAPPING+ACQUIRE
 935   MockUpdateRequest mock_update_request;
 936   expect_update_request(mock_update_request, 0);
 937   expect_add_event(mock_threads);
 938   std::map<std::string, Context*> peer_ack_ctxs;
 939   listener_acquire_images(mock_listener, global_image_ids,
 940                           &peer_ack_ctxs);
 941
 942   // set initial image list
 943   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
 944
 945   ASSERT_TRUE(wait_for_map_update(1));
 946   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 947
 948   // remote peer ACKs image acquire request -- completing action
 949   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
 950                          &peer_ack_ctxs);
 951   wait_for_scheduled_task();
 952
 953   mock_image_map->update_instances_added({m_local_instance_id});
 954
 955   std::set<std::string> shuffled_global_image_ids;
 956
 957   // RELEASE+UPDATE_MAPPING+ACQUIRE
 958   expect_add_event(mock_threads);
 959   expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids,
 960                                   &peer_ack_ctxs);
 961
 962   mock_image_map->update_instances_added({"9876"});
 963
 964   wait_for_scheduled_task();
 965   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 966
 967   update_map_and_acquire(mock_threads, mock_update_request,
 968                          mock_listener, shuffled_global_image_ids, 0,
 969                          &peer_ack_ctxs);
 970   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
 971                                 0, &peer_ack_ctxs);
 972
 973   // completion shuffle action for now (re)mapped images
 974   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
 975                          &peer_ack_ctxs);
 976   wait_for_scheduled_task();
 977
 978   shuffled_global_image_ids.clear();
 979
 980   // remove added instance
 981   expect_add_event(mock_threads);
 982   expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids,
 983                                   &peer_ack_ctxs);
 984
 985   mock_image_map->update_instances_removed({"9876"});
 986
 987   wait_for_scheduled_task();
 988   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 989
 990   update_map_and_acquire(mock_threads, mock_update_request,
 991                          mock_listener, shuffled_global_image_ids, 0,
 992                          &peer_ack_ctxs);
 993   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
 994                                 0, &peer_ack_ctxs);
 995
 996   // completion shuffle action for now (re)mapped images
 997   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
 998                          &peer_ack_ctxs);
 999
1000   wait_for_scheduled_task();
1001   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1002 }
1003
1004 TEST_F(TestMockImageMap, AddInstancePingPongImageTest) {
1005   EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle", "600"));
1006
1007   MockThreads mock_threads(m_threads);
1008   expect_work_queue(mock_threads);
1009
1010   InSequence seq;
1011
1012   std::set<std::string> global_image_ids{
1013     "global id 1", "global id 2", "global id 3", "global id 4", "global id 5",
1014     "global id 6", "global id 7", "global id 8", "global id 9", "global id 10",
1015     "global id 11", "global id 12", "global id 13", "global id 14"
1016   };
1017
1018   std::map<std::string, cls::rbd::MirrorImageMap> image_mapping;
1019   for (auto& global_image_id : global_image_ids) {
1020     image_mapping[global_image_id] = {m_local_instance_id, {}, {}};
1021   }
1022
1023   // ACQUIRE
1024   MockLoadRequest mock_load_request;
1025   EXPECT_CALL(mock_load_request, send()).WillOnce(
1026     Invoke([&mock_load_request, &image_mapping]() {
1027       *mock_load_request.image_map = image_mapping;
1028       mock_load_request.on_finish->complete(0);
1029     }));
1030
1031   expect_add_event(mock_threads);
1032   MockListener mock_listener(this);
1033   std::map<std::string, Context*> peer_ack_ctxs;
1034   listener_acquire_images(mock_listener, global_image_ids,
1035                           &peer_ack_ctxs);
1036
1037   std::unique_ptr<MockImageMap> mock_image_map{
1038     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1039                          mock_listener)};
1040
1041   C_SaferCond cond;
1042   mock_image_map->init(&cond);
1043   ASSERT_EQ(0, cond.wait());
1044
1045   mock_image_map->update_instances_added({m_local_instance_id});
1046
1047   std::set<std::string> global_image_ids_ack(global_image_ids);
1048
1049   // remote peer ACKs image acquire request -- completing action
1050   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1051   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1052                          &peer_ack_ctxs);
1053   wait_for_scheduled_task();
1054
1055   // RELEASE+UPDATE_MAPPING+ACQUIRE
1056   expect_add_event(mock_threads);
1057   MockUpdateRequest mock_update_request;
1058   expect_update_request(mock_update_request, 0);
1059   expect_add_event(mock_threads);
1060   listener_acquire_images(mock_listener, global_image_ids,
1061                           &peer_ack_ctxs);
1062
1063   // set initial image list
1064   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
1065
1066   ASSERT_TRUE(wait_for_map_update(1));
1067   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1068
1069   // remote peer ACKs image acquire request -- completing action
1070   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1071                          &peer_ack_ctxs);
1072   wait_for_scheduled_task();
1073
1074   std::set<std::string> shuffled_global_image_ids;
1075
1076   // RELEASE+UPDATE_MAPPING+ACQUIRE
1077   expect_add_event(mock_threads);
1078   expect_listener_images_unmapped(mock_listener, 7, &shuffled_global_image_ids,
1079                                   &peer_ack_ctxs);
1080
1081   mock_image_map->update_instances_added({"9876"});
1082
1083   wait_for_scheduled_task();
1084   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1085
1086   update_map_and_acquire(mock_threads, mock_update_request,
1087                          mock_listener, shuffled_global_image_ids, 0,
1088                          &peer_ack_ctxs);
1089   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1090                                 0, &peer_ack_ctxs);
1091
1092   // completion shuffle action for now (re)mapped images
1093   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1094                          &peer_ack_ctxs);
1095   wait_for_scheduled_task();
1096
1097   std::set<std::string> migrated_global_image_ids(shuffled_global_image_ids);
1098   shuffled_global_image_ids.clear();
1099
1100   // RELEASE+UPDATE_MAPPING+ACQUIRE
1101   expect_add_event(mock_threads);
1102   expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids,
1103                                   &peer_ack_ctxs);
1104
1105   // add another instance
1106   mock_image_map->update_instances_added({"5432"});
1107
1108   wait_for_scheduled_task();
1109   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1110
1111   update_map_and_acquire(mock_threads, mock_update_request,
1112                          mock_listener, shuffled_global_image_ids, 0,
1113                          &peer_ack_ctxs);
1114   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1115                                 0, &peer_ack_ctxs);
1116
1117   // completion shuffle action for now (re)mapped images
1118   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1119                          &peer_ack_ctxs);
1120
1121   // shuffle set should be distinct
1122   std::set<std::string> reshuffled;
1123   std::set_intersection(migrated_global_image_ids.begin(), migrated_global_image_ids.end(),
1124                         shuffled_global_image_ids.begin(), shuffled_global_image_ids.end(),
1125                         std::inserter(reshuffled, reshuffled.begin()));
1126   ASSERT_TRUE(reshuffled.empty());
1127
1128   wait_for_scheduled_task();
1129   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1130 }
1131
1132 TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) {
1133   MockThreads mock_threads(m_threads);
1134   expect_work_queue(mock_threads);
1135
1136   InSequence seq;
1137
1138   MockLoadRequest mock_load_request;
1139   expect_load_request(mock_load_request, 0);
1140
1141   MockListener mock_listener(this);
1142
1143   std::unique_ptr<MockImageMap> mock_image_map{
1144     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1145                          mock_listener)};
1146
1147   C_SaferCond cond;
1148   mock_image_map->init(&cond);
1149   ASSERT_EQ(0, cond.wait());
1150
1151   std::set<std::string> global_image_ids{
1152     "global id 1", "global id 2", "global id 3", "remote id 4",
1153   };
1154   std::set<std::string> global_image_ids_ack(global_image_ids);
1155
1156   std::set<std::string> remove_global_image_ids{
1157     "global id 1"
1158   };
1159   std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids);
1160
1161   expect_add_event(mock_threads);
1162   // UPDATE_MAPPING+ACQUIRE
1163   MockUpdateRequest mock_update_request;
1164   expect_update_request(mock_update_request, 0);
1165   expect_add_event(mock_threads);
1166   std::map<std::string, Context*> peer_ack_ctxs;
1167   listener_acquire_images(mock_listener, global_image_ids,
1168                           &peer_ack_ctxs);
1169
1170   // initial image list
1171   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
1172
1173   ASSERT_TRUE(wait_for_map_update(1));
1174   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1175
1176   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1177                          &peer_ack_ctxs);
1178   wait_for_scheduled_task();
1179
1180   mock_image_map->update_instances_added({m_local_instance_id});
1181
1182   std::set<std::string> shuffled_global_image_ids;
1183
1184   // RELEASE+UPDATE_MAPPING+ACQUIRE
1185   expect_add_event(mock_threads);
1186   expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids,
1187                                   &peer_ack_ctxs);
1188
1189   mock_image_map->update_instances_added({"9876"});
1190
1191   wait_for_scheduled_task();
1192   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1193
1194   update_map_and_acquire(mock_threads, mock_update_request,
1195                          mock_listener, shuffled_global_image_ids, 0,
1196                          &peer_ack_ctxs);
1197   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1198                                 0, &peer_ack_ctxs);
1199
1200   // completion shuffle action for now (re)mapped images
1201   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1202                          &peer_ack_ctxs);
1203   wait_for_scheduled_task();
1204
1205   std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids);
1206
1207   // RELEASE
1208
1209   std::map<std::string, Context*> peer_ack_remove_ctxs;
1210   listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids,
1211                          &peer_ack_remove_ctxs);
1212   expect_add_event(mock_threads);
1213   listener_release_images(mock_listener, shuffled_global_image_ids,
1214                           &peer_ack_ctxs);
1215   expect_add_event(mock_threads);
1216   expect_update_request(mock_update_request, 0);
1217   expect_add_event(mock_threads);
1218   expect_update_request(mock_update_request, 0);
1219
1220   mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids));
1221   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2));
1222
1223   // instance failed -- update policy for instance removal
1224   mock_image_map->update_instances_removed({"9876"});
1225
1226   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids,
1227                          -ENOENT, &peer_ack_remove_ctxs);
1228   remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids,
1229                        -EBLOCKLISTED, &peer_ack_ctxs);
1230
1231   wait_for_scheduled_task();
1232   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1233 }
1234
1235 TEST_F(TestMockImageMap, AddErrorAndRemoveImage) {
1236   MockThreads mock_threads(m_threads);
1237   expect_work_queue(mock_threads);
1238
1239   InSequence seq;
1240
1241   MockLoadRequest mock_load_request;
1242   expect_load_request(mock_load_request, 0);
1243
1244   MockListener mock_listener(this);
1245
1246   std::unique_ptr<MockImageMap> mock_image_map{
1247     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1248                          mock_listener)};
1249
1250   C_SaferCond cond;
1251   mock_image_map->init(&cond);
1252   ASSERT_EQ(0, cond.wait());
1253
1254   mock_image_map->update_instances_added({m_local_instance_id});
1255
1256   std::set<std::string> global_image_ids{
1257     "global id 1", "global id 2", "global id 3", "remote id 4",
1258   };
1259   std::set<std::string> global_image_ids_ack(global_image_ids);
1260
1261   // UPDATE_MAPPING+ACQUIRE
1262   expect_add_event(mock_threads);
1263   MockUpdateRequest mock_update_request;
1264   expect_update_request(mock_update_request, 0);
1265   expect_add_event(mock_threads);
1266   std::map<std::string, Context*> peer_ack_ctxs;
1267   listener_acquire_images(mock_listener, global_image_ids,
1268                           &peer_ack_ctxs);
1269
1270   // initial image list
1271   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
1272
1273   ASSERT_TRUE(wait_for_map_update(1));
1274   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1275
1276   // remote peer ACKs image acquire request
1277   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1278                          &peer_ack_ctxs);
1279   wait_for_scheduled_task();
1280
1281   std::set<std::string> shuffled_global_image_ids;
1282
1283   // RELEASE+UPDATE_MAPPING+ACQUIRE
1284   expect_add_event(mock_threads);
1285   expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids,
1286                                   &peer_ack_ctxs);
1287
1288   mock_image_map->update_instances_added({"9876"});
1289
1290   wait_for_scheduled_task();
1291   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1292
1293   update_map_and_acquire(mock_threads, mock_update_request,
1294                          mock_listener, shuffled_global_image_ids, 0,
1295                          &peer_ack_ctxs);
1296   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1297                                 0, &peer_ack_ctxs);
1298   wait_for_scheduled_task();
1299
1300   mock_image_map->update_instances_removed({"9876"});
1301
1302   std::set<std::string> released_global_image_ids;
1303   std::map<std::string, Context*> release_peer_ack_ctxs;
1304   expect_add_event(mock_threads);
1305   expect_listener_images_unmapped(mock_listener, 1, &released_global_image_ids,
1306                                   &release_peer_ack_ctxs);
1307   expect_add_event(mock_threads);
1308   expect_listener_images_unmapped(mock_listener, 1, &released_global_image_ids,
1309                                   &release_peer_ack_ctxs);
1310
1311   // instance blocklisted -- ACQUIRE request fails
1312   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids,
1313                          -EBLOCKLISTED, &peer_ack_ctxs);
1314   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1315
1316   std::map<std::string, Context*> remap_peer_ack_ctxs;
1317   update_map_and_acquire(mock_threads, mock_update_request,
1318                          mock_listener, shuffled_global_image_ids, 0,
1319                          &remap_peer_ack_ctxs);
1320
1321   // instance blocklisted -- RELEASE request fails
1322   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1323                                 -ENOENT, &release_peer_ack_ctxs);
1324   wait_for_scheduled_task();
1325
1326   // new peer acks acquire request
1327   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1328                          &remap_peer_ack_ctxs);
1329   wait_for_scheduled_task();
1330
1331   std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids);
1332
1333   // remove image
1334   std::map<std::string, Context*> peer_ack_remove_ctxs;
1335   listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids,
1336                          &peer_ack_remove_ctxs);
1337   expect_add_event(mock_threads);
1338   listener_release_images(mock_listener, shuffled_global_image_ids,
1339                           &peer_ack_ctxs);
1340   update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids, 0);
1341
1342   mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids));
1343   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2));
1344
1345   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids_ack, 0,
1346                          &peer_ack_remove_ctxs);
1347   remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0,
1348                        &peer_ack_ctxs);
1349
1350   wait_for_scheduled_task();
1351   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1352 }
1353
1354 TEST_F(TestMockImageMap, MirrorUUIDUpdated) {
1355   MockThreads mock_threads(m_threads);
1356   expect_work_queue(mock_threads);
1357
1358   InSequence seq;
1359
1360   MockLoadRequest mock_load_request;
1361   expect_load_request(mock_load_request, 0);
1362
1363   MockListener mock_listener(this);
1364
1365   std::unique_ptr<MockImageMap> mock_image_map{
1366     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1367                          mock_listener)};
1368
1369   C_SaferCond cond;
1370   mock_image_map->init(&cond);
1371   ASSERT_EQ(0, cond.wait());
1372
1373   // remote image set
1374   std::set<std::string> initial_remote_global_image_ids{
1375     "global id 1", "global id 2", "global id 3"
1376   };
1377   std::set<std::string> initial_remote_global_image_ids_ack(initial_remote_global_image_ids);
1378
1379   // remote/local images to remove
1380   std::set<std::string> remote_removed_global_image_ids{
1381     "global id 1", "global id 2", "global id 3"
1382   };
1383   std::set<std::string> remote_removed_global_image_ids_ack(remote_removed_global_image_ids);
1384
1385   std::set<std::string> remote_added_global_image_ids{
1386     "global id 1", "global id 2", "global id 3"
1387   };
1388   std::set<std::string> remote_added_global_image_ids_ack(remote_added_global_image_ids);
1389
1390   // UPDATE_MAPPING+ACQUIRE
1391   expect_add_event(mock_threads);
1392   MockUpdateRequest mock_update_request;
1393   expect_update_request(mock_update_request, 0);
1394   expect_add_event(mock_threads);
1395   std::map<std::string, Context*> peer_ack_ctxs;
1396   listener_acquire_images(mock_listener, initial_remote_global_image_ids,
1397                           &peer_ack_ctxs);
1398
1399   // initial remote image list
1400   mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {});
1401
1402   ASSERT_TRUE(wait_for_map_update(1));
1403   ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size()));
1404
1405   // remote peer ACKs image acquire request
1406   remote_peer_ack_nowait(mock_image_map.get(),
1407                          initial_remote_global_image_ids_ack, 0,
1408                          &peer_ack_ctxs);
1409   wait_for_scheduled_task();
1410
1411   // RELEASE+REMOVE_MAPPING
1412   std::map<std::string, Context*> peer_remove_ack_ctxs;
1413   listener_remove_images(mock_listener, "uuid1", remote_removed_global_image_ids,
1414                          &peer_remove_ack_ctxs);
1415   expect_add_event(mock_threads);
1416   listener_release_images(mock_listener, remote_removed_global_image_ids,
1417                           &peer_ack_ctxs);
1418   update_map_request(mock_threads, mock_update_request, remote_removed_global_image_ids, 0);
1419
1420   mock_image_map->update_images("uuid1", {}, std::move(remote_removed_global_image_ids));
1421   ASSERT_TRUE(wait_for_listener_notify(remote_removed_global_image_ids_ack.size() * 2));
1422
1423   remote_peer_ack_nowait(mock_image_map.get(),
1424                          remote_removed_global_image_ids_ack, 0,
1425                          &peer_remove_ack_ctxs);
1426   remote_peer_ack_wait(mock_image_map.get(),
1427                        remote_removed_global_image_ids_ack, 0,
1428                        &peer_ack_ctxs);
1429
1430   // UPDATE_MAPPING+ACQUIRE
1431   expect_add_event(mock_threads);
1432   expect_update_request(mock_update_request, 0);
1433   expect_add_event(mock_threads);
1434   listener_acquire_images(mock_listener, remote_added_global_image_ids,
1435                           &peer_ack_ctxs);
1436
1437   mock_image_map->update_images("uuid2", std::move(remote_added_global_image_ids), {});
1438
1439   ASSERT_TRUE(wait_for_map_update(1));
1440   ASSERT_TRUE(wait_for_listener_notify(remote_added_global_image_ids_ack.size()));
1441
1442   // remote peer ACKs image acquire request
1443   remote_peer_ack_nowait(mock_image_map.get(),
1444                          remote_added_global_image_ids_ack, 0,
1445                          &peer_ack_ctxs);
1446
1447   wait_for_scheduled_task();
1448   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1449 }
1450
1451 TEST_F(TestMockImageMap, RebalanceImageMap) {
1452   MockThreads mock_threads(m_threads);
1453   expect_work_queue(mock_threads);
1454
1455   InSequence seq;
1456
1457   MockLoadRequest mock_load_request;
1458   expect_load_request(mock_load_request, 0);
1459
1460   MockListener mock_listener(this);
1461
1462   std::unique_ptr<MockImageMap> mock_image_map{
1463     MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id,
1464                          mock_listener)};
1465
1466   C_SaferCond cond;
1467   mock_image_map->init(&cond);
1468   ASSERT_EQ(0, cond.wait());
1469
1470   std::set<std::string> global_image_ids{
1471     "global id 1", "global id 2", "global id 3", "global id 4", "global id 5",
1472     "global id 6", "global id 7", "global id 8", "global id 9", "global id 10",
1473   };
1474   std::set<std::string> global_image_ids_ack(global_image_ids);
1475
1476   // UPDATE_MAPPING+ACQUIRE
1477   expect_add_event(mock_threads);
1478   MockUpdateRequest mock_update_request;
1479   expect_update_request(mock_update_request, 0);
1480   expect_add_event(mock_threads);
1481   std::map<std::string, Context*> peer_ack_ctxs;
1482   listener_acquire_images(mock_listener, global_image_ids,
1483                           &peer_ack_ctxs);
1484
1485   // initial image list
1486   mock_image_map->update_images("", std::move(global_image_ids), {});
1487
1488   ASSERT_TRUE(wait_for_map_update(1));
1489   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
1490
1491   // remote peer ACKs image acquire request
1492   remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
1493                          &peer_ack_ctxs);
1494   wait_for_scheduled_task();
1495
1496   mock_image_map->update_instances_added({m_local_instance_id});
1497
1498   std::set<std::string> shuffled_global_image_ids;
1499
1500   // RELEASE+UPDATE_MAPPING+ACQUIRE
1501   expect_add_event(mock_threads);
1502   expect_listener_images_unmapped(mock_listener, 5, &shuffled_global_image_ids,
1503                                   &peer_ack_ctxs);
1504
1505   mock_image_map->update_instances_added({"9876"});
1506
1507   wait_for_scheduled_task();
1508   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1509
1510   update_map_and_acquire(mock_threads, mock_update_request,
1511                          mock_listener, shuffled_global_image_ids, 0,
1512                          &peer_ack_ctxs);
1513   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1514                                 0, &peer_ack_ctxs);
1515
1516   // completion shuffle action for now (re)mapped images
1517   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1518                          &peer_ack_ctxs);
1519
1520   wait_for_scheduled_task();
1521
1522   // remove all shuffled images -- make way for rebalance
1523   std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids);
1524
1525   // RELEASE+REMOVE_MAPPING
1526   expect_add_event(mock_threads);
1527   listener_release_images(mock_listener, shuffled_global_image_ids,
1528                           &peer_ack_ctxs);
1529   update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids,
1530                      0);
1531
1532   mock_image_map->update_images("", {}, std::move(shuffled_global_image_ids));
1533   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size()));
1534
1535   remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0,
1536                        &peer_ack_ctxs);
1537   wait_for_scheduled_task();
1538
1539   shuffled_global_image_ids.clear();
1540   shuffled_global_image_ids_ack.clear();
1541
1542   std::set<std::string> new_global_image_ids = {
1543     "global id 11"
1544   };
1545   std::set<std::string> new_global_image_ids_ack(new_global_image_ids);
1546
1547   expect_add_event(mock_threads);
1548   expect_update_request(mock_update_request, 0);
1549   expect_add_event(mock_threads);
1550   listener_acquire_images(mock_listener, new_global_image_ids, &peer_ack_ctxs);
1551
1552   expect_rebalance_event(mock_threads); // rebalance task
1553   expect_add_event(mock_threads);       // update task scheduled by
1554                                         // rebalance task
1555   expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids,
1556                                   &peer_ack_ctxs);
1557
1558   mock_image_map->update_images("", std::move(new_global_image_ids), {});
1559
1560   ASSERT_TRUE(wait_for_map_update(1));
1561   ASSERT_TRUE(wait_for_listener_notify(new_global_image_ids_ack.size()));
1562
1563   // set rebalance interval
1564   CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
1565   cct->_conf.set_val("rbd_mirror_image_policy_rebalance_timeout", "5");
1566   remote_peer_ack_nowait(mock_image_map.get(), new_global_image_ids_ack, 0,
1567                          &peer_ack_ctxs);
1568
1569   wait_for_scheduled_task();
1570   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
1571
1572   update_map_and_acquire(mock_threads, mock_update_request,
1573                          mock_listener, shuffled_global_image_ids, 0,
1574                          &peer_ack_ctxs);
1575   remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
1576                                 0, &peer_ack_ctxs);
1577
1578   // completion shuffle action for now (re)mapped images
1579   remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
1580                          &peer_ack_ctxs);
1581
1582   wait_for_scheduled_task();
1583   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
1584 }
1585
1586 } // namespace mirror
1587 } // namespace rbd