ceph/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "test/rbd_mirror/test_mock_fixture.h"
   5 #include "include/stringify.h"
   6 #include "tools/rbd_mirror/MirrorStatusUpdater.h"
   7 #include "tools/rbd_mirror/MirrorStatusWatcher.h"
   8 #include "tools/rbd_mirror/Threads.h"
   9 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
  10 #include "test/librbd/mock/MockImageCtx.h"
  11 #include "test/rbd_mirror/mock/MockContextWQ.h"
  12 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  13 #include <map>
  14 #include <string>
  15 #include <utility>
  16
  17 namespace librbd {
  18 namespace {
  19
  20 struct MockTestImageCtx : public MockImageCtx {
  21   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  22     : librbd::MockImageCtx(image_ctx) {
  23   }
  24 };
  25
  26 } // anonymous namespace
  27 } // namespace librbd
  28
  29 namespace rbd {
  30 namespace mirror {
  31
  32 template <>
  33 struct MirrorStatusWatcher<librbd::MockTestImageCtx> {
  34   static MirrorStatusWatcher* s_instance;
  35   static MirrorStatusWatcher* create(librados::IoCtx& io_ctx,
  36                                      MockContextWQ* mock_context_wq) {
  37     ceph_assert(s_instance != nullptr);
  38     return s_instance;
  39   }
  40
  41   MOCK_METHOD1(init, void(Context*));
  42   MOCK_METHOD1(shut_down, void(Context*));
  43
  44   MirrorStatusWatcher() {
  45     s_instance = this;
  46   }
  47 };
  48
  49 MirrorStatusWatcher<librbd::MockTestImageCtx>* MirrorStatusWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
  50
  51 template <>
  52 struct Threads<librbd::MockTestImageCtx> {
  53   MockSafeTimer *timer;
  54   ceph::mutex &timer_lock;
  55
  56   MockContextWQ *work_queue;
  57
  58   Threads(Threads<librbd::ImageCtx> *threads)
  59     : timer(new MockSafeTimer()),
  60       timer_lock(threads->timer_lock),
  61       work_queue(new MockContextWQ()) {
  62   }
  63   ~Threads() {
  64     delete timer;
  65     delete work_queue;
  66   }
  67 };
  68
  69 } // namespace mirror
  70 } // namespace rbd
  71
  72 #include "tools/rbd_mirror/MirrorStatusUpdater.cc"
  73
  74 namespace rbd {
  75 namespace mirror {
  76
  77 using ::testing::_;
  78 using ::testing::DoDefault;
  79 using ::testing::InSequence;
  80 using ::testing::Invoke;
  81 using ::testing::StrEq;
  82 using ::testing::Return;
  83 using ::testing::WithArg;
  84
  85 class TestMockMirrorStatusUpdater : public TestMockFixture {
  86 public:
  87   typedef MirrorStatusUpdater<librbd::MockTestImageCtx> MockMirrorStatusUpdater;
  88   typedef MirrorStatusWatcher<librbd::MockTestImageCtx> MockMirrorStatusWatcher;
  89   typedef Threads<librbd::MockTestImageCtx> MockThreads;
  90
  91   typedef std::map<std::string, cls::rbd::MirrorImageSiteStatus>
  92       MirrorImageSiteStatuses;
  93
  94   void SetUp() override {
  95     TestMockFixture::SetUp();
  96
  97     m_mock_local_io_ctx = &get_mock_io_ctx(m_local_io_ctx);
  98     m_mock_threads = new MockThreads(m_threads);
  99   }
 100
 101   void TearDown() override {
 102     delete m_mock_threads;
 103     TestMockFixture::TearDown();
 104   }
 105
 106   void expect_timer_add_event(Context** timer_event) {
 107     EXPECT_CALL(*m_mock_threads->timer, add_event_after(_, _))
 108       .WillOnce(WithArg<1>(Invoke([timer_event](Context *ctx) {
 109           *timer_event = ctx;
 110           return ctx;
 111         })));
 112   }
 113
 114   void expect_timer_cancel_event() {
 115     EXPECT_CALL(*m_mock_threads->timer, cancel_event(_))
 116       .WillOnce(Invoke([](Context* ctx) {
 117           delete ctx;
 118           return false;
 119         }));
 120   }
 121
 122   void expect_work_queue(bool async) {
 123     EXPECT_CALL(*m_mock_threads->work_queue, queue(_, _))
 124       .WillOnce(Invoke([this, async](Context *ctx, int r) {
 125           if (async) {
 126             m_threads->work_queue->queue(ctx, r);
 127           } else {
 128             ctx->complete(r);
 129           }
 130         }));
 131   }
 132
 133   void expect_mirror_status_watcher_init(
 134       MockMirrorStatusWatcher& mock_mirror_status_watcher, int r) {
 135     EXPECT_CALL(*mock_mirror_status_watcher.s_instance, init(_))
 136       .WillOnce(Invoke([this, r](Context* ctx) {
 137           m_threads->work_queue->queue(ctx, r);
 138         }));
 139   }
 140
 141   void expect_mirror_status_watcher_shut_down(
 142       MockMirrorStatusWatcher& mock_mirror_status_watcher, int r) {
 143     EXPECT_CALL(*mock_mirror_status_watcher.s_instance, shut_down(_))
 144       .WillOnce(Invoke([this, r](Context* ctx) {
 145           m_threads->work_queue->queue(ctx, r);
 146         }));
 147   }
 148
 149   void expect_mirror_status_update(
 150       const std::string& global_image_id,
 151       const cls::rbd::MirrorImageSiteStatus& mirror_image_status, int r) {
 152     EXPECT_CALL(*m_mock_local_io_ctx,
 153                 exec(RBD_MIRRORING, _, StrEq("rbd"),
 154                      StrEq("mirror_image_status_set"), _, _, _, _))
 155       .WillOnce(WithArg<4>(Invoke(
 156         [r, global_image_id, mirror_image_status](bufferlist& in_bl) {
 157           auto bl_it = in_bl.cbegin();
 158           std::string decode_global_image_id;
 159           decode(decode_global_image_id, bl_it);
 160           EXPECT_EQ(global_image_id, decode_global_image_id);
 161
 162           cls::rbd::MirrorImageSiteStatus decode_mirror_image_status;
 163           decode(decode_mirror_image_status, bl_it);
 164           EXPECT_EQ(mirror_image_status, decode_mirror_image_status);
 165           return r;
 166         })));
 167   }
 168
 169   void expect_mirror_status_update(
 170       const MirrorImageSiteStatuses& mirror_image_site_statuses,
 171       const std::string& mirror_uuid, int r) {
 172     EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _))
 173       .WillOnce(Invoke([this](auto&&... args) {
 174           int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...);
 175           m_mock_local_io_ctx->aio_flush();
 176           return r;
 177         }));
 178
 179     for (auto [global_image_id, mirror_image_status] :
 180            mirror_image_site_statuses) {
 181       mirror_image_status.mirror_uuid = mirror_uuid;
 182       expect_mirror_status_update(global_image_id, mirror_image_status, r);
 183       if (r < 0) {
 184         break;
 185       }
 186     }
 187   }
 188
 189   void fire_timer_event(Context** timer_event,
 190                         Context** update_task) {
 191     expect_timer_add_event(timer_event);
 192
 193     // timer queues the update task
 194     EXPECT_CALL(*m_mock_threads->work_queue, queue(_, _))
 195       .WillOnce(WithArg<0>(Invoke([update_task](Context* ctx) mutable {
 196           *update_task = ctx;
 197         })));
 198
 199     // fire the timer task
 200     {
 201       std::lock_guard timer_locker{m_mock_threads->timer_lock};
 202       ceph_assert(*timer_event != nullptr);
 203       (*timer_event)->complete(0);
 204     }
 205   }
 206
 207   void init_mirror_status_updater(
 208       MockMirrorStatusUpdater& mock_mirror_status_updater,
 209       MockMirrorStatusWatcher& mock_mirror_status_watcher,
 210       Context** timer_event) {
 211     expect_timer_add_event(timer_event);
 212     expect_mirror_status_watcher_init(mock_mirror_status_watcher, 0);
 213     expect_work_queue(true);
 214
 215     C_SaferCond ctx;
 216     mock_mirror_status_updater.init(&ctx);
 217     ASSERT_EQ(0, ctx.wait());
 218   }
 219
 220   void shut_down_mirror_status_updater(
 221       MockMirrorStatusUpdater& mock_mirror_status_updater,
 222       MockMirrorStatusWatcher& mock_mirror_status_watcher) {
 223     expect_timer_cancel_event();
 224     expect_mirror_status_watcher_shut_down(mock_mirror_status_watcher, 0);
 225     expect_work_queue(true);
 226
 227     C_SaferCond ctx;
 228     mock_mirror_status_updater.shut_down(&ctx);
 229     ASSERT_EQ(0, ctx.wait());
 230   }
 231
 232   librados::MockTestMemIoCtxImpl* m_mock_local_io_ctx = nullptr;
 233   MockThreads* m_mock_threads = nullptr;
 234 };
 235
 236 TEST_F(TestMockMirrorStatusUpdater, InitShutDown) {
 237   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 238                                                      m_mock_threads, "");
 239   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 240     new MockMirrorStatusWatcher();
 241
 242   Context* timer_event = nullptr;
 243   init_mirror_status_updater(mock_mirror_status_updater,
 244                              *mock_mirror_status_watcher, &timer_event);
 245
 246   shut_down_mirror_status_updater(mock_mirror_status_updater,
 247                                   *mock_mirror_status_watcher);
 248 }
 249
 250 TEST_F(TestMockMirrorStatusUpdater, InitStatusWatcherError) {
 251   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 252                                                      m_mock_threads, "");
 253   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 254     new MockMirrorStatusWatcher();
 255
 256   Context* timer_event = nullptr;
 257   expect_timer_add_event(&timer_event);
 258   expect_mirror_status_watcher_init(*mock_mirror_status_watcher, -EINVAL);
 259   expect_timer_cancel_event();
 260   expect_work_queue(true);
 261
 262   C_SaferCond ctx;
 263   mock_mirror_status_updater.init(&ctx);
 264   ASSERT_EQ(-EINVAL, ctx.wait());
 265 }
 266
 267 TEST_F(TestMockMirrorStatusUpdater, ShutDownStatusWatcherError) {
 268   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 269                                                      m_mock_threads, "");
 270   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 271     new MockMirrorStatusWatcher();
 272
 273   Context* timer_event = nullptr;
 274   init_mirror_status_updater(mock_mirror_status_updater,
 275                              *mock_mirror_status_watcher, &timer_event);
 276
 277   C_SaferCond on_shutdown;
 278   expect_timer_cancel_event();
 279   expect_mirror_status_watcher_shut_down(*mock_mirror_status_watcher, -EINVAL);
 280   expect_work_queue(true);
 281   mock_mirror_status_updater.shut_down(&on_shutdown);
 282
 283   ASSERT_EQ(-EINVAL, on_shutdown.wait());
 284 }
 285
 286 TEST_F(TestMockMirrorStatusUpdater, SmallBatch) {
 287   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 288                                                      m_mock_threads, "");
 289   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 290     new MockMirrorStatusWatcher();
 291
 292   InSequence seq;
 293
 294   Context* timer_event = nullptr;
 295   init_mirror_status_updater(mock_mirror_status_updater,
 296                              *mock_mirror_status_watcher, &timer_event);
 297
 298   MirrorImageSiteStatuses mirror_image_site_statuses;
 299   for (auto i = 0; i < 100; ++i) {
 300     auto pair = mirror_image_site_statuses.emplace(
 301       stringify(i), cls::rbd::MirrorImageSiteStatus{});
 302     mock_mirror_status_updater.set_mirror_image_status(pair.first->first,
 303                                                        pair.first->second,
 304                                                        false);
 305   }
 306
 307   Context* update_task = nullptr;
 308   fire_timer_event(&timer_event, &update_task);
 309
 310   expect_mirror_status_update(mirror_image_site_statuses, "", 0);
 311   update_task->complete(0);
 312
 313   shut_down_mirror_status_updater(mock_mirror_status_updater,
 314                                   *mock_mirror_status_watcher);
 315 }
 316
 317 TEST_F(TestMockMirrorStatusUpdater, LargeBatch) {
 318   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 319                                                      m_mock_threads, "");
 320   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 321     new MockMirrorStatusWatcher();
 322
 323   InSequence seq;
 324
 325   Context* timer_event = nullptr;
 326   init_mirror_status_updater(mock_mirror_status_updater,
 327                              *mock_mirror_status_watcher, &timer_event);
 328
 329   MirrorImageSiteStatuses mirror_image_site_statuses;
 330   for (auto i = 0; i < 200; ++i) {
 331     auto pair = mirror_image_site_statuses.emplace(
 332       stringify(i), cls::rbd::MirrorImageSiteStatus{});
 333     mock_mirror_status_updater.set_mirror_image_status(pair.first->first,
 334                                                        pair.first->second,
 335                                                        false);
 336   }
 337
 338   auto it_1 = mirror_image_site_statuses.begin();
 339   auto it_2 = mirror_image_site_statuses.begin();
 340   std::advance(it_2, 100);
 341   MirrorImageSiteStatuses mirror_image_site_statuses_1{it_1, it_2};
 342
 343   it_1 = it_2;
 344   std::advance(it_2, 100);
 345   MirrorImageSiteStatuses mirror_image_site_statuses_2{it_1, it_2};
 346
 347   Context* update_task = nullptr;
 348   fire_timer_event(&timer_event, &update_task);
 349
 350   expect_mirror_status_update(mirror_image_site_statuses_1, "", 0);
 351   expect_mirror_status_update(mirror_image_site_statuses_2, "", 0);
 352   update_task->complete(0);
 353
 354   shut_down_mirror_status_updater(mock_mirror_status_updater,
 355                                   *mock_mirror_status_watcher);
 356 }
 357
 358 TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) {
 359   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 360                                                      m_mock_threads, "");
 361   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 362     new MockMirrorStatusWatcher();
 363
 364   InSequence seq;
 365
 366   Context* timer_event = nullptr;
 367   init_mirror_status_updater(mock_mirror_status_updater,
 368                              *mock_mirror_status_watcher, &timer_event);
 369
 370   mock_mirror_status_updater.set_mirror_image_status("1", {}, false);
 371   mock_mirror_status_updater.set_mirror_image_status(
 372     "1", {"", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"},
 373     false);
 374
 375   Context* update_task = nullptr;
 376   fire_timer_event(&timer_event, &update_task);
 377
 378   expect_mirror_status_update(
 379     {{"1", cls::rbd::MirrorImageSiteStatus{
 380         "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}},
 381     "", 0);
 382   update_task->complete(0);
 383
 384   shut_down_mirror_status_updater(mock_mirror_status_updater,
 385                                   *mock_mirror_status_watcher);
 386 }
 387
 388 TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) {
 389   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 390                                                      m_mock_threads, "");
 391   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 392     new MockMirrorStatusWatcher();
 393
 394   InSequence seq;
 395
 396   Context* timer_event = nullptr;
 397   init_mirror_status_updater(mock_mirror_status_updater,
 398                              *mock_mirror_status_watcher, &timer_event);
 399
 400   mock_mirror_status_updater.set_mirror_image_status("1", {}, false);
 401
 402   Context* update_task = nullptr;
 403   fire_timer_event(&timer_event, &update_task);
 404
 405   EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _))
 406     .WillOnce(Invoke([this, &mock_mirror_status_updater](auto&&... args) {
 407         mock_mirror_status_updater.set_mirror_image_status(
 408           "1", {"", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING,
 409                 "description"},
 410           true);
 411
 412         int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...);
 413         m_mock_local_io_ctx->aio_flush();
 414         return r;
 415       }));
 416   expect_mirror_status_update("1", cls::rbd::MirrorImageSiteStatus{}, 0);
 417   expect_work_queue(false);
 418   expect_mirror_status_update(
 419     {{"1", cls::rbd::MirrorImageSiteStatus{
 420         "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}},
 421     "", 0);
 422
 423   update_task->complete(0);
 424
 425   shut_down_mirror_status_updater(mock_mirror_status_updater,
 426                                   *mock_mirror_status_watcher);
 427 }
 428
 429 TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) {
 430   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 431                                                      m_mock_threads, "");
 432   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 433     new MockMirrorStatusWatcher();
 434
 435   InSequence seq;
 436
 437   Context* timer_event = nullptr;
 438   init_mirror_status_updater(mock_mirror_status_updater,
 439                              *mock_mirror_status_watcher, &timer_event);
 440
 441   expect_work_queue(false);
 442   expect_mirror_status_update({{"1", cls::rbd::MirrorImageSiteStatus{}}},
 443                               "", 0);
 444   mock_mirror_status_updater.set_mirror_image_status("1", {}, true);
 445
 446   shut_down_mirror_status_updater(mock_mirror_status_updater,
 447                                   *mock_mirror_status_watcher);
 448 }
 449
 450 TEST_F(TestMockMirrorStatusUpdater, RemoveIdleStatus) {
 451   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 452                                                      m_mock_threads, "");
 453   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 454     new MockMirrorStatusWatcher();
 455
 456   InSequence seq;
 457
 458   Context* timer_event = nullptr;
 459   init_mirror_status_updater(mock_mirror_status_updater,
 460                              *mock_mirror_status_watcher, &timer_event);
 461
 462   mock_mirror_status_updater.set_mirror_image_status("1", {}, false);
 463
 464   C_SaferCond ctx;
 465   expect_work_queue(true);
 466   mock_mirror_status_updater.remove_mirror_image_status("1", &ctx);
 467   ASSERT_EQ(0, ctx.wait());
 468
 469   shut_down_mirror_status_updater(mock_mirror_status_updater,
 470                                   *mock_mirror_status_watcher);
 471 }
 472
 473 TEST_F(TestMockMirrorStatusUpdater, RemoveInFlightStatus) {
 474   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 475                                                      m_mock_threads, "");
 476   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 477     new MockMirrorStatusWatcher();
 478
 479   InSequence seq;
 480
 481   Context* timer_event = nullptr;
 482   init_mirror_status_updater(mock_mirror_status_updater,
 483                              *mock_mirror_status_watcher, &timer_event);
 484
 485   mock_mirror_status_updater.set_mirror_image_status("1", {}, false);
 486
 487   Context* update_task = nullptr;
 488   fire_timer_event(&timer_event, &update_task);
 489
 490   C_SaferCond on_removed;
 491   EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _))
 492     .WillOnce(Invoke(
 493       [this, &mock_mirror_status_updater, &on_removed](auto&&... args) {
 494         mock_mirror_status_updater.remove_mirror_image_status("1", &on_removed);
 495
 496         int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...);
 497         m_mock_local_io_ctx->aio_flush();
 498         return r;
 499       }));
 500   update_task->complete(0);
 501   ASSERT_EQ(0, on_removed.wait());
 502
 503   shut_down_mirror_status_updater(mock_mirror_status_updater,
 504                                   *mock_mirror_status_watcher);
 505 }
 506
 507 TEST_F(TestMockMirrorStatusUpdater, ShutDownWhileUpdating) {
 508   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 509                                                      m_mock_threads, "");
 510   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 511     new MockMirrorStatusWatcher();
 512
 513   InSequence seq;
 514
 515   Context* timer_event = nullptr;
 516   init_mirror_status_updater(mock_mirror_status_updater,
 517                              *mock_mirror_status_watcher, &timer_event);
 518
 519   mock_mirror_status_updater.set_mirror_image_status("1", {}, false);
 520
 521   Context* update_task = nullptr;
 522   fire_timer_event(&timer_event, &update_task);
 523
 524   C_SaferCond on_shutdown;
 525   EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _))
 526     .WillOnce(Invoke(
 527       [this, &mock_mirror_status_updater, &on_shutdown](auto&&... args) {
 528         mock_mirror_status_updater.shut_down(&on_shutdown);
 529         m_threads->work_queue->drain();
 530
 531         int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...);
 532         m_mock_local_io_ctx->aio_flush();
 533         return r;
 534       }));
 535
 536   expect_timer_cancel_event();
 537   expect_mirror_status_watcher_shut_down(*mock_mirror_status_watcher, 0);
 538
 539   update_task->complete(0);
 540   ASSERT_EQ(0, on_shutdown.wait());
 541 }
 542
 543 TEST_F(TestMockMirrorStatusUpdater, MirrorPeerSitePing) {
 544   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
 545                                                      m_mock_threads,
 546                                                      "mirror uuid");
 547   MockMirrorStatusWatcher* mock_mirror_status_watcher =
 548     new MockMirrorStatusWatcher();
 549
 550   InSequence seq;
 551
 552   Context* timer_event = nullptr;
 553   init_mirror_status_updater(mock_mirror_status_updater,
 554                              *mock_mirror_status_watcher, &timer_event);
 555
 556   MirrorImageSiteStatuses mirror_image_site_statuses;
 557   for (auto i = 0; i < 100; ++i) {
 558     auto pair = mirror_image_site_statuses.emplace(
 559       stringify(i), cls::rbd::MirrorImageSiteStatus{});
 560     mock_mirror_status_updater.set_mirror_image_status(pair.first->first,
 561                                                        pair.first->second,
 562                                                        false);
 563   }
 564
 565   Context* update_task = nullptr;
 566   fire_timer_event(&timer_event, &update_task);
 567
 568   expect_mirror_status_update(mirror_image_site_statuses, "mirror uuid", 0);
 569   update_task->complete(0);
 570
 571   shut_down_mirror_status_updater(mock_mirror_status_updater,
 572                                   *mock_mirror_status_watcher);
 573 }
 574
 575 } // namespace mirror
 576 } // namespace rbd