ceph/src/test/rbd_mirror/test_ImageReplayer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph distributed storage system
   5  *
   6  * Copyright (C) 2016 Mirantis Inc
   7  *
   8  * Author: Mykola Golub <mgolub@mirantis.com>
   9  *
  10  *  This library is free software; you can redistribute it and/or
  11  *  modify it under the terms of the GNU Lesser General Public
  12  *  License as published by the Free Software Foundation; either
  13  *  version 2.1 of the License, or (at your option) any later version.
  14  *
  15  */
  16
  17 #include "include/rados/librados.hpp"
  18 #include "include/rbd/librbd.hpp"
  19 #include "include/stringify.h"
  20 #include "test/rbd_mirror/test_fixture.h"
  21 #include "cls/journal/cls_journal_types.h"
  22 #include "cls/journal/cls_journal_client.h"
  23 #include "cls/rbd/cls_rbd_types.h"
  24 #include "cls/rbd/cls_rbd_client.h"
  25 #include "journal/Journaler.h"
  26 #include "librbd/ExclusiveLock.h"
  27 #include "librbd/ImageCtx.h"
  28 #include "librbd/ImageState.h"
  29 #include "librbd/Journal.h"
  30 #include "librbd/Operations.h"
  31 #include "librbd/Utils.h"
  32 #include "librbd/internal.h"
  33 #include "librbd/api/Mirror.h"
  34 #include "librbd/io/AioCompletion.h"
  35 #include "librbd/io/ImageRequestWQ.h"
  36 #include "librbd/io/ReadResult.h"
  37 #include "tools/rbd_mirror/types.h"
  38 #include "tools/rbd_mirror/ImageReplayer.h"
  39 #include "tools/rbd_mirror/InstanceWatcher.h"
  40 #include "tools/rbd_mirror/Threads.h"
  41 #include "tools/rbd_mirror/ImageDeleter.h"
  42
  43 #include "test/librados/test.h"
  44 #include "gtest/gtest.h"
  45
  46 using rbd::mirror::RadosRef;
  47
  48 void register_test_rbd_mirror() {
  49 }
  50
  51 #define TEST_IO_SIZE 512
  52 #define TEST_IO_COUNT 11
  53
  54 class TestImageReplayer : public ::rbd::mirror::TestFixture {
  55 public:
  56   struct C_WatchCtx : public librados::WatchCtx2 {
  57     TestImageReplayer *test;
  58     std::string oid;
  59     Mutex lock;
  60     Cond cond;
  61     bool notified;
  62
  63     C_WatchCtx(TestImageReplayer *test, const std::string &oid)
  64       : test(test), oid(oid), lock("C_WatchCtx::lock"), notified(false) {
  65     }
  66
  67     void handle_notify(uint64_t notify_id, uint64_t cookie,
  68                                uint64_t notifier_id, bufferlist& bl_) override {
  69       bufferlist bl;
  70       test->m_remote_ioctx.notify_ack(oid, notify_id, cookie, bl);
  71
  72       Mutex::Locker locker(lock);
  73       notified = true;
  74       cond.Signal();
  75     }
  76
  77     void handle_error(uint64_t cookie, int err) override {
  78       ASSERT_EQ(0, err);
  79     }
  80   };
  81
  82   TestImageReplayer()
  83     : m_local_cluster(new librados::Rados()), m_watch_handle(0)
  84   {
  85     EXPECT_EQ("", connect_cluster_pp(*m_local_cluster.get()));
  86     EXPECT_EQ(0, m_local_cluster->conf_set("rbd_cache", "false"));
  87     EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_poll_age", "1"));
  88
  89     m_local_pool_name = get_temp_pool_name();
  90     EXPECT_EQ(0, m_local_cluster->pool_create(m_local_pool_name.c_str()));
  91     EXPECT_EQ(0, m_local_cluster->ioctx_create(m_local_pool_name.c_str(),
  92                                               m_local_ioctx));
  93
  94     EXPECT_EQ("", connect_cluster_pp(m_remote_cluster));
  95     EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false"));
  96
  97     m_remote_pool_name = get_temp_pool_name();
  98     EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str()));
  99     m_remote_pool_id = m_remote_cluster.pool_lookup(m_remote_pool_name.c_str());
 100     EXPECT_GE(m_remote_pool_id, 0);
 101
 102     EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(),
 103                                                m_remote_ioctx));
 104     EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx,
 105                                                  RBD_MIRROR_MODE_POOL));
 106
 107     m_image_name = get_temp_image_name();
 108     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 109     features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 110     int order = 0;
 111     EXPECT_EQ(0, librbd::create(m_remote_ioctx, m_image_name.c_str(), 1 << 22,
 112                                 false, features, &order, 0, 0));
 113     m_remote_image_id = get_image_id(m_remote_ioctx, m_image_name);
 114
 115     m_threads = new rbd::mirror::Threads<>(reinterpret_cast<CephContext*>(
 116       m_local_ioctx.cct()));
 117
 118     m_image_deleter.reset(new rbd::mirror::ImageDeleter(m_threads->work_queue,
 119                                                         m_threads->timer,
 120                                                         &m_threads->timer_lock));
 121     m_instance_watcher = rbd::mirror::InstanceWatcher<>::create(
 122         m_local_ioctx, m_threads->work_queue, nullptr);
 123     m_instance_watcher->handle_acquire_leader();
 124   }
 125
 126   ~TestImageReplayer() override
 127   {
 128     unwatch();
 129
 130     m_instance_watcher->handle_release_leader();
 131
 132     delete m_replayer;
 133     delete m_instance_watcher;
 134     delete m_threads;
 135
 136     EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str()));
 137     EXPECT_EQ(0, m_local_cluster->pool_delete(m_local_pool_name.c_str()));
 138   }
 139
 140   template <typename ImageReplayerT = rbd::mirror::ImageReplayer<> >
 141   void create_replayer() {
 142     m_replayer = new ImageReplayerT(
 143         m_threads, m_image_deleter, m_instance_watcher,
 144         rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)),
 145         m_local_mirror_uuid, m_local_ioctx.get_id(), "global image id");
 146     m_replayer->add_remote_image(m_remote_mirror_uuid, m_remote_image_id,
 147                                  m_remote_ioctx);
 148   }
 149
 150   void start()
 151   {
 152     C_SaferCond cond;
 153     m_replayer->start(&cond);
 154     ASSERT_EQ(0, cond.wait());
 155
 156     ASSERT_EQ(0U, m_watch_handle);
 157     std::string oid = ::journal::Journaler::header_oid(m_remote_image_id);
 158     m_watch_ctx = new C_WatchCtx(this, oid);
 159     ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx));
 160   }
 161
 162   void unwatch() {
 163     if (m_watch_handle != 0) {
 164       m_remote_ioctx.unwatch2(m_watch_handle);
 165       delete m_watch_ctx;
 166       m_watch_ctx = nullptr;
 167       m_watch_handle = 0;
 168     }
 169   }
 170
 171   void stop()
 172   {
 173     unwatch();
 174
 175     C_SaferCond cond;
 176     m_replayer->stop(&cond);
 177     ASSERT_EQ(0, cond.wait());
 178   }
 179
 180   void bootstrap()
 181   {
 182     create_replayer<>();
 183
 184     start();
 185     wait_for_replay_complete();
 186     stop();
 187   }
 188
 189   std::string  get_temp_image_name()
 190   {
 191     return "image" + stringify(++_image_number);
 192   }
 193
 194   std::string get_image_id(librados::IoCtx &ioctx, const string &image_name)
 195   {
 196     std::string obj = librbd::util::id_obj_name(image_name);
 197     std::string id;
 198     EXPECT_EQ(0, librbd::cls_client::get_id(&ioctx, obj, &id));
 199     return id;
 200   }
 201
 202   void open_image(librados::IoCtx &ioctx, const std::string &image_name,
 203                   bool readonly, librbd::ImageCtx **ictxp)
 204   {
 205     librbd::ImageCtx *ictx = new librbd::ImageCtx(image_name.c_str(),
 206                                                   "", "", ioctx, readonly);
 207     EXPECT_EQ(0, ictx->state->open(false));
 208     *ictxp = ictx;
 209   }
 210
 211   void open_local_image(librbd::ImageCtx **ictxp)
 212   {
 213     open_image(m_local_ioctx, m_image_name, true, ictxp);
 214   }
 215
 216   void open_remote_image(librbd::ImageCtx **ictxp)
 217   {
 218     open_image(m_remote_ioctx, m_image_name, false, ictxp);
 219   }
 220
 221   void close_image(librbd::ImageCtx *ictx)
 222   {
 223     ictx->state->close();
 224   }
 225
 226   void get_commit_positions(cls::journal::ObjectPosition *master_position,
 227                             cls::journal::ObjectPosition *mirror_position)
 228   {
 229     std::string master_client_id = "";
 230     std::string mirror_client_id = m_local_mirror_uuid;
 231
 232     C_SaferCond cond;
 233     uint64_t minimum_set;
 234     uint64_t active_set;
 235     std::set<cls::journal::Client> registered_clients;
 236     std::string oid = ::journal::Journaler::header_oid(m_remote_image_id);
 237     cls::journal::client::get_mutable_metadata(m_remote_ioctx, oid,
 238                                                &minimum_set, &active_set,
 239                                                &registered_clients, &cond);
 240     ASSERT_EQ(0, cond.wait());
 241
 242     *master_position = cls::journal::ObjectPosition();
 243     *mirror_position = cls::journal::ObjectPosition();
 244
 245     std::set<cls::journal::Client>::const_iterator c;
 246     for (c = registered_clients.begin(); c != registered_clients.end(); ++c) {
 247       std::cout << __func__ << ": client: " << *c << std::endl;
 248       if (c->state != cls::journal::CLIENT_STATE_CONNECTED) {
 249         continue;
 250       }
 251       cls::journal::ObjectPositions object_positions =
 252         c->commit_position.object_positions;
 253       cls::journal::ObjectPositions::const_iterator p =
 254         object_positions.begin();
 255       if (p != object_positions.end()) {
 256         if (c->id == master_client_id) {
 257           ASSERT_EQ(cls::journal::ObjectPosition(), *master_position);
 258           *master_position = *p;
 259         } else if (c->id == mirror_client_id) {
 260           ASSERT_EQ(cls::journal::ObjectPosition(), *mirror_position);
 261           *mirror_position = *p;
 262         }
 263       }
 264     }
 265   }
 266
 267   bool wait_for_watcher_notify(int seconds)
 268   {
 269     if (m_watch_handle == 0) {
 270       return false;
 271     }
 272
 273     Mutex::Locker locker(m_watch_ctx->lock);
 274     while (!m_watch_ctx->notified) {
 275       if (m_watch_ctx->cond.WaitInterval(m_watch_ctx->lock,
 276                                          utime_t(seconds, 0)) != 0) {
 277         return false;
 278       }
 279     }
 280     m_watch_ctx->notified = false;
 281     return true;
 282   }
 283
 284   void wait_for_replay_complete()
 285   {
 286     cls::journal::ObjectPosition master_position;
 287     cls::journal::ObjectPosition mirror_position;
 288
 289     for (int i = 0; i < 100; i++) {
 290       printf("m_replayer->flush()\n");
 291       C_SaferCond cond;
 292       m_replayer->flush(&cond);
 293       ASSERT_EQ(0, cond.wait());
 294       get_commit_positions(&master_position, &mirror_position);
 295       if (master_position == mirror_position) {
 296         break;
 297       }
 298       wait_for_watcher_notify(1);
 299     }
 300
 301     ASSERT_EQ(master_position, mirror_position);
 302   }
 303
 304   void wait_for_stopped() {
 305     for (int i = 0; i < 100; i++) {
 306       if (m_replayer->is_stopped()) {
 307         break;
 308       }
 309       wait_for_watcher_notify(1);
 310     }
 311     ASSERT_TRUE(m_replayer->is_stopped());
 312   }
 313
 314   void write_test_data(librbd::ImageCtx *ictx, const char *test_data, off_t off,
 315                        size_t len)
 316   {
 317     size_t written;
 318     bufferlist bl;
 319     bl.append(std::string(test_data, len));
 320     written = ictx->io_work_queue->write(off, len, std::move(bl), 0);
 321     printf("wrote: %d\n", (int)written);
 322     ASSERT_EQ(len, written);
 323   }
 324
 325   void read_test_data(librbd::ImageCtx *ictx, const char *expected, off_t off,
 326                       size_t len)
 327   {
 328     ssize_t read;
 329     char *result = (char *)malloc(len + 1);
 330
 331     ASSERT_NE(static_cast<char *>(NULL), result);
 332     read = ictx->io_work_queue->read(
 333       off, len, librbd::io::ReadResult{result, len}, 0);
 334     printf("read: %d\n", (int)read);
 335     ASSERT_EQ(len, static_cast<size_t>(read));
 336     result[len] = '\0';
 337     if (memcmp(result, expected, len)) {
 338       printf("read: %s\nexpected: %s\n", result, expected);
 339       ASSERT_EQ(0, memcmp(result, expected, len));
 340     }
 341     free(result);
 342   }
 343
 344   void generate_test_data() {
 345     for (int i = 0; i < TEST_IO_SIZE; ++i) {
 346       m_test_data[i] = (char) (rand() % (126 - 33) + 33);
 347     }
 348     m_test_data[TEST_IO_SIZE] = '\0';
 349   }
 350
 351   void flush(librbd::ImageCtx *ictx)
 352   {
 353     C_SaferCond aio_flush_ctx;
 354     auto c = librbd::io::AioCompletion::create(&aio_flush_ctx);
 355     c->get();
 356     ictx->io_work_queue->aio_flush(c);
 357     ASSERT_EQ(0, c->wait_for_complete());
 358     c->put();
 359
 360     C_SaferCond journal_flush_ctx;
 361     ictx->journal->flush_commit_position(&journal_flush_ctx);
 362     ASSERT_EQ(0, journal_flush_ctx.wait());
 363
 364     printf("flushed\n");
 365   }
 366
 367   static int _image_number;
 368
 369   rbd::mirror::Threads<> *m_threads = nullptr;
 370   std::shared_ptr<rbd::mirror::ImageDeleter> m_image_deleter;
 371   std::shared_ptr<librados::Rados> m_local_cluster;
 372   librados::Rados m_remote_cluster;
 373   rbd::mirror::InstanceWatcher<> *m_instance_watcher;
 374   std::string m_local_mirror_uuid = "local mirror uuid";
 375   std::string m_remote_mirror_uuid = "remote mirror uuid";
 376   std::string m_local_pool_name, m_remote_pool_name;
 377   librados::IoCtx m_local_ioctx, m_remote_ioctx;
 378   std::string m_image_name;
 379   int64_t m_remote_pool_id;
 380   std::string m_remote_image_id;
 381   rbd::mirror::ImageReplayer<> *m_replayer;
 382   C_WatchCtx *m_watch_ctx;
 383   uint64_t m_watch_handle;
 384   char m_test_data[TEST_IO_SIZE + 1];
 385 };
 386
 387 int TestImageReplayer::_image_number;
 388
 389 TEST_F(TestImageReplayer, Bootstrap)
 390 {
 391   bootstrap();
 392 }
 393
 394 TEST_F(TestImageReplayer, BootstrapErrorLocalImageExists)
 395 {
 396   int order = 0;
 397   EXPECT_EQ(0, librbd::create(m_local_ioctx, m_image_name.c_str(), 1 << 22,
 398                               false, 0, &order, 0, 0));
 399
 400   create_replayer<>();
 401   C_SaferCond cond;
 402   m_replayer->start(&cond);
 403   ASSERT_EQ(-EEXIST, cond.wait());
 404 }
 405
 406 TEST_F(TestImageReplayer, BootstrapErrorNoJournal)
 407 {
 408   // disable remote image journaling
 409   librbd::ImageCtx *ictx;
 410   open_remote_image(&ictx);
 411   uint64_t features;
 412   ASSERT_EQ(0, librbd::get_features(ictx, &features));
 413   ASSERT_EQ(0, ictx->operations->update_features(RBD_FEATURE_JOURNALING,
 414                                                  false));
 415   close_image(ictx);
 416
 417   create_replayer<>();
 418   C_SaferCond cond;
 419   m_replayer->start(&cond);
 420   ASSERT_EQ(-ENOENT, cond.wait());
 421 }
 422
 423 TEST_F(TestImageReplayer, BootstrapErrorMirrorDisabled)
 424 {
 425   // disable remote image mirroring
 426   ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx,
 427                                                RBD_MIRROR_MODE_IMAGE));
 428   librbd::ImageCtx *ictx;
 429   open_remote_image(&ictx);
 430   ASSERT_EQ(0, librbd::api::Mirror<>::image_disable(ictx, true));
 431   close_image(ictx);
 432
 433   create_replayer<>();
 434   C_SaferCond cond;
 435   m_replayer->start(&cond);
 436   ASSERT_EQ(-ENOENT, cond.wait());
 437 }
 438
 439 TEST_F(TestImageReplayer, BootstrapMirrorDisabling)
 440 {
 441   // set remote image mirroring state to DISABLING
 442   ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx,
 443                                                RBD_MIRROR_MODE_IMAGE));
 444   librbd::ImageCtx *ictx;
 445   open_remote_image(&ictx);
 446   ASSERT_EQ(0, librbd::api::Mirror<>::image_enable(ictx, false));
 447   cls::rbd::MirrorImage mirror_image;
 448   ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&m_remote_ioctx, ictx->id,
 449                                                     &mirror_image));
 450   mirror_image.state = cls::rbd::MirrorImageState::MIRROR_IMAGE_STATE_DISABLING;
 451   ASSERT_EQ(0, librbd::cls_client::mirror_image_set(&m_remote_ioctx, ictx->id,
 452                                                     mirror_image));
 453   close_image(ictx);
 454
 455   create_replayer<>();
 456   C_SaferCond cond;
 457   m_replayer->start(&cond);
 458   ASSERT_EQ(0, cond.wait());
 459   ASSERT_TRUE(m_replayer->is_stopped());
 460 }
 461
 462 TEST_F(TestImageReplayer, BootstrapDemoted)
 463 {
 464   // demote remote image
 465   librbd::ImageCtx *ictx;
 466   open_remote_image(&ictx);
 467   ASSERT_EQ(0, librbd::api::Mirror<>::image_demote(ictx));
 468   close_image(ictx);
 469
 470   create_replayer<>();
 471   C_SaferCond cond;
 472   m_replayer->start(&cond);
 473   ASSERT_EQ(0, cond.wait());
 474   ASSERT_TRUE(m_replayer->is_stopped());
 475 }
 476
 477 TEST_F(TestImageReplayer, StartInterrupted)
 478 {
 479   create_replayer<>();
 480   C_SaferCond start_cond, stop_cond;
 481   m_replayer->start(&start_cond);
 482   m_replayer->stop(&stop_cond);
 483   int r = start_cond.wait();
 484   printf("start returned %d\n", r);
 485   // TODO: improve the test to avoid this race
 486   ASSERT_TRUE(r == -ECANCELED || r == 0);
 487   ASSERT_EQ(0, stop_cond.wait());
 488 }
 489
 490 TEST_F(TestImageReplayer, JournalReset)
 491 {
 492   bootstrap();
 493   delete m_replayer;
 494
 495   ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id));
 496
 497   // try to recover
 498   bootstrap();
 499 }
 500
 501 TEST_F(TestImageReplayer, ErrorNoJournal)
 502 {
 503   bootstrap();
 504
 505   // disable remote journal journaling
 506   // (reset before disabling, so it does not fail with EBUSY)
 507   ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id));
 508   librbd::ImageCtx *ictx;
 509   open_remote_image(&ictx);
 510   uint64_t features;
 511   ASSERT_EQ(0, librbd::get_features(ictx, &features));
 512   ASSERT_EQ(0, ictx->operations->update_features(RBD_FEATURE_JOURNALING,
 513                                                  false));
 514   close_image(ictx);
 515
 516   C_SaferCond cond;
 517   m_replayer->start(&cond);
 518   ASSERT_EQ(-ENOENT, cond.wait());
 519 }
 520
 521 TEST_F(TestImageReplayer, StartStop)
 522 {
 523   bootstrap();
 524
 525   start();
 526   wait_for_replay_complete();
 527   stop();
 528 }
 529
 530 TEST_F(TestImageReplayer, WriteAndStartReplay)
 531 {
 532   bootstrap();
 533
 534   // Write to remote image and start replay
 535
 536   librbd::ImageCtx *ictx;
 537
 538   generate_test_data();
 539   open_remote_image(&ictx);
 540   for (int i = 0; i < TEST_IO_COUNT; ++i) {
 541     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 542   }
 543   flush(ictx);
 544   close_image(ictx);
 545
 546   start();
 547   wait_for_replay_complete();
 548   stop();
 549
 550   open_local_image(&ictx);
 551   for (int i = 0; i < TEST_IO_COUNT; ++i) {
 552     read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 553   }
 554   close_image(ictx);
 555 }
 556
 557 TEST_F(TestImageReplayer, StartReplayAndWrite)
 558 {
 559   bootstrap();
 560
 561   // Start replay and write to remote image
 562
 563   librbd::ImageCtx *ictx;
 564
 565   start();
 566
 567   generate_test_data();
 568   open_remote_image(&ictx);
 569   for (int i = 0; i < TEST_IO_COUNT; ++i) {
 570     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 571   }
 572   flush(ictx);
 573
 574   wait_for_replay_complete();
 575
 576   for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
 577     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 578   }
 579   flush(ictx);
 580   close_image(ictx);
 581
 582   wait_for_replay_complete();
 583
 584   open_local_image(&ictx);
 585   for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) {
 586     read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 587   }
 588   close_image(ictx);
 589
 590   stop();
 591 }
 592
 593 TEST_F(TestImageReplayer, NextTag)
 594 {
 595   bootstrap();
 596
 597   // write, reopen, and write again to test switch to the next tag
 598
 599   librbd::ImageCtx *ictx;
 600
 601   start();
 602
 603   generate_test_data();
 604
 605   const int N = 10;
 606
 607   for (int j = 0; j < N; j++) {
 608     open_remote_image(&ictx);
 609     for (int i = j * TEST_IO_COUNT; i < (j + 1) * TEST_IO_COUNT; ++i) {
 610       write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 611     }
 612     close_image(ictx);
 613   }
 614
 615   wait_for_replay_complete();
 616
 617   open_local_image(&ictx);
 618   for (int i = 0; i < N * TEST_IO_COUNT; ++i) {
 619     read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 620   }
 621   close_image(ictx);
 622
 623   stop();
 624 }
 625
 626 TEST_F(TestImageReplayer, Resync)
 627 {
 628   bootstrap();
 629
 630   librbd::ImageCtx *ictx;
 631
 632   start();
 633
 634   generate_test_data();
 635
 636   open_remote_image(&ictx);
 637   for (int i = 0; i < TEST_IO_COUNT; ++i) {
 638     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 639   }
 640   flush(ictx);
 641
 642   wait_for_replay_complete();
 643
 644   for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
 645     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 646   }
 647   flush(ictx);
 648   close_image(ictx);
 649
 650   C_SaferCond ctx;
 651   m_replayer->resync_image(&ctx);
 652   ASSERT_EQ(0, ctx.wait());
 653
 654   C_SaferCond delete_ctx;
 655   m_image_deleter->wait_for_scheduled_deletion(
 656     m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx);
 657   EXPECT_EQ(0, delete_ctx.wait());
 658
 659   C_SaferCond cond;
 660   m_replayer->start(&cond);
 661   ASSERT_EQ(0, cond.wait());
 662
 663   ASSERT_TRUE(m_replayer->is_replaying());
 664
 665   wait_for_replay_complete();
 666
 667   open_local_image(&ictx);
 668   for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) {
 669     read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 670   }
 671   close_image(ictx);
 672
 673   stop();
 674 }
 675
 676 TEST_F(TestImageReplayer, Resync_While_Stop)
 677 {
 678
 679   bootstrap();
 680
 681   start();
 682
 683   generate_test_data();
 684
 685   librbd::ImageCtx *ictx;
 686   open_remote_image(&ictx);
 687   for (int i = 0; i < TEST_IO_COUNT; ++i) {
 688     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 689   }
 690   flush(ictx);
 691
 692   wait_for_replay_complete();
 693
 694   for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
 695     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 696   }
 697   flush(ictx);
 698   close_image(ictx);
 699
 700   wait_for_replay_complete();
 701
 702   C_SaferCond cond;
 703   m_replayer->stop(&cond);
 704   ASSERT_EQ(0, cond.wait());
 705
 706   open_local_image(&ictx);
 707   librbd::Journal<>::request_resync(ictx);
 708   close_image(ictx);
 709
 710   C_SaferCond cond2;
 711   m_replayer->start(&cond2);
 712   ASSERT_EQ(0, cond2.wait());
 713
 714   ASSERT_TRUE(m_replayer->is_stopped());
 715
 716   C_SaferCond delete_ctx;
 717   m_image_deleter->wait_for_scheduled_deletion(
 718     m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx);
 719   EXPECT_EQ(0, delete_ctx.wait());
 720
 721   C_SaferCond cond3;
 722   m_replayer->start(&cond3);
 723   ASSERT_EQ(0, cond3.wait());
 724
 725   ASSERT_TRUE(m_replayer->is_replaying());
 726
 727   wait_for_replay_complete();
 728
 729   open_local_image(&ictx);
 730   for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) {
 731     read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 732   }
 733   close_image(ictx);
 734
 735   stop();
 736 }
 737
 738 TEST_F(TestImageReplayer, Resync_StartInterrupted)
 739 {
 740
 741   bootstrap();
 742
 743   librbd::ImageCtx *ictx;
 744   open_local_image(&ictx);
 745   librbd::Journal<>::request_resync(ictx);
 746   close_image(ictx);
 747
 748   C_SaferCond cond;
 749   m_replayer->start(&cond);
 750   ASSERT_EQ(0, cond.wait());
 751
 752   ASSERT_TRUE(m_replayer->is_stopped());
 753
 754   C_SaferCond delete_ctx;
 755   m_image_deleter->wait_for_scheduled_deletion(
 756     m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx);
 757   EXPECT_EQ(0, delete_ctx.wait());
 758
 759   C_SaferCond cond2;
 760   m_replayer->start(&cond2);
 761   ASSERT_EQ(0, cond2.wait());
 762
 763   ASSERT_EQ(0U, m_watch_handle);
 764   std::string oid = ::journal::Journaler::header_oid(m_remote_image_id);
 765   m_watch_ctx = new C_WatchCtx(this, oid);
 766   ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx));
 767
 768   ASSERT_TRUE(m_replayer->is_replaying());
 769
 770   generate_test_data();
 771   open_remote_image(&ictx);
 772   for (int i = 0; i < TEST_IO_COUNT; ++i) {
 773     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 774   }
 775   flush(ictx);
 776
 777   wait_for_replay_complete();
 778
 779   for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
 780     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 781   }
 782   flush(ictx);
 783   close_image(ictx);
 784
 785   wait_for_replay_complete();
 786
 787   open_local_image(&ictx);
 788   for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) {
 789     read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 790   }
 791   close_image(ictx);
 792
 793   stop();
 794 }
 795
 796 TEST_F(TestImageReplayer, MultipleReplayFailures_SingleEpoch) {
 797   bootstrap();
 798
 799   // inject a snapshot that cannot be unprotected
 800   librbd::ImageCtx *ictx;
 801   open_image(m_local_ioctx, m_image_name, false, &ictx);
 802   ictx->features &= ~RBD_FEATURE_JOURNALING;
 803   ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
 804                                              "foo"));
 805   ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
 806                                               "foo"));
 807   ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN,
 808                                              {ictx->md_ctx.get_id(),
 809                                               ictx->id,
 810                                               ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), "foo"}]},
 811                                              "dummy child id"));
 812   close_image(ictx);
 813
 814   // race failed op shut down with new ops
 815   open_remote_image(&ictx);
 816   for (uint64_t i = 0; i < 10; ++i) {
 817     RWLock::RLocker owner_locker(ictx->owner_lock);
 818     C_SaferCond request_lock;
 819     ictx->exclusive_lock->acquire_lock(&request_lock);
 820     ASSERT_EQ(0, request_lock.wait());
 821
 822     C_SaferCond append_ctx;
 823     ictx->journal->append_op_event(
 824       i,
 825       librbd::journal::EventEntry{
 826         librbd::journal::SnapUnprotectEvent{i,
 827                                             cls::rbd::UserSnapshotNamespace(),
 828                                             "foo"}},
 829       &append_ctx);
 830     ASSERT_EQ(0, append_ctx.wait());
 831
 832     C_SaferCond commit_ctx;
 833     ictx->journal->commit_op_event(i, 0, &commit_ctx);
 834     ASSERT_EQ(0, commit_ctx.wait());
 835
 836     C_SaferCond release_ctx;
 837     ictx->exclusive_lock->release_lock(&release_ctx);
 838     ASSERT_EQ(0, release_ctx.wait());
 839   }
 840
 841   for (uint64_t i = 0; i < 5; ++i) {
 842     start();
 843     wait_for_stopped();
 844     unwatch();
 845   }
 846   close_image(ictx);
 847 }
 848
 849 TEST_F(TestImageReplayer, MultipleReplayFailures_MultiEpoch) {
 850   bootstrap();
 851
 852   // inject a snapshot that cannot be unprotected
 853   librbd::ImageCtx *ictx;
 854   open_image(m_local_ioctx, m_image_name, false, &ictx);
 855   ictx->features &= ~RBD_FEATURE_JOURNALING;
 856   ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
 857                                              "foo"));
 858   ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
 859                                               "foo"));
 860   ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN,
 861                                              {ictx->md_ctx.get_id(),
 862                                               ictx->id,
 863                                               ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(),
 864                                                               "foo"}]},
 865                                              "dummy child id"));
 866   close_image(ictx);
 867
 868   // race failed op shut down with new tag flush
 869   open_remote_image(&ictx);
 870   {
 871     RWLock::RLocker owner_locker(ictx->owner_lock);
 872     C_SaferCond request_lock;
 873     ictx->exclusive_lock->acquire_lock(&request_lock);
 874     ASSERT_EQ(0, request_lock.wait());
 875
 876     C_SaferCond append_ctx;
 877     ictx->journal->append_op_event(
 878       1U,
 879       librbd::journal::EventEntry{
 880         librbd::journal::SnapUnprotectEvent{1U,
 881                                             cls::rbd::UserSnapshotNamespace(),
 882                                             "foo"}},
 883       &append_ctx);
 884     ASSERT_EQ(0, append_ctx.wait());
 885
 886     C_SaferCond commit_ctx;
 887     ictx->journal->commit_op_event(1U, 0, &commit_ctx);
 888     ASSERT_EQ(0, commit_ctx.wait());
 889
 890     C_SaferCond release_ctx;
 891     ictx->exclusive_lock->release_lock(&release_ctx);
 892     ASSERT_EQ(0, release_ctx.wait());
 893   }
 894
 895   generate_test_data();
 896   write_test_data(ictx, m_test_data, 0, TEST_IO_SIZE);
 897
 898   for (uint64_t i = 0; i < 5; ++i) {
 899     start();
 900     wait_for_stopped();
 901     unwatch();
 902   }
 903   close_image(ictx);
 904 }
 905
 906 TEST_F(TestImageReplayer, Disconnect)
 907 {
 908   bootstrap();
 909
 910   // Make sure rbd_mirroring_resync_after_disconnect is not set
 911   EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "false"));
 912
 913   // Test start fails if disconnected
 914
 915   librbd::ImageCtx *ictx;
 916
 917   generate_test_data();
 918   open_remote_image(&ictx);
 919   for (int i = 0; i < TEST_IO_COUNT; ++i) {
 920     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 921   }
 922   flush(ictx);
 923   close_image(ictx);
 924
 925   std::string oid = ::journal::Journaler::header_oid(m_remote_image_id);
 926   ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx, oid,
 927         m_local_mirror_uuid, cls::journal::CLIENT_STATE_DISCONNECTED));
 928
 929   C_SaferCond cond1;
 930   m_replayer->start(&cond1);
 931   ASSERT_EQ(-ENOTCONN, cond1.wait());
 932
 933   // Test start succeeds after resync
 934
 935   open_local_image(&ictx);
 936   librbd::Journal<>::request_resync(ictx);
 937   close_image(ictx);
 938   C_SaferCond cond2;
 939   m_replayer->start(&cond2);
 940   ASSERT_EQ(-ENOTCONN, cond2.wait());
 941   C_SaferCond delete_cond;
 942   m_image_deleter->wait_for_scheduled_deletion(
 943     m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond);
 944   EXPECT_EQ(0, delete_cond.wait());
 945
 946   start();
 947   wait_for_replay_complete();
 948
 949   // Test replay stopped after disconnect
 950
 951   open_remote_image(&ictx);
 952   for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
 953     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
 954   }
 955   flush(ictx);
 956   close_image(ictx);
 957
 958   ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx, oid,
 959         m_local_mirror_uuid, cls::journal::CLIENT_STATE_DISCONNECTED));
 960   bufferlist bl;
 961   ASSERT_EQ(0, m_remote_ioctx.notify2(oid, bl, 5000, NULL));
 962
 963   wait_for_stopped();
 964
 965   // Test start fails after disconnect
 966
 967   C_SaferCond cond3;
 968   m_replayer->start(&cond3);
 969   ASSERT_EQ(-ENOTCONN, cond3.wait());
 970   C_SaferCond cond4;
 971   m_replayer->start(&cond4);
 972   ASSERT_EQ(-ENOTCONN, cond4.wait());
 973
 974   // Test automatic resync if rbd_mirroring_resync_after_disconnect is set
 975
 976   EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "true"));
 977
 978   // Resync is flagged on first start attempt
 979   C_SaferCond cond5;
 980   m_replayer->start(&cond5);
 981   ASSERT_EQ(-ENOTCONN, cond5.wait());
 982   C_SaferCond delete_cond1;
 983   m_image_deleter->wait_for_scheduled_deletion(
 984     m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond1);
 985   EXPECT_EQ(0, delete_cond1.wait());
 986
 987   C_SaferCond cond6;
 988   m_replayer->start(&cond6);
 989   ASSERT_EQ(0, cond6.wait());
 990   wait_for_replay_complete();
 991
 992   stop();
 993 }
 994
 995 TEST_F(TestImageReplayer, UpdateFeatures)
 996 {
 997   const uint64_t FEATURES_TO_UPDATE =
 998     RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF;
 999
1000   uint64_t features;
1001   librbd::ImageCtx *ictx;
1002
1003   // Make sure the features we will update are disabled initially
1004
1005   open_remote_image(&ictx);
1006   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1007   features &= FEATURES_TO_UPDATE;
1008   if (features) {
1009     ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE,
1010                                                    false));
1011   }
1012   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1013   ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1014   close_image(ictx);
1015
1016   bootstrap();
1017
1018   open_remote_image(&ictx);
1019   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1020   ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1021   close_image(ictx);
1022
1023   open_local_image(&ictx);
1024   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1025   ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1026   close_image(ictx);
1027
1028   // Start replay and update features
1029
1030   start();
1031
1032   open_remote_image(&ictx);
1033   ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE,
1034                                                  true));
1035   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1036   ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE);
1037   close_image(ictx);
1038
1039   wait_for_replay_complete();
1040
1041   open_local_image(&ictx);
1042   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1043   ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE);
1044   close_image(ictx);
1045
1046   open_remote_image(&ictx);
1047   ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE,
1048                                                  false));
1049   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1050   ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1051   close_image(ictx);
1052
1053   wait_for_replay_complete();
1054
1055   open_local_image(&ictx);
1056   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1057   ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1058   close_image(ictx);
1059
1060   // Test update_features error does not stop replication
1061
1062   open_remote_image(&ictx);
1063   ASSERT_EQ(0, librbd::get_features(ictx, &features));
1064   ASSERT_NE(0U, features & RBD_FEATURE_EXCLUSIVE_LOCK);
1065   ASSERT_EQ(-EINVAL, ictx->operations->update_features(RBD_FEATURE_EXCLUSIVE_LOCK,
1066                                                        false));
1067   generate_test_data();
1068   for (int i = 0; i < TEST_IO_COUNT; ++i) {
1069     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
1070   }
1071   flush(ictx);
1072   close_image(ictx);
1073
1074   wait_for_replay_complete();
1075
1076   open_local_image(&ictx);
1077   for (int i = 0; i < TEST_IO_COUNT; ++i) {
1078     read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
1079   }
1080   close_image(ictx);
1081
1082   stop();
1083 }
1084
1085 TEST_F(TestImageReplayer, MetadataSetRemove)
1086 {
1087   const std::string KEY = "test_key";
1088   const std::string VALUE = "test_value";
1089
1090   librbd::ImageCtx *ictx;
1091   std::string value;
1092
1093   bootstrap();
1094
1095   start();
1096
1097   // Test metadata_set replication
1098
1099   open_remote_image(&ictx);
1100   ASSERT_EQ(0, ictx->operations->metadata_set(KEY, VALUE));
1101   value.clear();
1102   ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value));
1103   ASSERT_EQ(VALUE, value);
1104   close_image(ictx);
1105
1106   wait_for_replay_complete();
1107
1108   open_local_image(&ictx);
1109   value.clear();
1110   ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value));
1111   ASSERT_EQ(VALUE, value);
1112   close_image(ictx);
1113
1114   // Test metadata_remove replication
1115
1116   open_remote_image(&ictx);
1117   ASSERT_EQ(0, ictx->operations->metadata_remove(KEY));
1118   ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value));
1119   close_image(ictx);
1120
1121   wait_for_replay_complete();
1122
1123   open_local_image(&ictx);
1124   ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value));
1125   close_image(ictx);
1126
1127   stop();
1128 }
1129
1130 TEST_F(TestImageReplayer, MirroringDelay)
1131 {
1132   const double DELAY = 10; // set less than wait_for_replay_complete timeout
1133
1134   librbd::ImageCtx *ictx;
1135   utime_t start_time;
1136   double delay;
1137
1138   bootstrap();
1139
1140   ASSERT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_replay_delay",
1141                                          stringify(DELAY).c_str()));
1142   open_local_image(&ictx);
1143   ASSERT_EQ(DELAY, ictx->mirroring_replay_delay);
1144   close_image(ictx);
1145
1146   start();
1147
1148   // Test delay
1149
1150   generate_test_data();
1151   open_remote_image(&ictx);
1152   start_time = ceph_clock_now();
1153   for (int i = 0; i < TEST_IO_COUNT; ++i) {
1154     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
1155   }
1156   flush(ictx);
1157   close_image(ictx);
1158
1159   wait_for_replay_complete();
1160   delay = ceph_clock_now() - start_time;
1161   ASSERT_GE(delay, DELAY);
1162
1163   // Test stop when delaying replay
1164
1165   open_remote_image(&ictx);
1166   start_time = ceph_clock_now();
1167   for (int i = 0; i < TEST_IO_COUNT; ++i) {
1168     write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
1169   }
1170   close_image(ictx);
1171
1172   sleep(DELAY / 2);
1173   stop();
1174   start();
1175
1176   wait_for_replay_complete();
1177   delay = ceph_clock_now() - start_time;
1178   ASSERT_GE(delay, DELAY);
1179
1180   stop();
1181 }