ceph/src/test/rbd_mirror/test_PoolWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "include/rados/librados.hpp"
   5 #include "include/rbd/librbd.hpp"
   6 #include "include/stringify.h"
   7 #include "test/rbd_mirror/test_fixture.h"
   8 #include "cls/rbd/cls_rbd_types.h"
   9 #include "cls/rbd/cls_rbd_client.h"
  10 #include "include/rbd_types.h"
  11 #include "librbd/internal.h"
  12 #include "librbd/ImageCtx.h"
  13 #include "librbd/ImageState.h"
  14 #include "librbd/Operations.h"
  15 #include "librbd/Utils.h"
  16 #include "librbd/api/Mirror.h"
  17 #include "common/Cond.h"
  18 #include "common/errno.h"
  19 #include "common/ceph_mutex.h"
  20 #include "tools/rbd_mirror/PoolWatcher.h"
  21 #include "tools/rbd_mirror/Threads.h"
  22 #include "tools/rbd_mirror/Types.h"
  23 #include "tools/rbd_mirror/pool_watcher/Types.h"
  24 #include "test/librados/test_cxx.h"
  25 #include "gtest/gtest.h"
  26 #include <boost/scope_exit.hpp>
  27 #include <iostream>
  28 #include <map>
  29 #include <memory>
  30 #include <set>
  31 #include <vector>
  32
  33 using namespace std::chrono_literals;
  34
  35 using rbd::mirror::ImageId;
  36 using rbd::mirror::ImageIds;
  37 using rbd::mirror::PoolWatcher;
  38 using rbd::mirror::PeerSpec;
  39 using rbd::mirror::RadosRef;
  40 using std::map;
  41 using std::set;
  42 using std::string;
  43
  44 void register_test_pool_watcher() {
  45 }
  46
  47 class TestPoolWatcher : public ::rbd::mirror::TestFixture {
  48 public:
  49
  50   TestPoolWatcher()
  51     : m_pool_watcher_listener(this),
  52       m_image_number(0), m_snap_number(0)
  53   {
  54     m_cluster = std::make_shared<librados::Rados>();
  55     EXPECT_EQ("", connect_cluster_pp(*m_cluster));
  56   }
  57
  58   void TearDown() override {
  59     if (m_pool_watcher) {
  60       C_SaferCond ctx;
  61       m_pool_watcher->shut_down(&ctx);
  62       EXPECT_EQ(0, ctx.wait());
  63     }
  64
  65     m_cluster->wait_for_latest_osdmap();
  66     for (auto& pool : m_pools) {
  67       EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str()));
  68     }
  69
  70     TestFixture::TearDown();
  71   }
  72
  73   struct PoolWatcherListener : public rbd::mirror::pool_watcher::Listener {
  74     TestPoolWatcher *test;
  75     ceph::condition_variable cond;
  76     ImageIds image_ids;
  77
  78     explicit PoolWatcherListener(TestPoolWatcher *test) : test(test) {
  79     }
  80
  81     void handle_update(const std::string &mirror_uuid,
  82                        ImageIds &&added_image_ids,
  83                        ImageIds &&removed_image_ids) override {
  84       std::lock_guard locker{test->m_lock};
  85       for (auto &image_id : removed_image_ids) {
  86         image_ids.erase(image_id);
  87       }
  88       image_ids.insert(added_image_ids.begin(), added_image_ids.end());
  89       cond.notify_all();
  90     }
  91   };
  92
  93   void create_pool(bool enable_mirroring, const PeerSpec &peer, string *name=nullptr) {
  94     string pool_name = get_temp_pool_name("test-rbd-mirror-");
  95     ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str()));
  96
  97     int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
  98     ASSERT_GE(pool_id, 0);
  99     m_pools.insert(pool_name);
 100
 101     librados::IoCtx ioctx;
 102     ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
 103     ioctx.application_enable("rbd", true);
 104
 105     m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, "mirror uuid",
 106                                            m_pool_watcher_listener));
 107
 108     if (enable_mirroring) {
 109       ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx,
 110                                                    RBD_MIRROR_MODE_POOL));
 111       std::string uuid;
 112       ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add(
 113         ioctx, &uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, peer.cluster_name,
 114         peer.client_name));
 115     }
 116     if (name != nullptr) {
 117       *name = pool_name;
 118     }
 119
 120     m_pool_watcher->init();
 121   }
 122
 123   string get_image_id(librados::IoCtx *ioctx, const string &image_name) {
 124     string obj = librbd::util::id_obj_name(image_name);
 125     string id;
 126     EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id));
 127     return id;
 128   }
 129
 130   void create_image(const string &pool_name, bool mirrored=true,
 131                     string *image_name=nullptr) {
 132     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 133     string name = "image" + stringify(++m_image_number);
 134     if (mirrored) {
 135       features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 136     }
 137
 138     librados::IoCtx ioctx;
 139     ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx));
 140     int order = 0;
 141     ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false,
 142                                 features, &order, 0, 0));
 143     if (mirrored) {
 144       librbd::Image image;
 145       librbd::RBD rbd;
 146       rbd.open(ioctx, image, name.c_str());
 147       image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL);
 148
 149       librbd::mirror_image_info_t mirror_image_info;
 150       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
 151                                                sizeof(mirror_image_info)));
 152       image.close();
 153
 154       m_mirrored_images.insert(ImageId(
 155         mirror_image_info.global_id, get_image_id(&ioctx, name)));
 156     }
 157     if (image_name != nullptr)
 158       *image_name = name;
 159   }
 160
 161   void clone_image(const string &parent_pool_name,
 162                    const string &parent_image_name,
 163                    const string &clone_pool_name,
 164                    bool mirrored=true,
 165                    string *image_name=nullptr) {
 166     librados::IoCtx pioctx, cioctx;
 167     ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx));
 168     ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx));
 169
 170     string snap_name = "snap" + stringify(++m_snap_number);
 171     {
 172       librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(),
 173                                                     "", "", pioctx, false);
 174       ictx->state->open(0);
 175       librbd::NoOpProgressContext prog_ctx;
 176       EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
 177                                                  snap_name, 0, prog_ctx));
 178       EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
 179                                                   snap_name));
 180       ictx->state->close();
 181     }
 182
 183     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 184     string name = "clone" + stringify(++m_image_number);
 185     if (mirrored) {
 186       features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 187     }
 188     int order = 0;
 189     librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(),
 190                   cioctx, name.c_str(), features, &order, 0, 0);
 191     if (mirrored) {
 192       librbd::Image image;
 193       librbd::RBD rbd;
 194       rbd.open(cioctx, image, name.c_str());
 195       image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL);
 196
 197       librbd::mirror_image_info_t mirror_image_info;
 198       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
 199                                                sizeof(mirror_image_info)));
 200       image.close();
 201
 202       m_mirrored_images.insert(ImageId(
 203         mirror_image_info.global_id, get_image_id(&cioctx, name)));
 204     }
 205     if (image_name != nullptr)
 206       *image_name = name;
 207   }
 208
 209   void check_images() {
 210     std::unique_lock l{m_lock};
 211     while (m_mirrored_images != m_pool_watcher_listener.image_ids) {
 212       if (m_pool_watcher_listener.cond.wait_for(l, 10s) == std::cv_status::timeout) {
 213         break;
 214       }
 215     }
 216
 217     ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids);
 218   }
 219
 220   ceph::mutex m_lock = ceph::make_mutex("TestPoolWatcherLock");
 221   RadosRef m_cluster;
 222   PoolWatcherListener m_pool_watcher_listener;
 223   std::unique_ptr<PoolWatcher<> > m_pool_watcher;
 224
 225   set<string> m_pools;
 226   ImageIds m_mirrored_images;
 227
 228   uint64_t m_image_number;
 229   uint64_t m_snap_number;
 230 };
 231
 232 TEST_F(TestPoolWatcher, EmptyPool) {
 233   string uuid1 = "00000000-0000-0000-0000-000000000001";
 234   PeerSpec site1(uuid1, "site1", "mirror1");
 235   create_pool(true, site1);
 236   check_images();
 237 }
 238
 239 TEST_F(TestPoolWatcher, ReplicatedPools) {
 240   string uuid1 = "00000000-0000-0000-0000-000000000001";
 241   PeerSpec site1(uuid1, "site1", "mirror1");
 242   string first_pool, local_pool, last_pool;
 243   create_pool(true, site1, &first_pool);
 244   check_images();
 245   create_image(first_pool);
 246   check_images();
 247   string parent_image, parent_image2;
 248   create_image(first_pool, true, &parent_image);
 249   check_images();
 250   clone_image(first_pool, parent_image, first_pool);
 251   check_images();
 252   clone_image(first_pool, parent_image, first_pool, true, &parent_image2);
 253   check_images();
 254   create_image(first_pool, false);
 255   check_images();
 256 }