ceph/src/test/rbd_mirror/test_PoolWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "include/rados/librados.hpp"
   5 #include "include/rbd/librbd.hpp"
   6 #include "include/stringify.h"
   7 #include "test/rbd_mirror/test_fixture.h"
   8 #include "cls/rbd/cls_rbd_types.h"
   9 #include "cls/rbd/cls_rbd_client.h"
  10 #include "include/rbd_types.h"
  11 #include "librbd/internal.h"
  12 #include "librbd/ImageCtx.h"
  13 #include "librbd/ImageState.h"
  14 #include "librbd/Operations.h"
  15 #include "librbd/Utils.h"
  16 #include "librbd/api/Mirror.h"
  17 #include "common/Cond.h"
  18 #include "common/errno.h"
  19 #include "common/Mutex.h"
  20 #include "tools/rbd_mirror/PoolWatcher.h"
  21 #include "tools/rbd_mirror/Threads.h"
  22 #include "tools/rbd_mirror/types.h"
  23 #include "test/librados/test.h"
  24 #include "gtest/gtest.h"
  25 #include <boost/scope_exit.hpp>
  26 #include <iostream>
  27 #include <map>
  28 #include <memory>
  29 #include <set>
  30 #include <vector>
  31
  32 using rbd::mirror::ImageId;
  33 using rbd::mirror::ImageIds;
  34 using rbd::mirror::PoolWatcher;
  35 using rbd::mirror::peer_t;
  36 using rbd::mirror::RadosRef;
  37 using std::map;
  38 using std::set;
  39 using std::string;
  40
  41 void register_test_pool_watcher() {
  42 }
  43
  44 class TestPoolWatcher : public ::rbd::mirror::TestFixture {
  45 public:
  46
  47   TestPoolWatcher()
  48     : m_lock("TestPoolWatcherLock"), m_pool_watcher_listener(this),
  49       m_image_number(0), m_snap_number(0)
  50   {
  51     m_cluster = std::make_shared<librados::Rados>();
  52     EXPECT_EQ("", connect_cluster_pp(*m_cluster));
  53   }
  54
  55   void TearDown() override {
  56     if (m_pool_watcher) {
  57       C_SaferCond ctx;
  58       m_pool_watcher->shut_down(&ctx);
  59       EXPECT_EQ(0, ctx.wait());
  60     }
  61
  62     m_cluster->wait_for_latest_osdmap();
  63     for (auto& pool : m_pools) {
  64       EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str()));
  65     }
  66
  67     TestFixture::TearDown();
  68   }
  69
  70   struct PoolWatcherListener : public PoolWatcher<>::Listener {
  71     TestPoolWatcher *test;
  72     Cond cond;
  73     ImageIds image_ids;
  74
  75     PoolWatcherListener(TestPoolWatcher *test) : test(test) {
  76     }
  77
  78     void handle_update(const std::string &mirror_uuid,
  79                        ImageIds &&added_image_ids,
  80                        ImageIds &&removed_image_ids) override {
  81       Mutex::Locker locker(test->m_lock);
  82       for (auto &image_id : removed_image_ids) {
  83         image_ids.erase(image_id);
  84       }
  85       image_ids.insert(added_image_ids.begin(), added_image_ids.end());
  86       cond.Signal();
  87     }
  88   };
  89
  90   void create_pool(bool enable_mirroring, const peer_t &peer, string *name=nullptr) {
  91     string pool_name = get_temp_pool_name("test-rbd-mirror-");
  92     ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str()));
  93
  94     int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
  95     ASSERT_GE(pool_id, 0);
  96     m_pools.insert(pool_name);
  97
  98     librados::IoCtx ioctx;
  99     ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
 100
 101     m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx,
 102                                            m_pool_watcher_listener));
 103
 104     if (enable_mirroring) {
 105       ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx,
 106                                                    RBD_MIRROR_MODE_POOL));
 107       std::string uuid;
 108       ASSERT_EQ(0, librbd::api::Mirror<>::peer_add(ioctx, &uuid,
 109                                                    peer.cluster_name,
 110                                                    peer.client_name));
 111     }
 112     if (name != nullptr) {
 113       *name = pool_name;
 114     }
 115
 116     m_pool_watcher->init();
 117   }
 118
 119   string get_image_id(librados::IoCtx *ioctx, const string &image_name) {
 120     string obj = librbd::util::id_obj_name(image_name);
 121     string id;
 122     EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id));
 123     return id;
 124   }
 125
 126   void create_image(const string &pool_name, bool mirrored=true,
 127                     string *image_name=nullptr) {
 128     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 129     string name = "image" + stringify(++m_image_number);
 130     if (mirrored) {
 131       features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 132     }
 133
 134     librados::IoCtx ioctx;
 135     ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx));
 136     int order = 0;
 137     ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false,
 138                                 features, &order, 0, 0));
 139     if (mirrored) {
 140       librbd::Image image;
 141       librbd::RBD rbd;
 142       rbd.open(ioctx, image, name.c_str());
 143       image.mirror_image_enable();
 144
 145       librbd::mirror_image_info_t mirror_image_info;
 146       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
 147                                                sizeof(mirror_image_info)));
 148       image.close();
 149
 150       m_mirrored_images.insert(ImageId(
 151         mirror_image_info.global_id, get_image_id(&ioctx, name)));
 152     }
 153     if (image_name != nullptr)
 154       *image_name = name;
 155   }
 156
 157   void clone_image(const string &parent_pool_name,
 158                    const string &parent_image_name,
 159                    const string &clone_pool_name,
 160                    bool mirrored=true,
 161                    string *image_name=nullptr) {
 162     librados::IoCtx pioctx, cioctx;
 163     ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx));
 164     ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx));
 165
 166     string snap_name = "snap" + stringify(++m_snap_number);
 167     {
 168       librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(),
 169                                                     "", "", pioctx, false);
 170       ictx->state->open(false);
 171       EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
 172                                                  snap_name.c_str()));
 173       EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
 174                                                   snap_name.c_str()));
 175       ictx->state->close();
 176     }
 177
 178     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 179     string name = "clone" + stringify(++m_image_number);
 180     if (mirrored) {
 181       features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 182     }
 183     int order = 0;
 184     librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(),
 185                   cioctx, name.c_str(), features, &order, 0, 0);
 186     if (mirrored) {
 187       librbd::Image image;
 188       librbd::RBD rbd;
 189       rbd.open(cioctx, image, name.c_str());
 190       image.mirror_image_enable();
 191
 192       librbd::mirror_image_info_t mirror_image_info;
 193       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
 194                                                sizeof(mirror_image_info)));
 195       image.close();
 196
 197       m_mirrored_images.insert(ImageId(
 198         mirror_image_info.global_id, get_image_id(&cioctx, name)));
 199     }
 200     if (image_name != nullptr)
 201       *image_name = name;
 202   }
 203
 204   void check_images() {
 205     Mutex::Locker l(m_lock);
 206     while (m_mirrored_images != m_pool_watcher_listener.image_ids) {
 207       if (m_pool_watcher_listener.cond.WaitInterval(
 208             m_lock, utime_t(10, 0)) != 0) {
 209         break;
 210       }
 211     }
 212
 213     ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids);
 214   }
 215
 216   Mutex m_lock;
 217   RadosRef m_cluster;
 218   PoolWatcherListener m_pool_watcher_listener;
 219   unique_ptr<PoolWatcher<> > m_pool_watcher;
 220
 221   set<string> m_pools;
 222   ImageIds m_mirrored_images;
 223
 224   uint64_t m_image_number;
 225   uint64_t m_snap_number;
 226 };
 227
 228 TEST_F(TestPoolWatcher, EmptyPool) {
 229   string uuid1 = "00000000-0000-0000-0000-000000000001";
 230   peer_t site1(uuid1, "site1", "mirror1");
 231   create_pool(true, site1);
 232   check_images();
 233 }
 234
 235 TEST_F(TestPoolWatcher, ReplicatedPools) {
 236   string uuid1 = "00000000-0000-0000-0000-000000000001";
 237   peer_t site1(uuid1, "site1", "mirror1");
 238   string first_pool, local_pool, last_pool;
 239   create_pool(true, site1, &first_pool);
 240   check_images();
 241   create_image(first_pool);
 242   check_images();
 243   string parent_image, parent_image2;
 244   create_image(first_pool, true, &parent_image);
 245   check_images();
 246   clone_image(first_pool, parent_image, first_pool);
 247   check_images();
 248   clone_image(first_pool, parent_image, first_pool, true, &parent_image2);
 249   check_images();
 250   create_image(first_pool, false);
 251   check_images();
 252 }