ceph/src/test/rbd_mirror/test_PoolWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "include/rados/librados.hpp"
   5 #include "include/rbd/librbd.hpp"
   6 #include "include/stringify.h"
   7 #include "test/rbd_mirror/test_fixture.h"
   8 #include "cls/rbd/cls_rbd_types.h"
   9 #include "cls/rbd/cls_rbd_client.h"
  10 #include "include/rbd_types.h"
  11 #include "librbd/internal.h"
  12 #include "librbd/ImageCtx.h"
  13 #include "librbd/ImageState.h"
  14 #include "librbd/Operations.h"
  15 #include "librbd/Utils.h"
  16 #include "librbd/api/Mirror.h"
  17 #include "common/Cond.h"
  18 #include "common/errno.h"
  19 #include "common/ceph_mutex.h"
  20 #include "tools/rbd_mirror/PoolWatcher.h"
  21 #include "tools/rbd_mirror/Threads.h"
  22 #include "tools/rbd_mirror/Types.h"
  23 #include "tools/rbd_mirror/pool_watcher/Types.h"
  24 #include "test/librados/test_cxx.h"
  25 #include "gtest/gtest.h"
  26 #include <boost/scope_exit.hpp>
  27 #include <iostream>
  28 #include <map>
  29 #include <memory>
  30 #include <set>
  31 #include <vector>
  32
  33 using rbd::mirror::ImageId;
  34 using rbd::mirror::ImageIds;
  35 using rbd::mirror::PoolWatcher;
  36 using rbd::mirror::PeerSpec;
  37 using rbd::mirror::RadosRef;
  38 using std::map;
  39 using std::set;
  40 using std::string;
  41
  42 void register_test_pool_watcher() {
  43 }
  44
  45 class TestPoolWatcher : public ::rbd::mirror::TestFixture {
  46 public:
  47
  48   TestPoolWatcher()
  49     : m_pool_watcher_listener(this),
  50       m_image_number(0), m_snap_number(0)
  51   {
  52     m_cluster = std::make_shared<librados::Rados>();
  53     EXPECT_EQ("", connect_cluster_pp(*m_cluster));
  54   }
  55
  56   void TearDown() override {
  57     if (m_pool_watcher) {
  58       C_SaferCond ctx;
  59       m_pool_watcher->shut_down(&ctx);
  60       EXPECT_EQ(0, ctx.wait());
  61     }
  62
  63     m_cluster->wait_for_latest_osdmap();
  64     for (auto& pool : m_pools) {
  65       EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str()));
  66     }
  67
  68     TestFixture::TearDown();
  69   }
  70
  71   struct PoolWatcherListener : public rbd::mirror::pool_watcher::Listener {
  72     TestPoolWatcher *test;
  73     ceph::condition_variable cond;
  74     ImageIds image_ids;
  75
  76     explicit PoolWatcherListener(TestPoolWatcher *test) : test(test) {
  77     }
  78
  79     void handle_update(const std::string &mirror_uuid,
  80                        ImageIds &&added_image_ids,
  81                        ImageIds &&removed_image_ids) override {
  82       std::lock_guard locker{test->m_lock};
  83       for (auto &image_id : removed_image_ids) {
  84         image_ids.erase(image_id);
  85       }
  86       image_ids.insert(added_image_ids.begin(), added_image_ids.end());
  87       cond.notify_all();
  88     }
  89   };
  90
  91   void create_pool(bool enable_mirroring, const PeerSpec &peer, string *name=nullptr) {
  92     string pool_name = get_temp_pool_name("test-rbd-mirror-");
  93     ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str()));
  94
  95     int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
  96     ASSERT_GE(pool_id, 0);
  97     m_pools.insert(pool_name);
  98
  99     librados::IoCtx ioctx;
 100     ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
 101     ioctx.application_enable("rbd", true);
 102
 103     m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, "mirror uuid",
 104                                            m_pool_watcher_listener));
 105
 106     if (enable_mirroring) {
 107       ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx,
 108                                                    RBD_MIRROR_MODE_POOL));
 109       std::string uuid;
 110       ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add(
 111         ioctx, &uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, peer.cluster_name,
 112         peer.client_name));
 113     }
 114     if (name != nullptr) {
 115       *name = pool_name;
 116     }
 117
 118     m_pool_watcher->init();
 119   }
 120
 121   string get_image_id(librados::IoCtx *ioctx, const string &image_name) {
 122     string obj = librbd::util::id_obj_name(image_name);
 123     string id;
 124     EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id));
 125     return id;
 126   }
 127
 128   void create_image(const string &pool_name, bool mirrored=true,
 129                     string *image_name=nullptr) {
 130     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 131     string name = "image" + stringify(++m_image_number);
 132     if (mirrored) {
 133       features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 134     }
 135
 136     librados::IoCtx ioctx;
 137     ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx));
 138     int order = 0;
 139     ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false,
 140                                 features, &order, 0, 0));
 141     if (mirrored) {
 142       librbd::Image image;
 143       librbd::RBD rbd;
 144       rbd.open(ioctx, image, name.c_str());
 145       image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL);
 146
 147       librbd::mirror_image_info_t mirror_image_info;
 148       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
 149                                                sizeof(mirror_image_info)));
 150       image.close();
 151
 152       m_mirrored_images.insert(ImageId(
 153         mirror_image_info.global_id, get_image_id(&ioctx, name)));
 154     }
 155     if (image_name != nullptr)
 156       *image_name = name;
 157   }
 158
 159   void clone_image(const string &parent_pool_name,
 160                    const string &parent_image_name,
 161                    const string &clone_pool_name,
 162                    bool mirrored=true,
 163                    string *image_name=nullptr) {
 164     librados::IoCtx pioctx, cioctx;
 165     ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx));
 166     ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx));
 167
 168     string snap_name = "snap" + stringify(++m_snap_number);
 169     {
 170       librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(),
 171                                                     "", "", pioctx, false);
 172       ictx->state->open(0);
 173       librbd::NoOpProgressContext prog_ctx;
 174       EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
 175                                                  snap_name, 0, prog_ctx));
 176       EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
 177                                                   snap_name));
 178       ictx->state->close();
 179     }
 180
 181     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 182     string name = "clone" + stringify(++m_image_number);
 183     if (mirrored) {
 184       features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 185     }
 186     int order = 0;
 187     librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(),
 188                   cioctx, name.c_str(), features, &order, 0, 0);
 189     if (mirrored) {
 190       librbd::Image image;
 191       librbd::RBD rbd;
 192       rbd.open(cioctx, image, name.c_str());
 193       image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL);
 194
 195       librbd::mirror_image_info_t mirror_image_info;
 196       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
 197                                                sizeof(mirror_image_info)));
 198       image.close();
 199
 200       m_mirrored_images.insert(ImageId(
 201         mirror_image_info.global_id, get_image_id(&cioctx, name)));
 202     }
 203     if (image_name != nullptr)
 204       *image_name = name;
 205   }
 206
 207   void check_images() {
 208     std::unique_lock l{m_lock};
 209     while (m_mirrored_images != m_pool_watcher_listener.image_ids) {
 210       if (m_pool_watcher_listener.cond.wait_for(l, 10s) == std::cv_status::timeout) {
 211         break;
 212       }
 213     }
 214
 215     ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids);
 216   }
 217
 218   ceph::mutex m_lock = ceph::make_mutex("TestPoolWatcherLock");
 219   RadosRef m_cluster;
 220   PoolWatcherListener m_pool_watcher_listener;
 221   unique_ptr<PoolWatcher<> > m_pool_watcher;
 222
 223   set<string> m_pools;
 224   ImageIds m_mirrored_images;
 225
 226   uint64_t m_image_number;
 227   uint64_t m_snap_number;
 228 };
 229
 230 TEST_F(TestPoolWatcher, EmptyPool) {
 231   string uuid1 = "00000000-0000-0000-0000-000000000001";
 232   PeerSpec site1(uuid1, "site1", "mirror1");
 233   create_pool(true, site1);
 234   check_images();
 235 }
 236
 237 TEST_F(TestPoolWatcher, ReplicatedPools) {
 238   string uuid1 = "00000000-0000-0000-0000-000000000001";
 239   PeerSpec site1(uuid1, "site1", "mirror1");
 240   string first_pool, local_pool, last_pool;
 241   create_pool(true, site1, &first_pool);
 242   check_images();
 243   create_image(first_pool);
 244   check_images();
 245   string parent_image, parent_image2;
 246   create_image(first_pool, true, &parent_image);
 247   check_images();
 248   clone_image(first_pool, parent_image, first_pool);
 249   check_images();
 250   clone_image(first_pool, parent_image, first_pool, true, &parent_image2);
 251   check_images();
 252   create_image(first_pool, false);
 253   check_images();
 254 }