[ceph.git] / ceph / src / test / rbd_mirror / test_PoolWatcher.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab

#include "include/rados/librados.hpp"
#include "include/rbd/librbd.hpp"
#include "include/stringify.h"
#include "test/rbd_mirror/test_fixture.h"
#include "cls/rbd/cls_rbd_types.h"
#include "cls/rbd/cls_rbd_client.h"
#include "include/rbd_types.h"
#include "librbd/internal.h"
#include "librbd/ImageCtx.h"
#include "librbd/ImageState.h"
#include "librbd/Operations.h"
#include "librbd/Utils.h"
#include "librbd/api/Mirror.h"
#include "common/Cond.h"
#include "common/errno.h"
#include "common/ceph_mutex.h"
#include "tools/rbd_mirror/PoolWatcher.h"
#include "tools/rbd_mirror/Threads.h"
#include "tools/rbd_mirror/Types.h"
#include "tools/rbd_mirror/pool_watcher/Types.h"
#include "test/librados/test_cxx.h"
#include "gtest/gtest.h"
#include <boost/scope_exit.hpp>
#include <iostream>
#include <map>
#include <memory>
#include <set>
#include <vector>

using namespace std::chrono_literals;

using rbd::mirror::ImageId;
using rbd::mirror::ImageIds;
using rbd::mirror::PoolWatcher;
using rbd::mirror::PeerSpec;
using rbd::mirror::RadosRef;
using std::map;
using std::set;
using std::string;

void register_test_pool_watcher() {
}

class TestPoolWatcher : public ::rbd::mirror::TestFixture {
public:

  TestPoolWatcher()
    : m_pool_watcher_listener(this),
      m_image_number(0), m_snap_number(0)
  {
    m_cluster = std::make_shared<librados::Rados>();
    EXPECT_EQ("", connect_cluster_pp(*m_cluster));
  }

  void TearDown() override {
    if (m_pool_watcher) {
      C_SaferCond ctx;
      m_pool_watcher->shut_down(&ctx);
      EXPECT_EQ(0, ctx.wait());
    }

    m_cluster->wait_for_latest_osdmap();
    for (auto& pool : m_pools) {
      EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str()));
    }

    TestFixture::TearDown();
  }

  struct PoolWatcherListener : public rbd::mirror::pool_watcher::Listener {
    TestPoolWatcher *test;
    ceph::condition_variable cond;
    ImageIds image_ids;

    explicit PoolWatcherListener(TestPoolWatcher *test) : test(test) {
    }

    void handle_update(const std::string &mirror_uuid,
                       ImageIds &&added_image_ids,
                       ImageIds &&removed_image_ids) override {
      std::lock_guard locker{test->m_lock};
      for (auto &image_id : removed_image_ids) {
        image_ids.erase(image_id);
      }
      image_ids.insert(added_image_ids.begin(), added_image_ids.end());
      cond.notify_all();
    }
  };

  void create_pool(bool enable_mirroring, const PeerSpec &peer, string *name=nullptr) {
    string pool_name = get_temp_pool_name("test-rbd-mirror-");
    ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str()));

    int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
    ASSERT_GE(pool_id, 0);
    m_pools.insert(pool_name);

    librados::IoCtx ioctx;
    ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
    ioctx.application_enable("rbd", true);

    m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, "mirror uuid",
                                           m_pool_watcher_listener));

    if (enable_mirroring) {
      ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx,
                                                   RBD_MIRROR_MODE_POOL));
      std::string uuid;
      ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add(
        ioctx, &uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, peer.cluster_name,
        peer.client_name));
    }
    if (name != nullptr) {
      *name = pool_name;
    }

    m_pool_watcher->init();
  }

  string get_image_id(librados::IoCtx *ioctx, const string &image_name) {
    string obj = librbd::util::id_obj_name(image_name);
    string id;
    EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id));
    return id;
  }

  void create_image(const string &pool_name, bool mirrored=true,
		    string *image_name=nullptr) {
    uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
    string name = "image" + stringify(++m_image_number);
    if (mirrored) {
      features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
    }

    librados::IoCtx ioctx;
    ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx));
    int order = 0;
    ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false,
				features, &order, 0, 0));
    if (mirrored) {
      librbd::Image image;
      librbd::RBD rbd;
      rbd.open(ioctx, image, name.c_str());
      image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL);

      librbd::mirror_image_info_t mirror_image_info;
      ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
                                               sizeof(mirror_image_info)));
      image.close();

      m_mirrored_images.insert(ImageId(
        mirror_image_info.global_id, get_image_id(&ioctx, name)));
    }
    if (image_name != nullptr)
      *image_name = name;
  }

  void clone_image(const string &parent_pool_name,
		   const string &parent_image_name,
		   const string &clone_pool_name,
		   bool mirrored=true,
		   string *image_name=nullptr) {
    librados::IoCtx pioctx, cioctx;
    ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx));
    ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx));

    string snap_name = "snap" + stringify(++m_snap_number);
    {
      librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(),
						    "", "", pioctx, false);
      ictx->state->open(0);
      librbd::NoOpProgressContext prog_ctx;
      EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
						 snap_name, 0, prog_ctx));
      EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
						  snap_name));
      ictx->state->close();
    }

    uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
    string name = "clone" + stringify(++m_image_number);
    if (mirrored) {
      features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
    }
    int order = 0;
    librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(),
		  cioctx, name.c_str(), features, &order, 0, 0);
    if (mirrored) {
      librbd::Image image;
      librbd::RBD rbd;
      rbd.open(cioctx, image, name.c_str());
      image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL);

      librbd::mirror_image_info_t mirror_image_info;
      ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
                                               sizeof(mirror_image_info)));
      image.close();

      m_mirrored_images.insert(ImageId(
        mirror_image_info.global_id, get_image_id(&cioctx, name)));
    }
    if (image_name != nullptr)
      *image_name = name;
  }

  void check_images() {
    std::unique_lock l{m_lock};
    while (m_mirrored_images != m_pool_watcher_listener.image_ids) {
      if (m_pool_watcher_listener.cond.wait_for(l, 10s) == std::cv_status::timeout) {
        break;
      }
    }

    ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids);
  }

  ceph::mutex m_lock = ceph::make_mutex("TestPoolWatcherLock");
  RadosRef m_cluster;
  PoolWatcherListener m_pool_watcher_listener;
  std::unique_ptr<PoolWatcher<> > m_pool_watcher;

  set<string> m_pools;
  ImageIds m_mirrored_images;

  uint64_t m_image_number;
  uint64_t m_snap_number;
};

TEST_F(TestPoolWatcher, EmptyPool) {
  string uuid1 = "00000000-0000-0000-0000-000000000001";
  PeerSpec site1(uuid1, "site1", "mirror1");
  create_pool(true, site1);
  check_images();
}

TEST_F(TestPoolWatcher, ReplicatedPools) {
  string uuid1 = "00000000-0000-0000-0000-000000000001";
  PeerSpec site1(uuid1, "site1", "mirror1");
  string first_pool, local_pool, last_pool;
  create_pool(true, site1, &first_pool);
  check_images();
  create_image(first_pool);
  check_images();
  string parent_image, parent_image2;
  create_image(first_pool, true, &parent_image);
  check_images();
  clone_image(first_pool, parent_image, first_pool);
  check_images();
  clone_image(first_pool, parent_image, first_pool, true, &parent_image2);
  check_images();
  create_image(first_pool, false);
  check_images();
}
Commit	Line	Data
7c673cae FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3
	4	#include "include/rados/librados.hpp"
	5	#include "include/rbd/librbd.hpp"
	6	#include "include/stringify.h"
	7	#include "test/rbd_mirror/test_fixture.h"
	8	#include "cls/rbd/cls_rbd_types.h"
	9	#include "cls/rbd/cls_rbd_client.h"
	10	#include "include/rbd_types.h"
	11	#include "librbd/internal.h"
	12	#include "librbd/ImageCtx.h"
	13	#include "librbd/ImageState.h"
	14	#include "librbd/Operations.h"
	15	#include "librbd/Utils.h"
	16	#include "librbd/api/Mirror.h"
	17	#include "common/Cond.h"
	18	#include "common/errno.h"
9f95a23c	19	#include "common/ceph_mutex.h"
7c673cae FG	20	#include "tools/rbd_mirror/PoolWatcher.h"
7c673cae FG	21	#include "tools/rbd_mirror/Threads.h"
11fdf7f2 TL	22	#include "tools/rbd_mirror/Types.h"
	23	#include "tools/rbd_mirror/pool_watcher/Types.h"
	24	#include "test/librados/test_cxx.h"
7c673cae FG	25	#include "gtest/gtest.h"
	26	#include <boost/scope_exit.hpp>
	27	#include <iostream>
	28	#include <map>
	29	#include <memory>
	30	#include <set>
	31	#include <vector>
	32
20effc67 TL	33	using namespace std::chrono_literals;
20effc67 TL	34
7c673cae FG	35	using rbd::mirror::ImageId;
	36	using rbd::mirror::ImageIds;
	37	using rbd::mirror::PoolWatcher;
11fdf7f2	38	using rbd::mirror::PeerSpec;
7c673cae FG	39	using rbd::mirror::RadosRef;
	40	using std::map;
	41	using std::set;
	42	using std::string;
	43
	44	void register_test_pool_watcher() {
	45	}
	46
	47	class TestPoolWatcher : public ::rbd::mirror::TestFixture {
	48	public:
	49
	50	TestPoolWatcher()
9f95a23c	51	: m_pool_watcher_listener(this),
7c673cae FG	52	m_image_number(0), m_snap_number(0)
	53	{
	54	m_cluster = std::make_shared<librados::Rados>();
	55	EXPECT_EQ("", connect_cluster_pp(*m_cluster));
	56	}
	57
	58	void TearDown() override {
	59	if (m_pool_watcher) {
	60	C_SaferCond ctx;
	61	m_pool_watcher->shut_down(&ctx);
	62	EXPECT_EQ(0, ctx.wait());
	63	}
	64
	65	m_cluster->wait_for_latest_osdmap();
	66	for (auto& pool : m_pools) {
	67	EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str()));
	68	}
	69
	70	TestFixture::TearDown();
	71	}
	72
11fdf7f2	73	struct PoolWatcherListener : public rbd::mirror::pool_watcher::Listener {
7c673cae	74	TestPoolWatcher *test;
9f95a23c	75	ceph::condition_variable cond;
7c673cae FG	76	ImageIds image_ids;
7c673cae FG	77
11fdf7f2	78	explicit PoolWatcherListener(TestPoolWatcher *test) : test(test) {
7c673cae FG	79	}
	80
	81	void handle_update(const std::string &mirror_uuid,
	82	ImageIds &&added_image_ids,
	83	ImageIds &&removed_image_ids) override {
9f95a23c	84	std::lock_guard locker{test->m_lock};
7c673cae FG	85	for (auto &image_id : removed_image_ids) {
	86	image_ids.erase(image_id);
	87	}
	88	image_ids.insert(added_image_ids.begin(), added_image_ids.end());
9f95a23c	89	cond.notify_all();
7c673cae FG	90	}
	91	};
	92
11fdf7f2	93	void create_pool(bool enable_mirroring, const PeerSpec &peer, string *name=nullptr) {
7c673cae FG	94	string pool_name = get_temp_pool_name("test-rbd-mirror-");
	95	ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str()));
	96
	97	int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
	98	ASSERT_GE(pool_id, 0);
	99	m_pools.insert(pool_name);
	100
	101	librados::IoCtx ioctx;
	102	ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
c07f9fc5	103	ioctx.application_enable("rbd", true);
7c673cae	104
9f95a23c	105	m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, "mirror uuid",
7c673cae FG	106	m_pool_watcher_listener));
	107
	108	if (enable_mirroring) {
	109	ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx,
	110	RBD_MIRROR_MODE_POOL));
	111	std::string uuid;
9f95a23c TL	112	ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add(
	113	ioctx, &uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, peer.cluster_name,
	114	peer.client_name));
7c673cae FG	115	}
	116	if (name != nullptr) {
	117	*name = pool_name;
	118	}
	119
	120	m_pool_watcher->init();
	121	}
	122
	123	string get_image_id(librados::IoCtx *ioctx, const string &image_name) {
	124	string obj = librbd::util::id_obj_name(image_name);
	125	string id;
	126	EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id));
	127	return id;
	128	}
	129
	130	void create_image(const string &pool_name, bool mirrored=true,
	131	string *image_name=nullptr) {
	132	uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
	133	string name = "image" + stringify(++m_image_number);
	134	if (mirrored) {
	135	features \|= RBD_FEATURE_EXCLUSIVE_LOCK \| RBD_FEATURE_JOURNALING;
	136	}
	137
	138	librados::IoCtx ioctx;
	139	ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx));
	140	int order = 0;
	141	ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false,
	142	features, &order, 0, 0));
	143	if (mirrored) {
	144	librbd::Image image;
	145	librbd::RBD rbd;
	146	rbd.open(ioctx, image, name.c_str());
9f95a23c	147	image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL);
7c673cae FG	148
	149	librbd::mirror_image_info_t mirror_image_info;
	150	ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
	151	sizeof(mirror_image_info)));
	152	image.close();
	153
	154	m_mirrored_images.insert(ImageId(
	155	mirror_image_info.global_id, get_image_id(&ioctx, name)));
	156	}
	157	if (image_name != nullptr)
	158	*image_name = name;
	159	}
	160
	161	void clone_image(const string &parent_pool_name,
	162	const string &parent_image_name,
	163	const string &clone_pool_name,
	164	bool mirrored=true,
	165	string *image_name=nullptr) {
	166	librados::IoCtx pioctx, cioctx;
	167	ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx));
	168	ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx));
	169
	170	string snap_name = "snap" + stringify(++m_snap_number);
	171	{
	172	librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(),
	173	"", "", pioctx, false);
11fdf7f2	174	ictx->state->open(0);
f67539c2	175	librbd::NoOpProgressContext prog_ctx;
7c673cae	176	EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
f67539c2	177	snap_name, 0, prog_ctx));
7c673cae	178	EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
f67539c2	179	snap_name));
7c673cae FG	180	ictx->state->close();
	181	}
	182
	183	uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
	184	string name = "clone" + stringify(++m_image_number);
	185	if (mirrored) {
	186	features \|= RBD_FEATURE_EXCLUSIVE_LOCK \| RBD_FEATURE_JOURNALING;
	187	}
	188	int order = 0;
	189	librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(),
	190	cioctx, name.c_str(), features, &order, 0, 0);
	191	if (mirrored) {
	192	librbd::Image image;
	193	librbd::RBD rbd;
	194	rbd.open(cioctx, image, name.c_str());
9f95a23c	195	image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL);
7c673cae FG	196
	197	librbd::mirror_image_info_t mirror_image_info;
	198	ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
	199	sizeof(mirror_image_info)));
	200	image.close();
	201
	202	m_mirrored_images.insert(ImageId(
	203	mirror_image_info.global_id, get_image_id(&cioctx, name)));
	204	}
	205	if (image_name != nullptr)
	206	*image_name = name;
	207	}
	208
	209	void check_images() {
9f95a23c	210	std::unique_lock l{m_lock};
7c673cae	211	while (m_mirrored_images != m_pool_watcher_listener.image_ids) {
9f95a23c	212	if (m_pool_watcher_listener.cond.wait_for(l, 10s) == std::cv_status::timeout) {
7c673cae FG	213	break;
	214	}
	215	}
	216
	217	ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids);
	218	}
	219
9f95a23c	220	ceph::mutex m_lock = ceph::make_mutex("TestPoolWatcherLock");
7c673cae FG	221	RadosRef m_cluster;
7c673cae FG	222	PoolWatcherListener m_pool_watcher_listener;
20effc67	223	std::unique_ptr<PoolWatcher<> > m_pool_watcher;
7c673cae FG	224
	225	set<string> m_pools;
	226	ImageIds m_mirrored_images;
	227
	228	uint64_t m_image_number;
	229	uint64_t m_snap_number;
	230	};
	231
	232	TEST_F(TestPoolWatcher, EmptyPool) {
	233	string uuid1 = "00000000-0000-0000-0000-000000000001";
11fdf7f2	234	PeerSpec site1(uuid1, "site1", "mirror1");
7c673cae FG	235	create_pool(true, site1);
	236	check_images();
	237	}
	238
	239	TEST_F(TestPoolWatcher, ReplicatedPools) {
	240	string uuid1 = "00000000-0000-0000-0000-000000000001";
11fdf7f2	241	PeerSpec site1(uuid1, "site1", "mirror1");
7c673cae FG	242	string first_pool, local_pool, last_pool;
	243	create_pool(true, site1, &first_pool);
	244	check_images();
	245	create_image(first_pool);
	246	check_images();
	247	string parent_image, parent_image2;
	248	create_image(first_pool, true, &parent_image);
	249	check_images();
	250	clone_image(first_pool, parent_image, first_pool);
	251	check_images();
	252	clone_image(first_pool, parent_image, first_pool, true, &parent_image2);
	253	check_images();
	254	create_image(first_pool, false);
	255	check_images();
	256	}