[ceph.git] / ceph / src / test / rbd_mirror / test_ClusterWatcher.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
#include "include/rados/librados.hpp"
#include "common/Cond.h"
#include "common/errno.h"
#include "common/ceph_mutex.h"
#include "librbd/internal.h"
#include "librbd/api/Mirror.h"
#include "tools/rbd_mirror/ClusterWatcher.h"
#include "tools/rbd_mirror/ServiceDaemon.h"
#include "tools/rbd_mirror/Types.h"
#include "test/rbd_mirror/test_fixture.h"
#include "test/librados/test_cxx.h"
#include "test/librbd/test_support.h"
#include "gtest/gtest.h"
#include <boost/scope_exit.hpp>
#include <iostream>
#include <map>
#include <memory>
#include <set>

using rbd::mirror::ClusterWatcher;
using rbd::mirror::PeerSpec;
using rbd::mirror::RadosRef;
using std::map;
using std::set;
using std::string;

void register_test_cluster_watcher() {
}

class TestClusterWatcher : public ::rbd::mirror::TestFixture {
public:

  TestClusterWatcher() {
    m_cluster = std::make_shared<librados::Rados>();
    EXPECT_EQ("", connect_cluster_pp(*m_cluster));
  }

  ~TestClusterWatcher() override {
    m_cluster->wait_for_latest_osdmap();
    for (auto& pool : m_pools) {
      EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str()));
    }
  }

  void SetUp() override {
    TestFixture::SetUp();
    m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context,
                                                            m_cluster,
                                                            m_threads));
    m_cluster_watcher.reset(new ClusterWatcher(m_cluster, m_lock,
                                               m_service_daemon.get()));
  }

  void TearDown() override {
    m_service_daemon.reset();
    m_cluster_watcher.reset();
    TestFixture::TearDown();
  }

  void create_pool(bool enable_mirroring, const PeerSpec &peer,
                   string *uuid = nullptr, string *name=nullptr) {
    string pool_name = get_temp_pool_name("test-rbd-mirror-");
    ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str()));

    int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
    ASSERT_GE(pool_id, 0);

    librados::IoCtx ioctx;
    ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
    ioctx.application_enable("rbd", true);

    m_pools.insert(pool_name);
    if (enable_mirroring) {
      ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx,
                                                   RBD_MIRROR_MODE_POOL));

      std::string gen_uuid;
      ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add(
                     ioctx, uuid != nullptr ? uuid : &gen_uuid,
                     RBD_MIRROR_PEER_DIRECTION_RX_TX,
                     peer.cluster_name, peer.client_name));
      m_pool_peers[pool_id].insert(peer);
    }
    if (name != nullptr) {
      *name = pool_name;
    }
  }

  void delete_pool(const string &name, const PeerSpec &peer) {
    int64_t pool_id = m_cluster->pool_lookup(name.c_str());
    ASSERT_GE(pool_id, 0);
    if (m_pool_peers.find(pool_id) != m_pool_peers.end()) {
      m_pool_peers[pool_id].erase(peer);
      if (m_pool_peers[pool_id].empty()) {
	m_pool_peers.erase(pool_id);
      }
    }
    m_pools.erase(name);
    ASSERT_EQ(0, m_cluster->pool_delete(name.c_str()));
  }

  void set_peer_config_key(const std::string& pool_name,
                           const PeerSpec &peer) {
    int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
    ASSERT_GE(pool_id, 0);

    std::string json =
      "{"
        "\\\"mon_host\\\": \\\"" + peer.mon_host + "\\\", "
        "\\\"key\\\": \\\"" + peer.key + "\\\""
      "}";

    bufferlist in_bl;
    ASSERT_EQ(0, m_cluster->mon_command(
      "{"
        "\"prefix\": \"config-key set\","
        "\"key\": \"" RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) +
          "/" + peer.uuid + "\","
        "\"val\": \"" + json + "\"" +
      "}", in_bl, nullptr, nullptr));
  }

  void create_cache_pool(const string &base_pool, string *cache_pool_name) {
    bufferlist inbl;
    *cache_pool_name = get_temp_pool_name("test-rbd-mirror-");
    ASSERT_EQ(0, m_cluster->pool_create(cache_pool_name->c_str()));

    ASSERT_EQ(0, m_cluster->mon_command(
      "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool +
      "\", \"tierpool\": \"" + *cache_pool_name +
      "\", \"force_nonempty\": \"--force-nonempty\" }",
      inbl, NULL, NULL));
    ASSERT_EQ(0, m_cluster->mon_command(
      "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool +
      "\", \"overlaypool\": \"" + *cache_pool_name + "\"}",
      inbl, NULL, NULL));
    ASSERT_EQ(0, m_cluster->mon_command(
      "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + *cache_pool_name +
      "\", \"mode\": \"writeback\"}",
      inbl, NULL, NULL));
    m_cluster->wait_for_latest_osdmap();
  }

  void remove_cache_pool(const string &base_pool, const string &cache_pool) {
    bufferlist inbl;
    // tear down tiers
    ASSERT_EQ(0, m_cluster->mon_command(
      "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool +
      "\"}",
      inbl, NULL, NULL));
    ASSERT_EQ(0, m_cluster->mon_command(
      "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool +
      "\", \"tierpool\": \"" + cache_pool + "\"}",
      inbl, NULL, NULL));
    m_cluster->wait_for_latest_osdmap();
    m_cluster->pool_delete(cache_pool.c_str());
  }

  void check_peers() {
    m_cluster_watcher->refresh_pools();
    std::lock_guard l{m_lock};
    ASSERT_EQ(m_pool_peers, m_cluster_watcher->get_pool_peers());
  }

  RadosRef m_cluster;
  ceph::mutex m_lock = ceph::make_mutex("TestClusterWatcherLock");
  std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon;
  std::unique_ptr<ClusterWatcher> m_cluster_watcher;

  set<string> m_pools;
  ClusterWatcher::PoolPeers m_pool_peers;
};

TEST_F(TestClusterWatcher, NoPools) {
  check_peers();
}

TEST_F(TestClusterWatcher, NoMirroredPools) {
  check_peers();
  create_pool(false, PeerSpec());
  check_peers();
  create_pool(false, PeerSpec());
  check_peers();
  create_pool(false, PeerSpec());
  check_peers();
}

TEST_F(TestClusterWatcher, ReplicatedPools) {
  PeerSpec site1("", "site1", "mirror1");
  PeerSpec site2("", "site2", "mirror2");
  string first_pool, last_pool;
  check_peers();
  create_pool(true, site1, &site1.uuid, &first_pool);
  check_peers();
  create_pool(false, PeerSpec());
  check_peers();
  create_pool(false, PeerSpec());
  check_peers();
  create_pool(false, PeerSpec());
  check_peers();
  create_pool(true, site2, &site2.uuid);
  check_peers();
  create_pool(true, site2, &site2.uuid);
  check_peers();
  create_pool(true, site2, &site2.uuid, &last_pool);
  check_peers();
  delete_pool(first_pool, site1);
  check_peers();
  delete_pool(last_pool, site2);
  check_peers();
}

TEST_F(TestClusterWatcher, CachePools) {
  PeerSpec site1("", "site1", "mirror1");
  string base1, base2, cache1, cache2;
  create_pool(true, site1, &site1.uuid, &base1);
  check_peers();

  create_cache_pool(base1, &cache1);
  BOOST_SCOPE_EXIT( base1, cache1, this_ ) {
    this_->remove_cache_pool(base1, cache1);
  } BOOST_SCOPE_EXIT_END;
  check_peers();

  create_pool(false, PeerSpec(), nullptr, &base2);
  create_cache_pool(base2, &cache2);
  BOOST_SCOPE_EXIT( base2, cache2, this_ ) {
    this_->remove_cache_pool(base2, cache2);
  } BOOST_SCOPE_EXIT_END;
  check_peers();
}

TEST_F(TestClusterWatcher, ConfigKey) {
  REQUIRE(!is_librados_test_stub(*m_cluster));

  std::string pool_name;
  check_peers();

  PeerSpec site1("", "site1", "mirror1");
  create_pool(true, site1, &site1.uuid, &pool_name);
  check_peers();

  PeerSpec site2("", "site2", "mirror2");
  site2.mon_host = "abc";
  site2.key = "xyz";
  create_pool(false, site2, &site2.uuid);
  set_peer_config_key(pool_name, site2);

  check_peers();
}

TEST_F(TestClusterWatcher, SiteName) {
  REQUIRE(!is_librados_test_stub(*m_cluster));

  std::string site_name;
  librbd::RBD rbd;
  ASSERT_EQ(0, rbd.mirror_site_name_get(*m_cluster, &site_name));

  m_cluster_watcher->refresh_pools();

  std::lock_guard l{m_lock};
  ASSERT_EQ(site_name, m_cluster_watcher->get_site_name());
}
Commit	Line	Data
7c673cae FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3	#include "include/rados/librados.hpp"
	4	#include "common/Cond.h"
	5	#include "common/errno.h"
9f95a23c	6	#include "common/ceph_mutex.h"
7c673cae FG	7	#include "librbd/internal.h"
	8	#include "librbd/api/Mirror.h"
	9	#include "tools/rbd_mirror/ClusterWatcher.h"
c07f9fc5	10	#include "tools/rbd_mirror/ServiceDaemon.h"
11fdf7f2	11	#include "tools/rbd_mirror/Types.h"
7c673cae	12	#include "test/rbd_mirror/test_fixture.h"
11fdf7f2 TL	13	#include "test/librados/test_cxx.h"
11fdf7f2 TL	14	#include "test/librbd/test_support.h"
7c673cae FG	15	#include "gtest/gtest.h"
	16	#include <boost/scope_exit.hpp>
	17	#include <iostream>
	18	#include <map>
	19	#include <memory>
	20	#include <set>
	21
	22	using rbd::mirror::ClusterWatcher;
11fdf7f2	23	using rbd::mirror::PeerSpec;
7c673cae FG	24	using rbd::mirror::RadosRef;
	25	using std::map;
	26	using std::set;
	27	using std::string;
	28
	29	void register_test_cluster_watcher() {
	30	}
	31
	32	class TestClusterWatcher : public ::rbd::mirror::TestFixture {
	33	public:
	34
9f95a23c	35	TestClusterWatcher() {
7c673cae FG	36	m_cluster = std::make_shared<librados::Rados>();
7c673cae FG	37	EXPECT_EQ("", connect_cluster_pp(*m_cluster));
7c673cae FG	38	}
	39
	40	~TestClusterWatcher() override {
	41	m_cluster->wait_for_latest_osdmap();
	42	for (auto& pool : m_pools) {
	43	EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str()));
	44	}
	45	}
	46
c07f9fc5 FG	47	void SetUp() override {
	48	TestFixture::SetUp();
	49	m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context,
	50	m_cluster,
	51	m_threads));
	52	m_cluster_watcher.reset(new ClusterWatcher(m_cluster, m_lock,
	53	m_service_daemon.get()));
	54	}
	55
	56	void TearDown() override {
	57	m_service_daemon.reset();
	58	m_cluster_watcher.reset();
	59	TestFixture::TearDown();
	60	}
	61
11fdf7f2	62	void create_pool(bool enable_mirroring, const PeerSpec &peer,
7c673cae FG	63	string uuid = nullptr, string name=nullptr) {
	64	string pool_name = get_temp_pool_name("test-rbd-mirror-");
	65	ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str()));
	66
	67	int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
	68	ASSERT_GE(pool_id, 0);
c07f9fc5 FG	69
	70	librados::IoCtx ioctx;
	71	ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
	72	ioctx.application_enable("rbd", true);
	73
7c673cae FG	74	m_pools.insert(pool_name);
7c673cae FG	75	if (enable_mirroring) {
7c673cae FG	76	ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx,
	77	RBD_MIRROR_MODE_POOL));
	78
	79	std::string gen_uuid;
9f95a23c TL	80	ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add(
	81	ioctx, uuid != nullptr ? uuid : &gen_uuid,
	82	RBD_MIRROR_PEER_DIRECTION_RX_TX,
	83	peer.cluster_name, peer.client_name));
7c673cae FG	84	m_pool_peers[pool_id].insert(peer);
	85	}
	86	if (name != nullptr) {
	87	*name = pool_name;
	88	}
	89	}
	90
11fdf7f2	91	void delete_pool(const string &name, const PeerSpec &peer) {
7c673cae FG	92	int64_t pool_id = m_cluster->pool_lookup(name.c_str());
	93	ASSERT_GE(pool_id, 0);
	94	if (m_pool_peers.find(pool_id) != m_pool_peers.end()) {
	95	m_pool_peers[pool_id].erase(peer);
	96	if (m_pool_peers[pool_id].empty()) {
	97	m_pool_peers.erase(pool_id);
	98	}
	99	}
	100	m_pools.erase(name);
	101	ASSERT_EQ(0, m_cluster->pool_delete(name.c_str()));
	102	}
	103
11fdf7f2 TL	104	void set_peer_config_key(const std::string& pool_name,
	105	const PeerSpec &peer) {
	106	int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
	107	ASSERT_GE(pool_id, 0);
	108
	109	std::string json =
	110	"{"
	111	"\\\"mon_host\\\": \\\"" + peer.mon_host + "\\\", "
	112	"\\\"key\\\": \\\"" + peer.key + "\\\""
	113	"}";
	114
	115	bufferlist in_bl;
	116	ASSERT_EQ(0, m_cluster->mon_command(
	117	"{"
	118	"\"prefix\": \"config-key set\","
	119	"\"key\": \"" RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) +
	120	"/" + peer.uuid + "\","
	121	"\"val\": \"" + json + "\"" +
	122	"}", in_bl, nullptr, nullptr));
	123	}
	124
7c673cae FG	125	void create_cache_pool(const string &base_pool, string *cache_pool_name) {
	126	bufferlist inbl;
	127	*cache_pool_name = get_temp_pool_name("test-rbd-mirror-");
	128	ASSERT_EQ(0, m_cluster->pool_create(cache_pool_name->c_str()));
	129
	130	ASSERT_EQ(0, m_cluster->mon_command(
	131	"{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool +
	132	"\", \"tierpool\": \"" + *cache_pool_name +
	133	"\", \"force_nonempty\": \"--force-nonempty\" }",
	134	inbl, NULL, NULL));
	135	ASSERT_EQ(0, m_cluster->mon_command(
	136	"{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool +
	137	"\", \"overlaypool\": \"" + *cache_pool_name + "\"}",
	138	inbl, NULL, NULL));
	139	ASSERT_EQ(0, m_cluster->mon_command(
	140	"{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + *cache_pool_name +
	141	"\", \"mode\": \"writeback\"}",
	142	inbl, NULL, NULL));
	143	m_cluster->wait_for_latest_osdmap();
	144	}
	145
	146	void remove_cache_pool(const string &base_pool, const string &cache_pool) {
	147	bufferlist inbl;
	148	// tear down tiers
	149	ASSERT_EQ(0, m_cluster->mon_command(
	150	"{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool +
	151	"\"}",
	152	inbl, NULL, NULL));
	153	ASSERT_EQ(0, m_cluster->mon_command(
	154	"{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool +
	155	"\", \"tierpool\": \"" + cache_pool + "\"}",
	156	inbl, NULL, NULL));
	157	m_cluster->wait_for_latest_osdmap();
	158	m_cluster->pool_delete(cache_pool.c_str());
	159	}
	160
	161	void check_peers() {
	162	m_cluster_watcher->refresh_pools();
9f95a23c	163	std::lock_guard l{m_lock};
7c673cae FG	164	ASSERT_EQ(m_pool_peers, m_cluster_watcher->get_pool_peers());
	165	}
	166
7c673cae	167	RadosRef m_cluster;
9f95a23c	168	ceph::mutex m_lock = ceph::make_mutex("TestClusterWatcherLock");
20effc67 TL	169	std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon;
20effc67 TL	170	std::unique_ptr<ClusterWatcher> m_cluster_watcher;
7c673cae FG	171
	172	set<string> m_pools;
	173	ClusterWatcher::PoolPeers m_pool_peers;
	174	};
	175
	176	TEST_F(TestClusterWatcher, NoPools) {
	177	check_peers();
	178	}
	179
	180	TEST_F(TestClusterWatcher, NoMirroredPools) {
	181	check_peers();
11fdf7f2	182	create_pool(false, PeerSpec());
7c673cae	183	check_peers();
11fdf7f2	184	create_pool(false, PeerSpec());
7c673cae	185	check_peers();
11fdf7f2	186	create_pool(false, PeerSpec());
7c673cae FG	187	check_peers();
	188	}
	189
	190	TEST_F(TestClusterWatcher, ReplicatedPools) {
11fdf7f2 TL	191	PeerSpec site1("", "site1", "mirror1");
11fdf7f2 TL	192	PeerSpec site2("", "site2", "mirror2");
7c673cae FG	193	string first_pool, last_pool;
	194	check_peers();
	195	create_pool(true, site1, &site1.uuid, &first_pool);
	196	check_peers();
11fdf7f2	197	create_pool(false, PeerSpec());
7c673cae	198	check_peers();
11fdf7f2	199	create_pool(false, PeerSpec());
7c673cae	200	check_peers();
11fdf7f2	201	create_pool(false, PeerSpec());
7c673cae FG	202	check_peers();
	203	create_pool(true, site2, &site2.uuid);
	204	check_peers();
	205	create_pool(true, site2, &site2.uuid);
	206	check_peers();
	207	create_pool(true, site2, &site2.uuid, &last_pool);
	208	check_peers();
	209	delete_pool(first_pool, site1);
	210	check_peers();
	211	delete_pool(last_pool, site2);
	212	check_peers();
	213	}
	214
	215	TEST_F(TestClusterWatcher, CachePools) {
11fdf7f2	216	PeerSpec site1("", "site1", "mirror1");
7c673cae FG	217	string base1, base2, cache1, cache2;
	218	create_pool(true, site1, &site1.uuid, &base1);
	219	check_peers();
	220
	221	create_cache_pool(base1, &cache1);
	222	BOOST_SCOPE_EXIT( base1, cache1, this_ ) {
	223	this_->remove_cache_pool(base1, cache1);
	224	} BOOST_SCOPE_EXIT_END;
	225	check_peers();
	226
11fdf7f2	227	create_pool(false, PeerSpec(), nullptr, &base2);
7c673cae FG	228	create_cache_pool(base2, &cache2);
	229	BOOST_SCOPE_EXIT( base2, cache2, this_ ) {
	230	this_->remove_cache_pool(base2, cache2);
	231	} BOOST_SCOPE_EXIT_END;
	232	check_peers();
	233	}
11fdf7f2 TL	234
	235	TEST_F(TestClusterWatcher, ConfigKey) {
	236	REQUIRE(!is_librados_test_stub(*m_cluster));
	237
	238	std::string pool_name;
	239	check_peers();
	240
	241	PeerSpec site1("", "site1", "mirror1");
	242	create_pool(true, site1, &site1.uuid, &pool_name);
	243	check_peers();
	244
	245	PeerSpec site2("", "site2", "mirror2");
	246	site2.mon_host = "abc";
	247	site2.key = "xyz";
	248	create_pool(false, site2, &site2.uuid);
	249	set_peer_config_key(pool_name, site2);
	250
	251	check_peers();
	252	}
9f95a23c TL	253
	254	TEST_F(TestClusterWatcher, SiteName) {
	255	REQUIRE(!is_librados_test_stub(*m_cluster));
	256
	257	std::string site_name;
	258	librbd::RBD rbd;
	259	ASSERT_EQ(0, rbd.mirror_site_name_get(*m_cluster, &site_name));
	260
	261	m_cluster_watcher->refresh_pools();
	262
	263	std::lock_guard l{m_lock};
	264	ASSERT_EQ(site_name, m_cluster_watcher->get_site_name());
	265	}