1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
3 #include "include/rados/librados.hpp"
4 #include "common/Cond.h"
5 #include "common/errno.h"
6 #include "common/Mutex.h"
7 #include "librbd/internal.h"
8 #include "librbd/api/Mirror.h"
9 #include "tools/rbd_mirror/ClusterWatcher.h"
10 #include "tools/rbd_mirror/ServiceDaemon.h"
11 #include "tools/rbd_mirror/Types.h"
12 #include "test/rbd_mirror/test_fixture.h"
13 #include "test/librados/test_cxx.h"
14 #include "test/librbd/test_support.h"
15 #include "gtest/gtest.h"
16 #include <boost/scope_exit.hpp>
22 using rbd::mirror::ClusterWatcher
;
23 using rbd::mirror::PeerSpec
;
24 using rbd::mirror::RadosRef
;
29 void register_test_cluster_watcher() {
32 class TestClusterWatcher
: public ::rbd::mirror::TestFixture
{
35 TestClusterWatcher() : m_lock("TestClusterWatcherLock")
37 m_cluster
= std::make_shared
<librados::Rados
>();
38 EXPECT_EQ("", connect_cluster_pp(*m_cluster
));
41 ~TestClusterWatcher() override
{
42 m_cluster
->wait_for_latest_osdmap();
43 for (auto& pool
: m_pools
) {
44 EXPECT_EQ(0, m_cluster
->pool_delete(pool
.c_str()));
48 void SetUp() override
{
50 m_service_daemon
.reset(new rbd::mirror::ServiceDaemon
<>(g_ceph_context
,
53 m_cluster_watcher
.reset(new ClusterWatcher(m_cluster
, m_lock
,
54 m_service_daemon
.get()));
57 void TearDown() override
{
58 m_service_daemon
.reset();
59 m_cluster_watcher
.reset();
60 TestFixture::TearDown();
63 void create_pool(bool enable_mirroring
, const PeerSpec
&peer
,
64 string
*uuid
= nullptr, string
*name
=nullptr) {
65 string pool_name
= get_temp_pool_name("test-rbd-mirror-");
66 ASSERT_EQ(0, m_cluster
->pool_create(pool_name
.c_str()));
68 int64_t pool_id
= m_cluster
->pool_lookup(pool_name
.c_str());
69 ASSERT_GE(pool_id
, 0);
71 librados::IoCtx ioctx
;
72 ASSERT_EQ(0, m_cluster
->ioctx_create2(pool_id
, ioctx
));
73 ioctx
.application_enable("rbd", true);
75 m_pools
.insert(pool_name
);
76 if (enable_mirroring
) {
77 ASSERT_EQ(0, librbd::api::Mirror
<>::mode_set(ioctx
,
78 RBD_MIRROR_MODE_POOL
));
81 ASSERT_EQ(0, librbd::api::Mirror
<>::peer_add(ioctx
,
82 uuid
!= nullptr ? uuid
:
86 m_pool_peers
[pool_id
].insert(peer
);
88 if (name
!= nullptr) {
93 void delete_pool(const string
&name
, const PeerSpec
&peer
) {
94 int64_t pool_id
= m_cluster
->pool_lookup(name
.c_str());
95 ASSERT_GE(pool_id
, 0);
96 if (m_pool_peers
.find(pool_id
) != m_pool_peers
.end()) {
97 m_pool_peers
[pool_id
].erase(peer
);
98 if (m_pool_peers
[pool_id
].empty()) {
99 m_pool_peers
.erase(pool_id
);
103 ASSERT_EQ(0, m_cluster
->pool_delete(name
.c_str()));
106 void set_peer_config_key(const std::string
& pool_name
,
107 const PeerSpec
&peer
) {
108 int64_t pool_id
= m_cluster
->pool_lookup(pool_name
.c_str());
109 ASSERT_GE(pool_id
, 0);
113 "\\\"mon_host\\\": \\\"" + peer
.mon_host
+ "\\\", "
114 "\\\"key\\\": \\\"" + peer
.key
+ "\\\""
118 ASSERT_EQ(0, m_cluster
->mon_command(
120 "\"prefix\": \"config-key set\","
121 "\"key\": \"" RBD_MIRROR_PEER_CONFIG_KEY_PREFIX
+ stringify(pool_id
) +
122 "/" + peer
.uuid
+ "\","
123 "\"val\": \"" + json
+ "\"" +
124 "}", in_bl
, nullptr, nullptr));
127 void create_cache_pool(const string
&base_pool
, string
*cache_pool_name
) {
129 *cache_pool_name
= get_temp_pool_name("test-rbd-mirror-");
130 ASSERT_EQ(0, m_cluster
->pool_create(cache_pool_name
->c_str()));
132 ASSERT_EQ(0, m_cluster
->mon_command(
133 "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool
+
134 "\", \"tierpool\": \"" + *cache_pool_name
+
135 "\", \"force_nonempty\": \"--force-nonempty\" }",
137 ASSERT_EQ(0, m_cluster
->mon_command(
138 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool
+
139 "\", \"overlaypool\": \"" + *cache_pool_name
+ "\"}",
141 ASSERT_EQ(0, m_cluster
->mon_command(
142 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + *cache_pool_name
+
143 "\", \"mode\": \"writeback\"}",
145 m_cluster
->wait_for_latest_osdmap();
148 void remove_cache_pool(const string
&base_pool
, const string
&cache_pool
) {
151 ASSERT_EQ(0, m_cluster
->mon_command(
152 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool
+
155 ASSERT_EQ(0, m_cluster
->mon_command(
156 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool
+
157 "\", \"tierpool\": \"" + cache_pool
+ "\"}",
159 m_cluster
->wait_for_latest_osdmap();
160 m_cluster
->pool_delete(cache_pool
.c_str());
164 m_cluster_watcher
->refresh_pools();
165 Mutex::Locker
l(m_lock
);
166 ASSERT_EQ(m_pool_peers
, m_cluster_watcher
->get_pool_peers());
171 unique_ptr
<rbd::mirror::ServiceDaemon
<>> m_service_daemon
;
172 unique_ptr
<ClusterWatcher
> m_cluster_watcher
;
175 ClusterWatcher::PoolPeers m_pool_peers
;
178 TEST_F(TestClusterWatcher
, NoPools
) {
182 TEST_F(TestClusterWatcher
, NoMirroredPools
) {
184 create_pool(false, PeerSpec());
186 create_pool(false, PeerSpec());
188 create_pool(false, PeerSpec());
192 TEST_F(TestClusterWatcher
, ReplicatedPools
) {
193 PeerSpec
site1("", "site1", "mirror1");
194 PeerSpec
site2("", "site2", "mirror2");
195 string first_pool
, last_pool
;
197 create_pool(true, site1
, &site1
.uuid
, &first_pool
);
199 create_pool(false, PeerSpec());
201 create_pool(false, PeerSpec());
203 create_pool(false, PeerSpec());
205 create_pool(true, site2
, &site2
.uuid
);
207 create_pool(true, site2
, &site2
.uuid
);
209 create_pool(true, site2
, &site2
.uuid
, &last_pool
);
211 delete_pool(first_pool
, site1
);
213 delete_pool(last_pool
, site2
);
217 TEST_F(TestClusterWatcher
, CachePools
) {
218 PeerSpec
site1("", "site1", "mirror1");
219 string base1
, base2
, cache1
, cache2
;
220 create_pool(true, site1
, &site1
.uuid
, &base1
);
223 create_cache_pool(base1
, &cache1
);
224 BOOST_SCOPE_EXIT( base1
, cache1
, this_
) {
225 this_
->remove_cache_pool(base1
, cache1
);
226 } BOOST_SCOPE_EXIT_END
;
229 create_pool(false, PeerSpec(), nullptr, &base2
);
230 create_cache_pool(base2
, &cache2
);
231 BOOST_SCOPE_EXIT( base2
, cache2
, this_
) {
232 this_
->remove_cache_pool(base2
, cache2
);
233 } BOOST_SCOPE_EXIT_END
;
237 TEST_F(TestClusterWatcher
, ConfigKey
) {
238 REQUIRE(!is_librados_test_stub(*m_cluster
));
240 std::string pool_name
;
243 PeerSpec
site1("", "site1", "mirror1");
244 create_pool(true, site1
, &site1
.uuid
, &pool_name
);
247 PeerSpec
site2("", "site2", "mirror2");
248 site2
.mon_host
= "abc";
250 create_pool(false, site2
, &site2
.uuid
);
251 set_peer_config_key(pool_name
, site2
);