]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | #include "include/rados/librados.hpp" | |
4 | #include "common/Cond.h" | |
5 | #include "common/errno.h" | |
6 | #include "common/Mutex.h" | |
7 | #include "librbd/internal.h" | |
8 | #include "librbd/api/Mirror.h" | |
9 | #include "tools/rbd_mirror/ClusterWatcher.h" | |
c07f9fc5 | 10 | #include "tools/rbd_mirror/ServiceDaemon.h" |
11fdf7f2 | 11 | #include "tools/rbd_mirror/Types.h" |
7c673cae | 12 | #include "test/rbd_mirror/test_fixture.h" |
11fdf7f2 TL |
13 | #include "test/librados/test_cxx.h" |
14 | #include "test/librbd/test_support.h" | |
7c673cae FG |
15 | #include "gtest/gtest.h" |
16 | #include <boost/scope_exit.hpp> | |
17 | #include <iostream> | |
18 | #include <map> | |
19 | #include <memory> | |
20 | #include <set> | |
21 | ||
22 | using rbd::mirror::ClusterWatcher; | |
11fdf7f2 | 23 | using rbd::mirror::PeerSpec; |
7c673cae FG |
24 | using rbd::mirror::RadosRef; |
25 | using std::map; | |
26 | using std::set; | |
27 | using std::string; | |
28 | ||
29 | void register_test_cluster_watcher() { | |
30 | } | |
31 | ||
32 | class TestClusterWatcher : public ::rbd::mirror::TestFixture { | |
33 | public: | |
34 | ||
35 | TestClusterWatcher() : m_lock("TestClusterWatcherLock") | |
36 | { | |
37 | m_cluster = std::make_shared<librados::Rados>(); | |
38 | EXPECT_EQ("", connect_cluster_pp(*m_cluster)); | |
7c673cae FG |
39 | } |
40 | ||
41 | ~TestClusterWatcher() override { | |
42 | m_cluster->wait_for_latest_osdmap(); | |
43 | for (auto& pool : m_pools) { | |
44 | EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str())); | |
45 | } | |
46 | } | |
47 | ||
c07f9fc5 FG |
48 | void SetUp() override { |
49 | TestFixture::SetUp(); | |
50 | m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, | |
51 | m_cluster, | |
52 | m_threads)); | |
53 | m_cluster_watcher.reset(new ClusterWatcher(m_cluster, m_lock, | |
54 | m_service_daemon.get())); | |
55 | } | |
56 | ||
57 | void TearDown() override { | |
58 | m_service_daemon.reset(); | |
59 | m_cluster_watcher.reset(); | |
60 | TestFixture::TearDown(); | |
61 | } | |
62 | ||
11fdf7f2 | 63 | void create_pool(bool enable_mirroring, const PeerSpec &peer, |
7c673cae FG |
64 | string *uuid = nullptr, string *name=nullptr) { |
65 | string pool_name = get_temp_pool_name("test-rbd-mirror-"); | |
66 | ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str())); | |
67 | ||
68 | int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); | |
69 | ASSERT_GE(pool_id, 0); | |
c07f9fc5 FG |
70 | |
71 | librados::IoCtx ioctx; | |
72 | ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx)); | |
73 | ioctx.application_enable("rbd", true); | |
74 | ||
7c673cae FG |
75 | m_pools.insert(pool_name); |
76 | if (enable_mirroring) { | |
7c673cae FG |
77 | ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx, |
78 | RBD_MIRROR_MODE_POOL)); | |
79 | ||
80 | std::string gen_uuid; | |
81 | ASSERT_EQ(0, librbd::api::Mirror<>::peer_add(ioctx, | |
82 | uuid != nullptr ? uuid : | |
83 | &gen_uuid, | |
84 | peer.cluster_name, | |
85 | peer.client_name)); | |
86 | m_pool_peers[pool_id].insert(peer); | |
87 | } | |
88 | if (name != nullptr) { | |
89 | *name = pool_name; | |
90 | } | |
91 | } | |
92 | ||
11fdf7f2 | 93 | void delete_pool(const string &name, const PeerSpec &peer) { |
7c673cae FG |
94 | int64_t pool_id = m_cluster->pool_lookup(name.c_str()); |
95 | ASSERT_GE(pool_id, 0); | |
96 | if (m_pool_peers.find(pool_id) != m_pool_peers.end()) { | |
97 | m_pool_peers[pool_id].erase(peer); | |
98 | if (m_pool_peers[pool_id].empty()) { | |
99 | m_pool_peers.erase(pool_id); | |
100 | } | |
101 | } | |
102 | m_pools.erase(name); | |
103 | ASSERT_EQ(0, m_cluster->pool_delete(name.c_str())); | |
104 | } | |
105 | ||
11fdf7f2 TL |
106 | void set_peer_config_key(const std::string& pool_name, |
107 | const PeerSpec &peer) { | |
108 | int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); | |
109 | ASSERT_GE(pool_id, 0); | |
110 | ||
111 | std::string json = | |
112 | "{" | |
113 | "\\\"mon_host\\\": \\\"" + peer.mon_host + "\\\", " | |
114 | "\\\"key\\\": \\\"" + peer.key + "\\\"" | |
115 | "}"; | |
116 | ||
117 | bufferlist in_bl; | |
118 | ASSERT_EQ(0, m_cluster->mon_command( | |
119 | "{" | |
120 | "\"prefix\": \"config-key set\"," | |
121 | "\"key\": \"" RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) + | |
122 | "/" + peer.uuid + "\"," | |
123 | "\"val\": \"" + json + "\"" + | |
124 | "}", in_bl, nullptr, nullptr)); | |
125 | } | |
126 | ||
7c673cae FG |
127 | void create_cache_pool(const string &base_pool, string *cache_pool_name) { |
128 | bufferlist inbl; | |
129 | *cache_pool_name = get_temp_pool_name("test-rbd-mirror-"); | |
130 | ASSERT_EQ(0, m_cluster->pool_create(cache_pool_name->c_str())); | |
131 | ||
132 | ASSERT_EQ(0, m_cluster->mon_command( | |
133 | "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool + | |
134 | "\", \"tierpool\": \"" + *cache_pool_name + | |
135 | "\", \"force_nonempty\": \"--force-nonempty\" }", | |
136 | inbl, NULL, NULL)); | |
137 | ASSERT_EQ(0, m_cluster->mon_command( | |
138 | "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool + | |
139 | "\", \"overlaypool\": \"" + *cache_pool_name + "\"}", | |
140 | inbl, NULL, NULL)); | |
141 | ASSERT_EQ(0, m_cluster->mon_command( | |
142 | "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + *cache_pool_name + | |
143 | "\", \"mode\": \"writeback\"}", | |
144 | inbl, NULL, NULL)); | |
145 | m_cluster->wait_for_latest_osdmap(); | |
146 | } | |
147 | ||
148 | void remove_cache_pool(const string &base_pool, const string &cache_pool) { | |
149 | bufferlist inbl; | |
150 | // tear down tiers | |
151 | ASSERT_EQ(0, m_cluster->mon_command( | |
152 | "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool + | |
153 | "\"}", | |
154 | inbl, NULL, NULL)); | |
155 | ASSERT_EQ(0, m_cluster->mon_command( | |
156 | "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool + | |
157 | "\", \"tierpool\": \"" + cache_pool + "\"}", | |
158 | inbl, NULL, NULL)); | |
159 | m_cluster->wait_for_latest_osdmap(); | |
160 | m_cluster->pool_delete(cache_pool.c_str()); | |
161 | } | |
162 | ||
163 | void check_peers() { | |
164 | m_cluster_watcher->refresh_pools(); | |
165 | Mutex::Locker l(m_lock); | |
166 | ASSERT_EQ(m_pool_peers, m_cluster_watcher->get_pool_peers()); | |
167 | } | |
168 | ||
7c673cae | 169 | RadosRef m_cluster; |
c07f9fc5 FG |
170 | Mutex m_lock; |
171 | unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon; | |
7c673cae FG |
172 | unique_ptr<ClusterWatcher> m_cluster_watcher; |
173 | ||
174 | set<string> m_pools; | |
175 | ClusterWatcher::PoolPeers m_pool_peers; | |
176 | }; | |
177 | ||
178 | TEST_F(TestClusterWatcher, NoPools) { | |
179 | check_peers(); | |
180 | } | |
181 | ||
182 | TEST_F(TestClusterWatcher, NoMirroredPools) { | |
183 | check_peers(); | |
11fdf7f2 | 184 | create_pool(false, PeerSpec()); |
7c673cae | 185 | check_peers(); |
11fdf7f2 | 186 | create_pool(false, PeerSpec()); |
7c673cae | 187 | check_peers(); |
11fdf7f2 | 188 | create_pool(false, PeerSpec()); |
7c673cae FG |
189 | check_peers(); |
190 | } | |
191 | ||
192 | TEST_F(TestClusterWatcher, ReplicatedPools) { | |
11fdf7f2 TL |
193 | PeerSpec site1("", "site1", "mirror1"); |
194 | PeerSpec site2("", "site2", "mirror2"); | |
7c673cae FG |
195 | string first_pool, last_pool; |
196 | check_peers(); | |
197 | create_pool(true, site1, &site1.uuid, &first_pool); | |
198 | check_peers(); | |
11fdf7f2 | 199 | create_pool(false, PeerSpec()); |
7c673cae | 200 | check_peers(); |
11fdf7f2 | 201 | create_pool(false, PeerSpec()); |
7c673cae | 202 | check_peers(); |
11fdf7f2 | 203 | create_pool(false, PeerSpec()); |
7c673cae FG |
204 | check_peers(); |
205 | create_pool(true, site2, &site2.uuid); | |
206 | check_peers(); | |
207 | create_pool(true, site2, &site2.uuid); | |
208 | check_peers(); | |
209 | create_pool(true, site2, &site2.uuid, &last_pool); | |
210 | check_peers(); | |
211 | delete_pool(first_pool, site1); | |
212 | check_peers(); | |
213 | delete_pool(last_pool, site2); | |
214 | check_peers(); | |
215 | } | |
216 | ||
217 | TEST_F(TestClusterWatcher, CachePools) { | |
11fdf7f2 | 218 | PeerSpec site1("", "site1", "mirror1"); |
7c673cae FG |
219 | string base1, base2, cache1, cache2; |
220 | create_pool(true, site1, &site1.uuid, &base1); | |
221 | check_peers(); | |
222 | ||
223 | create_cache_pool(base1, &cache1); | |
224 | BOOST_SCOPE_EXIT( base1, cache1, this_ ) { | |
225 | this_->remove_cache_pool(base1, cache1); | |
226 | } BOOST_SCOPE_EXIT_END; | |
227 | check_peers(); | |
228 | ||
11fdf7f2 | 229 | create_pool(false, PeerSpec(), nullptr, &base2); |
7c673cae FG |
230 | create_cache_pool(base2, &cache2); |
231 | BOOST_SCOPE_EXIT( base2, cache2, this_ ) { | |
232 | this_->remove_cache_pool(base2, cache2); | |
233 | } BOOST_SCOPE_EXIT_END; | |
234 | check_peers(); | |
235 | } | |
11fdf7f2 TL |
236 | |
237 | TEST_F(TestClusterWatcher, ConfigKey) { | |
238 | REQUIRE(!is_librados_test_stub(*m_cluster)); | |
239 | ||
240 | std::string pool_name; | |
241 | check_peers(); | |
242 | ||
243 | PeerSpec site1("", "site1", "mirror1"); | |
244 | create_pool(true, site1, &site1.uuid, &pool_name); | |
245 | check_peers(); | |
246 | ||
247 | PeerSpec site2("", "site2", "mirror2"); | |
248 | site2.mon_host = "abc"; | |
249 | site2.key = "xyz"; | |
250 | create_pool(false, site2, &site2.uuid); | |
251 | set_peer_config_key(pool_name, site2); | |
252 | ||
253 | check_peers(); | |
254 | } |