]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #include "include/rados/librados.hpp" | |
5 | #include "include/rbd/librbd.hpp" | |
6 | #include "include/stringify.h" | |
7 | #include "test/rbd_mirror/test_fixture.h" | |
8 | #include "cls/rbd/cls_rbd_types.h" | |
9 | #include "cls/rbd/cls_rbd_client.h" | |
10 | #include "include/rbd_types.h" | |
11 | #include "librbd/internal.h" | |
12 | #include "librbd/ImageCtx.h" | |
13 | #include "librbd/ImageState.h" | |
14 | #include "librbd/Operations.h" | |
15 | #include "librbd/Utils.h" | |
16 | #include "librbd/api/Mirror.h" | |
17 | #include "common/Cond.h" | |
18 | #include "common/errno.h" | |
9f95a23c | 19 | #include "common/ceph_mutex.h" |
7c673cae FG |
20 | #include "tools/rbd_mirror/PoolWatcher.h" |
21 | #include "tools/rbd_mirror/Threads.h" | |
11fdf7f2 TL |
22 | #include "tools/rbd_mirror/Types.h" |
23 | #include "tools/rbd_mirror/pool_watcher/Types.h" | |
24 | #include "test/librados/test_cxx.h" | |
7c673cae FG |
25 | #include "gtest/gtest.h" |
26 | #include <boost/scope_exit.hpp> | |
27 | #include <iostream> | |
28 | #include <map> | |
29 | #include <memory> | |
30 | #include <set> | |
31 | #include <vector> | |
32 | ||
33 | using rbd::mirror::ImageId; | |
34 | using rbd::mirror::ImageIds; | |
35 | using rbd::mirror::PoolWatcher; | |
11fdf7f2 | 36 | using rbd::mirror::PeerSpec; |
7c673cae FG |
37 | using rbd::mirror::RadosRef; |
38 | using std::map; | |
39 | using std::set; | |
40 | using std::string; | |
41 | ||
42 | void register_test_pool_watcher() { | |
43 | } | |
44 | ||
45 | class TestPoolWatcher : public ::rbd::mirror::TestFixture { | |
46 | public: | |
47 | ||
48 | TestPoolWatcher() | |
9f95a23c | 49 | : m_pool_watcher_listener(this), |
7c673cae FG |
50 | m_image_number(0), m_snap_number(0) |
51 | { | |
52 | m_cluster = std::make_shared<librados::Rados>(); | |
53 | EXPECT_EQ("", connect_cluster_pp(*m_cluster)); | |
54 | } | |
55 | ||
56 | void TearDown() override { | |
57 | if (m_pool_watcher) { | |
58 | C_SaferCond ctx; | |
59 | m_pool_watcher->shut_down(&ctx); | |
60 | EXPECT_EQ(0, ctx.wait()); | |
61 | } | |
62 | ||
63 | m_cluster->wait_for_latest_osdmap(); | |
64 | for (auto& pool : m_pools) { | |
65 | EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str())); | |
66 | } | |
67 | ||
68 | TestFixture::TearDown(); | |
69 | } | |
70 | ||
11fdf7f2 | 71 | struct PoolWatcherListener : public rbd::mirror::pool_watcher::Listener { |
7c673cae | 72 | TestPoolWatcher *test; |
9f95a23c | 73 | ceph::condition_variable cond; |
7c673cae FG |
74 | ImageIds image_ids; |
75 | ||
11fdf7f2 | 76 | explicit PoolWatcherListener(TestPoolWatcher *test) : test(test) { |
7c673cae FG |
77 | } |
78 | ||
79 | void handle_update(const std::string &mirror_uuid, | |
80 | ImageIds &&added_image_ids, | |
81 | ImageIds &&removed_image_ids) override { | |
9f95a23c | 82 | std::lock_guard locker{test->m_lock}; |
7c673cae FG |
83 | for (auto &image_id : removed_image_ids) { |
84 | image_ids.erase(image_id); | |
85 | } | |
86 | image_ids.insert(added_image_ids.begin(), added_image_ids.end()); | |
9f95a23c | 87 | cond.notify_all(); |
7c673cae FG |
88 | } |
89 | }; | |
90 | ||
11fdf7f2 | 91 | void create_pool(bool enable_mirroring, const PeerSpec &peer, string *name=nullptr) { |
7c673cae FG |
92 | string pool_name = get_temp_pool_name("test-rbd-mirror-"); |
93 | ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str())); | |
94 | ||
95 | int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); | |
96 | ASSERT_GE(pool_id, 0); | |
97 | m_pools.insert(pool_name); | |
98 | ||
99 | librados::IoCtx ioctx; | |
100 | ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx)); | |
c07f9fc5 | 101 | ioctx.application_enable("rbd", true); |
7c673cae | 102 | |
9f95a23c | 103 | m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, "mirror uuid", |
7c673cae FG |
104 | m_pool_watcher_listener)); |
105 | ||
106 | if (enable_mirroring) { | |
107 | ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx, | |
108 | RBD_MIRROR_MODE_POOL)); | |
109 | std::string uuid; | |
9f95a23c TL |
110 | ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add( |
111 | ioctx, &uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, peer.cluster_name, | |
112 | peer.client_name)); | |
7c673cae FG |
113 | } |
114 | if (name != nullptr) { | |
115 | *name = pool_name; | |
116 | } | |
117 | ||
118 | m_pool_watcher->init(); | |
119 | } | |
120 | ||
121 | string get_image_id(librados::IoCtx *ioctx, const string &image_name) { | |
122 | string obj = librbd::util::id_obj_name(image_name); | |
123 | string id; | |
124 | EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id)); | |
125 | return id; | |
126 | } | |
127 | ||
128 | void create_image(const string &pool_name, bool mirrored=true, | |
129 | string *image_name=nullptr) { | |
130 | uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); | |
131 | string name = "image" + stringify(++m_image_number); | |
132 | if (mirrored) { | |
133 | features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING; | |
134 | } | |
135 | ||
136 | librados::IoCtx ioctx; | |
137 | ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx)); | |
138 | int order = 0; | |
139 | ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false, | |
140 | features, &order, 0, 0)); | |
141 | if (mirrored) { | |
142 | librbd::Image image; | |
143 | librbd::RBD rbd; | |
144 | rbd.open(ioctx, image, name.c_str()); | |
9f95a23c | 145 | image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL); |
7c673cae FG |
146 | |
147 | librbd::mirror_image_info_t mirror_image_info; | |
148 | ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info, | |
149 | sizeof(mirror_image_info))); | |
150 | image.close(); | |
151 | ||
152 | m_mirrored_images.insert(ImageId( | |
153 | mirror_image_info.global_id, get_image_id(&ioctx, name))); | |
154 | } | |
155 | if (image_name != nullptr) | |
156 | *image_name = name; | |
157 | } | |
158 | ||
159 | void clone_image(const string &parent_pool_name, | |
160 | const string &parent_image_name, | |
161 | const string &clone_pool_name, | |
162 | bool mirrored=true, | |
163 | string *image_name=nullptr) { | |
164 | librados::IoCtx pioctx, cioctx; | |
165 | ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx)); | |
166 | ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx)); | |
167 | ||
168 | string snap_name = "snap" + stringify(++m_snap_number); | |
169 | { | |
170 | librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(), | |
171 | "", "", pioctx, false); | |
11fdf7f2 | 172 | ictx->state->open(0); |
7c673cae FG |
173 | EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), |
174 | snap_name.c_str())); | |
175 | EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), | |
176 | snap_name.c_str())); | |
177 | ictx->state->close(); | |
178 | } | |
179 | ||
180 | uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); | |
181 | string name = "clone" + stringify(++m_image_number); | |
182 | if (mirrored) { | |
183 | features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING; | |
184 | } | |
185 | int order = 0; | |
186 | librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(), | |
187 | cioctx, name.c_str(), features, &order, 0, 0); | |
188 | if (mirrored) { | |
189 | librbd::Image image; | |
190 | librbd::RBD rbd; | |
191 | rbd.open(cioctx, image, name.c_str()); | |
9f95a23c | 192 | image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL); |
7c673cae FG |
193 | |
194 | librbd::mirror_image_info_t mirror_image_info; | |
195 | ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info, | |
196 | sizeof(mirror_image_info))); | |
197 | image.close(); | |
198 | ||
199 | m_mirrored_images.insert(ImageId( | |
200 | mirror_image_info.global_id, get_image_id(&cioctx, name))); | |
201 | } | |
202 | if (image_name != nullptr) | |
203 | *image_name = name; | |
204 | } | |
205 | ||
206 | void check_images() { | |
9f95a23c | 207 | std::unique_lock l{m_lock}; |
7c673cae | 208 | while (m_mirrored_images != m_pool_watcher_listener.image_ids) { |
9f95a23c | 209 | if (m_pool_watcher_listener.cond.wait_for(l, 10s) == std::cv_status::timeout) { |
7c673cae FG |
210 | break; |
211 | } | |
212 | } | |
213 | ||
214 | ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids); | |
215 | } | |
216 | ||
9f95a23c | 217 | ceph::mutex m_lock = ceph::make_mutex("TestPoolWatcherLock"); |
7c673cae FG |
218 | RadosRef m_cluster; |
219 | PoolWatcherListener m_pool_watcher_listener; | |
220 | unique_ptr<PoolWatcher<> > m_pool_watcher; | |
221 | ||
222 | set<string> m_pools; | |
223 | ImageIds m_mirrored_images; | |
224 | ||
225 | uint64_t m_image_number; | |
226 | uint64_t m_snap_number; | |
227 | }; | |
228 | ||
229 | TEST_F(TestPoolWatcher, EmptyPool) { | |
230 | string uuid1 = "00000000-0000-0000-0000-000000000001"; | |
11fdf7f2 | 231 | PeerSpec site1(uuid1, "site1", "mirror1"); |
7c673cae FG |
232 | create_pool(true, site1); |
233 | check_images(); | |
234 | } | |
235 | ||
236 | TEST_F(TestPoolWatcher, ReplicatedPools) { | |
237 | string uuid1 = "00000000-0000-0000-0000-000000000001"; | |
11fdf7f2 | 238 | PeerSpec site1(uuid1, "site1", "mirror1"); |
7c673cae FG |
239 | string first_pool, local_pool, last_pool; |
240 | create_pool(true, site1, &first_pool); | |
241 | check_images(); | |
242 | create_image(first_pool); | |
243 | check_images(); | |
244 | string parent_image, parent_image2; | |
245 | create_image(first_pool, true, &parent_image); | |
246 | check_images(); | |
247 | clone_image(first_pool, parent_image, first_pool); | |
248 | check_images(); | |
249 | clone_image(first_pool, parent_image, first_pool, true, &parent_image2); | |
250 | check_images(); | |
251 | create_image(first_pool, false); | |
252 | check_images(); | |
253 | } |