]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #include "test/rbd_mirror/test_fixture.h" | |
5 | #include "include/stringify.h" | |
6 | #include "include/rbd/librbd.hpp" | |
9f95a23c | 7 | #include "common/Cond.h" |
7c673cae FG |
8 | #include "journal/Journaler.h" |
9 | #include "journal/Settings.h" | |
10 | #include "librbd/ExclusiveLock.h" | |
11 | #include "librbd/ImageCtx.h" | |
12 | #include "librbd/ImageState.h" | |
13 | #include "librbd/internal.h" | |
f67539c2 | 14 | #include "librbd/Journal.h" |
7c673cae | 15 | #include "librbd/Operations.h" |
f67539c2 | 16 | #include "librbd/api/Io.h" |
11fdf7f2 TL |
17 | #include "librbd/io/AioCompletion.h" |
18 | #include "librbd/io/ImageDispatchSpec.h" | |
7c673cae FG |
19 | #include "librbd/io/ReadResult.h" |
20 | #include "librbd/journal/Types.h" | |
21 | #include "tools/rbd_mirror/ImageSync.h" | |
31f18b77 | 22 | #include "tools/rbd_mirror/InstanceWatcher.h" |
7c673cae | 23 | #include "tools/rbd_mirror/Threads.h" |
9f95a23c TL |
24 | #include "tools/rbd_mirror/Throttler.h" |
25 | #include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" | |
7c673cae FG |
26 | |
27 | void register_test_image_sync() { | |
28 | } | |
29 | ||
30 | namespace rbd { | |
31 | namespace mirror { | |
32 | ||
33 | namespace { | |
34 | ||
11fdf7f2 TL |
35 | int flush(librbd::ImageCtx *image_ctx) { |
36 | C_SaferCond ctx; | |
494da23a | 37 | auto aio_comp = librbd::io::AioCompletion::create_and_start( |
11fdf7f2 | 38 | &ctx, image_ctx, librbd::io::AIO_TYPE_FLUSH); |
f67539c2 TL |
39 | auto req = librbd::io::ImageDispatchSpec::create_flush( |
40 | *image_ctx, librbd::io::IMAGE_DISPATCH_LAYER_INTERNAL_START, aio_comp, | |
41 | librbd::io::FLUSH_SOURCE_INTERNAL, {}); | |
11fdf7f2 | 42 | req->send(); |
11fdf7f2 TL |
43 | return ctx.wait(); |
44 | } | |
45 | ||
46 | void scribble(librbd::ImageCtx *image_ctx, int num_ops, uint64_t max_size) | |
7c673cae | 47 | { |
11fdf7f2 | 48 | max_size = std::min<uint64_t>(image_ctx->size, max_size); |
7c673cae FG |
49 | for (int i=0; i<num_ops; i++) { |
50 | uint64_t off = rand() % (image_ctx->size - max_size + 1); | |
51 | uint64_t len = 1 + rand() % max_size; | |
52 | ||
53 | if (rand() % 4 == 0) { | |
11fdf7f2 | 54 | ASSERT_EQ((int)len, |
f67539c2 TL |
55 | librbd::api::Io<>::discard( |
56 | *image_ctx, off, len, image_ctx->discard_granularity_bytes)); | |
7c673cae FG |
57 | } else { |
58 | bufferlist bl; | |
59 | bl.append(std::string(len, '1')); | |
f67539c2 TL |
60 | ASSERT_EQ((int)len, librbd::api::Io<>::write( |
61 | *image_ctx, off, len, std::move(bl), 0)); | |
7c673cae FG |
62 | } |
63 | } | |
64 | ||
9f95a23c | 65 | std::shared_lock owner_locker{image_ctx->owner_lock}; |
11fdf7f2 | 66 | ASSERT_EQ(0, flush(image_ctx)); |
7c673cae FG |
67 | } |
68 | ||
69 | } // anonymous namespace | |
70 | class TestImageSync : public TestFixture { | |
71 | public: | |
72 | ||
73 | void SetUp() override { | |
74 | TestFixture::SetUp(); | |
75 | create_and_open(m_local_io_ctx, &m_local_image_ctx); | |
76 | create_and_open(m_remote_io_ctx, &m_remote_image_ctx); | |
77 | ||
9f95a23c TL |
78 | auto cct = reinterpret_cast<CephContext*>(m_local_io_ctx.cct()); |
79 | m_image_sync_throttler = rbd::mirror::Throttler<>::create( | |
80 | cct, "rbd_mirror_concurrent_image_syncs"); | |
81 | ||
31f18b77 | 82 | m_instance_watcher = rbd::mirror::InstanceWatcher<>::create( |
f67539c2 | 83 | m_local_io_ctx, *m_threads->asio_engine, nullptr, m_image_sync_throttler); |
31f18b77 FG |
84 | m_instance_watcher->handle_acquire_leader(); |
85 | ||
f67539c2 TL |
86 | ContextWQ* context_wq; |
87 | librbd::Journal<>::get_work_queue(cct, &context_wq); | |
88 | ||
7c673cae | 89 | m_remote_journaler = new ::journal::Journaler( |
f67539c2 | 90 | context_wq, m_threads->timer, &m_threads->timer_lock, |
9f95a23c | 91 | m_remote_io_ctx, m_remote_image_ctx->id, "mirror-uuid", {}, nullptr); |
7c673cae FG |
92 | |
93 | m_client_meta = {"image-id"}; | |
94 | ||
95 | librbd::journal::ClientData client_data(m_client_meta); | |
96 | bufferlist client_data_bl; | |
11fdf7f2 | 97 | encode(client_data, client_data_bl); |
7c673cae FG |
98 | |
99 | ASSERT_EQ(0, m_remote_journaler->register_client(client_data_bl)); | |
9f95a23c TL |
100 | |
101 | m_state_builder = rbd::mirror::image_replayer::journal::StateBuilder< | |
102 | librbd::ImageCtx>::create("global image id"); | |
103 | m_state_builder->remote_journaler = m_remote_journaler; | |
104 | m_state_builder->remote_client_meta = m_client_meta; | |
105 | m_sync_point_handler = m_state_builder->create_sync_point_handler(); | |
7c673cae FG |
106 | } |
107 | ||
108 | void TearDown() override { | |
31f18b77 FG |
109 | m_instance_watcher->handle_release_leader(); |
110 | ||
9f95a23c TL |
111 | m_state_builder->remote_journaler = nullptr; |
112 | m_state_builder->destroy_sync_point_handler(); | |
113 | m_state_builder->destroy(); | |
114 | ||
7c673cae | 115 | delete m_remote_journaler; |
31f18b77 | 116 | delete m_instance_watcher; |
9f95a23c TL |
117 | delete m_image_sync_throttler; |
118 | ||
119 | TestFixture::TearDown(); | |
7c673cae FG |
120 | } |
121 | ||
122 | void create_and_open(librados::IoCtx &io_ctx, librbd::ImageCtx **image_ctx) { | |
123 | librbd::RBD rbd; | |
124 | ASSERT_EQ(0, create_image(rbd, io_ctx, m_image_name, m_image_size)); | |
125 | ASSERT_EQ(0, open_image(io_ctx, m_image_name, image_ctx)); | |
126 | ||
127 | C_SaferCond ctx; | |
128 | { | |
9f95a23c | 129 | std::shared_lock owner_locker{(*image_ctx)->owner_lock}; |
7c673cae FG |
130 | (*image_ctx)->exclusive_lock->try_acquire_lock(&ctx); |
131 | } | |
132 | ASSERT_EQ(0, ctx.wait()); | |
133 | ASSERT_TRUE((*image_ctx)->exclusive_lock->is_lock_owner()); | |
134 | } | |
135 | ||
136 | ImageSync<> *create_request(Context *ctx) { | |
9f95a23c TL |
137 | return new ImageSync<>(m_threads, m_local_image_ctx, m_remote_image_ctx, |
138 | "mirror-uuid", m_sync_point_handler, | |
139 | m_instance_watcher, nullptr, ctx); | |
7c673cae FG |
140 | } |
141 | ||
142 | librbd::ImageCtx *m_remote_image_ctx; | |
143 | librbd::ImageCtx *m_local_image_ctx; | |
9f95a23c | 144 | rbd::mirror::Throttler<> *m_image_sync_throttler; |
31f18b77 | 145 | rbd::mirror::InstanceWatcher<> *m_instance_watcher; |
7c673cae FG |
146 | ::journal::Journaler *m_remote_journaler; |
147 | librbd::journal::MirrorPeerClientMeta m_client_meta; | |
9f95a23c TL |
148 | rbd::mirror::image_replayer::journal::StateBuilder<librbd::ImageCtx>* m_state_builder = nullptr; |
149 | rbd::mirror::image_sync::SyncPointHandler* m_sync_point_handler = nullptr; | |
7c673cae FG |
150 | }; |
151 | ||
152 | TEST_F(TestImageSync, Empty) { | |
153 | C_SaferCond ctx; | |
154 | ImageSync<> *request = create_request(&ctx); | |
155 | request->send(); | |
156 | ASSERT_EQ(0, ctx.wait()); | |
157 | ||
158 | ASSERT_EQ(0U, m_client_meta.sync_points.size()); | |
159 | ASSERT_EQ(0, m_remote_image_ctx->state->refresh()); | |
160 | ASSERT_EQ(0U, m_remote_image_ctx->snap_ids.size()); | |
161 | ASSERT_EQ(0, m_local_image_ctx->state->refresh()); | |
162 | ASSERT_EQ(1U, m_local_image_ctx->snap_ids.size()); // deleted on journal replay | |
163 | } | |
164 | ||
165 | TEST_F(TestImageSync, Simple) { | |
166 | scribble(m_remote_image_ctx, 10, 102400); | |
167 | ||
168 | C_SaferCond ctx; | |
169 | ImageSync<> *request = create_request(&ctx); | |
170 | request->send(); | |
171 | ASSERT_EQ(0, ctx.wait()); | |
172 | ||
173 | int64_t object_size = std::min<int64_t>( | |
174 | m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); | |
175 | bufferlist read_remote_bl; | |
176 | read_remote_bl.append(std::string(object_size, '1')); | |
177 | bufferlist read_local_bl; | |
178 | read_local_bl.append(std::string(object_size, '1')); | |
179 | ||
180 | for (uint64_t offset = 0; offset < m_remote_image_ctx->size; | |
181 | offset += object_size) { | |
f67539c2 TL |
182 | ASSERT_LE(0, librbd::api::Io<>::read( |
183 | *m_remote_image_ctx, offset, object_size, | |
7c673cae | 184 | librbd::io::ReadResult{&read_remote_bl}, 0)); |
f67539c2 TL |
185 | ASSERT_LE(0, librbd::api::Io<>::read( |
186 | *m_local_image_ctx, offset, object_size, | |
7c673cae FG |
187 | librbd::io::ReadResult{&read_local_bl}, 0)); |
188 | ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); | |
189 | } | |
190 | } | |
191 | ||
192 | TEST_F(TestImageSync, Resize) { | |
193 | int64_t object_size = std::min<int64_t>( | |
194 | m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); | |
195 | ||
196 | uint64_t off = 0; | |
197 | uint64_t len = object_size / 10; | |
198 | ||
199 | bufferlist bl; | |
200 | bl.append(std::string(len, '1')); | |
f67539c2 TL |
201 | ASSERT_EQ((int)len, librbd::api::Io<>::write( |
202 | *m_remote_image_ctx, off, len, std::move(bl), 0)); | |
7c673cae | 203 | { |
9f95a23c | 204 | std::shared_lock owner_locker{m_remote_image_ctx->owner_lock}; |
11fdf7f2 | 205 | ASSERT_EQ(0, flush(m_remote_image_ctx)); |
7c673cae FG |
206 | } |
207 | ||
208 | ASSERT_EQ(0, create_snap(m_remote_image_ctx, "snap", nullptr)); | |
209 | ||
210 | uint64_t size = object_size - 1; | |
211 | librbd::NoOpProgressContext no_op_progress_ctx; | |
212 | ASSERT_EQ(0, m_remote_image_ctx->operations->resize(size, true, | |
213 | no_op_progress_ctx)); | |
214 | ||
215 | C_SaferCond ctx; | |
216 | ImageSync<> *request = create_request(&ctx); | |
217 | request->send(); | |
218 | ASSERT_EQ(0, ctx.wait()); | |
219 | ||
220 | bufferlist read_remote_bl; | |
221 | read_remote_bl.append(std::string(len, '\0')); | |
222 | bufferlist read_local_bl; | |
223 | read_local_bl.append(std::string(len, '\0')); | |
224 | ||
f67539c2 TL |
225 | ASSERT_LE(0, librbd::api::Io<>::read( |
226 | *m_remote_image_ctx, off, len, | |
227 | librbd::io::ReadResult{&read_remote_bl}, 0)); | |
228 | ASSERT_LE(0, librbd::api::Io<>::read( | |
229 | *m_local_image_ctx, off, len, | |
230 | librbd::io::ReadResult{&read_local_bl}, 0)); | |
7c673cae FG |
231 | |
232 | ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); | |
233 | } | |
234 | ||
235 | TEST_F(TestImageSync, Discard) { | |
236 | int64_t object_size = std::min<int64_t>( | |
237 | m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); | |
238 | ||
239 | uint64_t off = 0; | |
240 | uint64_t len = object_size / 10; | |
241 | ||
242 | bufferlist bl; | |
243 | bl.append(std::string(len, '1')); | |
f67539c2 TL |
244 | ASSERT_EQ((int)len, librbd::api::Io<>::write( |
245 | *m_remote_image_ctx, off, len, std::move(bl), 0)); | |
7c673cae | 246 | { |
9f95a23c | 247 | std::shared_lock owner_locker{m_remote_image_ctx->owner_lock}; |
11fdf7f2 | 248 | ASSERT_EQ(0, flush(m_remote_image_ctx)); |
7c673cae FG |
249 | } |
250 | ||
251 | ASSERT_EQ(0, create_snap(m_remote_image_ctx, "snap", nullptr)); | |
252 | ||
11fdf7f2 | 253 | ASSERT_EQ((int)len - 2, |
f67539c2 TL |
254 | librbd::api::Io<>::discard( |
255 | *m_remote_image_ctx, off + 1, len - 2, | |
256 | m_remote_image_ctx->discard_granularity_bytes)); | |
7c673cae | 257 | { |
9f95a23c | 258 | std::shared_lock owner_locker{m_remote_image_ctx->owner_lock}; |
11fdf7f2 | 259 | ASSERT_EQ(0, flush(m_remote_image_ctx)); |
7c673cae FG |
260 | } |
261 | ||
262 | C_SaferCond ctx; | |
263 | ImageSync<> *request = create_request(&ctx); | |
264 | request->send(); | |
265 | ASSERT_EQ(0, ctx.wait()); | |
266 | ||
267 | bufferlist read_remote_bl; | |
268 | read_remote_bl.append(std::string(object_size, '\0')); | |
269 | bufferlist read_local_bl; | |
270 | read_local_bl.append(std::string(object_size, '\0')); | |
271 | ||
f67539c2 TL |
272 | ASSERT_LE(0, librbd::api::Io<>::read( |
273 | *m_remote_image_ctx, off, len, | |
274 | librbd::io::ReadResult{&read_remote_bl}, 0)); | |
275 | ASSERT_LE(0, librbd::api::Io<>::read( | |
276 | *m_local_image_ctx, off, len, | |
277 | librbd::io::ReadResult{&read_local_bl}, 0)); | |
7c673cae FG |
278 | |
279 | ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); | |
280 | } | |
281 | ||
282 | TEST_F(TestImageSync, SnapshotStress) { | |
283 | std::list<std::string> snap_names; | |
284 | ||
285 | const int num_snaps = 4; | |
286 | for (int idx = 0; idx <= num_snaps; ++idx) { | |
287 | scribble(m_remote_image_ctx, 10, 102400); | |
288 | ||
289 | librbd::NoOpProgressContext no_op_progress_ctx; | |
290 | uint64_t size = 1 + rand() % m_image_size; | |
291 | ASSERT_EQ(0, m_remote_image_ctx->operations->resize(size, true, | |
292 | no_op_progress_ctx)); | |
293 | ASSERT_EQ(0, m_remote_image_ctx->state->refresh()); | |
294 | ||
295 | if (idx < num_snaps) { | |
296 | snap_names.push_back("snap" + stringify(idx + 1)); | |
297 | ASSERT_EQ(0, create_snap(m_remote_image_ctx, snap_names.back().c_str(), | |
298 | nullptr)); | |
299 | } else { | |
300 | snap_names.push_back(""); | |
301 | } | |
302 | } | |
303 | ||
304 | C_SaferCond ctx; | |
305 | ImageSync<> *request = create_request(&ctx); | |
306 | request->send(); | |
307 | ASSERT_EQ(0, ctx.wait()); | |
308 | ||
309 | int64_t object_size = std::min<int64_t>( | |
310 | m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); | |
311 | bufferlist read_remote_bl; | |
312 | read_remote_bl.append(std::string(object_size, '1')); | |
313 | bufferlist read_local_bl; | |
314 | read_local_bl.append(std::string(object_size, '1')); | |
315 | ||
316 | for (auto &snap_name : snap_names) { | |
11fdf7f2 TL |
317 | uint64_t remote_snap_id; |
318 | { | |
9f95a23c | 319 | std::shared_lock remote_image_locker{m_remote_image_ctx->image_lock}; |
11fdf7f2 TL |
320 | remote_snap_id = m_remote_image_ctx->get_snap_id( |
321 | cls::rbd::UserSnapshotNamespace{}, snap_name); | |
322 | } | |
323 | ||
7c673cae FG |
324 | uint64_t remote_size; |
325 | { | |
326 | C_SaferCond ctx; | |
11fdf7f2 | 327 | m_remote_image_ctx->state->snap_set(remote_snap_id, &ctx); |
7c673cae FG |
328 | ASSERT_EQ(0, ctx.wait()); |
329 | ||
9f95a23c | 330 | std::shared_lock remote_image_locker{m_remote_image_ctx->image_lock}; |
7c673cae FG |
331 | remote_size = m_remote_image_ctx->get_image_size( |
332 | m_remote_image_ctx->snap_id); | |
333 | } | |
334 | ||
11fdf7f2 TL |
335 | uint64_t local_snap_id; |
336 | { | |
9f95a23c | 337 | std::shared_lock image_locker{m_local_image_ctx->image_lock}; |
11fdf7f2 TL |
338 | local_snap_id = m_local_image_ctx->get_snap_id( |
339 | cls::rbd::UserSnapshotNamespace{}, snap_name); | |
340 | } | |
341 | ||
7c673cae FG |
342 | uint64_t local_size; |
343 | { | |
344 | C_SaferCond ctx; | |
11fdf7f2 | 345 | m_local_image_ctx->state->snap_set(local_snap_id, &ctx); |
7c673cae FG |
346 | ASSERT_EQ(0, ctx.wait()); |
347 | ||
9f95a23c | 348 | std::shared_lock image_locker{m_local_image_ctx->image_lock}; |
7c673cae FG |
349 | local_size = m_local_image_ctx->get_image_size( |
350 | m_local_image_ctx->snap_id); | |
31f18b77 | 351 | bool flags_set; |
91327a77 AA |
352 | ASSERT_EQ(0, m_local_image_ctx->test_flags(m_local_image_ctx->snap_id, |
353 | RBD_FLAG_OBJECT_MAP_INVALID, | |
9f95a23c | 354 | m_local_image_ctx->image_lock, |
31f18b77 FG |
355 | &flags_set)); |
356 | ASSERT_FALSE(flags_set); | |
7c673cae FG |
357 | } |
358 | ||
359 | ASSERT_EQ(remote_size, local_size); | |
360 | ||
361 | for (uint64_t offset = 0; offset < remote_size; offset += object_size) { | |
f67539c2 TL |
362 | ASSERT_LE(0, librbd::api::Io<>::read( |
363 | *m_remote_image_ctx, offset, object_size, | |
7c673cae | 364 | librbd::io::ReadResult{&read_remote_bl}, 0)); |
f67539c2 TL |
365 | ASSERT_LE(0, librbd::api::Io<>::read( |
366 | *m_local_image_ctx, offset, object_size, | |
7c673cae FG |
367 | librbd::io::ReadResult{&read_local_bl}, 0)); |
368 | ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); | |
369 | } | |
370 | } | |
371 | } | |
372 | ||
373 | } // namespace mirror | |
374 | } // namespace rbd |