]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph distributed storage system | |
5 | * | |
6 | * Copyright (C) 2016 Mirantis Inc | |
7 | * | |
8 | * Author: Mykola Golub <mgolub@mirantis.com> | |
9 | * | |
10 | * This library is free software; you can redistribute it and/or | |
11 | * modify it under the terms of the GNU Lesser General Public | |
12 | * License as published by the Free Software Foundation; either | |
13 | * version 2.1 of the License, or (at your option) any later version. | |
14 | * | |
15 | */ | |
16 | ||
17 | #include "include/rados/librados.hpp" | |
18 | #include "include/rbd/librbd.hpp" | |
19 | #include "include/stringify.h" | |
20 | #include "test/rbd_mirror/test_fixture.h" | |
21 | #include "cls/journal/cls_journal_types.h" | |
22 | #include "cls/journal/cls_journal_client.h" | |
23 | #include "cls/rbd/cls_rbd_types.h" | |
24 | #include "cls/rbd/cls_rbd_client.h" | |
25 | #include "journal/Journaler.h" | |
26 | #include "librbd/ExclusiveLock.h" | |
27 | #include "librbd/ImageCtx.h" | |
28 | #include "librbd/ImageState.h" | |
29 | #include "librbd/Journal.h" | |
30 | #include "librbd/Operations.h" | |
31 | #include "librbd/Utils.h" | |
32 | #include "librbd/internal.h" | |
33 | #include "librbd/api/Mirror.h" | |
34 | #include "librbd/io/AioCompletion.h" | |
35 | #include "librbd/io/ImageRequestWQ.h" | |
36 | #include "librbd/io/ReadResult.h" | |
37 | #include "tools/rbd_mirror/types.h" | |
c07f9fc5 | 38 | #include "tools/rbd_mirror/ImageDeleter.h" |
7c673cae | 39 | #include "tools/rbd_mirror/ImageReplayer.h" |
31f18b77 | 40 | #include "tools/rbd_mirror/InstanceWatcher.h" |
c07f9fc5 | 41 | #include "tools/rbd_mirror/ServiceDaemon.h" |
7c673cae | 42 | #include "tools/rbd_mirror/Threads.h" |
7c673cae FG |
43 | |
44 | #include "test/librados/test.h" | |
45 | #include "gtest/gtest.h" | |
46 | ||
47 | using rbd::mirror::RadosRef; | |
48 | ||
49 | void register_test_rbd_mirror() { | |
50 | } | |
51 | ||
52 | #define TEST_IO_SIZE 512 | |
53 | #define TEST_IO_COUNT 11 | |
54 | ||
55 | class TestImageReplayer : public ::rbd::mirror::TestFixture { | |
56 | public: | |
57 | struct C_WatchCtx : public librados::WatchCtx2 { | |
58 | TestImageReplayer *test; | |
59 | std::string oid; | |
60 | Mutex lock; | |
61 | Cond cond; | |
62 | bool notified; | |
63 | ||
64 | C_WatchCtx(TestImageReplayer *test, const std::string &oid) | |
65 | : test(test), oid(oid), lock("C_WatchCtx::lock"), notified(false) { | |
66 | } | |
67 | ||
68 | void handle_notify(uint64_t notify_id, uint64_t cookie, | |
69 | uint64_t notifier_id, bufferlist& bl_) override { | |
70 | bufferlist bl; | |
71 | test->m_remote_ioctx.notify_ack(oid, notify_id, cookie, bl); | |
72 | ||
73 | Mutex::Locker locker(lock); | |
74 | notified = true; | |
75 | cond.Signal(); | |
76 | } | |
77 | ||
78 | void handle_error(uint64_t cookie, int err) override { | |
79 | ASSERT_EQ(0, err); | |
80 | } | |
81 | }; | |
82 | ||
83 | TestImageReplayer() | |
84 | : m_local_cluster(new librados::Rados()), m_watch_handle(0) | |
85 | { | |
28e407b8 AA |
86 | EXPECT_EQ(0, g_ceph_context->_conf->set_val("rbd_mirror_journal_commit_age", |
87 | "0.1")); | |
88 | ||
7c673cae FG |
89 | EXPECT_EQ("", connect_cluster_pp(*m_local_cluster.get())); |
90 | EXPECT_EQ(0, m_local_cluster->conf_set("rbd_cache", "false")); | |
91 | EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_poll_age", "1")); | |
92 | ||
93 | m_local_pool_name = get_temp_pool_name(); | |
94 | EXPECT_EQ(0, m_local_cluster->pool_create(m_local_pool_name.c_str())); | |
95 | EXPECT_EQ(0, m_local_cluster->ioctx_create(m_local_pool_name.c_str(), | |
96 | m_local_ioctx)); | |
c07f9fc5 | 97 | m_local_ioctx.application_enable("rbd", true); |
7c673cae FG |
98 | |
99 | EXPECT_EQ("", connect_cluster_pp(m_remote_cluster)); | |
100 | EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false")); | |
101 | ||
102 | m_remote_pool_name = get_temp_pool_name(); | |
103 | EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str())); | |
104 | m_remote_pool_id = m_remote_cluster.pool_lookup(m_remote_pool_name.c_str()); | |
105 | EXPECT_GE(m_remote_pool_id, 0); | |
106 | ||
107 | EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(), | |
108 | m_remote_ioctx)); | |
c07f9fc5 FG |
109 | m_remote_ioctx.application_enable("rbd", true); |
110 | ||
7c673cae FG |
111 | EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, |
112 | RBD_MIRROR_MODE_POOL)); | |
113 | ||
114 | m_image_name = get_temp_image_name(); | |
115 | uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); | |
116 | features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING; | |
117 | int order = 0; | |
118 | EXPECT_EQ(0, librbd::create(m_remote_ioctx, m_image_name.c_str(), 1 << 22, | |
119 | false, features, &order, 0, 0)); | |
120 | m_remote_image_id = get_image_id(m_remote_ioctx, m_image_name); | |
d2e6a577 | 121 | m_global_image_id = get_global_image_id(m_remote_ioctx, m_remote_image_id); |
7c673cae | 122 | |
c07f9fc5 FG |
123 | m_threads.reset(new rbd::mirror::Threads<>(reinterpret_cast<CephContext*>( |
124 | m_local_ioctx.cct()))); | |
7c673cae | 125 | |
c07f9fc5 FG |
126 | m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, |
127 | m_local_cluster, | |
128 | m_threads.get())); | |
129 | m_image_deleter.reset(new rbd::mirror::ImageDeleter<>( | |
130 | m_threads->work_queue, m_threads->timer, &m_threads->timer_lock, | |
131 | m_service_daemon.get())); | |
31f18b77 FG |
132 | m_instance_watcher = rbd::mirror::InstanceWatcher<>::create( |
133 | m_local_ioctx, m_threads->work_queue, nullptr); | |
134 | m_instance_watcher->handle_acquire_leader(); | |
7c673cae FG |
135 | } |
136 | ||
31f18b77 | 137 | ~TestImageReplayer() override |
7c673cae FG |
138 | { |
139 | unwatch(); | |
140 | ||
31f18b77 FG |
141 | m_instance_watcher->handle_release_leader(); |
142 | ||
7c673cae | 143 | delete m_replayer; |
31f18b77 | 144 | delete m_instance_watcher; |
7c673cae FG |
145 | |
146 | EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str())); | |
147 | EXPECT_EQ(0, m_local_cluster->pool_delete(m_local_pool_name.c_str())); | |
28e407b8 AA |
148 | EXPECT_EQ(0, g_ceph_context->_conf->set_val("rbd_mirror_journal_commit_age", |
149 | "5")); | |
7c673cae FG |
150 | } |
151 | ||
152 | template <typename ImageReplayerT = rbd::mirror::ImageReplayer<> > | |
153 | void create_replayer() { | |
31f18b77 | 154 | m_replayer = new ImageReplayerT( |
c07f9fc5 | 155 | m_threads.get(), m_image_deleter.get(), m_instance_watcher, |
31f18b77 | 156 | rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)), |
d2e6a577 FG |
157 | m_local_mirror_uuid, m_local_ioctx.get_id(), m_global_image_id); |
158 | m_replayer->add_peer("peer uuid", m_remote_ioctx); | |
7c673cae FG |
159 | } |
160 | ||
161 | void start() | |
162 | { | |
163 | C_SaferCond cond; | |
164 | m_replayer->start(&cond); | |
165 | ASSERT_EQ(0, cond.wait()); | |
166 | ||
167 | ASSERT_EQ(0U, m_watch_handle); | |
168 | std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); | |
169 | m_watch_ctx = new C_WatchCtx(this, oid); | |
170 | ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx)); | |
171 | } | |
172 | ||
173 | void unwatch() { | |
174 | if (m_watch_handle != 0) { | |
175 | m_remote_ioctx.unwatch2(m_watch_handle); | |
176 | delete m_watch_ctx; | |
177 | m_watch_ctx = nullptr; | |
178 | m_watch_handle = 0; | |
179 | } | |
180 | } | |
181 | ||
182 | void stop() | |
183 | { | |
184 | unwatch(); | |
185 | ||
186 | C_SaferCond cond; | |
187 | m_replayer->stop(&cond); | |
188 | ASSERT_EQ(0, cond.wait()); | |
189 | } | |
190 | ||
191 | void bootstrap() | |
192 | { | |
193 | create_replayer<>(); | |
194 | ||
195 | start(); | |
196 | wait_for_replay_complete(); | |
197 | stop(); | |
198 | } | |
199 | ||
200 | std::string get_temp_image_name() | |
201 | { | |
202 | return "image" + stringify(++_image_number); | |
203 | } | |
204 | ||
205 | std::string get_image_id(librados::IoCtx &ioctx, const string &image_name) | |
206 | { | |
207 | std::string obj = librbd::util::id_obj_name(image_name); | |
208 | std::string id; | |
209 | EXPECT_EQ(0, librbd::cls_client::get_id(&ioctx, obj, &id)); | |
210 | return id; | |
211 | } | |
212 | ||
d2e6a577 FG |
213 | std::string get_global_image_id(librados::IoCtx& io_ctx, |
214 | const std::string& image_id) { | |
215 | cls::rbd::MirrorImage mirror_image; | |
216 | EXPECT_EQ(0, librbd::cls_client::mirror_image_get(&io_ctx, image_id, | |
217 | &mirror_image)); | |
218 | return mirror_image.global_image_id; | |
219 | } | |
220 | ||
7c673cae FG |
221 | void open_image(librados::IoCtx &ioctx, const std::string &image_name, |
222 | bool readonly, librbd::ImageCtx **ictxp) | |
223 | { | |
224 | librbd::ImageCtx *ictx = new librbd::ImageCtx(image_name.c_str(), | |
225 | "", "", ioctx, readonly); | |
226 | EXPECT_EQ(0, ictx->state->open(false)); | |
227 | *ictxp = ictx; | |
228 | } | |
229 | ||
230 | void open_local_image(librbd::ImageCtx **ictxp) | |
231 | { | |
232 | open_image(m_local_ioctx, m_image_name, true, ictxp); | |
233 | } | |
234 | ||
235 | void open_remote_image(librbd::ImageCtx **ictxp) | |
236 | { | |
237 | open_image(m_remote_ioctx, m_image_name, false, ictxp); | |
238 | } | |
239 | ||
240 | void close_image(librbd::ImageCtx *ictx) | |
241 | { | |
242 | ictx->state->close(); | |
243 | } | |
244 | ||
245 | void get_commit_positions(cls::journal::ObjectPosition *master_position, | |
246 | cls::journal::ObjectPosition *mirror_position) | |
247 | { | |
248 | std::string master_client_id = ""; | |
249 | std::string mirror_client_id = m_local_mirror_uuid; | |
250 | ||
251 | C_SaferCond cond; | |
252 | uint64_t minimum_set; | |
253 | uint64_t active_set; | |
254 | std::set<cls::journal::Client> registered_clients; | |
255 | std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); | |
256 | cls::journal::client::get_mutable_metadata(m_remote_ioctx, oid, | |
257 | &minimum_set, &active_set, | |
258 | ®istered_clients, &cond); | |
259 | ASSERT_EQ(0, cond.wait()); | |
260 | ||
261 | *master_position = cls::journal::ObjectPosition(); | |
262 | *mirror_position = cls::journal::ObjectPosition(); | |
263 | ||
264 | std::set<cls::journal::Client>::const_iterator c; | |
265 | for (c = registered_clients.begin(); c != registered_clients.end(); ++c) { | |
266 | std::cout << __func__ << ": client: " << *c << std::endl; | |
267 | if (c->state != cls::journal::CLIENT_STATE_CONNECTED) { | |
268 | continue; | |
269 | } | |
270 | cls::journal::ObjectPositions object_positions = | |
271 | c->commit_position.object_positions; | |
272 | cls::journal::ObjectPositions::const_iterator p = | |
273 | object_positions.begin(); | |
274 | if (p != object_positions.end()) { | |
275 | if (c->id == master_client_id) { | |
276 | ASSERT_EQ(cls::journal::ObjectPosition(), *master_position); | |
277 | *master_position = *p; | |
278 | } else if (c->id == mirror_client_id) { | |
279 | ASSERT_EQ(cls::journal::ObjectPosition(), *mirror_position); | |
280 | *mirror_position = *p; | |
281 | } | |
282 | } | |
283 | } | |
284 | } | |
285 | ||
286 | bool wait_for_watcher_notify(int seconds) | |
287 | { | |
288 | if (m_watch_handle == 0) { | |
289 | return false; | |
290 | } | |
291 | ||
292 | Mutex::Locker locker(m_watch_ctx->lock); | |
293 | while (!m_watch_ctx->notified) { | |
294 | if (m_watch_ctx->cond.WaitInterval(m_watch_ctx->lock, | |
295 | utime_t(seconds, 0)) != 0) { | |
296 | return false; | |
297 | } | |
298 | } | |
299 | m_watch_ctx->notified = false; | |
300 | return true; | |
301 | } | |
302 | ||
303 | void wait_for_replay_complete() | |
304 | { | |
305 | cls::journal::ObjectPosition master_position; | |
306 | cls::journal::ObjectPosition mirror_position; | |
307 | ||
308 | for (int i = 0; i < 100; i++) { | |
7c673cae FG |
309 | get_commit_positions(&master_position, &mirror_position); |
310 | if (master_position == mirror_position) { | |
311 | break; | |
312 | } | |
313 | wait_for_watcher_notify(1); | |
314 | } | |
315 | ||
316 | ASSERT_EQ(master_position, mirror_position); | |
317 | } | |
318 | ||
319 | void wait_for_stopped() { | |
320 | for (int i = 0; i < 100; i++) { | |
321 | if (m_replayer->is_stopped()) { | |
322 | break; | |
323 | } | |
324 | wait_for_watcher_notify(1); | |
325 | } | |
326 | ASSERT_TRUE(m_replayer->is_stopped()); | |
327 | } | |
328 | ||
329 | void write_test_data(librbd::ImageCtx *ictx, const char *test_data, off_t off, | |
330 | size_t len) | |
331 | { | |
332 | size_t written; | |
333 | bufferlist bl; | |
334 | bl.append(std::string(test_data, len)); | |
335 | written = ictx->io_work_queue->write(off, len, std::move(bl), 0); | |
336 | printf("wrote: %d\n", (int)written); | |
337 | ASSERT_EQ(len, written); | |
338 | } | |
339 | ||
340 | void read_test_data(librbd::ImageCtx *ictx, const char *expected, off_t off, | |
341 | size_t len) | |
342 | { | |
343 | ssize_t read; | |
344 | char *result = (char *)malloc(len + 1); | |
345 | ||
346 | ASSERT_NE(static_cast<char *>(NULL), result); | |
347 | read = ictx->io_work_queue->read( | |
348 | off, len, librbd::io::ReadResult{result, len}, 0); | |
349 | printf("read: %d\n", (int)read); | |
350 | ASSERT_EQ(len, static_cast<size_t>(read)); | |
351 | result[len] = '\0'; | |
352 | if (memcmp(result, expected, len)) { | |
353 | printf("read: %s\nexpected: %s\n", result, expected); | |
354 | ASSERT_EQ(0, memcmp(result, expected, len)); | |
355 | } | |
356 | free(result); | |
357 | } | |
358 | ||
359 | void generate_test_data() { | |
360 | for (int i = 0; i < TEST_IO_SIZE; ++i) { | |
361 | m_test_data[i] = (char) (rand() % (126 - 33) + 33); | |
362 | } | |
363 | m_test_data[TEST_IO_SIZE] = '\0'; | |
364 | } | |
365 | ||
366 | void flush(librbd::ImageCtx *ictx) | |
367 | { | |
368 | C_SaferCond aio_flush_ctx; | |
369 | auto c = librbd::io::AioCompletion::create(&aio_flush_ctx); | |
370 | c->get(); | |
371 | ictx->io_work_queue->aio_flush(c); | |
372 | ASSERT_EQ(0, c->wait_for_complete()); | |
373 | c->put(); | |
374 | ||
375 | C_SaferCond journal_flush_ctx; | |
376 | ictx->journal->flush_commit_position(&journal_flush_ctx); | |
377 | ASSERT_EQ(0, journal_flush_ctx.wait()); | |
378 | ||
379 | printf("flushed\n"); | |
380 | } | |
381 | ||
382 | static int _image_number; | |
383 | ||
7c673cae | 384 | std::shared_ptr<librados::Rados> m_local_cluster; |
c07f9fc5 FG |
385 | std::unique_ptr<rbd::mirror::Threads<>> m_threads; |
386 | std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon; | |
387 | std::unique_ptr<rbd::mirror::ImageDeleter<>> m_image_deleter; | |
7c673cae | 388 | librados::Rados m_remote_cluster; |
31f18b77 | 389 | rbd::mirror::InstanceWatcher<> *m_instance_watcher; |
7c673cae FG |
390 | std::string m_local_mirror_uuid = "local mirror uuid"; |
391 | std::string m_remote_mirror_uuid = "remote mirror uuid"; | |
392 | std::string m_local_pool_name, m_remote_pool_name; | |
393 | librados::IoCtx m_local_ioctx, m_remote_ioctx; | |
394 | std::string m_image_name; | |
395 | int64_t m_remote_pool_id; | |
396 | std::string m_remote_image_id; | |
d2e6a577 | 397 | std::string m_global_image_id; |
7c673cae FG |
398 | rbd::mirror::ImageReplayer<> *m_replayer; |
399 | C_WatchCtx *m_watch_ctx; | |
400 | uint64_t m_watch_handle; | |
401 | char m_test_data[TEST_IO_SIZE + 1]; | |
402 | }; | |
403 | ||
404 | int TestImageReplayer::_image_number; | |
405 | ||
406 | TEST_F(TestImageReplayer, Bootstrap) | |
407 | { | |
408 | bootstrap(); | |
409 | } | |
410 | ||
411 | TEST_F(TestImageReplayer, BootstrapErrorLocalImageExists) | |
412 | { | |
413 | int order = 0; | |
414 | EXPECT_EQ(0, librbd::create(m_local_ioctx, m_image_name.c_str(), 1 << 22, | |
415 | false, 0, &order, 0, 0)); | |
416 | ||
417 | create_replayer<>(); | |
418 | C_SaferCond cond; | |
419 | m_replayer->start(&cond); | |
420 | ASSERT_EQ(-EEXIST, cond.wait()); | |
421 | } | |
422 | ||
423 | TEST_F(TestImageReplayer, BootstrapErrorNoJournal) | |
424 | { | |
d2e6a577 | 425 | ASSERT_EQ(0, librbd::Journal<>::remove(m_remote_ioctx, m_remote_image_id)); |
7c673cae FG |
426 | |
427 | create_replayer<>(); | |
428 | C_SaferCond cond; | |
429 | m_replayer->start(&cond); | |
430 | ASSERT_EQ(-ENOENT, cond.wait()); | |
431 | } | |
432 | ||
433 | TEST_F(TestImageReplayer, BootstrapErrorMirrorDisabled) | |
434 | { | |
435 | // disable remote image mirroring | |
436 | ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, | |
437 | RBD_MIRROR_MODE_IMAGE)); | |
438 | librbd::ImageCtx *ictx; | |
439 | open_remote_image(&ictx); | |
440 | ASSERT_EQ(0, librbd::api::Mirror<>::image_disable(ictx, true)); | |
441 | close_image(ictx); | |
442 | ||
443 | create_replayer<>(); | |
444 | C_SaferCond cond; | |
445 | m_replayer->start(&cond); | |
446 | ASSERT_EQ(-ENOENT, cond.wait()); | |
447 | } | |
448 | ||
449 | TEST_F(TestImageReplayer, BootstrapMirrorDisabling) | |
450 | { | |
451 | // set remote image mirroring state to DISABLING | |
452 | ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, | |
453 | RBD_MIRROR_MODE_IMAGE)); | |
454 | librbd::ImageCtx *ictx; | |
455 | open_remote_image(&ictx); | |
456 | ASSERT_EQ(0, librbd::api::Mirror<>::image_enable(ictx, false)); | |
457 | cls::rbd::MirrorImage mirror_image; | |
458 | ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&m_remote_ioctx, ictx->id, | |
459 | &mirror_image)); | |
460 | mirror_image.state = cls::rbd::MirrorImageState::MIRROR_IMAGE_STATE_DISABLING; | |
461 | ASSERT_EQ(0, librbd::cls_client::mirror_image_set(&m_remote_ioctx, ictx->id, | |
462 | mirror_image)); | |
463 | close_image(ictx); | |
464 | ||
465 | create_replayer<>(); | |
466 | C_SaferCond cond; | |
467 | m_replayer->start(&cond); | |
c07f9fc5 | 468 | ASSERT_EQ(-EREMOTEIO, cond.wait()); |
7c673cae FG |
469 | ASSERT_TRUE(m_replayer->is_stopped()); |
470 | } | |
471 | ||
472 | TEST_F(TestImageReplayer, BootstrapDemoted) | |
473 | { | |
474 | // demote remote image | |
475 | librbd::ImageCtx *ictx; | |
476 | open_remote_image(&ictx); | |
477 | ASSERT_EQ(0, librbd::api::Mirror<>::image_demote(ictx)); | |
478 | close_image(ictx); | |
479 | ||
480 | create_replayer<>(); | |
481 | C_SaferCond cond; | |
482 | m_replayer->start(&cond); | |
c07f9fc5 | 483 | ASSERT_EQ(-EREMOTEIO, cond.wait()); |
7c673cae FG |
484 | ASSERT_TRUE(m_replayer->is_stopped()); |
485 | } | |
486 | ||
487 | TEST_F(TestImageReplayer, StartInterrupted) | |
488 | { | |
489 | create_replayer<>(); | |
490 | C_SaferCond start_cond, stop_cond; | |
491 | m_replayer->start(&start_cond); | |
492 | m_replayer->stop(&stop_cond); | |
493 | int r = start_cond.wait(); | |
494 | printf("start returned %d\n", r); | |
495 | // TODO: improve the test to avoid this race | |
496 | ASSERT_TRUE(r == -ECANCELED || r == 0); | |
497 | ASSERT_EQ(0, stop_cond.wait()); | |
498 | } | |
499 | ||
500 | TEST_F(TestImageReplayer, JournalReset) | |
501 | { | |
502 | bootstrap(); | |
31f18b77 FG |
503 | delete m_replayer; |
504 | ||
7c673cae | 505 | ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id)); |
31f18b77 | 506 | |
7c673cae FG |
507 | // try to recover |
508 | bootstrap(); | |
509 | } | |
510 | ||
511 | TEST_F(TestImageReplayer, ErrorNoJournal) | |
512 | { | |
513 | bootstrap(); | |
514 | ||
515 | // disable remote journal journaling | |
516 | // (reset before disabling, so it does not fail with EBUSY) | |
517 | ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id)); | |
518 | librbd::ImageCtx *ictx; | |
519 | open_remote_image(&ictx); | |
520 | uint64_t features; | |
521 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
522 | ASSERT_EQ(0, ictx->operations->update_features(RBD_FEATURE_JOURNALING, | |
523 | false)); | |
524 | close_image(ictx); | |
525 | ||
526 | C_SaferCond cond; | |
527 | m_replayer->start(&cond); | |
d2e6a577 | 528 | ASSERT_EQ(0, cond.wait()); |
7c673cae FG |
529 | } |
530 | ||
531 | TEST_F(TestImageReplayer, StartStop) | |
532 | { | |
533 | bootstrap(); | |
534 | ||
535 | start(); | |
536 | wait_for_replay_complete(); | |
537 | stop(); | |
538 | } | |
539 | ||
540 | TEST_F(TestImageReplayer, WriteAndStartReplay) | |
541 | { | |
542 | bootstrap(); | |
543 | ||
544 | // Write to remote image and start replay | |
545 | ||
546 | librbd::ImageCtx *ictx; | |
547 | ||
548 | generate_test_data(); | |
549 | open_remote_image(&ictx); | |
550 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
551 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
552 | } | |
553 | flush(ictx); | |
554 | close_image(ictx); | |
555 | ||
556 | start(); | |
557 | wait_for_replay_complete(); | |
558 | stop(); | |
559 | ||
560 | open_local_image(&ictx); | |
561 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
562 | read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
563 | } | |
564 | close_image(ictx); | |
565 | } | |
566 | ||
567 | TEST_F(TestImageReplayer, StartReplayAndWrite) | |
568 | { | |
569 | bootstrap(); | |
570 | ||
571 | // Start replay and write to remote image | |
572 | ||
573 | librbd::ImageCtx *ictx; | |
574 | ||
575 | start(); | |
576 | ||
577 | generate_test_data(); | |
578 | open_remote_image(&ictx); | |
579 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
580 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
581 | } | |
582 | flush(ictx); | |
583 | ||
584 | wait_for_replay_complete(); | |
585 | ||
586 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
587 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
588 | } | |
589 | flush(ictx); | |
590 | close_image(ictx); | |
591 | ||
592 | wait_for_replay_complete(); | |
593 | ||
594 | open_local_image(&ictx); | |
595 | for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { | |
596 | read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
597 | } | |
598 | close_image(ictx); | |
599 | ||
600 | stop(); | |
601 | } | |
602 | ||
603 | TEST_F(TestImageReplayer, NextTag) | |
604 | { | |
605 | bootstrap(); | |
606 | ||
607 | // write, reopen, and write again to test switch to the next tag | |
608 | ||
609 | librbd::ImageCtx *ictx; | |
610 | ||
611 | start(); | |
612 | ||
613 | generate_test_data(); | |
614 | ||
615 | const int N = 10; | |
616 | ||
617 | for (int j = 0; j < N; j++) { | |
618 | open_remote_image(&ictx); | |
619 | for (int i = j * TEST_IO_COUNT; i < (j + 1) * TEST_IO_COUNT; ++i) { | |
620 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
621 | } | |
622 | close_image(ictx); | |
623 | } | |
624 | ||
625 | wait_for_replay_complete(); | |
626 | ||
627 | open_local_image(&ictx); | |
628 | for (int i = 0; i < N * TEST_IO_COUNT; ++i) { | |
629 | read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
630 | } | |
631 | close_image(ictx); | |
632 | ||
633 | stop(); | |
634 | } | |
635 | ||
636 | TEST_F(TestImageReplayer, Resync) | |
637 | { | |
638 | bootstrap(); | |
639 | ||
640 | librbd::ImageCtx *ictx; | |
641 | ||
642 | start(); | |
643 | ||
644 | generate_test_data(); | |
645 | ||
646 | open_remote_image(&ictx); | |
647 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
648 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
649 | } | |
650 | flush(ictx); | |
651 | ||
652 | wait_for_replay_complete(); | |
653 | ||
654 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
655 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
656 | } | |
657 | flush(ictx); | |
658 | close_image(ictx); | |
659 | ||
660 | C_SaferCond ctx; | |
661 | m_replayer->resync_image(&ctx); | |
662 | ASSERT_EQ(0, ctx.wait()); | |
663 | ||
664 | C_SaferCond delete_ctx; | |
665 | m_image_deleter->wait_for_scheduled_deletion( | |
666 | m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx); | |
667 | EXPECT_EQ(0, delete_ctx.wait()); | |
668 | ||
669 | C_SaferCond cond; | |
670 | m_replayer->start(&cond); | |
671 | ASSERT_EQ(0, cond.wait()); | |
672 | ||
673 | ASSERT_TRUE(m_replayer->is_replaying()); | |
674 | ||
675 | wait_for_replay_complete(); | |
676 | ||
677 | open_local_image(&ictx); | |
678 | for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { | |
679 | read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
680 | } | |
681 | close_image(ictx); | |
682 | ||
683 | stop(); | |
684 | } | |
685 | ||
686 | TEST_F(TestImageReplayer, Resync_While_Stop) | |
687 | { | |
688 | ||
689 | bootstrap(); | |
690 | ||
691 | start(); | |
692 | ||
693 | generate_test_data(); | |
694 | ||
695 | librbd::ImageCtx *ictx; | |
696 | open_remote_image(&ictx); | |
697 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
698 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
699 | } | |
700 | flush(ictx); | |
701 | ||
702 | wait_for_replay_complete(); | |
703 | ||
704 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
705 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
706 | } | |
707 | flush(ictx); | |
708 | close_image(ictx); | |
709 | ||
710 | wait_for_replay_complete(); | |
711 | ||
712 | C_SaferCond cond; | |
713 | m_replayer->stop(&cond); | |
714 | ASSERT_EQ(0, cond.wait()); | |
715 | ||
716 | open_local_image(&ictx); | |
717 | librbd::Journal<>::request_resync(ictx); | |
718 | close_image(ictx); | |
719 | ||
720 | C_SaferCond cond2; | |
721 | m_replayer->start(&cond2); | |
722 | ASSERT_EQ(0, cond2.wait()); | |
723 | ||
724 | ASSERT_TRUE(m_replayer->is_stopped()); | |
725 | ||
726 | C_SaferCond delete_ctx; | |
727 | m_image_deleter->wait_for_scheduled_deletion( | |
728 | m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx); | |
729 | EXPECT_EQ(0, delete_ctx.wait()); | |
730 | ||
731 | C_SaferCond cond3; | |
732 | m_replayer->start(&cond3); | |
733 | ASSERT_EQ(0, cond3.wait()); | |
734 | ||
735 | ASSERT_TRUE(m_replayer->is_replaying()); | |
736 | ||
737 | wait_for_replay_complete(); | |
738 | ||
739 | open_local_image(&ictx); | |
740 | for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { | |
741 | read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
742 | } | |
743 | close_image(ictx); | |
744 | ||
745 | stop(); | |
746 | } | |
747 | ||
748 | TEST_F(TestImageReplayer, Resync_StartInterrupted) | |
749 | { | |
750 | ||
751 | bootstrap(); | |
752 | ||
753 | librbd::ImageCtx *ictx; | |
754 | open_local_image(&ictx); | |
755 | librbd::Journal<>::request_resync(ictx); | |
756 | close_image(ictx); | |
757 | ||
758 | C_SaferCond cond; | |
759 | m_replayer->start(&cond); | |
760 | ASSERT_EQ(0, cond.wait()); | |
761 | ||
762 | ASSERT_TRUE(m_replayer->is_stopped()); | |
763 | ||
764 | C_SaferCond delete_ctx; | |
765 | m_image_deleter->wait_for_scheduled_deletion( | |
766 | m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx); | |
767 | EXPECT_EQ(0, delete_ctx.wait()); | |
768 | ||
769 | C_SaferCond cond2; | |
770 | m_replayer->start(&cond2); | |
771 | ASSERT_EQ(0, cond2.wait()); | |
772 | ||
773 | ASSERT_EQ(0U, m_watch_handle); | |
774 | std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); | |
775 | m_watch_ctx = new C_WatchCtx(this, oid); | |
776 | ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx)); | |
777 | ||
778 | ASSERT_TRUE(m_replayer->is_replaying()); | |
779 | ||
31f18b77 | 780 | generate_test_data(); |
7c673cae FG |
781 | open_remote_image(&ictx); |
782 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
783 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
784 | } | |
785 | flush(ictx); | |
786 | ||
787 | wait_for_replay_complete(); | |
788 | ||
789 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
790 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
791 | } | |
792 | flush(ictx); | |
793 | close_image(ictx); | |
794 | ||
795 | wait_for_replay_complete(); | |
796 | ||
797 | open_local_image(&ictx); | |
798 | for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { | |
799 | read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
800 | } | |
801 | close_image(ictx); | |
802 | ||
803 | stop(); | |
804 | } | |
805 | ||
806 | TEST_F(TestImageReplayer, MultipleReplayFailures_SingleEpoch) { | |
807 | bootstrap(); | |
808 | ||
809 | // inject a snapshot that cannot be unprotected | |
810 | librbd::ImageCtx *ictx; | |
811 | open_image(m_local_ioctx, m_image_name, false, &ictx); | |
812 | ictx->features &= ~RBD_FEATURE_JOURNALING; | |
813 | ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), | |
814 | "foo")); | |
815 | ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), | |
816 | "foo")); | |
817 | ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN, | |
818 | {ictx->md_ctx.get_id(), | |
819 | ictx->id, | |
820 | ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), "foo"}]}, | |
821 | "dummy child id")); | |
822 | close_image(ictx); | |
823 | ||
824 | // race failed op shut down with new ops | |
825 | open_remote_image(&ictx); | |
826 | for (uint64_t i = 0; i < 10; ++i) { | |
827 | RWLock::RLocker owner_locker(ictx->owner_lock); | |
828 | C_SaferCond request_lock; | |
829 | ictx->exclusive_lock->acquire_lock(&request_lock); | |
830 | ASSERT_EQ(0, request_lock.wait()); | |
831 | ||
832 | C_SaferCond append_ctx; | |
833 | ictx->journal->append_op_event( | |
834 | i, | |
835 | librbd::journal::EventEntry{ | |
836 | librbd::journal::SnapUnprotectEvent{i, | |
837 | cls::rbd::UserSnapshotNamespace(), | |
838 | "foo"}}, | |
839 | &append_ctx); | |
840 | ASSERT_EQ(0, append_ctx.wait()); | |
841 | ||
842 | C_SaferCond commit_ctx; | |
843 | ictx->journal->commit_op_event(i, 0, &commit_ctx); | |
844 | ASSERT_EQ(0, commit_ctx.wait()); | |
845 | ||
846 | C_SaferCond release_ctx; | |
847 | ictx->exclusive_lock->release_lock(&release_ctx); | |
848 | ASSERT_EQ(0, release_ctx.wait()); | |
849 | } | |
850 | ||
851 | for (uint64_t i = 0; i < 5; ++i) { | |
852 | start(); | |
853 | wait_for_stopped(); | |
854 | unwatch(); | |
855 | } | |
31f18b77 | 856 | close_image(ictx); |
7c673cae FG |
857 | } |
858 | ||
859 | TEST_F(TestImageReplayer, MultipleReplayFailures_MultiEpoch) { | |
860 | bootstrap(); | |
861 | ||
862 | // inject a snapshot that cannot be unprotected | |
863 | librbd::ImageCtx *ictx; | |
864 | open_image(m_local_ioctx, m_image_name, false, &ictx); | |
865 | ictx->features &= ~RBD_FEATURE_JOURNALING; | |
866 | ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), | |
867 | "foo")); | |
868 | ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), | |
869 | "foo")); | |
870 | ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN, | |
871 | {ictx->md_ctx.get_id(), | |
872 | ictx->id, | |
873 | ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), | |
874 | "foo"}]}, | |
875 | "dummy child id")); | |
876 | close_image(ictx); | |
877 | ||
878 | // race failed op shut down with new tag flush | |
879 | open_remote_image(&ictx); | |
880 | { | |
881 | RWLock::RLocker owner_locker(ictx->owner_lock); | |
882 | C_SaferCond request_lock; | |
883 | ictx->exclusive_lock->acquire_lock(&request_lock); | |
884 | ASSERT_EQ(0, request_lock.wait()); | |
885 | ||
886 | C_SaferCond append_ctx; | |
887 | ictx->journal->append_op_event( | |
888 | 1U, | |
889 | librbd::journal::EventEntry{ | |
890 | librbd::journal::SnapUnprotectEvent{1U, | |
891 | cls::rbd::UserSnapshotNamespace(), | |
892 | "foo"}}, | |
893 | &append_ctx); | |
894 | ASSERT_EQ(0, append_ctx.wait()); | |
895 | ||
896 | C_SaferCond commit_ctx; | |
897 | ictx->journal->commit_op_event(1U, 0, &commit_ctx); | |
898 | ASSERT_EQ(0, commit_ctx.wait()); | |
899 | ||
900 | C_SaferCond release_ctx; | |
901 | ictx->exclusive_lock->release_lock(&release_ctx); | |
902 | ASSERT_EQ(0, release_ctx.wait()); | |
903 | } | |
904 | ||
31f18b77 | 905 | generate_test_data(); |
7c673cae FG |
906 | write_test_data(ictx, m_test_data, 0, TEST_IO_SIZE); |
907 | ||
908 | for (uint64_t i = 0; i < 5; ++i) { | |
909 | start(); | |
910 | wait_for_stopped(); | |
911 | unwatch(); | |
912 | } | |
913 | close_image(ictx); | |
914 | } | |
915 | ||
916 | TEST_F(TestImageReplayer, Disconnect) | |
917 | { | |
918 | bootstrap(); | |
919 | ||
920 | // Make sure rbd_mirroring_resync_after_disconnect is not set | |
921 | EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "false")); | |
922 | ||
923 | // Test start fails if disconnected | |
924 | ||
925 | librbd::ImageCtx *ictx; | |
926 | ||
927 | generate_test_data(); | |
928 | open_remote_image(&ictx); | |
929 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
930 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
931 | } | |
932 | flush(ictx); | |
933 | close_image(ictx); | |
934 | ||
935 | std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); | |
936 | ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx, oid, | |
937 | m_local_mirror_uuid, cls::journal::CLIENT_STATE_DISCONNECTED)); | |
938 | ||
939 | C_SaferCond cond1; | |
940 | m_replayer->start(&cond1); | |
941 | ASSERT_EQ(-ENOTCONN, cond1.wait()); | |
942 | ||
943 | // Test start succeeds after resync | |
944 | ||
945 | open_local_image(&ictx); | |
946 | librbd::Journal<>::request_resync(ictx); | |
947 | close_image(ictx); | |
948 | C_SaferCond cond2; | |
949 | m_replayer->start(&cond2); | |
d2e6a577 | 950 | ASSERT_EQ(0, cond2.wait()); |
7c673cae FG |
951 | C_SaferCond delete_cond; |
952 | m_image_deleter->wait_for_scheduled_deletion( | |
953 | m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond); | |
954 | EXPECT_EQ(0, delete_cond.wait()); | |
955 | ||
956 | start(); | |
957 | wait_for_replay_complete(); | |
958 | ||
959 | // Test replay stopped after disconnect | |
960 | ||
961 | open_remote_image(&ictx); | |
962 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
963 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
964 | } | |
965 | flush(ictx); | |
966 | close_image(ictx); | |
967 | ||
968 | ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx, oid, | |
969 | m_local_mirror_uuid, cls::journal::CLIENT_STATE_DISCONNECTED)); | |
970 | bufferlist bl; | |
971 | ASSERT_EQ(0, m_remote_ioctx.notify2(oid, bl, 5000, NULL)); | |
972 | ||
973 | wait_for_stopped(); | |
974 | ||
975 | // Test start fails after disconnect | |
976 | ||
977 | C_SaferCond cond3; | |
978 | m_replayer->start(&cond3); | |
979 | ASSERT_EQ(-ENOTCONN, cond3.wait()); | |
980 | C_SaferCond cond4; | |
981 | m_replayer->start(&cond4); | |
982 | ASSERT_EQ(-ENOTCONN, cond4.wait()); | |
983 | ||
984 | // Test automatic resync if rbd_mirroring_resync_after_disconnect is set | |
985 | ||
986 | EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "true")); | |
987 | ||
988 | // Resync is flagged on first start attempt | |
989 | C_SaferCond cond5; | |
990 | m_replayer->start(&cond5); | |
991 | ASSERT_EQ(-ENOTCONN, cond5.wait()); | |
992 | C_SaferCond delete_cond1; | |
993 | m_image_deleter->wait_for_scheduled_deletion( | |
994 | m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond1); | |
995 | EXPECT_EQ(0, delete_cond1.wait()); | |
996 | ||
997 | C_SaferCond cond6; | |
998 | m_replayer->start(&cond6); | |
999 | ASSERT_EQ(0, cond6.wait()); | |
1000 | wait_for_replay_complete(); | |
1001 | ||
1002 | stop(); | |
1003 | } | |
1004 | ||
1005 | TEST_F(TestImageReplayer, UpdateFeatures) | |
1006 | { | |
1007 | const uint64_t FEATURES_TO_UPDATE = | |
1008 | RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF; | |
1009 | ||
1010 | uint64_t features; | |
1011 | librbd::ImageCtx *ictx; | |
1012 | ||
1013 | // Make sure the features we will update are disabled initially | |
1014 | ||
1015 | open_remote_image(&ictx); | |
1016 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1017 | features &= FEATURES_TO_UPDATE; | |
1018 | if (features) { | |
1019 | ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, | |
1020 | false)); | |
1021 | } | |
1022 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1023 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
1024 | close_image(ictx); | |
1025 | ||
1026 | bootstrap(); | |
1027 | ||
1028 | open_remote_image(&ictx); | |
1029 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1030 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
1031 | close_image(ictx); | |
1032 | ||
1033 | open_local_image(&ictx); | |
1034 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1035 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
1036 | close_image(ictx); | |
1037 | ||
1038 | // Start replay and update features | |
1039 | ||
1040 | start(); | |
1041 | ||
1042 | open_remote_image(&ictx); | |
1043 | ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, | |
1044 | true)); | |
1045 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1046 | ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE); | |
1047 | close_image(ictx); | |
1048 | ||
1049 | wait_for_replay_complete(); | |
1050 | ||
1051 | open_local_image(&ictx); | |
1052 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1053 | ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE); | |
1054 | close_image(ictx); | |
1055 | ||
1056 | open_remote_image(&ictx); | |
1057 | ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, | |
1058 | false)); | |
1059 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1060 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
1061 | close_image(ictx); | |
1062 | ||
1063 | wait_for_replay_complete(); | |
1064 | ||
1065 | open_local_image(&ictx); | |
1066 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1067 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
1068 | close_image(ictx); | |
1069 | ||
1070 | // Test update_features error does not stop replication | |
1071 | ||
1072 | open_remote_image(&ictx); | |
1073 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1074 | ASSERT_NE(0U, features & RBD_FEATURE_EXCLUSIVE_LOCK); | |
1075 | ASSERT_EQ(-EINVAL, ictx->operations->update_features(RBD_FEATURE_EXCLUSIVE_LOCK, | |
1076 | false)); | |
1077 | generate_test_data(); | |
1078 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
1079 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
1080 | } | |
1081 | flush(ictx); | |
1082 | close_image(ictx); | |
1083 | ||
1084 | wait_for_replay_complete(); | |
1085 | ||
1086 | open_local_image(&ictx); | |
1087 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
1088 | read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
1089 | } | |
1090 | close_image(ictx); | |
1091 | ||
1092 | stop(); | |
1093 | } | |
1094 | ||
1095 | TEST_F(TestImageReplayer, MetadataSetRemove) | |
1096 | { | |
1097 | const std::string KEY = "test_key"; | |
1098 | const std::string VALUE = "test_value"; | |
1099 | ||
1100 | librbd::ImageCtx *ictx; | |
1101 | std::string value; | |
1102 | ||
1103 | bootstrap(); | |
1104 | ||
1105 | start(); | |
1106 | ||
1107 | // Test metadata_set replication | |
1108 | ||
1109 | open_remote_image(&ictx); | |
1110 | ASSERT_EQ(0, ictx->operations->metadata_set(KEY, VALUE)); | |
1111 | value.clear(); | |
1112 | ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value)); | |
1113 | ASSERT_EQ(VALUE, value); | |
1114 | close_image(ictx); | |
1115 | ||
1116 | wait_for_replay_complete(); | |
1117 | ||
1118 | open_local_image(&ictx); | |
1119 | value.clear(); | |
1120 | ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value)); | |
1121 | ASSERT_EQ(VALUE, value); | |
1122 | close_image(ictx); | |
1123 | ||
1124 | // Test metadata_remove replication | |
1125 | ||
1126 | open_remote_image(&ictx); | |
1127 | ASSERT_EQ(0, ictx->operations->metadata_remove(KEY)); | |
1128 | ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value)); | |
1129 | close_image(ictx); | |
1130 | ||
1131 | wait_for_replay_complete(); | |
1132 | ||
1133 | open_local_image(&ictx); | |
1134 | ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value)); | |
1135 | close_image(ictx); | |
1136 | ||
1137 | stop(); | |
1138 | } | |
1139 | ||
1140 | TEST_F(TestImageReplayer, MirroringDelay) | |
1141 | { | |
1142 | const double DELAY = 10; // set less than wait_for_replay_complete timeout | |
1143 | ||
1144 | librbd::ImageCtx *ictx; | |
1145 | utime_t start_time; | |
1146 | double delay; | |
1147 | ||
1148 | bootstrap(); | |
1149 | ||
1150 | ASSERT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_replay_delay", | |
1151 | stringify(DELAY).c_str())); | |
1152 | open_local_image(&ictx); | |
1153 | ASSERT_EQ(DELAY, ictx->mirroring_replay_delay); | |
1154 | close_image(ictx); | |
1155 | ||
1156 | start(); | |
1157 | ||
1158 | // Test delay | |
1159 | ||
1160 | generate_test_data(); | |
1161 | open_remote_image(&ictx); | |
1162 | start_time = ceph_clock_now(); | |
1163 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
1164 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
1165 | } | |
1166 | flush(ictx); | |
1167 | close_image(ictx); | |
1168 | ||
1169 | wait_for_replay_complete(); | |
1170 | delay = ceph_clock_now() - start_time; | |
1171 | ASSERT_GE(delay, DELAY); | |
1172 | ||
1173 | // Test stop when delaying replay | |
1174 | ||
1175 | open_remote_image(&ictx); | |
1176 | start_time = ceph_clock_now(); | |
1177 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
1178 | write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); | |
1179 | } | |
31f18b77 | 1180 | close_image(ictx); |
7c673cae FG |
1181 | |
1182 | sleep(DELAY / 2); | |
1183 | stop(); | |
1184 | start(); | |
1185 | ||
1186 | wait_for_replay_complete(); | |
1187 | delay = ceph_clock_now() - start_time; | |
1188 | ASSERT_GE(delay, DELAY); | |
1189 | ||
1190 | stop(); | |
1191 | } |