1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph distributed storage system
6 * Copyright (C) 2016 Mirantis Inc
8 * Author: Mykola Golub <mgolub@mirantis.com>
10 * This library is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
17 #include "include/rados/librados.hpp"
18 #include "include/rbd/librbd.hpp"
19 #include "include/stringify.h"
20 #include "test/rbd_mirror/test_fixture.h"
21 #include "cls/journal/cls_journal_types.h"
22 #include "cls/journal/cls_journal_client.h"
23 #include "cls/rbd/cls_rbd_types.h"
24 #include "cls/rbd/cls_rbd_client.h"
25 #include "journal/Journaler.h"
26 #include "librbd/ExclusiveLock.h"
27 #include "librbd/ImageCtx.h"
28 #include "librbd/ImageState.h"
29 #include "librbd/Journal.h"
30 #include "librbd/Operations.h"
31 #include "librbd/Utils.h"
32 #include "librbd/internal.h"
33 #include "librbd/api/Mirror.h"
34 #include "librbd/io/AioCompletion.h"
35 #include "librbd/io/ImageRequestWQ.h"
36 #include "librbd/io/ReadResult.h"
37 #include "tools/rbd_mirror/types.h"
38 #include "tools/rbd_mirror/ImageDeleter.h"
39 #include "tools/rbd_mirror/ImageReplayer.h"
40 #include "tools/rbd_mirror/InstanceWatcher.h"
41 #include "tools/rbd_mirror/ServiceDaemon.h"
42 #include "tools/rbd_mirror/Threads.h"
44 #include "test/librados/test.h"
45 #include "gtest/gtest.h"
47 using rbd::mirror::RadosRef
;
49 void register_test_rbd_mirror() {
52 #define TEST_IO_SIZE 512
53 #define TEST_IO_COUNT 11
55 class TestImageReplayer
: public ::rbd::mirror::TestFixture
{
57 struct C_WatchCtx
: public librados::WatchCtx2
{
58 TestImageReplayer
*test
;
64 C_WatchCtx(TestImageReplayer
*test
, const std::string
&oid
)
65 : test(test
), oid(oid
), lock("C_WatchCtx::lock"), notified(false) {
68 void handle_notify(uint64_t notify_id
, uint64_t cookie
,
69 uint64_t notifier_id
, bufferlist
& bl_
) override
{
71 test
->m_remote_ioctx
.notify_ack(oid
, notify_id
, cookie
, bl
);
73 Mutex::Locker
locker(lock
);
78 void handle_error(uint64_t cookie
, int err
) override
{
84 : m_local_cluster(new librados::Rados()), m_watch_handle(0)
86 EXPECT_EQ(0, g_ceph_context
->_conf
->set_val("rbd_mirror_journal_commit_age",
89 EXPECT_EQ("", connect_cluster_pp(*m_local_cluster
.get()));
90 EXPECT_EQ(0, m_local_cluster
->conf_set("rbd_cache", "false"));
91 EXPECT_EQ(0, m_local_cluster
->conf_set("rbd_mirror_journal_poll_age", "1"));
93 m_local_pool_name
= get_temp_pool_name();
94 EXPECT_EQ(0, m_local_cluster
->pool_create(m_local_pool_name
.c_str()));
95 EXPECT_EQ(0, m_local_cluster
->ioctx_create(m_local_pool_name
.c_str(),
97 m_local_ioctx
.application_enable("rbd", true);
99 EXPECT_EQ("", connect_cluster_pp(m_remote_cluster
));
100 EXPECT_EQ(0, m_remote_cluster
.conf_set("rbd_cache", "false"));
102 m_remote_pool_name
= get_temp_pool_name();
103 EXPECT_EQ(0, m_remote_cluster
.pool_create(m_remote_pool_name
.c_str()));
104 m_remote_pool_id
= m_remote_cluster
.pool_lookup(m_remote_pool_name
.c_str());
105 EXPECT_GE(m_remote_pool_id
, 0);
107 EXPECT_EQ(0, m_remote_cluster
.ioctx_create(m_remote_pool_name
.c_str(),
109 m_remote_ioctx
.application_enable("rbd", true);
111 EXPECT_EQ(0, librbd::api::Mirror
<>::mode_set(m_remote_ioctx
,
112 RBD_MIRROR_MODE_POOL
));
114 m_image_name
= get_temp_image_name();
115 uint64_t features
= librbd::util::get_rbd_default_features(g_ceph_context
);
116 features
|= RBD_FEATURE_EXCLUSIVE_LOCK
| RBD_FEATURE_JOURNALING
;
118 EXPECT_EQ(0, librbd::create(m_remote_ioctx
, m_image_name
.c_str(), 1 << 22,
119 false, features
, &order
, 0, 0));
120 m_remote_image_id
= get_image_id(m_remote_ioctx
, m_image_name
);
121 m_global_image_id
= get_global_image_id(m_remote_ioctx
, m_remote_image_id
);
123 m_threads
.reset(new rbd::mirror::Threads
<>(reinterpret_cast<CephContext
*>(
124 m_local_ioctx
.cct())));
126 m_service_daemon
.reset(new rbd::mirror::ServiceDaemon
<>(g_ceph_context
,
129 m_image_deleter
.reset(new rbd::mirror::ImageDeleter
<>(
130 m_threads
->work_queue
, m_threads
->timer
, &m_threads
->timer_lock
,
131 m_service_daemon
.get()));
132 m_instance_watcher
= rbd::mirror::InstanceWatcher
<>::create(
133 m_local_ioctx
, m_threads
->work_queue
, nullptr);
134 m_instance_watcher
->handle_acquire_leader();
137 ~TestImageReplayer() override
141 m_instance_watcher
->handle_release_leader();
144 delete m_instance_watcher
;
146 EXPECT_EQ(0, m_remote_cluster
.pool_delete(m_remote_pool_name
.c_str()));
147 EXPECT_EQ(0, m_local_cluster
->pool_delete(m_local_pool_name
.c_str()));
148 EXPECT_EQ(0, g_ceph_context
->_conf
->set_val("rbd_mirror_journal_commit_age",
152 template <typename ImageReplayerT
= rbd::mirror::ImageReplayer
<> >
153 void create_replayer() {
154 m_replayer
= new ImageReplayerT(
155 m_threads
.get(), m_image_deleter
.get(), m_instance_watcher
,
156 rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx
)),
157 m_local_mirror_uuid
, m_local_ioctx
.get_id(), m_global_image_id
);
158 m_replayer
->add_peer("peer uuid", m_remote_ioctx
);
164 m_replayer
->start(&cond
);
165 ASSERT_EQ(0, cond
.wait());
167 ASSERT_EQ(0U, m_watch_handle
);
168 std::string oid
= ::journal::Journaler::header_oid(m_remote_image_id
);
169 m_watch_ctx
= new C_WatchCtx(this, oid
);
170 ASSERT_EQ(0, m_remote_ioctx
.watch2(oid
, &m_watch_handle
, m_watch_ctx
));
174 if (m_watch_handle
!= 0) {
175 m_remote_ioctx
.unwatch2(m_watch_handle
);
177 m_watch_ctx
= nullptr;
187 m_replayer
->stop(&cond
);
188 ASSERT_EQ(0, cond
.wait());
196 wait_for_replay_complete();
200 std::string
get_temp_image_name()
202 return "image" + stringify(++_image_number
);
205 std::string
get_image_id(librados::IoCtx
&ioctx
, const string
&image_name
)
207 std::string obj
= librbd::util::id_obj_name(image_name
);
209 EXPECT_EQ(0, librbd::cls_client::get_id(&ioctx
, obj
, &id
));
213 std::string
get_global_image_id(librados::IoCtx
& io_ctx
,
214 const std::string
& image_id
) {
215 cls::rbd::MirrorImage mirror_image
;
216 EXPECT_EQ(0, librbd::cls_client::mirror_image_get(&io_ctx
, image_id
,
218 return mirror_image
.global_image_id
;
221 void open_image(librados::IoCtx
&ioctx
, const std::string
&image_name
,
222 bool readonly
, librbd::ImageCtx
**ictxp
)
224 librbd::ImageCtx
*ictx
= new librbd::ImageCtx(image_name
.c_str(),
225 "", "", ioctx
, readonly
);
226 EXPECT_EQ(0, ictx
->state
->open(false));
230 void open_local_image(librbd::ImageCtx
**ictxp
)
232 open_image(m_local_ioctx
, m_image_name
, true, ictxp
);
235 void open_remote_image(librbd::ImageCtx
**ictxp
)
237 open_image(m_remote_ioctx
, m_image_name
, false, ictxp
);
240 void close_image(librbd::ImageCtx
*ictx
)
242 ictx
->state
->close();
245 void get_commit_positions(cls::journal::ObjectPosition
*master_position
,
246 cls::journal::ObjectPosition
*mirror_position
)
248 std::string master_client_id
= "";
249 std::string mirror_client_id
= m_local_mirror_uuid
;
252 uint64_t minimum_set
;
254 std::set
<cls::journal::Client
> registered_clients
;
255 std::string oid
= ::journal::Journaler::header_oid(m_remote_image_id
);
256 cls::journal::client::get_mutable_metadata(m_remote_ioctx
, oid
,
257 &minimum_set
, &active_set
,
258 ®istered_clients
, &cond
);
259 ASSERT_EQ(0, cond
.wait());
261 *master_position
= cls::journal::ObjectPosition();
262 *mirror_position
= cls::journal::ObjectPosition();
264 std::set
<cls::journal::Client
>::const_iterator c
;
265 for (c
= registered_clients
.begin(); c
!= registered_clients
.end(); ++c
) {
266 std::cout
<< __func__
<< ": client: " << *c
<< std::endl
;
267 if (c
->state
!= cls::journal::CLIENT_STATE_CONNECTED
) {
270 cls::journal::ObjectPositions object_positions
=
271 c
->commit_position
.object_positions
;
272 cls::journal::ObjectPositions::const_iterator p
=
273 object_positions
.begin();
274 if (p
!= object_positions
.end()) {
275 if (c
->id
== master_client_id
) {
276 ASSERT_EQ(cls::journal::ObjectPosition(), *master_position
);
277 *master_position
= *p
;
278 } else if (c
->id
== mirror_client_id
) {
279 ASSERT_EQ(cls::journal::ObjectPosition(), *mirror_position
);
280 *mirror_position
= *p
;
286 bool wait_for_watcher_notify(int seconds
)
288 if (m_watch_handle
== 0) {
292 Mutex::Locker
locker(m_watch_ctx
->lock
);
293 while (!m_watch_ctx
->notified
) {
294 if (m_watch_ctx
->cond
.WaitInterval(m_watch_ctx
->lock
,
295 utime_t(seconds
, 0)) != 0) {
299 m_watch_ctx
->notified
= false;
303 void wait_for_replay_complete()
305 cls::journal::ObjectPosition master_position
;
306 cls::journal::ObjectPosition mirror_position
;
308 for (int i
= 0; i
< 100; i
++) {
309 get_commit_positions(&master_position
, &mirror_position
);
310 if (master_position
== mirror_position
) {
313 wait_for_watcher_notify(1);
316 ASSERT_EQ(master_position
, mirror_position
);
319 void wait_for_stopped() {
320 for (int i
= 0; i
< 100; i
++) {
321 if (m_replayer
->is_stopped()) {
324 wait_for_watcher_notify(1);
326 ASSERT_TRUE(m_replayer
->is_stopped());
329 void write_test_data(librbd::ImageCtx
*ictx
, const char *test_data
, off_t off
,
334 bl
.append(std::string(test_data
, len
));
335 written
= ictx
->io_work_queue
->write(off
, len
, std::move(bl
), 0);
336 printf("wrote: %d\n", (int)written
);
337 ASSERT_EQ(len
, written
);
340 void read_test_data(librbd::ImageCtx
*ictx
, const char *expected
, off_t off
,
344 char *result
= (char *)malloc(len
+ 1);
346 ASSERT_NE(static_cast<char *>(NULL
), result
);
347 read
= ictx
->io_work_queue
->read(
348 off
, len
, librbd::io::ReadResult
{result
, len
}, 0);
349 printf("read: %d\n", (int)read
);
350 ASSERT_EQ(len
, static_cast<size_t>(read
));
352 if (memcmp(result
, expected
, len
)) {
353 printf("read: %s\nexpected: %s\n", result
, expected
);
354 ASSERT_EQ(0, memcmp(result
, expected
, len
));
359 void generate_test_data() {
360 for (int i
= 0; i
< TEST_IO_SIZE
; ++i
) {
361 m_test_data
[i
] = (char) (rand() % (126 - 33) + 33);
363 m_test_data
[TEST_IO_SIZE
] = '\0';
366 void flush(librbd::ImageCtx
*ictx
)
368 C_SaferCond aio_flush_ctx
;
369 auto c
= librbd::io::AioCompletion::create(&aio_flush_ctx
);
371 ictx
->io_work_queue
->aio_flush(c
);
372 ASSERT_EQ(0, c
->wait_for_complete());
375 C_SaferCond journal_flush_ctx
;
376 ictx
->journal
->flush_commit_position(&journal_flush_ctx
);
377 ASSERT_EQ(0, journal_flush_ctx
.wait());
382 static int _image_number
;
384 std::shared_ptr
<librados::Rados
> m_local_cluster
;
385 std::unique_ptr
<rbd::mirror::Threads
<>> m_threads
;
386 std::unique_ptr
<rbd::mirror::ServiceDaemon
<>> m_service_daemon
;
387 std::unique_ptr
<rbd::mirror::ImageDeleter
<>> m_image_deleter
;
388 librados::Rados m_remote_cluster
;
389 rbd::mirror::InstanceWatcher
<> *m_instance_watcher
;
390 std::string m_local_mirror_uuid
= "local mirror uuid";
391 std::string m_remote_mirror_uuid
= "remote mirror uuid";
392 std::string m_local_pool_name
, m_remote_pool_name
;
393 librados::IoCtx m_local_ioctx
, m_remote_ioctx
;
394 std::string m_image_name
;
395 int64_t m_remote_pool_id
;
396 std::string m_remote_image_id
;
397 std::string m_global_image_id
;
398 rbd::mirror::ImageReplayer
<> *m_replayer
;
399 C_WatchCtx
*m_watch_ctx
;
400 uint64_t m_watch_handle
;
401 char m_test_data
[TEST_IO_SIZE
+ 1];
404 int TestImageReplayer::_image_number
;
406 TEST_F(TestImageReplayer
, Bootstrap
)
411 TEST_F(TestImageReplayer
, BootstrapErrorLocalImageExists
)
414 EXPECT_EQ(0, librbd::create(m_local_ioctx
, m_image_name
.c_str(), 1 << 22,
415 false, 0, &order
, 0, 0));
419 m_replayer
->start(&cond
);
420 ASSERT_EQ(-EEXIST
, cond
.wait());
423 TEST_F(TestImageReplayer
, BootstrapErrorNoJournal
)
425 ASSERT_EQ(0, librbd::Journal
<>::remove(m_remote_ioctx
, m_remote_image_id
));
429 m_replayer
->start(&cond
);
430 ASSERT_EQ(-ENOENT
, cond
.wait());
433 TEST_F(TestImageReplayer
, BootstrapErrorMirrorDisabled
)
435 // disable remote image mirroring
436 ASSERT_EQ(0, librbd::api::Mirror
<>::mode_set(m_remote_ioctx
,
437 RBD_MIRROR_MODE_IMAGE
));
438 librbd::ImageCtx
*ictx
;
439 open_remote_image(&ictx
);
440 ASSERT_EQ(0, librbd::api::Mirror
<>::image_disable(ictx
, true));
445 m_replayer
->start(&cond
);
446 ASSERT_EQ(-ENOENT
, cond
.wait());
449 TEST_F(TestImageReplayer
, BootstrapMirrorDisabling
)
451 // set remote image mirroring state to DISABLING
452 ASSERT_EQ(0, librbd::api::Mirror
<>::mode_set(m_remote_ioctx
,
453 RBD_MIRROR_MODE_IMAGE
));
454 librbd::ImageCtx
*ictx
;
455 open_remote_image(&ictx
);
456 ASSERT_EQ(0, librbd::api::Mirror
<>::image_enable(ictx
, false));
457 cls::rbd::MirrorImage mirror_image
;
458 ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&m_remote_ioctx
, ictx
->id
,
460 mirror_image
.state
= cls::rbd::MirrorImageState::MIRROR_IMAGE_STATE_DISABLING
;
461 ASSERT_EQ(0, librbd::cls_client::mirror_image_set(&m_remote_ioctx
, ictx
->id
,
467 m_replayer
->start(&cond
);
468 ASSERT_EQ(-EREMOTEIO
, cond
.wait());
469 ASSERT_TRUE(m_replayer
->is_stopped());
472 TEST_F(TestImageReplayer
, BootstrapDemoted
)
474 // demote remote image
475 librbd::ImageCtx
*ictx
;
476 open_remote_image(&ictx
);
477 ASSERT_EQ(0, librbd::api::Mirror
<>::image_demote(ictx
));
482 m_replayer
->start(&cond
);
483 ASSERT_EQ(-EREMOTEIO
, cond
.wait());
484 ASSERT_TRUE(m_replayer
->is_stopped());
487 TEST_F(TestImageReplayer
, StartInterrupted
)
490 C_SaferCond start_cond
, stop_cond
;
491 m_replayer
->start(&start_cond
);
492 m_replayer
->stop(&stop_cond
);
493 int r
= start_cond
.wait();
494 printf("start returned %d\n", r
);
495 // TODO: improve the test to avoid this race
496 ASSERT_TRUE(r
== -ECANCELED
|| r
== 0);
497 ASSERT_EQ(0, stop_cond
.wait());
500 TEST_F(TestImageReplayer
, JournalReset
)
505 ASSERT_EQ(0, librbd::Journal
<>::reset(m_remote_ioctx
, m_remote_image_id
));
511 TEST_F(TestImageReplayer
, ErrorNoJournal
)
515 // disable remote journal journaling
516 // (reset before disabling, so it does not fail with EBUSY)
517 ASSERT_EQ(0, librbd::Journal
<>::reset(m_remote_ioctx
, m_remote_image_id
));
518 librbd::ImageCtx
*ictx
;
519 open_remote_image(&ictx
);
521 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
522 ASSERT_EQ(0, ictx
->operations
->update_features(RBD_FEATURE_JOURNALING
,
527 m_replayer
->start(&cond
);
528 ASSERT_EQ(0, cond
.wait());
531 TEST_F(TestImageReplayer
, StartStop
)
536 wait_for_replay_complete();
540 TEST_F(TestImageReplayer
, WriteAndStartReplay
)
544 // Write to remote image and start replay
546 librbd::ImageCtx
*ictx
;
548 generate_test_data();
549 open_remote_image(&ictx
);
550 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
551 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
557 wait_for_replay_complete();
560 open_local_image(&ictx
);
561 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
562 read_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
567 TEST_F(TestImageReplayer
, StartReplayAndWrite
)
571 // Start replay and write to remote image
573 librbd::ImageCtx
*ictx
;
577 generate_test_data();
578 open_remote_image(&ictx
);
579 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
580 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
584 wait_for_replay_complete();
586 for (int i
= TEST_IO_COUNT
; i
< 2 * TEST_IO_COUNT
; ++i
) {
587 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
592 wait_for_replay_complete();
594 open_local_image(&ictx
);
595 for (int i
= 0; i
< 2 * TEST_IO_COUNT
; ++i
) {
596 read_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
603 TEST_F(TestImageReplayer
, NextTag
)
607 // write, reopen, and write again to test switch to the next tag
609 librbd::ImageCtx
*ictx
;
613 generate_test_data();
617 for (int j
= 0; j
< N
; j
++) {
618 open_remote_image(&ictx
);
619 for (int i
= j
* TEST_IO_COUNT
; i
< (j
+ 1) * TEST_IO_COUNT
; ++i
) {
620 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
625 wait_for_replay_complete();
627 open_local_image(&ictx
);
628 for (int i
= 0; i
< N
* TEST_IO_COUNT
; ++i
) {
629 read_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
636 TEST_F(TestImageReplayer
, Resync
)
640 librbd::ImageCtx
*ictx
;
644 generate_test_data();
646 open_remote_image(&ictx
);
647 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
648 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
652 wait_for_replay_complete();
654 for (int i
= TEST_IO_COUNT
; i
< 2 * TEST_IO_COUNT
; ++i
) {
655 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
661 m_replayer
->resync_image(&ctx
);
662 ASSERT_EQ(0, ctx
.wait());
664 C_SaferCond delete_ctx
;
665 m_image_deleter
->wait_for_scheduled_deletion(
666 m_local_ioctx
.get_id(), m_replayer
->get_global_image_id(), &delete_ctx
);
667 EXPECT_EQ(0, delete_ctx
.wait());
670 m_replayer
->start(&cond
);
671 ASSERT_EQ(0, cond
.wait());
673 ASSERT_TRUE(m_replayer
->is_replaying());
675 wait_for_replay_complete();
677 open_local_image(&ictx
);
678 for (int i
= 0; i
< 2 * TEST_IO_COUNT
; ++i
) {
679 read_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
686 TEST_F(TestImageReplayer
, Resync_While_Stop
)
693 generate_test_data();
695 librbd::ImageCtx
*ictx
;
696 open_remote_image(&ictx
);
697 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
698 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
702 wait_for_replay_complete();
704 for (int i
= TEST_IO_COUNT
; i
< 2 * TEST_IO_COUNT
; ++i
) {
705 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
710 wait_for_replay_complete();
713 m_replayer
->stop(&cond
);
714 ASSERT_EQ(0, cond
.wait());
716 open_local_image(&ictx
);
717 librbd::Journal
<>::request_resync(ictx
);
721 m_replayer
->start(&cond2
);
722 ASSERT_EQ(0, cond2
.wait());
724 ASSERT_TRUE(m_replayer
->is_stopped());
726 C_SaferCond delete_ctx
;
727 m_image_deleter
->wait_for_scheduled_deletion(
728 m_local_ioctx
.get_id(), m_replayer
->get_global_image_id(), &delete_ctx
);
729 EXPECT_EQ(0, delete_ctx
.wait());
732 m_replayer
->start(&cond3
);
733 ASSERT_EQ(0, cond3
.wait());
735 ASSERT_TRUE(m_replayer
->is_replaying());
737 wait_for_replay_complete();
739 open_local_image(&ictx
);
740 for (int i
= 0; i
< 2 * TEST_IO_COUNT
; ++i
) {
741 read_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
748 TEST_F(TestImageReplayer
, Resync_StartInterrupted
)
753 librbd::ImageCtx
*ictx
;
754 open_local_image(&ictx
);
755 librbd::Journal
<>::request_resync(ictx
);
759 m_replayer
->start(&cond
);
760 ASSERT_EQ(0, cond
.wait());
762 ASSERT_TRUE(m_replayer
->is_stopped());
764 C_SaferCond delete_ctx
;
765 m_image_deleter
->wait_for_scheduled_deletion(
766 m_local_ioctx
.get_id(), m_replayer
->get_global_image_id(), &delete_ctx
);
767 EXPECT_EQ(0, delete_ctx
.wait());
770 m_replayer
->start(&cond2
);
771 ASSERT_EQ(0, cond2
.wait());
773 ASSERT_EQ(0U, m_watch_handle
);
774 std::string oid
= ::journal::Journaler::header_oid(m_remote_image_id
);
775 m_watch_ctx
= new C_WatchCtx(this, oid
);
776 ASSERT_EQ(0, m_remote_ioctx
.watch2(oid
, &m_watch_handle
, m_watch_ctx
));
778 ASSERT_TRUE(m_replayer
->is_replaying());
780 generate_test_data();
781 open_remote_image(&ictx
);
782 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
783 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
787 wait_for_replay_complete();
789 for (int i
= TEST_IO_COUNT
; i
< 2 * TEST_IO_COUNT
; ++i
) {
790 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
795 wait_for_replay_complete();
797 open_local_image(&ictx
);
798 for (int i
= 0; i
< 2 * TEST_IO_COUNT
; ++i
) {
799 read_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
806 TEST_F(TestImageReplayer
, MultipleReplayFailures_SingleEpoch
) {
809 // inject a snapshot that cannot be unprotected
810 librbd::ImageCtx
*ictx
;
811 open_image(m_local_ioctx
, m_image_name
, false, &ictx
);
812 ictx
->features
&= ~RBD_FEATURE_JOURNALING
;
813 ASSERT_EQ(0, ictx
->operations
->snap_create(cls::rbd::UserSnapshotNamespace(),
815 ASSERT_EQ(0, ictx
->operations
->snap_protect(cls::rbd::UserSnapshotNamespace(),
817 ASSERT_EQ(0, librbd::cls_client::add_child(&ictx
->md_ctx
, RBD_CHILDREN
,
818 {ictx
->md_ctx
.get_id(),
820 ictx
->snap_ids
[{cls::rbd::UserSnapshotNamespace(), "foo"}]},
824 // race failed op shut down with new ops
825 open_remote_image(&ictx
);
826 for (uint64_t i
= 0; i
< 10; ++i
) {
827 RWLock::RLocker
owner_locker(ictx
->owner_lock
);
828 C_SaferCond request_lock
;
829 ictx
->exclusive_lock
->acquire_lock(&request_lock
);
830 ASSERT_EQ(0, request_lock
.wait());
832 C_SaferCond append_ctx
;
833 ictx
->journal
->append_op_event(
835 librbd::journal::EventEntry
{
836 librbd::journal::SnapUnprotectEvent
{i
,
837 cls::rbd::UserSnapshotNamespace(),
840 ASSERT_EQ(0, append_ctx
.wait());
842 C_SaferCond commit_ctx
;
843 ictx
->journal
->commit_op_event(i
, 0, &commit_ctx
);
844 ASSERT_EQ(0, commit_ctx
.wait());
846 C_SaferCond release_ctx
;
847 ictx
->exclusive_lock
->release_lock(&release_ctx
);
848 ASSERT_EQ(0, release_ctx
.wait());
851 for (uint64_t i
= 0; i
< 5; ++i
) {
859 TEST_F(TestImageReplayer
, MultipleReplayFailures_MultiEpoch
) {
862 // inject a snapshot that cannot be unprotected
863 librbd::ImageCtx
*ictx
;
864 open_image(m_local_ioctx
, m_image_name
, false, &ictx
);
865 ictx
->features
&= ~RBD_FEATURE_JOURNALING
;
866 ASSERT_EQ(0, ictx
->operations
->snap_create(cls::rbd::UserSnapshotNamespace(),
868 ASSERT_EQ(0, ictx
->operations
->snap_protect(cls::rbd::UserSnapshotNamespace(),
870 ASSERT_EQ(0, librbd::cls_client::add_child(&ictx
->md_ctx
, RBD_CHILDREN
,
871 {ictx
->md_ctx
.get_id(),
873 ictx
->snap_ids
[{cls::rbd::UserSnapshotNamespace(),
878 // race failed op shut down with new tag flush
879 open_remote_image(&ictx
);
881 RWLock::RLocker
owner_locker(ictx
->owner_lock
);
882 C_SaferCond request_lock
;
883 ictx
->exclusive_lock
->acquire_lock(&request_lock
);
884 ASSERT_EQ(0, request_lock
.wait());
886 C_SaferCond append_ctx
;
887 ictx
->journal
->append_op_event(
889 librbd::journal::EventEntry
{
890 librbd::journal::SnapUnprotectEvent
{1U,
891 cls::rbd::UserSnapshotNamespace(),
894 ASSERT_EQ(0, append_ctx
.wait());
896 C_SaferCond commit_ctx
;
897 ictx
->journal
->commit_op_event(1U, 0, &commit_ctx
);
898 ASSERT_EQ(0, commit_ctx
.wait());
900 C_SaferCond release_ctx
;
901 ictx
->exclusive_lock
->release_lock(&release_ctx
);
902 ASSERT_EQ(0, release_ctx
.wait());
905 generate_test_data();
906 write_test_data(ictx
, m_test_data
, 0, TEST_IO_SIZE
);
908 for (uint64_t i
= 0; i
< 5; ++i
) {
916 TEST_F(TestImageReplayer
, Disconnect
)
920 // Make sure rbd_mirroring_resync_after_disconnect is not set
921 EXPECT_EQ(0, m_local_cluster
->conf_set("rbd_mirroring_resync_after_disconnect", "false"));
923 // Test start fails if disconnected
925 librbd::ImageCtx
*ictx
;
927 generate_test_data();
928 open_remote_image(&ictx
);
929 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
930 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
935 std::string oid
= ::journal::Journaler::header_oid(m_remote_image_id
);
936 ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx
, oid
,
937 m_local_mirror_uuid
, cls::journal::CLIENT_STATE_DISCONNECTED
));
940 m_replayer
->start(&cond1
);
941 ASSERT_EQ(-ENOTCONN
, cond1
.wait());
943 // Test start succeeds after resync
945 open_local_image(&ictx
);
946 librbd::Journal
<>::request_resync(ictx
);
949 m_replayer
->start(&cond2
);
950 ASSERT_EQ(0, cond2
.wait());
951 C_SaferCond delete_cond
;
952 m_image_deleter
->wait_for_scheduled_deletion(
953 m_local_ioctx
.get_id(), m_replayer
->get_global_image_id(), &delete_cond
);
954 EXPECT_EQ(0, delete_cond
.wait());
957 wait_for_replay_complete();
959 // Test replay stopped after disconnect
961 open_remote_image(&ictx
);
962 for (int i
= TEST_IO_COUNT
; i
< 2 * TEST_IO_COUNT
; ++i
) {
963 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
968 ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx
, oid
,
969 m_local_mirror_uuid
, cls::journal::CLIENT_STATE_DISCONNECTED
));
971 ASSERT_EQ(0, m_remote_ioctx
.notify2(oid
, bl
, 5000, NULL
));
975 // Test start fails after disconnect
978 m_replayer
->start(&cond3
);
979 ASSERT_EQ(-ENOTCONN
, cond3
.wait());
981 m_replayer
->start(&cond4
);
982 ASSERT_EQ(-ENOTCONN
, cond4
.wait());
984 // Test automatic resync if rbd_mirroring_resync_after_disconnect is set
986 EXPECT_EQ(0, m_local_cluster
->conf_set("rbd_mirroring_resync_after_disconnect", "true"));
988 // Resync is flagged on first start attempt
990 m_replayer
->start(&cond5
);
991 ASSERT_EQ(-ENOTCONN
, cond5
.wait());
992 C_SaferCond delete_cond1
;
993 m_image_deleter
->wait_for_scheduled_deletion(
994 m_local_ioctx
.get_id(), m_replayer
->get_global_image_id(), &delete_cond1
);
995 EXPECT_EQ(0, delete_cond1
.wait());
998 m_replayer
->start(&cond6
);
999 ASSERT_EQ(0, cond6
.wait());
1000 wait_for_replay_complete();
1005 TEST_F(TestImageReplayer
, UpdateFeatures
)
1007 const uint64_t FEATURES_TO_UPDATE
=
1008 RBD_FEATURE_OBJECT_MAP
| RBD_FEATURE_FAST_DIFF
;
1011 librbd::ImageCtx
*ictx
;
1013 // Make sure the features we will update are disabled initially
1015 open_remote_image(&ictx
);
1016 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1017 features
&= FEATURES_TO_UPDATE
;
1019 ASSERT_EQ(0, ictx
->operations
->update_features(FEATURES_TO_UPDATE
,
1022 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1023 ASSERT_EQ(0U, features
& FEATURES_TO_UPDATE
);
1028 open_remote_image(&ictx
);
1029 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1030 ASSERT_EQ(0U, features
& FEATURES_TO_UPDATE
);
1033 open_local_image(&ictx
);
1034 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1035 ASSERT_EQ(0U, features
& FEATURES_TO_UPDATE
);
1038 // Start replay and update features
1042 open_remote_image(&ictx
);
1043 ASSERT_EQ(0, ictx
->operations
->update_features(FEATURES_TO_UPDATE
,
1045 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1046 ASSERT_EQ(FEATURES_TO_UPDATE
, features
& FEATURES_TO_UPDATE
);
1049 wait_for_replay_complete();
1051 open_local_image(&ictx
);
1052 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1053 ASSERT_EQ(FEATURES_TO_UPDATE
, features
& FEATURES_TO_UPDATE
);
1056 open_remote_image(&ictx
);
1057 ASSERT_EQ(0, ictx
->operations
->update_features(FEATURES_TO_UPDATE
,
1059 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1060 ASSERT_EQ(0U, features
& FEATURES_TO_UPDATE
);
1063 wait_for_replay_complete();
1065 open_local_image(&ictx
);
1066 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1067 ASSERT_EQ(0U, features
& FEATURES_TO_UPDATE
);
1070 // Test update_features error does not stop replication
1072 open_remote_image(&ictx
);
1073 ASSERT_EQ(0, librbd::get_features(ictx
, &features
));
1074 ASSERT_NE(0U, features
& RBD_FEATURE_EXCLUSIVE_LOCK
);
1075 ASSERT_EQ(-EINVAL
, ictx
->operations
->update_features(RBD_FEATURE_EXCLUSIVE_LOCK
,
1077 generate_test_data();
1078 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
1079 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
1084 wait_for_replay_complete();
1086 open_local_image(&ictx
);
1087 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
1088 read_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
1095 TEST_F(TestImageReplayer
, MetadataSetRemove
)
1097 const std::string KEY
= "test_key";
1098 const std::string VALUE
= "test_value";
1100 librbd::ImageCtx
*ictx
;
1107 // Test metadata_set replication
1109 open_remote_image(&ictx
);
1110 ASSERT_EQ(0, ictx
->operations
->metadata_set(KEY
, VALUE
));
1112 ASSERT_EQ(0, librbd::metadata_get(ictx
, KEY
, &value
));
1113 ASSERT_EQ(VALUE
, value
);
1116 wait_for_replay_complete();
1118 open_local_image(&ictx
);
1120 ASSERT_EQ(0, librbd::metadata_get(ictx
, KEY
, &value
));
1121 ASSERT_EQ(VALUE
, value
);
1124 // Test metadata_remove replication
1126 open_remote_image(&ictx
);
1127 ASSERT_EQ(0, ictx
->operations
->metadata_remove(KEY
));
1128 ASSERT_EQ(-ENOENT
, librbd::metadata_get(ictx
, KEY
, &value
));
1131 wait_for_replay_complete();
1133 open_local_image(&ictx
);
1134 ASSERT_EQ(-ENOENT
, librbd::metadata_get(ictx
, KEY
, &value
));
1140 TEST_F(TestImageReplayer
, MirroringDelay
)
1142 const double DELAY
= 10; // set less than wait_for_replay_complete timeout
1144 librbd::ImageCtx
*ictx
;
1150 ASSERT_EQ(0, m_local_cluster
->conf_set("rbd_mirroring_replay_delay",
1151 stringify(DELAY
).c_str()));
1152 open_local_image(&ictx
);
1153 ASSERT_EQ(DELAY
, ictx
->mirroring_replay_delay
);
1160 generate_test_data();
1161 open_remote_image(&ictx
);
1162 start_time
= ceph_clock_now();
1163 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
1164 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
1169 wait_for_replay_complete();
1170 delay
= ceph_clock_now() - start_time
;
1171 ASSERT_GE(delay
, DELAY
);
1173 // Test stop when delaying replay
1175 open_remote_image(&ictx
);
1176 start_time
= ceph_clock_now();
1177 for (int i
= 0; i
< TEST_IO_COUNT
; ++i
) {
1178 write_test_data(ictx
, m_test_data
, TEST_IO_SIZE
* i
, TEST_IO_SIZE
);
1186 wait_for_replay_complete();
1187 delay
= ceph_clock_now() - start_time
;
1188 ASSERT_GE(delay
, DELAY
);