]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph distributed storage system | |
5 | * | |
6 | * Copyright (C) 2016 Mirantis Inc | |
7 | * | |
8 | * Author: Mykola Golub <mgolub@mirantis.com> | |
9 | * | |
10 | * This library is free software; you can redistribute it and/or | |
11 | * modify it under the terms of the GNU Lesser General Public | |
12 | * License as published by the Free Software Foundation; either | |
13 | * version 2.1 of the License, or (at your option) any later version. | |
14 | * | |
15 | */ | |
16 | ||
17 | #include "include/rados/librados.hpp" | |
18 | #include "include/rbd/librbd.hpp" | |
19 | #include "include/stringify.h" | |
9f95a23c | 20 | #include "test/librbd/test_support.h" |
7c673cae FG |
21 | #include "test/rbd_mirror/test_fixture.h" |
22 | #include "cls/journal/cls_journal_types.h" | |
23 | #include "cls/journal/cls_journal_client.h" | |
24 | #include "cls/rbd/cls_rbd_types.h" | |
25 | #include "cls/rbd/cls_rbd_client.h" | |
26 | #include "journal/Journaler.h" | |
27 | #include "librbd/ExclusiveLock.h" | |
28 | #include "librbd/ImageCtx.h" | |
29 | #include "librbd/ImageState.h" | |
30 | #include "librbd/Journal.h" | |
31 | #include "librbd/Operations.h" | |
32 | #include "librbd/Utils.h" | |
33 | #include "librbd/internal.h" | |
f67539c2 | 34 | #include "librbd/api/Io.h" |
7c673cae | 35 | #include "librbd/api/Mirror.h" |
9f95a23c | 36 | #include "librbd/api/Snapshot.h" |
7c673cae | 37 | #include "librbd/io/AioCompletion.h" |
7c673cae | 38 | #include "librbd/io/ReadResult.h" |
7c673cae | 39 | #include "tools/rbd_mirror/ImageReplayer.h" |
31f18b77 | 40 | #include "tools/rbd_mirror/InstanceWatcher.h" |
9f95a23c TL |
41 | #include "tools/rbd_mirror/MirrorStatusUpdater.h" |
42 | #include "tools/rbd_mirror/PoolMetaCache.h" | |
7c673cae | 43 | #include "tools/rbd_mirror/Threads.h" |
9f95a23c | 44 | #include "tools/rbd_mirror/Throttler.h" |
11fdf7f2 | 45 | #include "tools/rbd_mirror/Types.h" |
7c673cae | 46 | |
11fdf7f2 | 47 | #include "test/librados/test_cxx.h" |
7c673cae FG |
48 | #include "gtest/gtest.h" |
49 | ||
7c673cae FG |
50 | void register_test_rbd_mirror() { |
51 | } | |
52 | ||
53 | #define TEST_IO_SIZE 512 | |
54 | #define TEST_IO_COUNT 11 | |
55 | ||
9f95a23c TL |
56 | namespace rbd { |
57 | namespace mirror { | |
58 | ||
59 | template <typename T> | |
60 | class TestImageReplayer : public TestFixture { | |
7c673cae | 61 | public: |
9f95a23c TL |
62 | static const cls::rbd::MirrorImageMode MIRROR_IMAGE_MODE = |
63 | T::MIRROR_IMAGE_MODE; | |
64 | static const uint64_t FEATURES = T::FEATURES; | |
65 | ||
7c673cae FG |
66 | struct C_WatchCtx : public librados::WatchCtx2 { |
67 | TestImageReplayer *test; | |
68 | std::string oid; | |
9f95a23c TL |
69 | ceph::mutex lock = ceph::make_mutex("C_WatchCtx::lock"); |
70 | ceph::condition_variable cond; | |
7c673cae FG |
71 | bool notified; |
72 | ||
73 | C_WatchCtx(TestImageReplayer *test, const std::string &oid) | |
9f95a23c | 74 | : test(test), oid(oid), notified(false) { |
7c673cae FG |
75 | } |
76 | ||
77 | void handle_notify(uint64_t notify_id, uint64_t cookie, | |
78 | uint64_t notifier_id, bufferlist& bl_) override { | |
79 | bufferlist bl; | |
80 | test->m_remote_ioctx.notify_ack(oid, notify_id, cookie, bl); | |
81 | ||
9f95a23c | 82 | std::lock_guard locker{lock}; |
7c673cae | 83 | notified = true; |
9f95a23c | 84 | cond.notify_all(); |
7c673cae FG |
85 | } |
86 | ||
87 | void handle_error(uint64_t cookie, int err) override { | |
88 | ASSERT_EQ(0, err); | |
89 | } | |
90 | }; | |
91 | ||
92 | TestImageReplayer() | |
93 | : m_local_cluster(new librados::Rados()), m_watch_handle(0) | |
94 | { | |
95 | EXPECT_EQ("", connect_cluster_pp(*m_local_cluster.get())); | |
96 | EXPECT_EQ(0, m_local_cluster->conf_set("rbd_cache", "false")); | |
97 | EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_poll_age", "1")); | |
11fdf7f2 TL |
98 | EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_commit_age", |
99 | "0.1")); | |
7c673cae FG |
100 | m_local_pool_name = get_temp_pool_name(); |
101 | EXPECT_EQ(0, m_local_cluster->pool_create(m_local_pool_name.c_str())); | |
102 | EXPECT_EQ(0, m_local_cluster->ioctx_create(m_local_pool_name.c_str(), | |
103 | m_local_ioctx)); | |
c07f9fc5 | 104 | m_local_ioctx.application_enable("rbd", true); |
7c673cae FG |
105 | |
106 | EXPECT_EQ("", connect_cluster_pp(m_remote_cluster)); | |
107 | EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false")); | |
108 | ||
109 | m_remote_pool_name = get_temp_pool_name(); | |
110 | EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str())); | |
111 | m_remote_pool_id = m_remote_cluster.pool_lookup(m_remote_pool_name.c_str()); | |
112 | EXPECT_GE(m_remote_pool_id, 0); | |
113 | ||
114 | EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(), | |
115 | m_remote_ioctx)); | |
c07f9fc5 FG |
116 | m_remote_ioctx.application_enable("rbd", true); |
117 | ||
9f95a23c TL |
118 | // make snap id debugging easier when local/remote have different mappings |
119 | uint64_t snap_id; | |
120 | EXPECT_EQ(0, m_remote_ioctx.selfmanaged_snap_create(&snap_id)); | |
121 | ||
122 | uint64_t features = FEATURES; | |
123 | if (MIRROR_IMAGE_MODE == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { | |
124 | EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, | |
125 | RBD_MIRROR_MODE_POOL)); | |
126 | EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_local_ioctx, | |
127 | RBD_MIRROR_MODE_POOL)); | |
128 | } else { | |
129 | EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, | |
130 | RBD_MIRROR_MODE_IMAGE)); | |
131 | EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_local_ioctx, | |
132 | RBD_MIRROR_MODE_IMAGE)); | |
133 | ||
134 | ||
135 | uuid_d uuid_gen; | |
136 | uuid_gen.generate_random(); | |
137 | std::string remote_peer_uuid = uuid_gen.to_string(); | |
138 | ||
139 | EXPECT_EQ(0, librbd::cls_client::mirror_peer_add( | |
140 | &m_remote_ioctx, {remote_peer_uuid, | |
141 | cls::rbd::MIRROR_PEER_DIRECTION_RX_TX, | |
142 | "siteA", "client", m_local_mirror_uuid})); | |
143 | ||
144 | m_pool_meta_cache.set_remote_pool_meta( | |
145 | m_remote_ioctx.get_id(), {m_remote_mirror_uuid, remote_peer_uuid}); | |
146 | } | |
147 | ||
148 | EXPECT_EQ(0, librbd::api::Mirror<>::uuid_get(m_remote_ioctx, | |
149 | &m_remote_mirror_uuid)); | |
150 | EXPECT_EQ(0, librbd::api::Mirror<>::uuid_get(m_local_ioctx, | |
151 | &m_local_mirror_uuid)); | |
7c673cae FG |
152 | |
153 | m_image_name = get_temp_image_name(); | |
7c673cae FG |
154 | int order = 0; |
155 | EXPECT_EQ(0, librbd::create(m_remote_ioctx, m_image_name.c_str(), 1 << 22, | |
156 | false, features, &order, 0, 0)); | |
9f95a23c TL |
157 | |
158 | if (MIRROR_IMAGE_MODE != cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { | |
159 | librbd::ImageCtx* remote_image_ctx; | |
160 | open_remote_image(&remote_image_ctx); | |
161 | EXPECT_EQ(0, | |
162 | librbd::api::Mirror<>::image_enable( | |
163 | remote_image_ctx, | |
164 | static_cast<rbd_mirror_image_mode_t>(MIRROR_IMAGE_MODE), | |
165 | false)); | |
166 | close_image(remote_image_ctx); | |
167 | } | |
168 | ||
7c673cae | 169 | m_remote_image_id = get_image_id(m_remote_ioctx, m_image_name); |
d2e6a577 | 170 | m_global_image_id = get_global_image_id(m_remote_ioctx, m_remote_image_id); |
7c673cae | 171 | |
9f95a23c | 172 | auto cct = reinterpret_cast<CephContext*>(m_local_ioctx.cct()); |
f67539c2 | 173 | m_threads.reset(new Threads<>(m_local_cluster)); |
7c673cae | 174 | |
9f95a23c TL |
175 | m_image_sync_throttler.reset(new Throttler<>( |
176 | cct, "rbd_mirror_concurrent_image_syncs")); | |
11fdf7f2 | 177 | |
9f95a23c | 178 | m_instance_watcher = InstanceWatcher<>::create( |
f67539c2 | 179 | m_local_ioctx, *m_threads->asio_engine, nullptr, |
9f95a23c | 180 | m_image_sync_throttler.get()); |
31f18b77 | 181 | m_instance_watcher->handle_acquire_leader(); |
9f95a23c TL |
182 | |
183 | EXPECT_EQ(0, m_local_ioctx.create(RBD_MIRRORING, false)); | |
184 | ||
185 | m_local_status_updater = MirrorStatusUpdater<>::create( | |
186 | m_local_ioctx, m_threads.get(), ""); | |
187 | C_SaferCond status_updater_ctx; | |
188 | m_local_status_updater->init(&status_updater_ctx); | |
189 | EXPECT_EQ(0, status_updater_ctx.wait()); | |
7c673cae FG |
190 | } |
191 | ||
31f18b77 | 192 | ~TestImageReplayer() override |
7c673cae FG |
193 | { |
194 | unwatch(); | |
195 | ||
31f18b77 FG |
196 | m_instance_watcher->handle_release_leader(); |
197 | ||
7c673cae | 198 | delete m_replayer; |
31f18b77 | 199 | delete m_instance_watcher; |
7c673cae | 200 | |
9f95a23c TL |
201 | C_SaferCond status_updater_ctx; |
202 | m_local_status_updater->shut_down(&status_updater_ctx); | |
203 | EXPECT_EQ(0, status_updater_ctx.wait()); | |
204 | delete m_local_status_updater; | |
205 | ||
7c673cae FG |
206 | EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str())); |
207 | EXPECT_EQ(0, m_local_cluster->pool_delete(m_local_pool_name.c_str())); | |
208 | } | |
209 | ||
7c673cae | 210 | void create_replayer() { |
9f95a23c TL |
211 | m_replayer = new ImageReplayer<>(m_local_ioctx, m_local_mirror_uuid, |
212 | m_global_image_id, m_threads.get(), | |
213 | m_instance_watcher, m_local_status_updater, | |
214 | nullptr, &m_pool_meta_cache); | |
215 | m_replayer->add_peer({"peer uuid", m_remote_ioctx, | |
216 | {m_remote_mirror_uuid, "remote mirror peer uuid"}, | |
217 | nullptr}); | |
7c673cae FG |
218 | } |
219 | ||
220 | void start() | |
221 | { | |
222 | C_SaferCond cond; | |
223 | m_replayer->start(&cond); | |
224 | ASSERT_EQ(0, cond.wait()); | |
225 | ||
9f95a23c TL |
226 | create_watch_ctx(); |
227 | } | |
228 | ||
229 | void create_watch_ctx() { | |
230 | std::string oid; | |
231 | if (MIRROR_IMAGE_MODE == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { | |
232 | oid = ::journal::Journaler::header_oid(m_remote_image_id); | |
233 | } else { | |
234 | oid = librbd::util::header_name(m_remote_image_id); | |
235 | } | |
236 | ||
7c673cae | 237 | ASSERT_EQ(0U, m_watch_handle); |
9f95a23c | 238 | ASSERT_TRUE(m_watch_ctx == nullptr); |
7c673cae FG |
239 | m_watch_ctx = new C_WatchCtx(this, oid); |
240 | ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx)); | |
241 | } | |
242 | ||
243 | void unwatch() { | |
244 | if (m_watch_handle != 0) { | |
245 | m_remote_ioctx.unwatch2(m_watch_handle); | |
246 | delete m_watch_ctx; | |
247 | m_watch_ctx = nullptr; | |
248 | m_watch_handle = 0; | |
249 | } | |
250 | } | |
251 | ||
252 | void stop() | |
253 | { | |
254 | unwatch(); | |
255 | ||
256 | C_SaferCond cond; | |
257 | m_replayer->stop(&cond); | |
258 | ASSERT_EQ(0, cond.wait()); | |
259 | } | |
260 | ||
261 | void bootstrap() | |
262 | { | |
9f95a23c | 263 | create_replayer(); |
7c673cae FG |
264 | |
265 | start(); | |
266 | wait_for_replay_complete(); | |
267 | stop(); | |
268 | } | |
269 | ||
270 | std::string get_temp_image_name() | |
271 | { | |
272 | return "image" + stringify(++_image_number); | |
273 | } | |
274 | ||
20effc67 | 275 | std::string get_image_id(librados::IoCtx &ioctx, const std::string &image_name) |
7c673cae FG |
276 | { |
277 | std::string obj = librbd::util::id_obj_name(image_name); | |
278 | std::string id; | |
279 | EXPECT_EQ(0, librbd::cls_client::get_id(&ioctx, obj, &id)); | |
280 | return id; | |
281 | } | |
282 | ||
d2e6a577 FG |
283 | std::string get_global_image_id(librados::IoCtx& io_ctx, |
284 | const std::string& image_id) { | |
285 | cls::rbd::MirrorImage mirror_image; | |
286 | EXPECT_EQ(0, librbd::cls_client::mirror_image_get(&io_ctx, image_id, | |
287 | &mirror_image)); | |
288 | return mirror_image.global_image_id; | |
289 | } | |
290 | ||
7c673cae FG |
291 | void open_image(librados::IoCtx &ioctx, const std::string &image_name, |
292 | bool readonly, librbd::ImageCtx **ictxp) | |
293 | { | |
294 | librbd::ImageCtx *ictx = new librbd::ImageCtx(image_name.c_str(), | |
295 | "", "", ioctx, readonly); | |
11fdf7f2 | 296 | EXPECT_EQ(0, ictx->state->open(0)); |
7c673cae FG |
297 | *ictxp = ictx; |
298 | } | |
299 | ||
300 | void open_local_image(librbd::ImageCtx **ictxp) | |
301 | { | |
302 | open_image(m_local_ioctx, m_image_name, true, ictxp); | |
303 | } | |
304 | ||
305 | void open_remote_image(librbd::ImageCtx **ictxp) | |
306 | { | |
307 | open_image(m_remote_ioctx, m_image_name, false, ictxp); | |
308 | } | |
309 | ||
310 | void close_image(librbd::ImageCtx *ictx) | |
311 | { | |
312 | ictx->state->close(); | |
313 | } | |
314 | ||
315 | void get_commit_positions(cls::journal::ObjectPosition *master_position, | |
316 | cls::journal::ObjectPosition *mirror_position) | |
317 | { | |
318 | std::string master_client_id = ""; | |
319 | std::string mirror_client_id = m_local_mirror_uuid; | |
320 | ||
9f95a23c TL |
321 | m_replayer->flush(); |
322 | ||
7c673cae FG |
323 | C_SaferCond cond; |
324 | uint64_t minimum_set; | |
325 | uint64_t active_set; | |
326 | std::set<cls::journal::Client> registered_clients; | |
327 | std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); | |
328 | cls::journal::client::get_mutable_metadata(m_remote_ioctx, oid, | |
329 | &minimum_set, &active_set, | |
330 | ®istered_clients, &cond); | |
331 | ASSERT_EQ(0, cond.wait()); | |
332 | ||
333 | *master_position = cls::journal::ObjectPosition(); | |
334 | *mirror_position = cls::journal::ObjectPosition(); | |
335 | ||
336 | std::set<cls::journal::Client>::const_iterator c; | |
337 | for (c = registered_clients.begin(); c != registered_clients.end(); ++c) { | |
338 | std::cout << __func__ << ": client: " << *c << std::endl; | |
339 | if (c->state != cls::journal::CLIENT_STATE_CONNECTED) { | |
340 | continue; | |
341 | } | |
342 | cls::journal::ObjectPositions object_positions = | |
343 | c->commit_position.object_positions; | |
344 | cls::journal::ObjectPositions::const_iterator p = | |
345 | object_positions.begin(); | |
346 | if (p != object_positions.end()) { | |
347 | if (c->id == master_client_id) { | |
348 | ASSERT_EQ(cls::journal::ObjectPosition(), *master_position); | |
349 | *master_position = *p; | |
350 | } else if (c->id == mirror_client_id) { | |
351 | ASSERT_EQ(cls::journal::ObjectPosition(), *mirror_position); | |
352 | *mirror_position = *p; | |
353 | } | |
354 | } | |
355 | } | |
356 | } | |
357 | ||
358 | bool wait_for_watcher_notify(int seconds) | |
359 | { | |
360 | if (m_watch_handle == 0) { | |
361 | return false; | |
362 | } | |
363 | ||
9f95a23c | 364 | std::unique_lock locker{m_watch_ctx->lock}; |
7c673cae | 365 | while (!m_watch_ctx->notified) { |
9f95a23c TL |
366 | if (m_watch_ctx->cond.wait_for(locker, |
367 | std::chrono::seconds(seconds)) == | |
368 | std::cv_status::timeout) { | |
7c673cae FG |
369 | return false; |
370 | } | |
371 | } | |
372 | m_watch_ctx->notified = false; | |
373 | return true; | |
374 | } | |
375 | ||
9f95a23c TL |
376 | int get_last_mirror_snapshot(librados::IoCtx& io_ctx, |
377 | const std::string& image_id, | |
378 | uint64_t* mirror_snap_id, | |
379 | cls::rbd::MirrorSnapshotNamespace* mirror_ns) { | |
380 | auto header_oid = librbd::util::header_name(image_id); | |
381 | ::SnapContext snapc; | |
382 | int r = librbd::cls_client::get_snapcontext(&io_ctx, header_oid, &snapc); | |
383 | if (r < 0) { | |
384 | return r; | |
385 | } | |
386 | ||
387 | // stored in reverse order | |
388 | for (auto snap_id : snapc.snaps) { | |
389 | cls::rbd::SnapshotInfo snap_info; | |
390 | r = librbd::cls_client::snapshot_get(&io_ctx, header_oid, snap_id, | |
391 | &snap_info); | |
392 | if (r < 0) { | |
393 | return r; | |
394 | } | |
395 | ||
396 | auto ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( | |
397 | &snap_info.snapshot_namespace); | |
398 | if (ns != nullptr) { | |
399 | *mirror_snap_id = snap_id; | |
400 | *mirror_ns = *ns; | |
401 | return 0; | |
402 | } | |
403 | } | |
404 | ||
405 | return -ENOENT; | |
406 | } | |
407 | ||
408 | void wait_for_journal_synced() { | |
7c673cae FG |
409 | cls::journal::ObjectPosition master_position; |
410 | cls::journal::ObjectPosition mirror_position; | |
7c673cae | 411 | for (int i = 0; i < 100; i++) { |
7c673cae FG |
412 | get_commit_positions(&master_position, &mirror_position); |
413 | if (master_position == mirror_position) { | |
9f95a23c | 414 | break; |
7c673cae FG |
415 | } |
416 | wait_for_watcher_notify(1); | |
417 | } | |
418 | ||
419 | ASSERT_EQ(master_position, mirror_position); | |
420 | } | |
421 | ||
9f95a23c TL |
422 | void wait_for_snapshot_synced() { |
423 | uint64_t remote_snap_id = CEPH_NOSNAP; | |
424 | cls::rbd::MirrorSnapshotNamespace remote_mirror_ns; | |
425 | ASSERT_EQ(0, get_last_mirror_snapshot(m_remote_ioctx, m_remote_image_id, | |
426 | &remote_snap_id, &remote_mirror_ns)); | |
427 | ||
428 | std::cout << "remote_snap_id=" << remote_snap_id << std::endl; | |
429 | ||
430 | std::string local_image_id; | |
431 | ASSERT_EQ(0, librbd::cls_client::mirror_image_get_image_id( | |
432 | &m_local_ioctx, m_global_image_id, &local_image_id)); | |
433 | ||
434 | uint64_t local_snap_id = CEPH_NOSNAP; | |
435 | cls::rbd::MirrorSnapshotNamespace local_mirror_ns; | |
436 | for (int i = 0; i < 100; i++) { | |
437 | int r = get_last_mirror_snapshot(m_local_ioctx, local_image_id, | |
438 | &local_snap_id, &local_mirror_ns); | |
439 | if (r == 0 && | |
440 | ((remote_mirror_ns.state == | |
441 | cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY && | |
442 | local_mirror_ns.state == | |
443 | cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) || | |
444 | (remote_mirror_ns.state == | |
445 | cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED && | |
446 | local_mirror_ns.state == | |
447 | cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED)) && | |
448 | local_mirror_ns.primary_mirror_uuid == m_remote_mirror_uuid && | |
449 | local_mirror_ns.primary_snap_id == remote_snap_id && | |
450 | local_mirror_ns.complete) { | |
451 | ||
452 | std::cout << "local_snap_id=" << local_snap_id << ", " | |
453 | << "local_snap_ns=" << local_mirror_ns << std::endl; | |
454 | return; | |
455 | } | |
456 | ||
457 | wait_for_watcher_notify(1); | |
458 | } | |
459 | ||
460 | ADD_FAILURE() << "failed to locate matching snapshot: " | |
461 | << "remote_snap_id=" << remote_snap_id << ", " | |
462 | << "remote_snap_ns=" << remote_mirror_ns << ", " | |
463 | << "local_snap_id=" << local_snap_id << ", " | |
464 | << "local_snap_ns=" << local_mirror_ns; | |
465 | } | |
466 | ||
467 | void wait_for_replay_complete() | |
468 | { | |
469 | if (MIRROR_IMAGE_MODE == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { | |
470 | wait_for_journal_synced(); | |
471 | } else { | |
472 | wait_for_snapshot_synced(); | |
473 | } | |
474 | } | |
475 | ||
7c673cae FG |
476 | void wait_for_stopped() { |
477 | for (int i = 0; i < 100; i++) { | |
478 | if (m_replayer->is_stopped()) { | |
479 | break; | |
480 | } | |
481 | wait_for_watcher_notify(1); | |
482 | } | |
483 | ASSERT_TRUE(m_replayer->is_stopped()); | |
484 | } | |
485 | ||
486 | void write_test_data(librbd::ImageCtx *ictx, const char *test_data, off_t off, | |
487 | size_t len) | |
488 | { | |
489 | size_t written; | |
490 | bufferlist bl; | |
491 | bl.append(std::string(test_data, len)); | |
f67539c2 | 492 | written = librbd::api::Io<>::write(*ictx, off, len, std::move(bl), 0); |
7c673cae FG |
493 | printf("wrote: %d\n", (int)written); |
494 | ASSERT_EQ(len, written); | |
495 | } | |
496 | ||
497 | void read_test_data(librbd::ImageCtx *ictx, const char *expected, off_t off, | |
498 | size_t len) | |
499 | { | |
500 | ssize_t read; | |
501 | char *result = (char *)malloc(len + 1); | |
502 | ||
503 | ASSERT_NE(static_cast<char *>(NULL), result); | |
f67539c2 TL |
504 | read = librbd::api::Io<>::read( |
505 | *ictx, off, len, librbd::io::ReadResult{result, len}, 0); | |
7c673cae FG |
506 | printf("read: %d\n", (int)read); |
507 | ASSERT_EQ(len, static_cast<size_t>(read)); | |
508 | result[len] = '\0'; | |
509 | if (memcmp(result, expected, len)) { | |
510 | printf("read: %s\nexpected: %s\n", result, expected); | |
511 | ASSERT_EQ(0, memcmp(result, expected, len)); | |
512 | } | |
513 | free(result); | |
514 | } | |
515 | ||
516 | void generate_test_data() { | |
517 | for (int i = 0; i < TEST_IO_SIZE; ++i) { | |
518 | m_test_data[i] = (char) (rand() % (126 - 33) + 33); | |
519 | } | |
520 | m_test_data[TEST_IO_SIZE] = '\0'; | |
521 | } | |
522 | ||
523 | void flush(librbd::ImageCtx *ictx) | |
524 | { | |
525 | C_SaferCond aio_flush_ctx; | |
526 | auto c = librbd::io::AioCompletion::create(&aio_flush_ctx); | |
527 | c->get(); | |
f67539c2 | 528 | librbd::api::Io<>::aio_flush(*ictx, c, true); |
7c673cae FG |
529 | ASSERT_EQ(0, c->wait_for_complete()); |
530 | c->put(); | |
531 | ||
9f95a23c TL |
532 | if (MIRROR_IMAGE_MODE == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { |
533 | C_SaferCond journal_flush_ctx; | |
534 | ictx->journal->flush_commit_position(&journal_flush_ctx); | |
535 | ASSERT_EQ(0, journal_flush_ctx.wait()); | |
536 | } else { | |
537 | uint64_t snap_id = CEPH_NOSNAP; | |
538 | ASSERT_EQ(0, librbd::api::Mirror<>::image_snapshot_create( | |
f67539c2 | 539 | ictx, 0, &snap_id)); |
9f95a23c | 540 | } |
7c673cae FG |
541 | |
542 | printf("flushed\n"); | |
543 | } | |
544 | ||
545 | static int _image_number; | |
546 | ||
9f95a23c TL |
547 | PoolMetaCache m_pool_meta_cache{g_ceph_context}; |
548 | ||
7c673cae | 549 | std::shared_ptr<librados::Rados> m_local_cluster; |
9f95a23c TL |
550 | std::unique_ptr<Threads<>> m_threads; |
551 | std::unique_ptr<Throttler<>> m_image_sync_throttler; | |
7c673cae | 552 | librados::Rados m_remote_cluster; |
9f95a23c TL |
553 | InstanceWatcher<> *m_instance_watcher; |
554 | MirrorStatusUpdater<> *m_local_status_updater; | |
7c673cae FG |
555 | std::string m_local_mirror_uuid = "local mirror uuid"; |
556 | std::string m_remote_mirror_uuid = "remote mirror uuid"; | |
557 | std::string m_local_pool_name, m_remote_pool_name; | |
558 | librados::IoCtx m_local_ioctx, m_remote_ioctx; | |
559 | std::string m_image_name; | |
560 | int64_t m_remote_pool_id; | |
561 | std::string m_remote_image_id; | |
d2e6a577 | 562 | std::string m_global_image_id; |
9f95a23c TL |
563 | ImageReplayer<> *m_replayer = nullptr; |
564 | C_WatchCtx *m_watch_ctx = nullptr; | |
565 | uint64_t m_watch_handle = 0; | |
7c673cae | 566 | char m_test_data[TEST_IO_SIZE + 1]; |
11fdf7f2 | 567 | std::string m_journal_commit_age; |
7c673cae FG |
568 | }; |
569 | ||
9f95a23c TL |
570 | template <typename T> |
571 | int TestImageReplayer<T>::_image_number; | |
7c673cae | 572 | |
9f95a23c TL |
573 | template <cls::rbd::MirrorImageMode _mirror_image_mode, uint64_t _features> |
574 | class TestImageReplayerParams { | |
575 | public: | |
576 | static const cls::rbd::MirrorImageMode MIRROR_IMAGE_MODE = _mirror_image_mode; | |
577 | static const uint64_t FEATURES = _features; | |
578 | }; | |
579 | ||
580 | typedef ::testing::Types<TestImageReplayerParams< | |
581 | cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, 125>, | |
582 | TestImageReplayerParams< | |
583 | cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, 1>, | |
584 | TestImageReplayerParams< | |
585 | cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, 5>, | |
586 | TestImageReplayerParams< | |
587 | cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, 61>, | |
588 | TestImageReplayerParams< | |
589 | cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, 125>> | |
590 | TestImageReplayerTypes; | |
591 | ||
592 | TYPED_TEST_SUITE(TestImageReplayer, TestImageReplayerTypes); | |
593 | ||
594 | TYPED_TEST(TestImageReplayer, Bootstrap) | |
7c673cae | 595 | { |
9f95a23c | 596 | this->bootstrap(); |
7c673cae FG |
597 | } |
598 | ||
9f95a23c TL |
599 | typedef TestImageReplayer<TestImageReplayerParams< |
600 | cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, 125>> TestImageReplayerJournal; | |
601 | ||
602 | TYPED_TEST(TestImageReplayer, BootstrapErrorLocalImageExists) | |
7c673cae FG |
603 | { |
604 | int order = 0; | |
9f95a23c TL |
605 | EXPECT_EQ(0, librbd::create(this->m_local_ioctx, this->m_image_name.c_str(), |
606 | 1 << 22, false, 0, &order, 0, 0)); | |
7c673cae | 607 | |
9f95a23c | 608 | this->create_replayer(); |
7c673cae | 609 | C_SaferCond cond; |
9f95a23c | 610 | this->m_replayer->start(&cond); |
7c673cae FG |
611 | ASSERT_EQ(-EEXIST, cond.wait()); |
612 | } | |
613 | ||
9f95a23c | 614 | TEST_F(TestImageReplayerJournal, BootstrapErrorNoJournal) |
7c673cae | 615 | { |
9f95a23c TL |
616 | ASSERT_EQ(0, librbd::Journal<>::remove(this->m_remote_ioctx, |
617 | this->m_remote_image_id)); | |
7c673cae | 618 | |
9f95a23c | 619 | this->create_replayer(); |
7c673cae | 620 | C_SaferCond cond; |
9f95a23c | 621 | this->m_replayer->start(&cond); |
7c673cae FG |
622 | ASSERT_EQ(-ENOENT, cond.wait()); |
623 | } | |
624 | ||
9f95a23c | 625 | TYPED_TEST(TestImageReplayer, BootstrapErrorMirrorDisabled) |
7c673cae FG |
626 | { |
627 | // disable remote image mirroring | |
9f95a23c | 628 | ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(this->m_remote_ioctx, |
7c673cae FG |
629 | RBD_MIRROR_MODE_IMAGE)); |
630 | librbd::ImageCtx *ictx; | |
9f95a23c | 631 | this->open_remote_image(&ictx); |
7c673cae | 632 | ASSERT_EQ(0, librbd::api::Mirror<>::image_disable(ictx, true)); |
9f95a23c | 633 | this->close_image(ictx); |
7c673cae | 634 | |
9f95a23c | 635 | this->create_replayer(); |
7c673cae | 636 | C_SaferCond cond; |
9f95a23c | 637 | this->m_replayer->start(&cond); |
7c673cae FG |
638 | ASSERT_EQ(-ENOENT, cond.wait()); |
639 | } | |
640 | ||
9f95a23c | 641 | TYPED_TEST(TestImageReplayer, BootstrapMirrorDisabling) |
7c673cae FG |
642 | { |
643 | // set remote image mirroring state to DISABLING | |
9f95a23c TL |
644 | if (gtest_TypeParam_::MIRROR_IMAGE_MODE == |
645 | cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { | |
646 | ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(this->m_remote_ioctx, | |
647 | RBD_MIRROR_MODE_IMAGE)); | |
648 | librbd::ImageCtx *ictx; | |
649 | this->open_remote_image(&ictx); | |
650 | ASSERT_EQ(0, librbd::api::Mirror<>::image_enable( | |
651 | ictx, RBD_MIRROR_IMAGE_MODE_JOURNAL, false)); | |
652 | this->close_image(ictx); | |
653 | } | |
654 | ||
7c673cae | 655 | cls::rbd::MirrorImage mirror_image; |
9f95a23c TL |
656 | ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&this->m_remote_ioctx, |
657 | this->m_remote_image_id, | |
7c673cae FG |
658 | &mirror_image)); |
659 | mirror_image.state = cls::rbd::MirrorImageState::MIRROR_IMAGE_STATE_DISABLING; | |
9f95a23c TL |
660 | ASSERT_EQ(0, librbd::cls_client::mirror_image_set(&this->m_remote_ioctx, |
661 | this->m_remote_image_id, | |
7c673cae | 662 | mirror_image)); |
7c673cae | 663 | |
9f95a23c | 664 | this->create_replayer(); |
7c673cae | 665 | C_SaferCond cond; |
9f95a23c | 666 | this->m_replayer->start(&cond); |
a4b75251 | 667 | ASSERT_EQ(-ENOENT, cond.wait()); |
9f95a23c | 668 | ASSERT_TRUE(this->m_replayer->is_stopped()); |
7c673cae FG |
669 | } |
670 | ||
9f95a23c | 671 | TYPED_TEST(TestImageReplayer, BootstrapDemoted) |
7c673cae FG |
672 | { |
673 | // demote remote image | |
674 | librbd::ImageCtx *ictx; | |
9f95a23c | 675 | this->open_remote_image(&ictx); |
7c673cae | 676 | ASSERT_EQ(0, librbd::api::Mirror<>::image_demote(ictx)); |
9f95a23c | 677 | this->close_image(ictx); |
7c673cae | 678 | |
9f95a23c | 679 | this->create_replayer(); |
7c673cae | 680 | C_SaferCond cond; |
9f95a23c | 681 | this->m_replayer->start(&cond); |
c07f9fc5 | 682 | ASSERT_EQ(-EREMOTEIO, cond.wait()); |
9f95a23c | 683 | ASSERT_TRUE(this->m_replayer->is_stopped()); |
7c673cae FG |
684 | } |
685 | ||
9f95a23c | 686 | TYPED_TEST(TestImageReplayer, StartInterrupted) |
7c673cae | 687 | { |
9f95a23c | 688 | this->create_replayer(); |
7c673cae | 689 | C_SaferCond start_cond, stop_cond; |
9f95a23c TL |
690 | this->m_replayer->start(&start_cond); |
691 | this->m_replayer->stop(&stop_cond); | |
7c673cae FG |
692 | int r = start_cond.wait(); |
693 | printf("start returned %d\n", r); | |
694 | // TODO: improve the test to avoid this race | |
695 | ASSERT_TRUE(r == -ECANCELED || r == 0); | |
696 | ASSERT_EQ(0, stop_cond.wait()); | |
697 | } | |
698 | ||
9f95a23c | 699 | TEST_F(TestImageReplayerJournal, JournalReset) |
7c673cae | 700 | { |
9f95a23c TL |
701 | this->bootstrap(); |
702 | delete this->m_replayer; | |
31f18b77 | 703 | |
9f95a23c TL |
704 | ASSERT_EQ(0, librbd::Journal<>::reset(this->m_remote_ioctx, |
705 | this->m_remote_image_id)); | |
31f18b77 | 706 | |
7c673cae | 707 | // try to recover |
9f95a23c | 708 | this->bootstrap(); |
7c673cae FG |
709 | } |
710 | ||
9f95a23c | 711 | TEST_F(TestImageReplayerJournal, ErrorNoJournal) |
7c673cae | 712 | { |
9f95a23c | 713 | this->bootstrap(); |
7c673cae FG |
714 | |
715 | // disable remote journal journaling | |
716 | // (reset before disabling, so it does not fail with EBUSY) | |
9f95a23c TL |
717 | ASSERT_EQ(0, librbd::Journal<>::reset(this->m_remote_ioctx, |
718 | this->m_remote_image_id)); | |
7c673cae | 719 | librbd::ImageCtx *ictx; |
9f95a23c | 720 | this->open_remote_image(&ictx); |
7c673cae FG |
721 | uint64_t features; |
722 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
723 | ASSERT_EQ(0, ictx->operations->update_features(RBD_FEATURE_JOURNALING, | |
724 | false)); | |
9f95a23c | 725 | this->close_image(ictx); |
7c673cae FG |
726 | |
727 | C_SaferCond cond; | |
9f95a23c | 728 | this->m_replayer->start(&cond); |
d2e6a577 | 729 | ASSERT_EQ(0, cond.wait()); |
7c673cae FG |
730 | } |
731 | ||
9f95a23c | 732 | TYPED_TEST(TestImageReplayer, StartStop) |
7c673cae | 733 | { |
9f95a23c | 734 | this->bootstrap(); |
7c673cae | 735 | |
9f95a23c TL |
736 | this->start(); |
737 | this->wait_for_replay_complete(); | |
738 | this->stop(); | |
7c673cae FG |
739 | } |
740 | ||
9f95a23c | 741 | TYPED_TEST(TestImageReplayer, WriteAndStartReplay) |
7c673cae | 742 | { |
9f95a23c | 743 | this->bootstrap(); |
7c673cae FG |
744 | |
745 | // Write to remote image and start replay | |
746 | ||
747 | librbd::ImageCtx *ictx; | |
748 | ||
9f95a23c TL |
749 | this->generate_test_data(); |
750 | this->open_remote_image(&ictx); | |
7c673cae | 751 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
752 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
753 | TEST_IO_SIZE); | |
7c673cae | 754 | } |
9f95a23c TL |
755 | this->flush(ictx); |
756 | this->close_image(ictx); | |
7c673cae | 757 | |
9f95a23c TL |
758 | this->start(); |
759 | this->wait_for_replay_complete(); | |
760 | this->stop(); | |
7c673cae | 761 | |
9f95a23c | 762 | this->open_local_image(&ictx); |
7c673cae | 763 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
764 | this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
765 | TEST_IO_SIZE); | |
7c673cae | 766 | } |
9f95a23c | 767 | this->close_image(ictx); |
7c673cae FG |
768 | } |
769 | ||
9f95a23c | 770 | TYPED_TEST(TestImageReplayer, StartReplayAndWrite) |
7c673cae | 771 | { |
9f95a23c | 772 | this->bootstrap(); |
7c673cae FG |
773 | |
774 | // Start replay and write to remote image | |
775 | ||
776 | librbd::ImageCtx *ictx; | |
777 | ||
9f95a23c | 778 | this->start(); |
7c673cae | 779 | |
9f95a23c TL |
780 | this->generate_test_data(); |
781 | this->open_remote_image(&ictx); | |
7c673cae | 782 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
783 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
784 | TEST_IO_SIZE); | |
7c673cae | 785 | } |
9f95a23c | 786 | this->flush(ictx); |
7c673cae | 787 | |
9f95a23c | 788 | this->wait_for_replay_complete(); |
7c673cae FG |
789 | |
790 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
9f95a23c TL |
791 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
792 | TEST_IO_SIZE); | |
7c673cae | 793 | } |
9f95a23c TL |
794 | this->flush(ictx); |
795 | this->close_image(ictx); | |
7c673cae | 796 | |
9f95a23c | 797 | this->wait_for_replay_complete(); |
7c673cae | 798 | |
9f95a23c | 799 | this->open_local_image(&ictx); |
7c673cae | 800 | for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { |
9f95a23c TL |
801 | this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
802 | TEST_IO_SIZE); | |
7c673cae | 803 | } |
9f95a23c | 804 | this->close_image(ictx); |
7c673cae | 805 | |
9f95a23c | 806 | this->stop(); |
7c673cae FG |
807 | } |
808 | ||
9f95a23c | 809 | TEST_F(TestImageReplayerJournal, NextTag) |
7c673cae | 810 | { |
9f95a23c | 811 | this->bootstrap(); |
7c673cae FG |
812 | |
813 | // write, reopen, and write again to test switch to the next tag | |
814 | ||
815 | librbd::ImageCtx *ictx; | |
816 | ||
9f95a23c | 817 | this->start(); |
7c673cae | 818 | |
9f95a23c | 819 | this->generate_test_data(); |
7c673cae FG |
820 | |
821 | const int N = 10; | |
822 | ||
823 | for (int j = 0; j < N; j++) { | |
9f95a23c | 824 | this->open_remote_image(&ictx); |
7c673cae | 825 | for (int i = j * TEST_IO_COUNT; i < (j + 1) * TEST_IO_COUNT; ++i) { |
9f95a23c TL |
826 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
827 | TEST_IO_SIZE); | |
7c673cae | 828 | } |
9f95a23c | 829 | this->close_image(ictx); |
7c673cae FG |
830 | } |
831 | ||
9f95a23c | 832 | this->wait_for_replay_complete(); |
7c673cae | 833 | |
9f95a23c | 834 | this->open_local_image(&ictx); |
7c673cae | 835 | for (int i = 0; i < N * TEST_IO_COUNT; ++i) { |
9f95a23c TL |
836 | this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
837 | TEST_IO_SIZE); | |
7c673cae | 838 | } |
9f95a23c | 839 | this->close_image(ictx); |
7c673cae | 840 | |
9f95a23c | 841 | this->stop(); |
7c673cae FG |
842 | } |
843 | ||
9f95a23c | 844 | TYPED_TEST(TestImageReplayer, Resync) |
7c673cae | 845 | { |
9f95a23c | 846 | this->bootstrap(); |
7c673cae FG |
847 | |
848 | librbd::ImageCtx *ictx; | |
849 | ||
9f95a23c | 850 | this->start(); |
7c673cae | 851 | |
9f95a23c | 852 | this->generate_test_data(); |
7c673cae | 853 | |
9f95a23c | 854 | this->open_remote_image(&ictx); |
7c673cae | 855 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
856 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
857 | TEST_IO_SIZE); | |
7c673cae | 858 | } |
9f95a23c | 859 | this->flush(ictx); |
7c673cae | 860 | |
9f95a23c | 861 | this->wait_for_replay_complete(); |
7c673cae FG |
862 | |
863 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
9f95a23c TL |
864 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
865 | TEST_IO_SIZE); | |
7c673cae | 866 | } |
9f95a23c TL |
867 | this->flush(ictx); |
868 | this->close_image(ictx); | |
7c673cae | 869 | |
9f95a23c TL |
870 | this->open_local_image(&ictx); |
871 | EXPECT_EQ(0, librbd::api::Mirror<>::image_resync(ictx)); | |
872 | this->close_image(ictx); | |
7c673cae | 873 | |
9f95a23c | 874 | this->wait_for_stopped(); |
7c673cae FG |
875 | |
876 | C_SaferCond cond; | |
9f95a23c | 877 | this->m_replayer->start(&cond); |
7c673cae FG |
878 | ASSERT_EQ(0, cond.wait()); |
879 | ||
9f95a23c TL |
880 | ASSERT_TRUE(this->m_replayer->is_replaying()); |
881 | this->wait_for_replay_complete(); | |
7c673cae | 882 | |
9f95a23c | 883 | this->open_local_image(&ictx); |
7c673cae | 884 | for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { |
9f95a23c TL |
885 | this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
886 | TEST_IO_SIZE); | |
7c673cae | 887 | } |
9f95a23c | 888 | this->close_image(ictx); |
7c673cae | 889 | |
9f95a23c | 890 | this->stop(); |
7c673cae FG |
891 | } |
892 | ||
9f95a23c | 893 | TYPED_TEST(TestImageReplayer, Resync_While_Stop) |
7c673cae | 894 | { |
9f95a23c | 895 | this->bootstrap(); |
7c673cae | 896 | |
9f95a23c | 897 | this->start(); |
7c673cae | 898 | |
9f95a23c | 899 | this->generate_test_data(); |
7c673cae FG |
900 | |
901 | librbd::ImageCtx *ictx; | |
9f95a23c | 902 | this->open_remote_image(&ictx); |
7c673cae | 903 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
904 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
905 | TEST_IO_SIZE); | |
7c673cae | 906 | } |
9f95a23c | 907 | this->flush(ictx); |
7c673cae | 908 | |
9f95a23c | 909 | this->wait_for_replay_complete(); |
7c673cae FG |
910 | |
911 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
9f95a23c TL |
912 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
913 | TEST_IO_SIZE); | |
7c673cae | 914 | } |
9f95a23c TL |
915 | this->flush(ictx); |
916 | this->close_image(ictx); | |
7c673cae | 917 | |
9f95a23c | 918 | this->wait_for_replay_complete(); |
7c673cae FG |
919 | |
920 | C_SaferCond cond; | |
9f95a23c | 921 | this->m_replayer->stop(&cond); |
7c673cae FG |
922 | ASSERT_EQ(0, cond.wait()); |
923 | ||
9f95a23c TL |
924 | this->open_local_image(&ictx); |
925 | EXPECT_EQ(0, librbd::api::Mirror<>::image_resync(ictx)); | |
926 | this->close_image(ictx); | |
7c673cae FG |
927 | |
928 | C_SaferCond cond2; | |
9f95a23c | 929 | this->m_replayer->start(&cond2); |
7c673cae FG |
930 | ASSERT_EQ(0, cond2.wait()); |
931 | ||
9f95a23c | 932 | ASSERT_TRUE(this->m_replayer->is_stopped()); |
7c673cae | 933 | |
7c673cae | 934 | C_SaferCond cond3; |
9f95a23c | 935 | this->m_replayer->start(&cond3); |
7c673cae FG |
936 | ASSERT_EQ(0, cond3.wait()); |
937 | ||
9f95a23c | 938 | ASSERT_TRUE(this->m_replayer->is_replaying()); |
7c673cae | 939 | |
9f95a23c | 940 | this->wait_for_replay_complete(); |
7c673cae | 941 | |
9f95a23c | 942 | this->open_local_image(&ictx); |
7c673cae | 943 | for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { |
9f95a23c TL |
944 | this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
945 | TEST_IO_SIZE); | |
7c673cae | 946 | } |
9f95a23c | 947 | this->close_image(ictx); |
7c673cae | 948 | |
9f95a23c | 949 | this->stop(); |
7c673cae FG |
950 | } |
951 | ||
9f95a23c | 952 | TYPED_TEST(TestImageReplayer, Resync_StartInterrupted) |
7c673cae | 953 | { |
9f95a23c | 954 | this->bootstrap(); |
7c673cae FG |
955 | |
956 | librbd::ImageCtx *ictx; | |
9f95a23c TL |
957 | this->open_local_image(&ictx); |
958 | EXPECT_EQ(0, librbd::api::Mirror<>::image_resync(ictx)); | |
959 | this->close_image(ictx); | |
7c673cae FG |
960 | |
961 | C_SaferCond cond; | |
9f95a23c | 962 | this->m_replayer->start(&cond); |
7c673cae FG |
963 | ASSERT_EQ(0, cond.wait()); |
964 | ||
9f95a23c | 965 | ASSERT_TRUE(this->m_replayer->is_stopped()); |
7c673cae | 966 | |
7c673cae | 967 | C_SaferCond cond2; |
9f95a23c | 968 | this->m_replayer->start(&cond2); |
7c673cae FG |
969 | ASSERT_EQ(0, cond2.wait()); |
970 | ||
9f95a23c | 971 | this->create_watch_ctx(); |
7c673cae | 972 | |
9f95a23c | 973 | ASSERT_TRUE(this->m_replayer->is_replaying()); |
7c673cae | 974 | |
9f95a23c TL |
975 | this->generate_test_data(); |
976 | this->open_remote_image(&ictx); | |
7c673cae | 977 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
978 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
979 | TEST_IO_SIZE); | |
7c673cae | 980 | } |
9f95a23c | 981 | this->flush(ictx); |
7c673cae | 982 | |
9f95a23c | 983 | this->wait_for_replay_complete(); |
7c673cae FG |
984 | |
985 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { | |
9f95a23c TL |
986 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
987 | TEST_IO_SIZE); | |
7c673cae | 988 | } |
9f95a23c TL |
989 | this->flush(ictx); |
990 | this->close_image(ictx); | |
7c673cae | 991 | |
9f95a23c | 992 | this->wait_for_replay_complete(); |
7c673cae | 993 | |
9f95a23c | 994 | this->open_local_image(&ictx); |
7c673cae | 995 | for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { |
9f95a23c TL |
996 | this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
997 | TEST_IO_SIZE); | |
7c673cae | 998 | } |
9f95a23c | 999 | this->close_image(ictx); |
7c673cae | 1000 | |
9f95a23c | 1001 | this->stop(); |
7c673cae FG |
1002 | } |
1003 | ||
9f95a23c TL |
1004 | TEST_F(TestImageReplayerJournal, MultipleReplayFailures_SingleEpoch) { |
1005 | this->bootstrap(); | |
7c673cae FG |
1006 | |
1007 | // inject a snapshot that cannot be unprotected | |
1008 | librbd::ImageCtx *ictx; | |
9f95a23c | 1009 | this->open_image(this->m_local_ioctx, this->m_image_name, false, &ictx); |
7c673cae | 1010 | ictx->features &= ~RBD_FEATURE_JOURNALING; |
f67539c2 | 1011 | librbd::NoOpProgressContext prog_ctx; |
7c673cae | 1012 | ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), |
f67539c2 | 1013 | "foo", 0, prog_ctx)); |
7c673cae FG |
1014 | ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), |
1015 | "foo")); | |
1016 | ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN, | |
11fdf7f2 | 1017 | {ictx->md_ctx.get_id(), "", |
7c673cae FG |
1018 | ictx->id, |
1019 | ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), "foo"}]}, | |
1020 | "dummy child id")); | |
9f95a23c | 1021 | this->close_image(ictx); |
7c673cae FG |
1022 | |
1023 | // race failed op shut down with new ops | |
9f95a23c | 1024 | this->open_remote_image(&ictx); |
7c673cae | 1025 | for (uint64_t i = 0; i < 10; ++i) { |
9f95a23c | 1026 | std::shared_lock owner_locker{ictx->owner_lock}; |
7c673cae FG |
1027 | C_SaferCond request_lock; |
1028 | ictx->exclusive_lock->acquire_lock(&request_lock); | |
1029 | ASSERT_EQ(0, request_lock.wait()); | |
1030 | ||
1031 | C_SaferCond append_ctx; | |
1032 | ictx->journal->append_op_event( | |
1033 | i, | |
1034 | librbd::journal::EventEntry{ | |
1035 | librbd::journal::SnapUnprotectEvent{i, | |
1036 | cls::rbd::UserSnapshotNamespace(), | |
1037 | "foo"}}, | |
1038 | &append_ctx); | |
1039 | ASSERT_EQ(0, append_ctx.wait()); | |
1040 | ||
1041 | C_SaferCond commit_ctx; | |
1042 | ictx->journal->commit_op_event(i, 0, &commit_ctx); | |
1043 | ASSERT_EQ(0, commit_ctx.wait()); | |
1044 | ||
1045 | C_SaferCond release_ctx; | |
1046 | ictx->exclusive_lock->release_lock(&release_ctx); | |
1047 | ASSERT_EQ(0, release_ctx.wait()); | |
1048 | } | |
1049 | ||
1050 | for (uint64_t i = 0; i < 5; ++i) { | |
9f95a23c TL |
1051 | this->start(); |
1052 | this->wait_for_stopped(); | |
1053 | this->unwatch(); | |
7c673cae | 1054 | } |
9f95a23c | 1055 | this->close_image(ictx); |
7c673cae FG |
1056 | } |
1057 | ||
9f95a23c TL |
1058 | TEST_F(TestImageReplayerJournal, MultipleReplayFailures_MultiEpoch) { |
1059 | this->bootstrap(); | |
7c673cae FG |
1060 | |
1061 | // inject a snapshot that cannot be unprotected | |
1062 | librbd::ImageCtx *ictx; | |
9f95a23c | 1063 | this->open_image(this->m_local_ioctx, this->m_image_name, false, &ictx); |
7c673cae | 1064 | ictx->features &= ~RBD_FEATURE_JOURNALING; |
f67539c2 | 1065 | librbd::NoOpProgressContext prog_ctx; |
7c673cae | 1066 | ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), |
f67539c2 | 1067 | "foo", 0, prog_ctx)); |
7c673cae FG |
1068 | ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), |
1069 | "foo")); | |
1070 | ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN, | |
11fdf7f2 | 1071 | {ictx->md_ctx.get_id(), "", |
7c673cae FG |
1072 | ictx->id, |
1073 | ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), | |
1074 | "foo"}]}, | |
1075 | "dummy child id")); | |
9f95a23c | 1076 | this->close_image(ictx); |
7c673cae FG |
1077 | |
1078 | // race failed op shut down with new tag flush | |
9f95a23c | 1079 | this->open_remote_image(&ictx); |
7c673cae | 1080 | { |
9f95a23c | 1081 | std::shared_lock owner_locker{ictx->owner_lock}; |
7c673cae FG |
1082 | C_SaferCond request_lock; |
1083 | ictx->exclusive_lock->acquire_lock(&request_lock); | |
1084 | ASSERT_EQ(0, request_lock.wait()); | |
1085 | ||
1086 | C_SaferCond append_ctx; | |
1087 | ictx->journal->append_op_event( | |
1088 | 1U, | |
1089 | librbd::journal::EventEntry{ | |
1090 | librbd::journal::SnapUnprotectEvent{1U, | |
1091 | cls::rbd::UserSnapshotNamespace(), | |
1092 | "foo"}}, | |
1093 | &append_ctx); | |
1094 | ASSERT_EQ(0, append_ctx.wait()); | |
1095 | ||
1096 | C_SaferCond commit_ctx; | |
1097 | ictx->journal->commit_op_event(1U, 0, &commit_ctx); | |
1098 | ASSERT_EQ(0, commit_ctx.wait()); | |
1099 | ||
1100 | C_SaferCond release_ctx; | |
1101 | ictx->exclusive_lock->release_lock(&release_ctx); | |
1102 | ASSERT_EQ(0, release_ctx.wait()); | |
1103 | } | |
1104 | ||
9f95a23c TL |
1105 | this->generate_test_data(); |
1106 | this->write_test_data(ictx, this->m_test_data, 0, TEST_IO_SIZE); | |
7c673cae FG |
1107 | |
1108 | for (uint64_t i = 0; i < 5; ++i) { | |
9f95a23c TL |
1109 | this->start(); |
1110 | this->wait_for_stopped(); | |
1111 | this->unwatch(); | |
7c673cae | 1112 | } |
9f95a23c | 1113 | this->close_image(ictx); |
7c673cae FG |
1114 | } |
1115 | ||
9f95a23c | 1116 | TEST_F(TestImageReplayerJournal, Disconnect) |
7c673cae | 1117 | { |
9f95a23c | 1118 | this->bootstrap(); |
7c673cae FG |
1119 | |
1120 | // Make sure rbd_mirroring_resync_after_disconnect is not set | |
9f95a23c | 1121 | EXPECT_EQ(0, this->m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "false")); |
7c673cae FG |
1122 | |
1123 | // Test start fails if disconnected | |
1124 | ||
1125 | librbd::ImageCtx *ictx; | |
1126 | ||
9f95a23c TL |
1127 | this->generate_test_data(); |
1128 | this->open_remote_image(&ictx); | |
7c673cae | 1129 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
1130 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
1131 | TEST_IO_SIZE); | |
7c673cae | 1132 | } |
9f95a23c TL |
1133 | this->flush(ictx); |
1134 | this->close_image(ictx); | |
7c673cae | 1135 | |
9f95a23c TL |
1136 | std::string oid = ::journal::Journaler::header_oid(this->m_remote_image_id); |
1137 | ASSERT_EQ(0, | |
1138 | cls::journal::client::client_update_state(this->m_remote_ioctx, | |
1139 | oid, this->m_local_mirror_uuid, | |
1140 | cls::journal::CLIENT_STATE_DISCONNECTED)); | |
7c673cae FG |
1141 | |
1142 | C_SaferCond cond1; | |
9f95a23c | 1143 | this->m_replayer->start(&cond1); |
7c673cae FG |
1144 | ASSERT_EQ(-ENOTCONN, cond1.wait()); |
1145 | ||
1146 | // Test start succeeds after resync | |
1147 | ||
9f95a23c | 1148 | this->open_local_image(&ictx); |
7c673cae | 1149 | librbd::Journal<>::request_resync(ictx); |
9f95a23c | 1150 | this->close_image(ictx); |
7c673cae | 1151 | C_SaferCond cond2; |
9f95a23c | 1152 | this->m_replayer->start(&cond2); |
d2e6a577 | 1153 | ASSERT_EQ(0, cond2.wait()); |
7c673cae | 1154 | |
9f95a23c TL |
1155 | this->start(); |
1156 | this->wait_for_replay_complete(); | |
7c673cae FG |
1157 | |
1158 | // Test replay stopped after disconnect | |
1159 | ||
9f95a23c | 1160 | this->open_remote_image(&ictx); |
7c673cae | 1161 | for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { |
9f95a23c TL |
1162 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
1163 | TEST_IO_SIZE); | |
7c673cae | 1164 | } |
9f95a23c TL |
1165 | this->flush(ictx); |
1166 | this->close_image(ictx); | |
7c673cae | 1167 | |
9f95a23c TL |
1168 | ASSERT_EQ(0, |
1169 | cls::journal::client::client_update_state(this->m_remote_ioctx, oid, | |
1170 | this->m_local_mirror_uuid, | |
1171 | cls::journal::CLIENT_STATE_DISCONNECTED)); | |
7c673cae | 1172 | bufferlist bl; |
9f95a23c | 1173 | ASSERT_EQ(0, this->m_remote_ioctx.notify2(oid, bl, 5000, NULL)); |
7c673cae | 1174 | |
9f95a23c | 1175 | this->wait_for_stopped(); |
7c673cae FG |
1176 | |
1177 | // Test start fails after disconnect | |
1178 | ||
1179 | C_SaferCond cond3; | |
9f95a23c | 1180 | this->m_replayer->start(&cond3); |
7c673cae FG |
1181 | ASSERT_EQ(-ENOTCONN, cond3.wait()); |
1182 | C_SaferCond cond4; | |
9f95a23c | 1183 | this->m_replayer->start(&cond4); |
7c673cae FG |
1184 | ASSERT_EQ(-ENOTCONN, cond4.wait()); |
1185 | ||
1186 | // Test automatic resync if rbd_mirroring_resync_after_disconnect is set | |
1187 | ||
9f95a23c | 1188 | EXPECT_EQ(0, this->m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "true")); |
7c673cae FG |
1189 | |
1190 | // Resync is flagged on first start attempt | |
1191 | C_SaferCond cond5; | |
9f95a23c | 1192 | this->m_replayer->start(&cond5); |
7c673cae | 1193 | ASSERT_EQ(-ENOTCONN, cond5.wait()); |
7c673cae FG |
1194 | |
1195 | C_SaferCond cond6; | |
9f95a23c | 1196 | this->m_replayer->start(&cond6); |
7c673cae | 1197 | ASSERT_EQ(0, cond6.wait()); |
9f95a23c | 1198 | this->wait_for_replay_complete(); |
7c673cae | 1199 | |
9f95a23c | 1200 | this->stop(); |
7c673cae FG |
1201 | } |
1202 | ||
9f95a23c | 1203 | TEST_F(TestImageReplayerJournal, UpdateFeatures) |
7c673cae | 1204 | { |
9f95a23c | 1205 | // TODO add support to snapshot-based mirroring |
7c673cae FG |
1206 | const uint64_t FEATURES_TO_UPDATE = |
1207 | RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF; | |
1208 | ||
1209 | uint64_t features; | |
1210 | librbd::ImageCtx *ictx; | |
1211 | ||
1212 | // Make sure the features we will update are disabled initially | |
1213 | ||
9f95a23c | 1214 | this->open_remote_image(&ictx); |
7c673cae FG |
1215 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); |
1216 | features &= FEATURES_TO_UPDATE; | |
1217 | if (features) { | |
1218 | ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, | |
1219 | false)); | |
1220 | } | |
1221 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1222 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
9f95a23c | 1223 | this->close_image(ictx); |
7c673cae | 1224 | |
9f95a23c | 1225 | this->bootstrap(); |
7c673cae | 1226 | |
9f95a23c | 1227 | this->open_remote_image(&ictx); |
7c673cae FG |
1228 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); |
1229 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
9f95a23c | 1230 | this->close_image(ictx); |
7c673cae | 1231 | |
9f95a23c | 1232 | this->open_local_image(&ictx); |
7c673cae FG |
1233 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); |
1234 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
9f95a23c | 1235 | this->close_image(ictx); |
7c673cae FG |
1236 | |
1237 | // Start replay and update features | |
1238 | ||
9f95a23c | 1239 | this->start(); |
7c673cae | 1240 | |
9f95a23c | 1241 | this->open_remote_image(&ictx); |
7c673cae FG |
1242 | ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, |
1243 | true)); | |
1244 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1245 | ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE); | |
9f95a23c | 1246 | this->close_image(ictx); |
7c673cae | 1247 | |
9f95a23c | 1248 | this->wait_for_replay_complete(); |
7c673cae | 1249 | |
9f95a23c | 1250 | this->open_local_image(&ictx); |
7c673cae FG |
1251 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); |
1252 | ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE); | |
9f95a23c | 1253 | this->close_image(ictx); |
7c673cae | 1254 | |
9f95a23c | 1255 | this->open_remote_image(&ictx); |
7c673cae FG |
1256 | ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, |
1257 | false)); | |
1258 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); | |
1259 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
9f95a23c | 1260 | this->close_image(ictx); |
7c673cae | 1261 | |
9f95a23c | 1262 | this->wait_for_replay_complete(); |
7c673cae | 1263 | |
9f95a23c | 1264 | this->open_local_image(&ictx); |
7c673cae FG |
1265 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); |
1266 | ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); | |
9f95a23c | 1267 | this->close_image(ictx); |
7c673cae FG |
1268 | |
1269 | // Test update_features error does not stop replication | |
1270 | ||
9f95a23c | 1271 | this->open_remote_image(&ictx); |
7c673cae FG |
1272 | ASSERT_EQ(0, librbd::get_features(ictx, &features)); |
1273 | ASSERT_NE(0U, features & RBD_FEATURE_EXCLUSIVE_LOCK); | |
1274 | ASSERT_EQ(-EINVAL, ictx->operations->update_features(RBD_FEATURE_EXCLUSIVE_LOCK, | |
1275 | false)); | |
9f95a23c | 1276 | this->generate_test_data(); |
7c673cae | 1277 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
1278 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
1279 | TEST_IO_SIZE); | |
7c673cae | 1280 | } |
9f95a23c TL |
1281 | this->flush(ictx); |
1282 | this->close_image(ictx); | |
7c673cae | 1283 | |
9f95a23c | 1284 | this->wait_for_replay_complete(); |
7c673cae | 1285 | |
9f95a23c | 1286 | this->open_local_image(&ictx); |
7c673cae | 1287 | for (int i = 0; i < TEST_IO_COUNT; ++i) { |
9f95a23c TL |
1288 | this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
1289 | TEST_IO_SIZE); | |
7c673cae | 1290 | } |
9f95a23c | 1291 | this->close_image(ictx); |
7c673cae | 1292 | |
9f95a23c | 1293 | this->stop(); |
7c673cae FG |
1294 | } |
1295 | ||
9f95a23c | 1296 | TEST_F(TestImageReplayerJournal, MetadataSetRemove) |
7c673cae | 1297 | { |
9f95a23c | 1298 | // TODO add support to snapshot-based mirroring |
7c673cae FG |
1299 | const std::string KEY = "test_key"; |
1300 | const std::string VALUE = "test_value"; | |
1301 | ||
1302 | librbd::ImageCtx *ictx; | |
1303 | std::string value; | |
1304 | ||
9f95a23c | 1305 | this->bootstrap(); |
7c673cae | 1306 | |
9f95a23c | 1307 | this->start(); |
7c673cae FG |
1308 | |
1309 | // Test metadata_set replication | |
1310 | ||
9f95a23c | 1311 | this->open_remote_image(&ictx); |
7c673cae FG |
1312 | ASSERT_EQ(0, ictx->operations->metadata_set(KEY, VALUE)); |
1313 | value.clear(); | |
1314 | ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value)); | |
1315 | ASSERT_EQ(VALUE, value); | |
9f95a23c | 1316 | this->close_image(ictx); |
7c673cae | 1317 | |
9f95a23c | 1318 | this->wait_for_replay_complete(); |
7c673cae | 1319 | |
9f95a23c | 1320 | this->open_local_image(&ictx); |
7c673cae FG |
1321 | value.clear(); |
1322 | ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value)); | |
1323 | ASSERT_EQ(VALUE, value); | |
9f95a23c | 1324 | this->close_image(ictx); |
7c673cae FG |
1325 | |
1326 | // Test metadata_remove replication | |
1327 | ||
9f95a23c | 1328 | this->open_remote_image(&ictx); |
7c673cae FG |
1329 | ASSERT_EQ(0, ictx->operations->metadata_remove(KEY)); |
1330 | ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value)); | |
9f95a23c | 1331 | this->close_image(ictx); |
7c673cae | 1332 | |
9f95a23c | 1333 | this->wait_for_replay_complete(); |
7c673cae | 1334 | |
9f95a23c | 1335 | this->open_local_image(&ictx); |
7c673cae | 1336 | ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value)); |
9f95a23c | 1337 | this->close_image(ictx); |
7c673cae | 1338 | |
9f95a23c | 1339 | this->stop(); |
7c673cae FG |
1340 | } |
1341 | ||
9f95a23c | 1342 | TEST_F(TestImageReplayerJournal, MirroringDelay) |
7c673cae | 1343 | { |
9f95a23c | 1344 | // TODO add support to snapshot-based mirroring |
7c673cae FG |
1345 | const double DELAY = 10; // set less than wait_for_replay_complete timeout |
1346 | ||
1347 | librbd::ImageCtx *ictx; | |
1348 | utime_t start_time; | |
1349 | double delay; | |
1350 | ||
9f95a23c | 1351 | this->bootstrap(); |
7c673cae | 1352 | |
9f95a23c | 1353 | ASSERT_EQ(0, this->m_local_cluster->conf_set("rbd_mirroring_replay_delay", |
7c673cae | 1354 | stringify(DELAY).c_str())); |
9f95a23c | 1355 | this->open_local_image(&ictx); |
7c673cae | 1356 | ASSERT_EQ(DELAY, ictx->mirroring_replay_delay); |
9f95a23c | 1357 | this->close_image(ictx); |
7c673cae | 1358 | |
9f95a23c | 1359 | this->start(); |
7c673cae FG |
1360 | |
1361 | // Test delay | |
1362 | ||
9f95a23c TL |
1363 | this->generate_test_data(); |
1364 | this->open_remote_image(&ictx); | |
7c673cae FG |
1365 | start_time = ceph_clock_now(); |
1366 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
9f95a23c TL |
1367 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
1368 | TEST_IO_SIZE); | |
7c673cae | 1369 | } |
9f95a23c TL |
1370 | this->flush(ictx); |
1371 | this->close_image(ictx); | |
7c673cae | 1372 | |
9f95a23c | 1373 | this->wait_for_replay_complete(); |
7c673cae FG |
1374 | delay = ceph_clock_now() - start_time; |
1375 | ASSERT_GE(delay, DELAY); | |
1376 | ||
1377 | // Test stop when delaying replay | |
1378 | ||
9f95a23c | 1379 | this->open_remote_image(&ictx); |
7c673cae FG |
1380 | start_time = ceph_clock_now(); |
1381 | for (int i = 0; i < TEST_IO_COUNT; ++i) { | |
9f95a23c TL |
1382 | this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, |
1383 | TEST_IO_SIZE); | |
7c673cae | 1384 | } |
9f95a23c | 1385 | this->close_image(ictx); |
7c673cae FG |
1386 | |
1387 | sleep(DELAY / 2); | |
9f95a23c TL |
1388 | this->stop(); |
1389 | this->start(); | |
7c673cae | 1390 | |
9f95a23c | 1391 | this->wait_for_replay_complete(); |
7c673cae FG |
1392 | delay = ceph_clock_now() - start_time; |
1393 | ASSERT_GE(delay, DELAY); | |
1394 | ||
9f95a23c | 1395 | this->stop(); |
7c673cae | 1396 | } |
9f95a23c TL |
1397 | |
1398 | TYPED_TEST(TestImageReplayer, ImageRename) { | |
1399 | this->create_replayer(); | |
1400 | this->start(); | |
1401 | ||
1402 | librbd::ImageCtx* remote_image_ctx = nullptr; | |
1403 | this->open_remote_image(&remote_image_ctx); | |
1404 | auto image_name = this->get_temp_image_name(); | |
1405 | ASSERT_EQ(0, remote_image_ctx->operations->rename(image_name.c_str())); | |
1406 | this->flush(remote_image_ctx); | |
1407 | ||
1408 | this->wait_for_replay_complete(); | |
1409 | ||
1410 | librbd::ImageCtx* local_image_ctx = nullptr; | |
1411 | this->open_image(this->m_local_ioctx, image_name, true, &local_image_ctx); | |
1412 | ASSERT_EQ(image_name, local_image_ctx->name); | |
1413 | ||
1414 | this->close_image(local_image_ctx); | |
1415 | this->close_image(remote_image_ctx); | |
1416 | this->stop(); | |
1417 | } | |
1418 | ||
1419 | TYPED_TEST(TestImageReplayer, UpdateFeatures) { | |
1420 | const uint64_t FEATURES_TO_UPDATE = | |
1421 | RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF | RBD_FEATURE_DEEP_FLATTEN; | |
1422 | REQUIRE((this->FEATURES & FEATURES_TO_UPDATE) == FEATURES_TO_UPDATE); | |
1423 | ||
1424 | librbd::ImageCtx* remote_image_ctx = nullptr; | |
1425 | this->open_remote_image(&remote_image_ctx); | |
1426 | ||
1427 | ASSERT_EQ(0, remote_image_ctx->operations->update_features( | |
1428 | (RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF), false)); | |
1429 | this->flush(remote_image_ctx); | |
1430 | ||
1431 | this->create_replayer(); | |
1432 | this->start(); | |
1433 | this->wait_for_replay_complete(); | |
1434 | ||
1435 | librbd::ImageCtx* local_image_ctx = nullptr; | |
1436 | this->open_local_image(&local_image_ctx); | |
1437 | ASSERT_EQ(0U, local_image_ctx->features & ( | |
1438 | RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF)); | |
1439 | ||
1440 | // enable object-map/fast-diff | |
1441 | ASSERT_EQ(0, remote_image_ctx->operations->update_features( | |
1442 | (RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF), true)); | |
1443 | this->flush(remote_image_ctx); | |
1444 | this->wait_for_replay_complete(); | |
1445 | ||
1446 | ASSERT_EQ(0, local_image_ctx->state->refresh()); | |
1447 | ASSERT_EQ(RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF, | |
1448 | local_image_ctx->features & ( | |
1449 | RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF)); | |
1450 | ||
1451 | // disable deep-flatten | |
1452 | ASSERT_EQ(0, remote_image_ctx->operations->update_features( | |
1453 | RBD_FEATURE_DEEP_FLATTEN, false)); | |
1454 | this->flush(remote_image_ctx); | |
1455 | this->wait_for_replay_complete(); | |
1456 | ||
1457 | ASSERT_EQ(0, local_image_ctx->state->refresh()); | |
1458 | ASSERT_EQ(0, local_image_ctx->features & RBD_FEATURE_DEEP_FLATTEN); | |
1459 | ||
1460 | this->close_image(local_image_ctx); | |
1461 | this->close_image(remote_image_ctx); | |
1462 | this->stop(); | |
1463 | } | |
1464 | ||
1465 | TYPED_TEST(TestImageReplayer, SnapshotUnprotect) { | |
1466 | librbd::ImageCtx* remote_image_ctx = nullptr; | |
1467 | this->open_remote_image(&remote_image_ctx); | |
1468 | ||
1469 | // create a protected snapshot | |
f67539c2 | 1470 | librbd::NoOpProgressContext prog_ctx; |
9f95a23c | 1471 | ASSERT_EQ(0, remote_image_ctx->operations->snap_create( |
f67539c2 | 1472 | cls::rbd::UserSnapshotNamespace{}, "snap1", 0, prog_ctx)); |
9f95a23c TL |
1473 | ASSERT_EQ(0, remote_image_ctx->operations->snap_protect( |
1474 | cls::rbd::UserSnapshotNamespace{}, "snap1")); | |
1475 | this->flush(remote_image_ctx); | |
1476 | ||
1477 | this->create_replayer(); | |
1478 | this->start(); | |
1479 | this->wait_for_replay_complete(); | |
1480 | ||
1481 | librbd::ImageCtx* local_image_ctx = nullptr; | |
1482 | this->open_local_image(&local_image_ctx); | |
1483 | auto local_snap_id_it = local_image_ctx->snap_ids.find({ | |
1484 | {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); | |
1485 | ASSERT_NE(local_image_ctx->snap_ids.end(), local_snap_id_it); | |
1486 | auto local_snap_id = local_snap_id_it->second; | |
1487 | auto local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); | |
1488 | ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); | |
1489 | ASSERT_EQ(RBD_PROTECTION_STATUS_PROTECTED, | |
1490 | local_snap_info_it->second.protection_status); | |
1491 | ||
1492 | // unprotect the snapshot | |
1493 | ASSERT_EQ(0, remote_image_ctx->operations->snap_unprotect( | |
1494 | cls::rbd::UserSnapshotNamespace{}, "snap1")); | |
1495 | this->flush(remote_image_ctx); | |
1496 | this->wait_for_replay_complete(); | |
1497 | ||
1498 | ASSERT_EQ(0, local_image_ctx->state->refresh()); | |
1499 | local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); | |
1500 | ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); | |
1501 | ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, | |
1502 | local_snap_info_it->second.protection_status); | |
1503 | ||
1504 | this->close_image(local_image_ctx); | |
1505 | this->close_image(remote_image_ctx); | |
1506 | this->stop(); | |
1507 | } | |
1508 | ||
1509 | TYPED_TEST(TestImageReplayer, SnapshotProtect) { | |
1510 | librbd::ImageCtx* remote_image_ctx = nullptr; | |
1511 | this->open_remote_image(&remote_image_ctx); | |
1512 | ||
1513 | // create an unprotected snapshot | |
f67539c2 | 1514 | librbd::NoOpProgressContext prog_ctx; |
9f95a23c | 1515 | ASSERT_EQ(0, remote_image_ctx->operations->snap_create( |
f67539c2 | 1516 | cls::rbd::UserSnapshotNamespace{}, "snap1", 0, prog_ctx)); |
9f95a23c TL |
1517 | this->flush(remote_image_ctx); |
1518 | ||
1519 | this->create_replayer(); | |
1520 | this->start(); | |
1521 | this->wait_for_replay_complete(); | |
1522 | ||
1523 | librbd::ImageCtx* local_image_ctx = nullptr; | |
1524 | this->open_local_image(&local_image_ctx); | |
1525 | auto local_snap_id_it = local_image_ctx->snap_ids.find({ | |
1526 | {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); | |
1527 | ASSERT_NE(local_image_ctx->snap_ids.end(), local_snap_id_it); | |
1528 | auto local_snap_id = local_snap_id_it->second; | |
1529 | auto local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); | |
1530 | ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); | |
1531 | ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, | |
1532 | local_snap_info_it->second.protection_status); | |
1533 | ||
1534 | // protect the snapshot | |
1535 | ASSERT_EQ(0, remote_image_ctx->operations->snap_protect( | |
1536 | cls::rbd::UserSnapshotNamespace{}, "snap1")); | |
1537 | this->flush(remote_image_ctx); | |
1538 | this->wait_for_replay_complete(); | |
1539 | ||
1540 | ASSERT_EQ(0, local_image_ctx->state->refresh()); | |
1541 | local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); | |
1542 | ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); | |
1543 | ASSERT_EQ(RBD_PROTECTION_STATUS_PROTECTED, | |
1544 | local_snap_info_it->second.protection_status); | |
1545 | ||
1546 | this->close_image(local_image_ctx); | |
1547 | this->close_image(remote_image_ctx); | |
1548 | this->stop(); | |
1549 | } | |
1550 | ||
1551 | TYPED_TEST(TestImageReplayer, SnapshotRemove) { | |
1552 | librbd::ImageCtx* remote_image_ctx = nullptr; | |
1553 | this->open_remote_image(&remote_image_ctx); | |
1554 | ||
1555 | // create a user snapshot | |
f67539c2 | 1556 | librbd::NoOpProgressContext prog_ctx; |
9f95a23c | 1557 | ASSERT_EQ(0, remote_image_ctx->operations->snap_create( |
f67539c2 | 1558 | cls::rbd::UserSnapshotNamespace{}, "snap1", 0, prog_ctx)); |
9f95a23c TL |
1559 | this->flush(remote_image_ctx); |
1560 | ||
1561 | this->create_replayer(); | |
1562 | this->start(); | |
1563 | this->wait_for_replay_complete(); | |
1564 | ||
1565 | librbd::ImageCtx* local_image_ctx = nullptr; | |
1566 | this->open_local_image(&local_image_ctx); | |
1567 | auto local_snap_id_it = local_image_ctx->snap_ids.find({ | |
1568 | {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); | |
1569 | ASSERT_NE(local_image_ctx->snap_ids.end(), local_snap_id_it); | |
1570 | ||
1571 | // remove the snapshot | |
1572 | ASSERT_EQ(0, remote_image_ctx->operations->snap_remove( | |
1573 | cls::rbd::UserSnapshotNamespace{}, "snap1")); | |
1574 | this->flush(remote_image_ctx); | |
1575 | this->wait_for_replay_complete(); | |
1576 | ||
1577 | ASSERT_EQ(0, local_image_ctx->state->refresh()); | |
1578 | local_snap_id_it = local_image_ctx->snap_ids.find({ | |
1579 | {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); | |
1580 | ASSERT_EQ(local_image_ctx->snap_ids.end(), local_snap_id_it); | |
1581 | ||
1582 | this->close_image(local_image_ctx); | |
1583 | this->close_image(remote_image_ctx); | |
1584 | this->stop(); | |
1585 | } | |
1586 | ||
1587 | TYPED_TEST(TestImageReplayer, SnapshotRename) { | |
1588 | librbd::ImageCtx* remote_image_ctx = nullptr; | |
1589 | this->open_remote_image(&remote_image_ctx); | |
1590 | ||
1591 | // create a user snapshot | |
f67539c2 | 1592 | librbd::NoOpProgressContext prog_ctx; |
9f95a23c | 1593 | ASSERT_EQ(0, remote_image_ctx->operations->snap_create( |
f67539c2 | 1594 | cls::rbd::UserSnapshotNamespace{}, "snap1", 0, prog_ctx)); |
9f95a23c TL |
1595 | this->flush(remote_image_ctx); |
1596 | ||
1597 | this->create_replayer(); | |
1598 | this->start(); | |
1599 | this->wait_for_replay_complete(); | |
1600 | ||
1601 | librbd::ImageCtx* local_image_ctx = nullptr; | |
1602 | this->open_local_image(&local_image_ctx); | |
1603 | auto local_snap_id_it = local_image_ctx->snap_ids.find({ | |
1604 | {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); | |
1605 | ASSERT_NE(local_image_ctx->snap_ids.end(), local_snap_id_it); | |
1606 | auto local_snap_id = local_snap_id_it->second; | |
1607 | auto local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); | |
1608 | ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); | |
1609 | ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, | |
1610 | local_snap_info_it->second.protection_status); | |
1611 | ||
1612 | // rename the snapshot | |
1613 | ASSERT_EQ(0, remote_image_ctx->operations->snap_rename( | |
1614 | "snap1", "snap1-renamed")); | |
1615 | this->flush(remote_image_ctx); | |
1616 | this->wait_for_replay_complete(); | |
1617 | ||
1618 | ASSERT_EQ(0, local_image_ctx->state->refresh()); | |
1619 | local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); | |
1620 | ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); | |
1621 | ASSERT_EQ("snap1-renamed", local_snap_info_it->second.name); | |
1622 | ||
1623 | this->close_image(local_image_ctx); | |
1624 | this->close_image(remote_image_ctx); | |
1625 | this->stop(); | |
1626 | } | |
1627 | ||
1628 | TYPED_TEST(TestImageReplayer, SnapshotLimit) { | |
1629 | librbd::ImageCtx* remote_image_ctx = nullptr; | |
1630 | this->open_remote_image(&remote_image_ctx); | |
1631 | ||
1632 | this->create_replayer(); | |
1633 | this->start(); | |
1634 | this->wait_for_replay_complete(); | |
1635 | ||
1636 | // update the snap limit | |
1637 | ASSERT_EQ(0, librbd::api::Snapshot<>::set_limit(remote_image_ctx, 123U)); | |
1638 | this->flush(remote_image_ctx); | |
1639 | this->wait_for_replay_complete(); | |
1640 | ||
1641 | librbd::ImageCtx* local_image_ctx = nullptr; | |
1642 | this->open_local_image(&local_image_ctx); | |
1643 | uint64_t local_snap_limit; | |
1644 | ASSERT_EQ(0, librbd::api::Snapshot<>::get_limit(local_image_ctx, | |
1645 | &local_snap_limit)); | |
1646 | ASSERT_EQ(123U, local_snap_limit); | |
1647 | ||
1648 | // update the limit again | |
1649 | ASSERT_EQ(0, librbd::api::Snapshot<>::set_limit( | |
1650 | remote_image_ctx, std::numeric_limits<uint64_t>::max())); | |
1651 | this->flush(remote_image_ctx); | |
1652 | this->wait_for_replay_complete(); | |
1653 | ||
1654 | ASSERT_EQ(0, librbd::api::Snapshot<>::get_limit(local_image_ctx, | |
1655 | &local_snap_limit)); | |
1656 | ASSERT_EQ(std::numeric_limits<uint64_t>::max(), local_snap_limit); | |
1657 | ||
1658 | this->close_image(local_image_ctx); | |
1659 | this->close_image(remote_image_ctx); | |
1660 | this->stop(); | |
1661 | } | |
1662 | ||
1663 | } // namespace mirror | |
1664 | } // namespace rbd |