]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #include "librados/AioCompletionImpl.h" | |
5 | #include "librbd/ManagedLock.h" | |
11fdf7f2 | 6 | #include "test/librados/test_cxx.h" |
7c673cae FG |
7 | #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" |
8 | #include "test/librados_test_stub/MockTestMemRadosClient.h" | |
9 | #include "test/librbd/mock/MockImageCtx.h" | |
10 | #include "test/rbd_mirror/test_mock_fixture.h" | |
11 | #include "tools/rbd_mirror/InstanceReplayer.h" | |
12 | #include "tools/rbd_mirror/InstanceWatcher.h" | |
13 | #include "tools/rbd_mirror/Threads.h" | |
14 | ||
15 | namespace librbd { | |
16 | ||
17 | namespace { | |
18 | ||
19 | struct MockTestImageCtx : public MockImageCtx { | |
20 | MockTestImageCtx(librbd::ImageCtx &image_ctx) | |
21 | : librbd::MockImageCtx(image_ctx) { | |
22 | } | |
23 | }; | |
24 | ||
25 | } // anonymous namespace | |
26 | ||
27 | template <> | |
28 | struct ManagedLock<MockTestImageCtx> { | |
29 | static ManagedLock* s_instance; | |
30 | ||
f67539c2 TL |
31 | static ManagedLock *create(librados::IoCtx& ioctx, |
32 | librbd::AsioEngine& asio_engine, | |
7c673cae FG |
33 | const std::string& oid, librbd::Watcher *watcher, |
34 | managed_lock::Mode mode, | |
f67539c2 TL |
35 | bool blocklist_on_break_lock, |
36 | uint32_t blocklist_expire_seconds) { | |
11fdf7f2 | 37 | ceph_assert(s_instance != nullptr); |
7c673cae FG |
38 | return s_instance; |
39 | } | |
40 | ||
41 | ManagedLock() { | |
11fdf7f2 | 42 | ceph_assert(s_instance == nullptr); |
7c673cae FG |
43 | s_instance = this; |
44 | } | |
45 | ||
46 | ~ManagedLock() { | |
11fdf7f2 | 47 | ceph_assert(s_instance == this); |
7c673cae FG |
48 | s_instance = nullptr; |
49 | } | |
50 | ||
51 | MOCK_METHOD0(destroy, void()); | |
52 | MOCK_METHOD1(shut_down, void(Context *)); | |
53 | MOCK_METHOD1(acquire_lock, void(Context *)); | |
54 | MOCK_METHOD2(get_locker, void(managed_lock::Locker *, Context *)); | |
55 | MOCK_METHOD3(break_lock, void(const managed_lock::Locker &, bool, Context *)); | |
56 | }; | |
57 | ||
58 | ManagedLock<MockTestImageCtx> *ManagedLock<MockTestImageCtx>::s_instance = nullptr; | |
59 | ||
60 | } // namespace librbd | |
61 | ||
62 | namespace rbd { | |
63 | namespace mirror { | |
64 | ||
65 | template <> | |
66 | struct Threads<librbd::MockTestImageCtx> { | |
9f95a23c | 67 | ceph::mutex &timer_lock; |
7c673cae | 68 | SafeTimer *timer; |
f67539c2 TL |
69 | librbd::asio::ContextWQ *work_queue; |
70 | librbd::AsioEngine* asio_engine; | |
7c673cae FG |
71 | |
72 | Threads(Threads<librbd::ImageCtx> *threads) | |
73 | : timer_lock(threads->timer_lock), timer(threads->timer), | |
f67539c2 | 74 | work_queue(threads->work_queue), asio_engine(threads->asio_engine) { |
7c673cae FG |
75 | } |
76 | }; | |
77 | ||
78 | template <> | |
79 | struct InstanceReplayer<librbd::MockTestImageCtx> { | |
d2e6a577 | 80 | MOCK_METHOD3(acquire_image, void(InstanceWatcher<librbd::MockTestImageCtx> *, |
7c673cae | 81 | const std::string &, Context *)); |
d2e6a577 FG |
82 | MOCK_METHOD2(release_image, void(const std::string &, Context *)); |
83 | MOCK_METHOD3(remove_peer_image, void(const std::string&, const std::string&, | |
84 | Context *)); | |
7c673cae FG |
85 | }; |
86 | ||
31f18b77 | 87 | template <> |
9f95a23c TL |
88 | struct Throttler<librbd::MockTestImageCtx> { |
89 | static Throttler* s_instance; | |
31f18b77 | 90 | |
9f95a23c | 91 | Throttler() { |
11fdf7f2 | 92 | ceph_assert(s_instance == nullptr); |
31f18b77 FG |
93 | s_instance = this; |
94 | } | |
95 | ||
9f95a23c | 96 | virtual ~Throttler() { |
11fdf7f2 | 97 | ceph_assert(s_instance == this); |
31f18b77 FG |
98 | s_instance = nullptr; |
99 | } | |
100 | ||
9f95a23c TL |
101 | MOCK_METHOD3(start_op, void(const std::string &, const std::string &, |
102 | Context *)); | |
103 | MOCK_METHOD2(finish_op, void(const std::string &, const std::string &)); | |
104 | MOCK_METHOD2(drain, void(const std::string &, int)); | |
31f18b77 FG |
105 | }; |
106 | ||
9f95a23c | 107 | Throttler<librbd::MockTestImageCtx>* Throttler<librbd::MockTestImageCtx>::s_instance = nullptr; |
31f18b77 | 108 | |
7c673cae FG |
109 | } // namespace mirror |
110 | } // namespace rbd | |
111 | ||
112 | // template definitions | |
113 | #include "tools/rbd_mirror/InstanceWatcher.cc" | |
114 | ||
115 | namespace rbd { | |
116 | namespace mirror { | |
117 | ||
118 | using ::testing::_; | |
119 | using ::testing::InSequence; | |
120 | using ::testing::Invoke; | |
121 | using ::testing::Return; | |
122 | using ::testing::StrEq; | |
123 | using ::testing::WithArg; | |
124 | ||
125 | class TestMockInstanceWatcher : public TestMockFixture { | |
126 | public: | |
127 | typedef librbd::ManagedLock<librbd::MockTestImageCtx> MockManagedLock; | |
128 | typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer; | |
129 | typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; | |
130 | typedef Threads<librbd::MockTestImageCtx> MockThreads; | |
131 | ||
132 | std::string m_instance_id; | |
133 | std::string m_oid; | |
134 | MockThreads *m_mock_threads; | |
135 | ||
136 | void SetUp() override { | |
137 | TestFixture::SetUp(); | |
138 | m_local_io_ctx.remove(RBD_MIRROR_LEADER); | |
139 | EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true)); | |
140 | ||
141 | m_instance_id = stringify(m_local_io_ctx.get_instance_id()); | |
142 | m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id; | |
143 | ||
144 | m_mock_threads = new MockThreads(m_threads); | |
145 | } | |
146 | ||
147 | void TearDown() override { | |
148 | delete m_mock_threads; | |
149 | TestMockFixture::TearDown(); | |
150 | } | |
151 | ||
152 | void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) { | |
153 | EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _)); | |
154 | } | |
155 | ||
156 | void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx, | |
157 | const std::string &instance_id) { | |
158 | std::string oid = RBD_MIRROR_INSTANCE_PREFIX + instance_id; | |
159 | EXPECT_CALL(mock_io_ctx, aio_watch(oid, _, _, _)); | |
160 | } | |
161 | ||
162 | void expect_unregister_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) { | |
163 | EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _)); | |
164 | } | |
165 | ||
166 | void expect_register_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx, | |
167 | int r) { | |
168 | EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"), | |
f67539c2 | 169 | StrEq("mirror_instances_add"), _, _, _, _)) |
7c673cae FG |
170 | .WillOnce(Return(r)); |
171 | } | |
172 | ||
173 | void expect_unregister_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx, | |
174 | int r) { | |
175 | EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"), | |
f67539c2 | 176 | StrEq("mirror_instances_remove"), _, _, _, _)) |
7c673cae FG |
177 | .WillOnce(Return(r)); |
178 | } | |
179 | ||
180 | void expect_acquire_lock(MockManagedLock &mock_managed_lock, int r) { | |
181 | EXPECT_CALL(mock_managed_lock, acquire_lock(_)) | |
182 | .WillOnce(CompleteContext(r)); | |
183 | } | |
184 | ||
185 | void expect_release_lock(MockManagedLock &mock_managed_lock, int r) { | |
186 | EXPECT_CALL(mock_managed_lock, shut_down(_)).WillOnce(CompleteContext(r)); | |
187 | } | |
188 | ||
189 | void expect_destroy_lock(MockManagedLock &mock_managed_lock, | |
190 | Context *ctx = nullptr) { | |
191 | EXPECT_CALL(mock_managed_lock, destroy()) | |
192 | .WillOnce(Invoke([ctx]() { | |
193 | if (ctx != nullptr) { | |
194 | ctx->complete(0); | |
195 | } | |
196 | })); | |
197 | } | |
198 | ||
199 | void expect_get_locker(MockManagedLock &mock_managed_lock, | |
200 | const librbd::managed_lock::Locker &locker, int r) { | |
201 | EXPECT_CALL(mock_managed_lock, get_locker(_, _)) | |
202 | .WillOnce(Invoke([r, locker](librbd::managed_lock::Locker *out, | |
203 | Context *ctx) { | |
204 | if (r == 0) { | |
205 | *out = locker; | |
206 | } | |
207 | ctx->complete(r); | |
208 | })); | |
209 | } | |
210 | ||
211 | void expect_break_lock(MockManagedLock &mock_managed_lock, | |
212 | const librbd::managed_lock::Locker &locker, int r) { | |
213 | EXPECT_CALL(mock_managed_lock, break_lock(locker, true, _)) | |
214 | .WillOnce(WithArg<2>(CompleteContext(r))); | |
215 | } | |
216 | }; | |
217 | ||
218 | TEST_F(TestMockInstanceWatcher, InitShutdown) { | |
219 | MockManagedLock mock_managed_lock; | |
220 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
221 | ||
222 | auto instance_watcher = new MockInstanceWatcher( | |
f67539c2 | 223 | m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, |
9f95a23c | 224 | m_instance_id); |
7c673cae FG |
225 | InSequence seq; |
226 | ||
227 | // Init | |
228 | expect_register_instance(mock_io_ctx, 0); | |
229 | expect_register_watch(mock_io_ctx); | |
230 | expect_acquire_lock(mock_managed_lock, 0); | |
231 | ASSERT_EQ(0, instance_watcher->init()); | |
232 | ||
233 | // Shutdown | |
234 | expect_release_lock(mock_managed_lock, 0); | |
235 | expect_unregister_watch(mock_io_ctx); | |
236 | expect_unregister_instance(mock_io_ctx, 0); | |
237 | instance_watcher->shut_down(); | |
238 | ||
239 | expect_destroy_lock(mock_managed_lock); | |
240 | delete instance_watcher; | |
241 | } | |
242 | ||
243 | TEST_F(TestMockInstanceWatcher, InitError) { | |
244 | MockManagedLock mock_managed_lock; | |
245 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
246 | ||
247 | auto instance_watcher = new MockInstanceWatcher( | |
f67539c2 | 248 | m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, |
9f95a23c | 249 | m_instance_id); |
7c673cae FG |
250 | InSequence seq; |
251 | ||
252 | expect_register_instance(mock_io_ctx, 0); | |
253 | expect_register_watch(mock_io_ctx); | |
254 | expect_acquire_lock(mock_managed_lock, -EINVAL); | |
255 | expect_unregister_watch(mock_io_ctx); | |
256 | expect_unregister_instance(mock_io_ctx, 0); | |
257 | ||
258 | ASSERT_EQ(-EINVAL, instance_watcher->init()); | |
259 | ||
260 | expect_destroy_lock(mock_managed_lock); | |
261 | delete instance_watcher; | |
262 | } | |
263 | ||
264 | TEST_F(TestMockInstanceWatcher, ShutdownError) { | |
265 | MockManagedLock mock_managed_lock; | |
266 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
267 | ||
268 | auto instance_watcher = new MockInstanceWatcher( | |
f67539c2 | 269 | m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, |
9f95a23c | 270 | m_instance_id); |
7c673cae FG |
271 | InSequence seq; |
272 | ||
273 | // Init | |
274 | expect_register_instance(mock_io_ctx, 0); | |
275 | expect_register_watch(mock_io_ctx); | |
276 | expect_acquire_lock(mock_managed_lock, 0); | |
277 | ASSERT_EQ(0, instance_watcher->init()); | |
278 | ||
279 | // Shutdown | |
280 | expect_release_lock(mock_managed_lock, -EINVAL); | |
281 | expect_unregister_watch(mock_io_ctx); | |
282 | expect_unregister_instance(mock_io_ctx, 0); | |
283 | instance_watcher->shut_down(); | |
284 | ||
285 | expect_destroy_lock(mock_managed_lock); | |
286 | delete instance_watcher; | |
287 | } | |
288 | ||
289 | ||
290 | TEST_F(TestMockInstanceWatcher, Remove) { | |
291 | MockManagedLock mock_managed_lock; | |
292 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
293 | librbd::managed_lock::Locker | |
294 | locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123}; | |
295 | ||
296 | InSequence seq; | |
297 | ||
298 | expect_get_locker(mock_managed_lock, locker, 0); | |
299 | expect_break_lock(mock_managed_lock, locker, 0); | |
300 | expect_unregister_instance(mock_io_ctx, 0); | |
301 | C_SaferCond on_destroy; | |
302 | expect_destroy_lock(mock_managed_lock, &on_destroy); | |
303 | ||
304 | C_SaferCond on_remove; | |
305 | MockInstanceWatcher::remove_instance(m_local_io_ctx, | |
f67539c2 | 306 | *m_mock_threads->asio_engine, |
7c673cae FG |
307 | "instance_id", &on_remove); |
308 | ASSERT_EQ(0, on_remove.wait()); | |
309 | ASSERT_EQ(0, on_destroy.wait()); | |
310 | } | |
311 | ||
312 | TEST_F(TestMockInstanceWatcher, RemoveNoent) { | |
313 | MockManagedLock mock_managed_lock; | |
314 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
315 | ||
316 | InSequence seq; | |
317 | ||
318 | expect_get_locker(mock_managed_lock, librbd::managed_lock::Locker(), -ENOENT); | |
319 | expect_unregister_instance(mock_io_ctx, 0); | |
320 | C_SaferCond on_destroy; | |
321 | expect_destroy_lock(mock_managed_lock, &on_destroy); | |
322 | ||
323 | C_SaferCond on_remove; | |
324 | MockInstanceWatcher::remove_instance(m_local_io_ctx, | |
f67539c2 | 325 | *m_mock_threads->asio_engine, |
7c673cae FG |
326 | "instance_id", &on_remove); |
327 | ASSERT_EQ(0, on_remove.wait()); | |
328 | ASSERT_EQ(0, on_destroy.wait()); | |
329 | } | |
330 | ||
331 | TEST_F(TestMockInstanceWatcher, ImageAcquireRelease) { | |
332 | MockManagedLock mock_managed_lock; | |
333 | ||
334 | librados::IoCtx& io_ctx1 = m_local_io_ctx; | |
335 | std::string instance_id1 = m_instance_id; | |
336 | librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); | |
337 | MockInstanceReplayer mock_instance_replayer1; | |
338 | auto instance_watcher1 = MockInstanceWatcher::create( | |
f67539c2 | 339 | io_ctx1, *m_mock_threads->asio_engine, &mock_instance_replayer1, nullptr); |
7c673cae FG |
340 | |
341 | librados::Rados cluster; | |
342 | librados::IoCtx io_ctx2; | |
343 | EXPECT_EQ("", connect_cluster_pp(cluster)); | |
344 | EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); | |
345 | std::string instance_id2 = stringify(io_ctx2.get_instance_id()); | |
346 | librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); | |
347 | MockInstanceReplayer mock_instance_replayer2; | |
348 | auto instance_watcher2 = MockInstanceWatcher::create( | |
f67539c2 | 349 | io_ctx2, *m_mock_threads->asio_engine, &mock_instance_replayer2, nullptr); |
7c673cae FG |
350 | |
351 | InSequence seq; | |
352 | ||
353 | // Init instance watcher 1 | |
354 | expect_register_instance(mock_io_ctx1, 0); | |
355 | expect_register_watch(mock_io_ctx1, instance_id1); | |
356 | expect_acquire_lock(mock_managed_lock, 0); | |
357 | ASSERT_EQ(0, instance_watcher1->init()); | |
358 | ||
359 | // Init instance watcher 2 | |
360 | expect_register_instance(mock_io_ctx2, 0); | |
361 | expect_register_watch(mock_io_ctx2, instance_id2); | |
362 | expect_acquire_lock(mock_managed_lock, 0); | |
363 | ASSERT_EQ(0, instance_watcher2->init()); | |
364 | ||
365 | // Acquire Image on the the same instance | |
31f18b77 | 366 | EXPECT_CALL(mock_instance_replayer1, acquire_image(instance_watcher1, "gid", |
d2e6a577 FG |
367 | _)) |
368 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
7c673cae | 369 | C_SaferCond on_acquire1; |
d2e6a577 | 370 | instance_watcher1->notify_image_acquire(instance_id1, "gid", &on_acquire1); |
7c673cae FG |
371 | ASSERT_EQ(0, on_acquire1.wait()); |
372 | ||
373 | // Acquire Image on the other instance | |
31f18b77 | 374 | EXPECT_CALL(mock_instance_replayer2, acquire_image(instance_watcher2, "gid", |
d2e6a577 FG |
375 | _)) |
376 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
7c673cae | 377 | C_SaferCond on_acquire2; |
d2e6a577 | 378 | instance_watcher1->notify_image_acquire(instance_id2, "gid", &on_acquire2); |
7c673cae FG |
379 | ASSERT_EQ(0, on_acquire2.wait()); |
380 | ||
381 | // Release Image on the the same instance | |
d2e6a577 FG |
382 | EXPECT_CALL(mock_instance_replayer1, release_image("gid", _)) |
383 | .WillOnce(WithArg<1>(CompleteContext(0))); | |
7c673cae | 384 | C_SaferCond on_release1; |
d2e6a577 | 385 | instance_watcher1->notify_image_release(instance_id1, "gid", &on_release1); |
7c673cae FG |
386 | ASSERT_EQ(0, on_release1.wait()); |
387 | ||
388 | // Release Image on the other instance | |
d2e6a577 FG |
389 | EXPECT_CALL(mock_instance_replayer2, release_image("gid", _)) |
390 | .WillOnce(WithArg<1>(CompleteContext(0))); | |
7c673cae | 391 | C_SaferCond on_release2; |
d2e6a577 | 392 | instance_watcher1->notify_image_release(instance_id2, "gid", &on_release2); |
7c673cae FG |
393 | ASSERT_EQ(0, on_release2.wait()); |
394 | ||
395 | // Shutdown instance watcher 1 | |
396 | expect_release_lock(mock_managed_lock, 0); | |
397 | expect_unregister_watch(mock_io_ctx1); | |
398 | expect_unregister_instance(mock_io_ctx1, 0); | |
399 | instance_watcher1->shut_down(); | |
400 | ||
401 | expect_destroy_lock(mock_managed_lock); | |
402 | delete instance_watcher1; | |
403 | ||
404 | // Shutdown instance watcher 2 | |
405 | expect_release_lock(mock_managed_lock, 0); | |
406 | expect_unregister_watch(mock_io_ctx2); | |
407 | expect_unregister_instance(mock_io_ctx2, 0); | |
408 | instance_watcher2->shut_down(); | |
409 | ||
410 | expect_destroy_lock(mock_managed_lock); | |
411 | delete instance_watcher2; | |
412 | } | |
413 | ||
d2e6a577 FG |
414 | TEST_F(TestMockInstanceWatcher, PeerImageRemoved) { |
415 | MockManagedLock mock_managed_lock; | |
416 | ||
417 | librados::IoCtx& io_ctx1 = m_local_io_ctx; | |
418 | std::string instance_id1 = m_instance_id; | |
419 | librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); | |
420 | MockInstanceReplayer mock_instance_replayer1; | |
421 | auto instance_watcher1 = MockInstanceWatcher::create( | |
f67539c2 | 422 | io_ctx1, *m_mock_threads->asio_engine, &mock_instance_replayer1, nullptr); |
d2e6a577 FG |
423 | |
424 | librados::Rados cluster; | |
425 | librados::IoCtx io_ctx2; | |
426 | EXPECT_EQ("", connect_cluster_pp(cluster)); | |
427 | EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); | |
428 | std::string instance_id2 = stringify(io_ctx2.get_instance_id()); | |
429 | librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); | |
430 | MockInstanceReplayer mock_instance_replayer2; | |
431 | auto instance_watcher2 = MockInstanceWatcher::create( | |
f67539c2 | 432 | io_ctx2, *m_mock_threads->asio_engine, &mock_instance_replayer2, nullptr); |
d2e6a577 FG |
433 | |
434 | InSequence seq; | |
435 | ||
436 | // Init instance watcher 1 | |
437 | expect_register_instance(mock_io_ctx1, 0); | |
438 | expect_register_watch(mock_io_ctx1, instance_id1); | |
439 | expect_acquire_lock(mock_managed_lock, 0); | |
440 | ASSERT_EQ(0, instance_watcher1->init()); | |
441 | ||
442 | // Init instance watcher 2 | |
443 | expect_register_instance(mock_io_ctx2, 0); | |
444 | expect_register_watch(mock_io_ctx2, instance_id2); | |
445 | expect_acquire_lock(mock_managed_lock, 0); | |
446 | ASSERT_EQ(0, instance_watcher2->init()); | |
447 | ||
448 | // Peer Image Removed on the same instance | |
449 | EXPECT_CALL(mock_instance_replayer1, remove_peer_image("gid", "uuid", _)) | |
450 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
451 | C_SaferCond on_removed1; | |
452 | instance_watcher1->notify_peer_image_removed(instance_id1, "gid", "uuid", | |
453 | &on_removed1); | |
454 | ASSERT_EQ(0, on_removed1.wait()); | |
455 | ||
456 | // Peer Image Removed on the other instance | |
457 | EXPECT_CALL(mock_instance_replayer2, remove_peer_image("gid", "uuid", _)) | |
458 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
459 | C_SaferCond on_removed2; | |
460 | instance_watcher1->notify_peer_image_removed(instance_id2, "gid", "uuid", | |
461 | &on_removed2); | |
462 | ASSERT_EQ(0, on_removed2.wait()); | |
463 | ||
464 | // Shutdown instance watcher 1 | |
465 | expect_release_lock(mock_managed_lock, 0); | |
466 | expect_unregister_watch(mock_io_ctx1); | |
467 | expect_unregister_instance(mock_io_ctx1, 0); | |
468 | instance_watcher1->shut_down(); | |
469 | ||
470 | expect_destroy_lock(mock_managed_lock); | |
471 | delete instance_watcher1; | |
472 | ||
473 | // Shutdown instance watcher 2 | |
474 | expect_release_lock(mock_managed_lock, 0); | |
475 | expect_unregister_watch(mock_io_ctx2); | |
476 | expect_unregister_instance(mock_io_ctx2, 0); | |
477 | instance_watcher2->shut_down(); | |
478 | ||
479 | expect_destroy_lock(mock_managed_lock); | |
480 | delete instance_watcher2; | |
481 | } | |
482 | ||
7c673cae FG |
483 | TEST_F(TestMockInstanceWatcher, ImageAcquireReleaseCancel) { |
484 | MockManagedLock mock_managed_lock; | |
485 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
486 | ||
487 | auto instance_watcher = new MockInstanceWatcher( | |
f67539c2 | 488 | m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, |
9f95a23c | 489 | m_instance_id); |
7c673cae FG |
490 | InSequence seq; |
491 | ||
492 | // Init | |
493 | expect_register_instance(mock_io_ctx, 0); | |
494 | expect_register_watch(mock_io_ctx); | |
495 | expect_acquire_lock(mock_managed_lock, 0); | |
496 | ASSERT_EQ(0, instance_watcher->init()); | |
497 | ||
498 | // Send Acquire Image and cancel | |
499 | EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) | |
500 | .WillOnce(Invoke( | |
501 | [this, instance_watcher, &mock_io_ctx]( | |
502 | const std::string& o, librados::AioCompletionImpl *c, | |
503 | bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { | |
504 | c->get(); | |
9f95a23c | 505 | auto ctx = new LambdaContext( |
7c673cae FG |
506 | [instance_watcher, &mock_io_ctx, c, pbl](int r) { |
507 | instance_watcher->cancel_notify_requests("other"); | |
11fdf7f2 | 508 | encode(librbd::watcher::NotifyResponse(), *pbl); |
7c673cae FG |
509 | mock_io_ctx.get_mock_rados_client()-> |
510 | finish_aio_completion(c, -ETIMEDOUT); | |
511 | }); | |
512 | m_threads->work_queue->queue(ctx, 0); | |
513 | })); | |
514 | ||
515 | C_SaferCond on_acquire; | |
d2e6a577 | 516 | instance_watcher->notify_image_acquire("other", "gid", &on_acquire); |
7c673cae FG |
517 | ASSERT_EQ(-ECANCELED, on_acquire.wait()); |
518 | ||
519 | // Send Release Image and cancel | |
520 | EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) | |
521 | .WillOnce(Invoke( | |
522 | [this, instance_watcher, &mock_io_ctx]( | |
523 | const std::string& o, librados::AioCompletionImpl *c, | |
524 | bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { | |
525 | c->get(); | |
9f95a23c | 526 | auto ctx = new LambdaContext( |
7c673cae FG |
527 | [instance_watcher, &mock_io_ctx, c, pbl](int r) { |
528 | instance_watcher->cancel_notify_requests("other"); | |
11fdf7f2 | 529 | encode(librbd::watcher::NotifyResponse(), *pbl); |
7c673cae FG |
530 | mock_io_ctx.get_mock_rados_client()-> |
531 | finish_aio_completion(c, -ETIMEDOUT); | |
532 | }); | |
533 | m_threads->work_queue->queue(ctx, 0); | |
534 | })); | |
535 | ||
536 | C_SaferCond on_release; | |
d2e6a577 | 537 | instance_watcher->notify_image_release("other", "gid", &on_release); |
7c673cae FG |
538 | ASSERT_EQ(-ECANCELED, on_release.wait()); |
539 | ||
540 | // Shutdown | |
541 | expect_release_lock(mock_managed_lock, 0); | |
542 | expect_unregister_watch(mock_io_ctx); | |
543 | expect_unregister_instance(mock_io_ctx, 0); | |
544 | instance_watcher->shut_down(); | |
545 | ||
546 | expect_destroy_lock(mock_managed_lock); | |
547 | delete instance_watcher; | |
548 | } | |
549 | ||
11fdf7f2 TL |
550 | TEST_F(TestMockInstanceWatcher, PeerImageAcquireWatchDNE) { |
551 | MockManagedLock mock_managed_lock; | |
552 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
553 | ||
554 | MockInstanceReplayer mock_instance_replayer; | |
555 | auto instance_watcher = new MockInstanceWatcher( | |
f67539c2 | 556 | m_local_io_ctx, *m_mock_threads->asio_engine, &mock_instance_replayer, |
9f95a23c | 557 | nullptr, m_instance_id); |
11fdf7f2 TL |
558 | InSequence seq; |
559 | ||
560 | // Init | |
561 | expect_register_instance(mock_io_ctx, 0); | |
562 | expect_register_watch(mock_io_ctx); | |
563 | expect_acquire_lock(mock_managed_lock, 0); | |
564 | ASSERT_EQ(0, instance_watcher->init()); | |
565 | ||
f67539c2 | 566 | // Acquire image on dead (blocklisted) instance |
11fdf7f2 TL |
567 | C_SaferCond on_acquire; |
568 | instance_watcher->notify_image_acquire("dead instance", "global image id", | |
569 | &on_acquire); | |
570 | ASSERT_EQ(-ENOENT, on_acquire.wait()); | |
571 | ||
572 | // Shutdown | |
573 | expect_release_lock(mock_managed_lock, 0); | |
574 | expect_unregister_watch(mock_io_ctx); | |
575 | expect_unregister_instance(mock_io_ctx, 0); | |
576 | instance_watcher->shut_down(); | |
577 | ||
578 | expect_destroy_lock(mock_managed_lock); | |
579 | delete instance_watcher; | |
580 | } | |
581 | ||
582 | TEST_F(TestMockInstanceWatcher, PeerImageReleaseWatchDNE) { | |
583 | MockManagedLock mock_managed_lock; | |
584 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
585 | ||
586 | MockInstanceReplayer mock_instance_replayer; | |
587 | auto instance_watcher = new MockInstanceWatcher( | |
f67539c2 | 588 | m_local_io_ctx, *m_mock_threads->asio_engine, &mock_instance_replayer, |
9f95a23c | 589 | nullptr, m_instance_id); |
11fdf7f2 TL |
590 | InSequence seq; |
591 | ||
592 | // Init | |
593 | expect_register_instance(mock_io_ctx, 0); | |
594 | expect_register_watch(mock_io_ctx); | |
595 | expect_acquire_lock(mock_managed_lock, 0); | |
596 | ASSERT_EQ(0, instance_watcher->init()); | |
597 | ||
f67539c2 | 598 | // Release image on dead (blocklisted) instance |
11fdf7f2 TL |
599 | C_SaferCond on_acquire; |
600 | instance_watcher->notify_image_release("dead instance", "global image id", | |
601 | &on_acquire); | |
602 | ASSERT_EQ(-ENOENT, on_acquire.wait()); | |
603 | ||
604 | // Shutdown | |
605 | expect_release_lock(mock_managed_lock, 0); | |
606 | expect_unregister_watch(mock_io_ctx); | |
607 | expect_unregister_instance(mock_io_ctx, 0); | |
608 | instance_watcher->shut_down(); | |
609 | ||
610 | expect_destroy_lock(mock_managed_lock); | |
611 | delete instance_watcher; | |
612 | } | |
613 | ||
d2e6a577 FG |
614 | TEST_F(TestMockInstanceWatcher, PeerImageRemovedCancel) { |
615 | MockManagedLock mock_managed_lock; | |
616 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
617 | ||
618 | auto instance_watcher = new MockInstanceWatcher( | |
f67539c2 | 619 | m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, |
9f95a23c | 620 | m_instance_id); |
d2e6a577 FG |
621 | InSequence seq; |
622 | ||
623 | // Init | |
624 | expect_register_instance(mock_io_ctx, 0); | |
625 | expect_register_watch(mock_io_ctx); | |
626 | expect_acquire_lock(mock_managed_lock, 0); | |
627 | ASSERT_EQ(0, instance_watcher->init()); | |
628 | ||
629 | // Send Acquire Image and cancel | |
630 | EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) | |
631 | .WillOnce(Invoke( | |
632 | [this, instance_watcher, &mock_io_ctx]( | |
633 | const std::string& o, librados::AioCompletionImpl *c, | |
634 | bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { | |
635 | c->get(); | |
9f95a23c | 636 | auto ctx = new LambdaContext( |
d2e6a577 FG |
637 | [instance_watcher, &mock_io_ctx, c, pbl](int r) { |
638 | instance_watcher->cancel_notify_requests("other"); | |
11fdf7f2 | 639 | encode(librbd::watcher::NotifyResponse(), *pbl); |
d2e6a577 FG |
640 | mock_io_ctx.get_mock_rados_client()-> |
641 | finish_aio_completion(c, -ETIMEDOUT); | |
642 | }); | |
643 | m_threads->work_queue->queue(ctx, 0); | |
644 | })); | |
645 | ||
646 | C_SaferCond on_acquire; | |
647 | instance_watcher->notify_peer_image_removed("other", "gid", "uuid", | |
648 | &on_acquire); | |
649 | ASSERT_EQ(-ECANCELED, on_acquire.wait()); | |
650 | ||
651 | // Shutdown | |
652 | expect_release_lock(mock_managed_lock, 0); | |
653 | expect_unregister_watch(mock_io_ctx); | |
654 | expect_unregister_instance(mock_io_ctx, 0); | |
655 | instance_watcher->shut_down(); | |
656 | ||
657 | expect_destroy_lock(mock_managed_lock); | |
658 | delete instance_watcher; | |
659 | } | |
660 | ||
31f18b77 FG |
661 | class TestMockInstanceWatcher_NotifySync : public TestMockInstanceWatcher { |
662 | public: | |
9f95a23c | 663 | typedef Throttler<librbd::MockTestImageCtx> MockThrottler; |
31f18b77 FG |
664 | |
665 | MockManagedLock mock_managed_lock; | |
9f95a23c | 666 | MockThrottler mock_image_sync_throttler; |
31f18b77 FG |
667 | std::string instance_id1; |
668 | std::string instance_id2; | |
669 | ||
670 | librados::Rados cluster; | |
671 | librados::IoCtx io_ctx2; | |
672 | ||
673 | MockInstanceWatcher *instance_watcher1; | |
674 | MockInstanceWatcher *instance_watcher2; | |
675 | ||
676 | void SetUp() override { | |
677 | TestMockInstanceWatcher::SetUp(); | |
678 | ||
679 | instance_id1 = m_instance_id; | |
680 | librados::IoCtx& io_ctx1 = m_local_io_ctx; | |
681 | librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); | |
682 | instance_watcher1 = MockInstanceWatcher::create(io_ctx1, | |
f67539c2 | 683 | *m_mock_threads->asio_engine, |
9f95a23c TL |
684 | nullptr, |
685 | &mock_image_sync_throttler); | |
31f18b77 FG |
686 | EXPECT_EQ("", connect_cluster_pp(cluster)); |
687 | EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); | |
688 | instance_id2 = stringify(io_ctx2.get_instance_id()); | |
689 | librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); | |
690 | instance_watcher2 = MockInstanceWatcher::create(io_ctx2, | |
f67539c2 | 691 | *m_mock_threads->asio_engine, |
9f95a23c TL |
692 | nullptr, |
693 | &mock_image_sync_throttler); | |
31f18b77 FG |
694 | InSequence seq; |
695 | ||
696 | // Init instance watcher 1 (leader) | |
697 | expect_register_instance(mock_io_ctx1, 0); | |
698 | expect_register_watch(mock_io_ctx1, instance_id1); | |
699 | expect_acquire_lock(mock_managed_lock, 0); | |
700 | EXPECT_EQ(0, instance_watcher1->init()); | |
701 | instance_watcher1->handle_acquire_leader(); | |
702 | ||
703 | // Init instance watcher 2 | |
704 | expect_register_instance(mock_io_ctx2, 0); | |
705 | expect_register_watch(mock_io_ctx2, instance_id2); | |
706 | expect_acquire_lock(mock_managed_lock, 0); | |
707 | EXPECT_EQ(0, instance_watcher2->init()); | |
708 | instance_watcher2->handle_update_leader(instance_id1); | |
709 | } | |
710 | ||
711 | void TearDown() override { | |
712 | librados::IoCtx& io_ctx1 = m_local_io_ctx; | |
713 | librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); | |
714 | librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); | |
715 | ||
716 | InSequence seq; | |
717 | ||
9f95a23c | 718 | expect_throttler_drain(); |
31f18b77 FG |
719 | instance_watcher1->handle_release_leader(); |
720 | ||
721 | // Shutdown instance watcher 1 | |
722 | expect_release_lock(mock_managed_lock, 0); | |
723 | expect_unregister_watch(mock_io_ctx1); | |
724 | expect_unregister_instance(mock_io_ctx1, 0); | |
725 | instance_watcher1->shut_down(); | |
726 | ||
727 | expect_destroy_lock(mock_managed_lock); | |
728 | delete instance_watcher1; | |
729 | ||
730 | // Shutdown instance watcher 2 | |
731 | expect_release_lock(mock_managed_lock, 0); | |
732 | expect_unregister_watch(mock_io_ctx2); | |
733 | expect_unregister_instance(mock_io_ctx2, 0); | |
734 | instance_watcher2->shut_down(); | |
735 | ||
736 | expect_destroy_lock(mock_managed_lock); | |
737 | delete instance_watcher2; | |
738 | ||
739 | TestMockInstanceWatcher::TearDown(); | |
740 | } | |
741 | ||
31f18b77 FG |
742 | void expect_throttler_start_op(const std::string &sync_id, |
743 | Context *on_call = nullptr, | |
744 | Context **on_start_ctx = nullptr) { | |
9f95a23c | 745 | EXPECT_CALL(mock_image_sync_throttler, start_op("", sync_id, _)) |
31f18b77 | 746 | .WillOnce(Invoke([on_call, on_start_ctx] (const std::string &, |
9f95a23c | 747 | const std::string &, |
31f18b77 | 748 | Context *ctx) { |
31f18b77 FG |
749 | if (on_start_ctx != nullptr) { |
750 | *on_start_ctx = ctx; | |
751 | } else { | |
752 | ctx->complete(0); | |
753 | } | |
11fdf7f2 TL |
754 | if (on_call != nullptr) { |
755 | on_call->complete(0); | |
756 | } | |
31f18b77 FG |
757 | })); |
758 | } | |
759 | ||
760 | void expect_throttler_finish_op(const std::string &sync_id, | |
761 | Context *on_finish) { | |
9f95a23c TL |
762 | EXPECT_CALL(mock_image_sync_throttler, finish_op("", "sync_id")) |
763 | .WillOnce(Invoke([on_finish](const std::string &, const std::string &) { | |
31f18b77 FG |
764 | on_finish->complete(0); |
765 | })); | |
766 | } | |
9f95a23c TL |
767 | |
768 | void expect_throttler_drain() { | |
769 | EXPECT_CALL(mock_image_sync_throttler, drain("", -ESTALE)); | |
770 | } | |
31f18b77 FG |
771 | }; |
772 | ||
773 | TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnLeader) { | |
774 | InSequence seq; | |
775 | ||
776 | expect_throttler_start_op("sync_id"); | |
777 | C_SaferCond on_start; | |
778 | instance_watcher1->notify_sync_request("sync_id", &on_start); | |
779 | ASSERT_EQ(0, on_start.wait()); | |
780 | ||
781 | C_SaferCond on_finish; | |
782 | expect_throttler_finish_op("sync_id", &on_finish); | |
783 | instance_watcher1->notify_sync_complete("sync_id"); | |
784 | ASSERT_EQ(0, on_finish.wait()); | |
785 | } | |
786 | ||
787 | TEST_F(TestMockInstanceWatcher_NotifySync, CancelStartedOnLeader) { | |
788 | InSequence seq; | |
789 | ||
790 | expect_throttler_start_op("sync_id"); | |
791 | C_SaferCond on_start; | |
792 | instance_watcher1->notify_sync_request("sync_id", &on_start); | |
793 | ASSERT_EQ(0, on_start.wait()); | |
794 | ||
795 | ASSERT_FALSE(instance_watcher1->cancel_sync_request("sync_id")); | |
796 | ||
797 | C_SaferCond on_finish; | |
798 | expect_throttler_finish_op("sync_id", &on_finish); | |
799 | instance_watcher1->notify_sync_complete("sync_id"); | |
800 | ASSERT_EQ(0, on_finish.wait()); | |
801 | } | |
802 | ||
803 | TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnNonLeader) { | |
804 | InSequence seq; | |
805 | ||
806 | expect_throttler_start_op("sync_id"); | |
807 | C_SaferCond on_start; | |
808 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
809 | ASSERT_EQ(0, on_start.wait()); | |
810 | ||
811 | C_SaferCond on_finish; | |
812 | expect_throttler_finish_op("sync_id", &on_finish); | |
813 | instance_watcher2->notify_sync_complete("sync_id"); | |
814 | ASSERT_EQ(0, on_finish.wait()); | |
815 | } | |
816 | ||
817 | TEST_F(TestMockInstanceWatcher_NotifySync, CancelStartedOnNonLeader) { | |
818 | InSequence seq; | |
819 | ||
820 | expect_throttler_start_op("sync_id"); | |
821 | C_SaferCond on_start; | |
822 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
823 | ASSERT_EQ(0, on_start.wait()); | |
824 | ||
825 | ASSERT_FALSE(instance_watcher2->cancel_sync_request("sync_id")); | |
826 | ||
827 | C_SaferCond on_finish; | |
828 | expect_throttler_finish_op("sync_id", &on_finish); | |
829 | instance_watcher2->notify_sync_complete("sync_id"); | |
830 | ASSERT_EQ(0, on_finish.wait()); | |
831 | } | |
832 | ||
833 | TEST_F(TestMockInstanceWatcher_NotifySync, CancelWaitingOnNonLeader) { | |
834 | InSequence seq; | |
835 | ||
836 | C_SaferCond on_start_op_called; | |
837 | Context *on_start_ctx; | |
838 | expect_throttler_start_op("sync_id", &on_start_op_called, | |
839 | &on_start_ctx); | |
840 | C_SaferCond on_start; | |
841 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
842 | ASSERT_EQ(0, on_start_op_called.wait()); | |
843 | ||
844 | ASSERT_TRUE(instance_watcher2->cancel_sync_request("sync_id")); | |
845 | // emulate watcher timeout | |
846 | on_start_ctx->complete(-ETIMEDOUT); | |
847 | ASSERT_EQ(-ECANCELED, on_start.wait()); | |
848 | } | |
849 | ||
850 | TEST_F(TestMockInstanceWatcher_NotifySync, InFlightPrevNotification) { | |
851 | // start sync when previous notification is still in flight | |
852 | ||
853 | InSequence seq; | |
854 | ||
855 | expect_throttler_start_op("sync_id"); | |
856 | C_SaferCond on_start1; | |
857 | instance_watcher2->notify_sync_request("sync_id", &on_start1); | |
858 | ASSERT_EQ(0, on_start1.wait()); | |
859 | ||
860 | C_SaferCond on_start2; | |
9f95a23c TL |
861 | EXPECT_CALL(mock_image_sync_throttler, finish_op("", "sync_id")) |
862 | .WillOnce(Invoke([this, &on_start2](const std::string &, | |
863 | const std::string &) { | |
31f18b77 FG |
864 | instance_watcher2->notify_sync_request("sync_id", &on_start2); |
865 | })); | |
866 | expect_throttler_start_op("sync_id"); | |
867 | instance_watcher2->notify_sync_complete("sync_id"); | |
868 | ||
869 | ASSERT_EQ(0, on_start2.wait()); | |
870 | C_SaferCond on_finish; | |
871 | expect_throttler_finish_op("sync_id", &on_finish); | |
872 | instance_watcher2->notify_sync_complete("sync_id"); | |
873 | ASSERT_EQ(0, on_finish.wait()); | |
874 | } | |
875 | ||
876 | TEST_F(TestMockInstanceWatcher_NotifySync, NoInFlightReleaseAcquireLeader) { | |
877 | InSequence seq; | |
878 | ||
9f95a23c | 879 | expect_throttler_drain(); |
31f18b77 FG |
880 | instance_watcher1->handle_release_leader(); |
881 | instance_watcher1->handle_acquire_leader(); | |
882 | } | |
883 | ||
884 | TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnLeaderReleaseLeader) { | |
885 | InSequence seq; | |
886 | ||
9f95a23c | 887 | expect_throttler_drain(); |
31f18b77 FG |
888 | instance_watcher1->handle_release_leader(); |
889 | instance_watcher2->handle_acquire_leader(); | |
890 | ||
891 | expect_throttler_start_op("sync_id"); | |
892 | C_SaferCond on_start; | |
893 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
894 | ASSERT_EQ(0, on_start.wait()); | |
9f95a23c | 895 | expect_throttler_drain(); |
31f18b77 FG |
896 | instance_watcher2->handle_release_leader(); |
897 | instance_watcher2->notify_sync_complete("sync_id"); | |
898 | ||
899 | instance_watcher1->handle_acquire_leader(); | |
900 | } | |
901 | ||
902 | TEST_F(TestMockInstanceWatcher_NotifySync, WaitingOnLeaderReleaseLeader) { | |
903 | InSequence seq; | |
904 | ||
905 | C_SaferCond on_start_op_called; | |
906 | Context *on_start_ctx; | |
11fdf7f2 | 907 | expect_throttler_start_op("sync_id", &on_start_op_called, &on_start_ctx); |
31f18b77 FG |
908 | C_SaferCond on_start; |
909 | instance_watcher1->notify_sync_request("sync_id", &on_start); | |
910 | ASSERT_EQ(0, on_start_op_called.wait()); | |
911 | ||
9f95a23c | 912 | expect_throttler_drain(); |
31f18b77 | 913 | instance_watcher1->handle_release_leader(); |
9f95a23c TL |
914 | // emulate throttler queue drain on leader release |
915 | on_start_ctx->complete(-ESTALE); | |
916 | ||
11fdf7f2 | 917 | expect_throttler_start_op("sync_id"); |
31f18b77 FG |
918 | instance_watcher2->handle_acquire_leader(); |
919 | instance_watcher1->handle_update_leader(instance_id2); | |
920 | ||
31f18b77 FG |
921 | ASSERT_EQ(0, on_start.wait()); |
922 | C_SaferCond on_finish; | |
923 | expect_throttler_finish_op("sync_id", &on_finish); | |
924 | instance_watcher1->notify_sync_complete("sync_id"); | |
925 | ASSERT_EQ(0, on_finish.wait()); | |
926 | ||
9f95a23c | 927 | expect_throttler_drain(); |
31f18b77 FG |
928 | instance_watcher2->handle_release_leader(); |
929 | instance_watcher1->handle_acquire_leader(); | |
930 | } | |
931 | ||
932 | TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnNonLeaderAcquireLeader) { | |
933 | InSequence seq; | |
934 | ||
9f95a23c | 935 | expect_throttler_drain(); |
31f18b77 FG |
936 | instance_watcher1->handle_release_leader(); |
937 | instance_watcher2->handle_acquire_leader(); | |
938 | instance_watcher1->handle_update_leader(instance_id2); | |
939 | ||
940 | expect_throttler_start_op("sync_id"); | |
941 | C_SaferCond on_start; | |
942 | instance_watcher1->notify_sync_request("sync_id", &on_start); | |
943 | ASSERT_EQ(0, on_start.wait()); | |
944 | ||
9f95a23c | 945 | expect_throttler_drain(); |
31f18b77 FG |
946 | instance_watcher2->handle_release_leader(); |
947 | instance_watcher1->handle_acquire_leader(); | |
11fdf7f2 | 948 | instance_watcher2->handle_update_leader(instance_id1); |
31f18b77 FG |
949 | |
950 | instance_watcher1->notify_sync_complete("sync_id"); | |
951 | } | |
952 | ||
953 | TEST_F(TestMockInstanceWatcher_NotifySync, WaitingOnNonLeaderAcquireLeader) { | |
954 | InSequence seq; | |
955 | ||
956 | C_SaferCond on_start_op_called; | |
957 | Context *on_start_ctx; | |
958 | expect_throttler_start_op("sync_id", &on_start_op_called, | |
959 | &on_start_ctx); | |
960 | C_SaferCond on_start; | |
961 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
962 | ASSERT_EQ(0, on_start_op_called.wait()); | |
963 | ||
9f95a23c | 964 | expect_throttler_drain(); |
31f18b77 | 965 | instance_watcher1->handle_release_leader(); |
9f95a23c TL |
966 | // emulate throttler queue drain on leader release |
967 | on_start_ctx->complete(-ESTALE); | |
31f18b77 | 968 | |
9f95a23c TL |
969 | EXPECT_CALL(mock_image_sync_throttler, start_op("", "sync_id", _)) |
970 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
31f18b77 FG |
971 | instance_watcher2->handle_acquire_leader(); |
972 | instance_watcher1->handle_update_leader(instance_id2); | |
973 | ||
974 | ASSERT_EQ(0, on_start.wait()); | |
975 | ||
976 | C_SaferCond on_finish; | |
977 | expect_throttler_finish_op("sync_id", &on_finish); | |
978 | instance_watcher2->notify_sync_complete("sync_id"); | |
979 | ASSERT_EQ(0, on_finish.wait()); | |
980 | ||
9f95a23c | 981 | expect_throttler_drain(); |
31f18b77 FG |
982 | instance_watcher2->handle_release_leader(); |
983 | instance_watcher1->handle_acquire_leader(); | |
984 | } | |
985 | ||
7c673cae FG |
986 | } // namespace mirror |
987 | } // namespace rbd |