]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #include "librados/AioCompletionImpl.h" | |
5 | #include "librbd/ManagedLock.h" | |
11fdf7f2 | 6 | #include "test/librados/test_cxx.h" |
7c673cae FG |
7 | #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" |
8 | #include "test/librados_test_stub/MockTestMemRadosClient.h" | |
9 | #include "test/librbd/mock/MockImageCtx.h" | |
10 | #include "test/rbd_mirror/test_mock_fixture.h" | |
11 | #include "tools/rbd_mirror/InstanceReplayer.h" | |
31f18b77 | 12 | #include "tools/rbd_mirror/ImageSyncThrottler.h" |
7c673cae FG |
13 | #include "tools/rbd_mirror/InstanceWatcher.h" |
14 | #include "tools/rbd_mirror/Threads.h" | |
15 | ||
16 | namespace librbd { | |
17 | ||
18 | namespace { | |
19 | ||
20 | struct MockTestImageCtx : public MockImageCtx { | |
21 | MockTestImageCtx(librbd::ImageCtx &image_ctx) | |
22 | : librbd::MockImageCtx(image_ctx) { | |
23 | } | |
24 | }; | |
25 | ||
26 | } // anonymous namespace | |
27 | ||
28 | template <> | |
29 | struct ManagedLock<MockTestImageCtx> { | |
30 | static ManagedLock* s_instance; | |
31 | ||
32 | static ManagedLock *create(librados::IoCtx& ioctx, ContextWQ *work_queue, | |
33 | const std::string& oid, librbd::Watcher *watcher, | |
34 | managed_lock::Mode mode, | |
35 | bool blacklist_on_break_lock, | |
36 | uint32_t blacklist_expire_seconds) { | |
11fdf7f2 | 37 | ceph_assert(s_instance != nullptr); |
7c673cae FG |
38 | return s_instance; |
39 | } | |
40 | ||
41 | ManagedLock() { | |
11fdf7f2 | 42 | ceph_assert(s_instance == nullptr); |
7c673cae FG |
43 | s_instance = this; |
44 | } | |
45 | ||
46 | ~ManagedLock() { | |
11fdf7f2 | 47 | ceph_assert(s_instance == this); |
7c673cae FG |
48 | s_instance = nullptr; |
49 | } | |
50 | ||
51 | MOCK_METHOD0(destroy, void()); | |
52 | MOCK_METHOD1(shut_down, void(Context *)); | |
53 | MOCK_METHOD1(acquire_lock, void(Context *)); | |
54 | MOCK_METHOD2(get_locker, void(managed_lock::Locker *, Context *)); | |
55 | MOCK_METHOD3(break_lock, void(const managed_lock::Locker &, bool, Context *)); | |
56 | }; | |
57 | ||
58 | ManagedLock<MockTestImageCtx> *ManagedLock<MockTestImageCtx>::s_instance = nullptr; | |
59 | ||
60 | } // namespace librbd | |
61 | ||
62 | namespace rbd { | |
63 | namespace mirror { | |
64 | ||
65 | template <> | |
66 | struct Threads<librbd::MockTestImageCtx> { | |
67 | Mutex &timer_lock; | |
68 | SafeTimer *timer; | |
69 | ContextWQ *work_queue; | |
70 | ||
71 | Threads(Threads<librbd::ImageCtx> *threads) | |
72 | : timer_lock(threads->timer_lock), timer(threads->timer), | |
73 | work_queue(threads->work_queue) { | |
74 | } | |
75 | }; | |
76 | ||
77 | template <> | |
78 | struct InstanceReplayer<librbd::MockTestImageCtx> { | |
d2e6a577 | 79 | MOCK_METHOD3(acquire_image, void(InstanceWatcher<librbd::MockTestImageCtx> *, |
7c673cae | 80 | const std::string &, Context *)); |
d2e6a577 FG |
81 | MOCK_METHOD2(release_image, void(const std::string &, Context *)); |
82 | MOCK_METHOD3(remove_peer_image, void(const std::string&, const std::string&, | |
83 | Context *)); | |
7c673cae FG |
84 | }; |
85 | ||
31f18b77 FG |
86 | template <> |
87 | struct ImageSyncThrottler<librbd::MockTestImageCtx> { | |
88 | static ImageSyncThrottler* s_instance; | |
89 | ||
11fdf7f2 TL |
90 | static ImageSyncThrottler *create(CephContext *cct) { |
91 | ceph_assert(s_instance != nullptr); | |
31f18b77 FG |
92 | return s_instance; |
93 | } | |
94 | ||
95 | ImageSyncThrottler() { | |
11fdf7f2 | 96 | ceph_assert(s_instance == nullptr); |
31f18b77 FG |
97 | s_instance = this; |
98 | } | |
99 | ||
100 | virtual ~ImageSyncThrottler() { | |
11fdf7f2 | 101 | ceph_assert(s_instance == this); |
31f18b77 FG |
102 | s_instance = nullptr; |
103 | } | |
104 | ||
105 | MOCK_METHOD0(destroy, void()); | |
106 | MOCK_METHOD1(drain, void(int)); | |
107 | MOCK_METHOD2(start_op, void(const std::string &, Context *)); | |
108 | MOCK_METHOD1(finish_op, void(const std::string &)); | |
109 | }; | |
110 | ||
111 | ImageSyncThrottler<librbd::MockTestImageCtx>* ImageSyncThrottler<librbd::MockTestImageCtx>::s_instance = nullptr; | |
112 | ||
7c673cae FG |
113 | } // namespace mirror |
114 | } // namespace rbd | |
115 | ||
116 | // template definitions | |
117 | #include "tools/rbd_mirror/InstanceWatcher.cc" | |
118 | ||
119 | namespace rbd { | |
120 | namespace mirror { | |
121 | ||
122 | using ::testing::_; | |
123 | using ::testing::InSequence; | |
124 | using ::testing::Invoke; | |
125 | using ::testing::Return; | |
126 | using ::testing::StrEq; | |
127 | using ::testing::WithArg; | |
128 | ||
129 | class TestMockInstanceWatcher : public TestMockFixture { | |
130 | public: | |
131 | typedef librbd::ManagedLock<librbd::MockTestImageCtx> MockManagedLock; | |
132 | typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer; | |
133 | typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; | |
134 | typedef Threads<librbd::MockTestImageCtx> MockThreads; | |
135 | ||
136 | std::string m_instance_id; | |
137 | std::string m_oid; | |
138 | MockThreads *m_mock_threads; | |
139 | ||
140 | void SetUp() override { | |
141 | TestFixture::SetUp(); | |
142 | m_local_io_ctx.remove(RBD_MIRROR_LEADER); | |
143 | EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true)); | |
144 | ||
145 | m_instance_id = stringify(m_local_io_ctx.get_instance_id()); | |
146 | m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id; | |
147 | ||
148 | m_mock_threads = new MockThreads(m_threads); | |
149 | } | |
150 | ||
151 | void TearDown() override { | |
152 | delete m_mock_threads; | |
153 | TestMockFixture::TearDown(); | |
154 | } | |
155 | ||
156 | void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) { | |
157 | EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _)); | |
158 | } | |
159 | ||
160 | void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx, | |
161 | const std::string &instance_id) { | |
162 | std::string oid = RBD_MIRROR_INSTANCE_PREFIX + instance_id; | |
163 | EXPECT_CALL(mock_io_ctx, aio_watch(oid, _, _, _)); | |
164 | } | |
165 | ||
166 | void expect_unregister_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) { | |
167 | EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _)); | |
168 | } | |
169 | ||
170 | void expect_register_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx, | |
171 | int r) { | |
172 | EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"), | |
173 | StrEq("mirror_instances_add"), _, _, _)) | |
174 | .WillOnce(Return(r)); | |
175 | } | |
176 | ||
177 | void expect_unregister_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx, | |
178 | int r) { | |
179 | EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"), | |
180 | StrEq("mirror_instances_remove"), _, _, _)) | |
181 | .WillOnce(Return(r)); | |
182 | } | |
183 | ||
184 | void expect_acquire_lock(MockManagedLock &mock_managed_lock, int r) { | |
185 | EXPECT_CALL(mock_managed_lock, acquire_lock(_)) | |
186 | .WillOnce(CompleteContext(r)); | |
187 | } | |
188 | ||
189 | void expect_release_lock(MockManagedLock &mock_managed_lock, int r) { | |
190 | EXPECT_CALL(mock_managed_lock, shut_down(_)).WillOnce(CompleteContext(r)); | |
191 | } | |
192 | ||
193 | void expect_destroy_lock(MockManagedLock &mock_managed_lock, | |
194 | Context *ctx = nullptr) { | |
195 | EXPECT_CALL(mock_managed_lock, destroy()) | |
196 | .WillOnce(Invoke([ctx]() { | |
197 | if (ctx != nullptr) { | |
198 | ctx->complete(0); | |
199 | } | |
200 | })); | |
201 | } | |
202 | ||
203 | void expect_get_locker(MockManagedLock &mock_managed_lock, | |
204 | const librbd::managed_lock::Locker &locker, int r) { | |
205 | EXPECT_CALL(mock_managed_lock, get_locker(_, _)) | |
206 | .WillOnce(Invoke([r, locker](librbd::managed_lock::Locker *out, | |
207 | Context *ctx) { | |
208 | if (r == 0) { | |
209 | *out = locker; | |
210 | } | |
211 | ctx->complete(r); | |
212 | })); | |
213 | } | |
214 | ||
215 | void expect_break_lock(MockManagedLock &mock_managed_lock, | |
216 | const librbd::managed_lock::Locker &locker, int r) { | |
217 | EXPECT_CALL(mock_managed_lock, break_lock(locker, true, _)) | |
218 | .WillOnce(WithArg<2>(CompleteContext(r))); | |
219 | } | |
220 | }; | |
221 | ||
222 | TEST_F(TestMockInstanceWatcher, InitShutdown) { | |
223 | MockManagedLock mock_managed_lock; | |
224 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
225 | ||
226 | auto instance_watcher = new MockInstanceWatcher( | |
227 | m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); | |
228 | InSequence seq; | |
229 | ||
230 | // Init | |
231 | expect_register_instance(mock_io_ctx, 0); | |
232 | expect_register_watch(mock_io_ctx); | |
233 | expect_acquire_lock(mock_managed_lock, 0); | |
234 | ASSERT_EQ(0, instance_watcher->init()); | |
235 | ||
236 | // Shutdown | |
237 | expect_release_lock(mock_managed_lock, 0); | |
238 | expect_unregister_watch(mock_io_ctx); | |
239 | expect_unregister_instance(mock_io_ctx, 0); | |
240 | instance_watcher->shut_down(); | |
241 | ||
242 | expect_destroy_lock(mock_managed_lock); | |
243 | delete instance_watcher; | |
244 | } | |
245 | ||
246 | TEST_F(TestMockInstanceWatcher, InitError) { | |
247 | MockManagedLock mock_managed_lock; | |
248 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
249 | ||
250 | auto instance_watcher = new MockInstanceWatcher( | |
251 | m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); | |
252 | InSequence seq; | |
253 | ||
254 | expect_register_instance(mock_io_ctx, 0); | |
255 | expect_register_watch(mock_io_ctx); | |
256 | expect_acquire_lock(mock_managed_lock, -EINVAL); | |
257 | expect_unregister_watch(mock_io_ctx); | |
258 | expect_unregister_instance(mock_io_ctx, 0); | |
259 | ||
260 | ASSERT_EQ(-EINVAL, instance_watcher->init()); | |
261 | ||
262 | expect_destroy_lock(mock_managed_lock); | |
263 | delete instance_watcher; | |
264 | } | |
265 | ||
266 | TEST_F(TestMockInstanceWatcher, ShutdownError) { | |
267 | MockManagedLock mock_managed_lock; | |
268 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
269 | ||
270 | auto instance_watcher = new MockInstanceWatcher( | |
271 | m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); | |
272 | InSequence seq; | |
273 | ||
274 | // Init | |
275 | expect_register_instance(mock_io_ctx, 0); | |
276 | expect_register_watch(mock_io_ctx); | |
277 | expect_acquire_lock(mock_managed_lock, 0); | |
278 | ASSERT_EQ(0, instance_watcher->init()); | |
279 | ||
280 | // Shutdown | |
281 | expect_release_lock(mock_managed_lock, -EINVAL); | |
282 | expect_unregister_watch(mock_io_ctx); | |
283 | expect_unregister_instance(mock_io_ctx, 0); | |
284 | instance_watcher->shut_down(); | |
285 | ||
286 | expect_destroy_lock(mock_managed_lock); | |
287 | delete instance_watcher; | |
288 | } | |
289 | ||
290 | ||
291 | TEST_F(TestMockInstanceWatcher, Remove) { | |
292 | MockManagedLock mock_managed_lock; | |
293 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
294 | librbd::managed_lock::Locker | |
295 | locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123}; | |
296 | ||
297 | InSequence seq; | |
298 | ||
299 | expect_get_locker(mock_managed_lock, locker, 0); | |
300 | expect_break_lock(mock_managed_lock, locker, 0); | |
301 | expect_unregister_instance(mock_io_ctx, 0); | |
302 | C_SaferCond on_destroy; | |
303 | expect_destroy_lock(mock_managed_lock, &on_destroy); | |
304 | ||
305 | C_SaferCond on_remove; | |
306 | MockInstanceWatcher::remove_instance(m_local_io_ctx, | |
307 | m_mock_threads->work_queue, | |
308 | "instance_id", &on_remove); | |
309 | ASSERT_EQ(0, on_remove.wait()); | |
310 | ASSERT_EQ(0, on_destroy.wait()); | |
311 | } | |
312 | ||
313 | TEST_F(TestMockInstanceWatcher, RemoveNoent) { | |
314 | MockManagedLock mock_managed_lock; | |
315 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
316 | ||
317 | InSequence seq; | |
318 | ||
319 | expect_get_locker(mock_managed_lock, librbd::managed_lock::Locker(), -ENOENT); | |
320 | expect_unregister_instance(mock_io_ctx, 0); | |
321 | C_SaferCond on_destroy; | |
322 | expect_destroy_lock(mock_managed_lock, &on_destroy); | |
323 | ||
324 | C_SaferCond on_remove; | |
325 | MockInstanceWatcher::remove_instance(m_local_io_ctx, | |
326 | m_mock_threads->work_queue, | |
327 | "instance_id", &on_remove); | |
328 | ASSERT_EQ(0, on_remove.wait()); | |
329 | ASSERT_EQ(0, on_destroy.wait()); | |
330 | } | |
331 | ||
332 | TEST_F(TestMockInstanceWatcher, ImageAcquireRelease) { | |
333 | MockManagedLock mock_managed_lock; | |
334 | ||
335 | librados::IoCtx& io_ctx1 = m_local_io_ctx; | |
336 | std::string instance_id1 = m_instance_id; | |
337 | librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); | |
338 | MockInstanceReplayer mock_instance_replayer1; | |
339 | auto instance_watcher1 = MockInstanceWatcher::create( | |
340 | io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1); | |
341 | ||
342 | librados::Rados cluster; | |
343 | librados::IoCtx io_ctx2; | |
344 | EXPECT_EQ("", connect_cluster_pp(cluster)); | |
345 | EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); | |
346 | std::string instance_id2 = stringify(io_ctx2.get_instance_id()); | |
347 | librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); | |
348 | MockInstanceReplayer mock_instance_replayer2; | |
349 | auto instance_watcher2 = MockInstanceWatcher::create( | |
350 | io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2); | |
351 | ||
352 | InSequence seq; | |
353 | ||
354 | // Init instance watcher 1 | |
355 | expect_register_instance(mock_io_ctx1, 0); | |
356 | expect_register_watch(mock_io_ctx1, instance_id1); | |
357 | expect_acquire_lock(mock_managed_lock, 0); | |
358 | ASSERT_EQ(0, instance_watcher1->init()); | |
359 | ||
360 | // Init instance watcher 2 | |
361 | expect_register_instance(mock_io_ctx2, 0); | |
362 | expect_register_watch(mock_io_ctx2, instance_id2); | |
363 | expect_acquire_lock(mock_managed_lock, 0); | |
364 | ASSERT_EQ(0, instance_watcher2->init()); | |
365 | ||
366 | // Acquire Image on the the same instance | |
31f18b77 | 367 | EXPECT_CALL(mock_instance_replayer1, acquire_image(instance_watcher1, "gid", |
d2e6a577 FG |
368 | _)) |
369 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
7c673cae | 370 | C_SaferCond on_acquire1; |
d2e6a577 | 371 | instance_watcher1->notify_image_acquire(instance_id1, "gid", &on_acquire1); |
7c673cae FG |
372 | ASSERT_EQ(0, on_acquire1.wait()); |
373 | ||
374 | // Acquire Image on the other instance | |
31f18b77 | 375 | EXPECT_CALL(mock_instance_replayer2, acquire_image(instance_watcher2, "gid", |
d2e6a577 FG |
376 | _)) |
377 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
7c673cae | 378 | C_SaferCond on_acquire2; |
d2e6a577 | 379 | instance_watcher1->notify_image_acquire(instance_id2, "gid", &on_acquire2); |
7c673cae FG |
380 | ASSERT_EQ(0, on_acquire2.wait()); |
381 | ||
382 | // Release Image on the the same instance | |
d2e6a577 FG |
383 | EXPECT_CALL(mock_instance_replayer1, release_image("gid", _)) |
384 | .WillOnce(WithArg<1>(CompleteContext(0))); | |
7c673cae | 385 | C_SaferCond on_release1; |
d2e6a577 | 386 | instance_watcher1->notify_image_release(instance_id1, "gid", &on_release1); |
7c673cae FG |
387 | ASSERT_EQ(0, on_release1.wait()); |
388 | ||
389 | // Release Image on the other instance | |
d2e6a577 FG |
390 | EXPECT_CALL(mock_instance_replayer2, release_image("gid", _)) |
391 | .WillOnce(WithArg<1>(CompleteContext(0))); | |
7c673cae | 392 | C_SaferCond on_release2; |
d2e6a577 | 393 | instance_watcher1->notify_image_release(instance_id2, "gid", &on_release2); |
7c673cae FG |
394 | ASSERT_EQ(0, on_release2.wait()); |
395 | ||
396 | // Shutdown instance watcher 1 | |
397 | expect_release_lock(mock_managed_lock, 0); | |
398 | expect_unregister_watch(mock_io_ctx1); | |
399 | expect_unregister_instance(mock_io_ctx1, 0); | |
400 | instance_watcher1->shut_down(); | |
401 | ||
402 | expect_destroy_lock(mock_managed_lock); | |
403 | delete instance_watcher1; | |
404 | ||
405 | // Shutdown instance watcher 2 | |
406 | expect_release_lock(mock_managed_lock, 0); | |
407 | expect_unregister_watch(mock_io_ctx2); | |
408 | expect_unregister_instance(mock_io_ctx2, 0); | |
409 | instance_watcher2->shut_down(); | |
410 | ||
411 | expect_destroy_lock(mock_managed_lock); | |
412 | delete instance_watcher2; | |
413 | } | |
414 | ||
d2e6a577 FG |
415 | TEST_F(TestMockInstanceWatcher, PeerImageRemoved) { |
416 | MockManagedLock mock_managed_lock; | |
417 | ||
418 | librados::IoCtx& io_ctx1 = m_local_io_ctx; | |
419 | std::string instance_id1 = m_instance_id; | |
420 | librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); | |
421 | MockInstanceReplayer mock_instance_replayer1; | |
422 | auto instance_watcher1 = MockInstanceWatcher::create( | |
423 | io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1); | |
424 | ||
425 | librados::Rados cluster; | |
426 | librados::IoCtx io_ctx2; | |
427 | EXPECT_EQ("", connect_cluster_pp(cluster)); | |
428 | EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); | |
429 | std::string instance_id2 = stringify(io_ctx2.get_instance_id()); | |
430 | librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); | |
431 | MockInstanceReplayer mock_instance_replayer2; | |
432 | auto instance_watcher2 = MockInstanceWatcher::create( | |
433 | io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2); | |
434 | ||
435 | InSequence seq; | |
436 | ||
437 | // Init instance watcher 1 | |
438 | expect_register_instance(mock_io_ctx1, 0); | |
439 | expect_register_watch(mock_io_ctx1, instance_id1); | |
440 | expect_acquire_lock(mock_managed_lock, 0); | |
441 | ASSERT_EQ(0, instance_watcher1->init()); | |
442 | ||
443 | // Init instance watcher 2 | |
444 | expect_register_instance(mock_io_ctx2, 0); | |
445 | expect_register_watch(mock_io_ctx2, instance_id2); | |
446 | expect_acquire_lock(mock_managed_lock, 0); | |
447 | ASSERT_EQ(0, instance_watcher2->init()); | |
448 | ||
449 | // Peer Image Removed on the same instance | |
450 | EXPECT_CALL(mock_instance_replayer1, remove_peer_image("gid", "uuid", _)) | |
451 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
452 | C_SaferCond on_removed1; | |
453 | instance_watcher1->notify_peer_image_removed(instance_id1, "gid", "uuid", | |
454 | &on_removed1); | |
455 | ASSERT_EQ(0, on_removed1.wait()); | |
456 | ||
457 | // Peer Image Removed on the other instance | |
458 | EXPECT_CALL(mock_instance_replayer2, remove_peer_image("gid", "uuid", _)) | |
459 | .WillOnce(WithArg<2>(CompleteContext(0))); | |
460 | C_SaferCond on_removed2; | |
461 | instance_watcher1->notify_peer_image_removed(instance_id2, "gid", "uuid", | |
462 | &on_removed2); | |
463 | ASSERT_EQ(0, on_removed2.wait()); | |
464 | ||
465 | // Shutdown instance watcher 1 | |
466 | expect_release_lock(mock_managed_lock, 0); | |
467 | expect_unregister_watch(mock_io_ctx1); | |
468 | expect_unregister_instance(mock_io_ctx1, 0); | |
469 | instance_watcher1->shut_down(); | |
470 | ||
471 | expect_destroy_lock(mock_managed_lock); | |
472 | delete instance_watcher1; | |
473 | ||
474 | // Shutdown instance watcher 2 | |
475 | expect_release_lock(mock_managed_lock, 0); | |
476 | expect_unregister_watch(mock_io_ctx2); | |
477 | expect_unregister_instance(mock_io_ctx2, 0); | |
478 | instance_watcher2->shut_down(); | |
479 | ||
480 | expect_destroy_lock(mock_managed_lock); | |
481 | delete instance_watcher2; | |
482 | } | |
483 | ||
7c673cae FG |
484 | TEST_F(TestMockInstanceWatcher, ImageAcquireReleaseCancel) { |
485 | MockManagedLock mock_managed_lock; | |
486 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
487 | ||
488 | auto instance_watcher = new MockInstanceWatcher( | |
489 | m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); | |
490 | InSequence seq; | |
491 | ||
492 | // Init | |
493 | expect_register_instance(mock_io_ctx, 0); | |
494 | expect_register_watch(mock_io_ctx); | |
495 | expect_acquire_lock(mock_managed_lock, 0); | |
496 | ASSERT_EQ(0, instance_watcher->init()); | |
497 | ||
498 | // Send Acquire Image and cancel | |
499 | EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) | |
500 | .WillOnce(Invoke( | |
501 | [this, instance_watcher, &mock_io_ctx]( | |
502 | const std::string& o, librados::AioCompletionImpl *c, | |
503 | bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { | |
504 | c->get(); | |
505 | auto ctx = new FunctionContext( | |
506 | [instance_watcher, &mock_io_ctx, c, pbl](int r) { | |
507 | instance_watcher->cancel_notify_requests("other"); | |
11fdf7f2 | 508 | encode(librbd::watcher::NotifyResponse(), *pbl); |
7c673cae FG |
509 | mock_io_ctx.get_mock_rados_client()-> |
510 | finish_aio_completion(c, -ETIMEDOUT); | |
511 | }); | |
512 | m_threads->work_queue->queue(ctx, 0); | |
513 | })); | |
514 | ||
515 | C_SaferCond on_acquire; | |
d2e6a577 | 516 | instance_watcher->notify_image_acquire("other", "gid", &on_acquire); |
7c673cae FG |
517 | ASSERT_EQ(-ECANCELED, on_acquire.wait()); |
518 | ||
519 | // Send Release Image and cancel | |
520 | EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) | |
521 | .WillOnce(Invoke( | |
522 | [this, instance_watcher, &mock_io_ctx]( | |
523 | const std::string& o, librados::AioCompletionImpl *c, | |
524 | bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { | |
525 | c->get(); | |
526 | auto ctx = new FunctionContext( | |
527 | [instance_watcher, &mock_io_ctx, c, pbl](int r) { | |
528 | instance_watcher->cancel_notify_requests("other"); | |
11fdf7f2 | 529 | encode(librbd::watcher::NotifyResponse(), *pbl); |
7c673cae FG |
530 | mock_io_ctx.get_mock_rados_client()-> |
531 | finish_aio_completion(c, -ETIMEDOUT); | |
532 | }); | |
533 | m_threads->work_queue->queue(ctx, 0); | |
534 | })); | |
535 | ||
536 | C_SaferCond on_release; | |
d2e6a577 | 537 | instance_watcher->notify_image_release("other", "gid", &on_release); |
7c673cae FG |
538 | ASSERT_EQ(-ECANCELED, on_release.wait()); |
539 | ||
540 | // Shutdown | |
541 | expect_release_lock(mock_managed_lock, 0); | |
542 | expect_unregister_watch(mock_io_ctx); | |
543 | expect_unregister_instance(mock_io_ctx, 0); | |
544 | instance_watcher->shut_down(); | |
545 | ||
546 | expect_destroy_lock(mock_managed_lock); | |
547 | delete instance_watcher; | |
548 | } | |
549 | ||
11fdf7f2 TL |
550 | TEST_F(TestMockInstanceWatcher, PeerImageAcquireWatchDNE) { |
551 | MockManagedLock mock_managed_lock; | |
552 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
553 | ||
554 | MockInstanceReplayer mock_instance_replayer; | |
555 | auto instance_watcher = new MockInstanceWatcher( | |
556 | m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer, | |
557 | m_instance_id); | |
558 | InSequence seq; | |
559 | ||
560 | // Init | |
561 | expect_register_instance(mock_io_ctx, 0); | |
562 | expect_register_watch(mock_io_ctx); | |
563 | expect_acquire_lock(mock_managed_lock, 0); | |
564 | ASSERT_EQ(0, instance_watcher->init()); | |
565 | ||
566 | // Acquire image on dead (blacklisted) instance | |
567 | C_SaferCond on_acquire; | |
568 | instance_watcher->notify_image_acquire("dead instance", "global image id", | |
569 | &on_acquire); | |
570 | ASSERT_EQ(-ENOENT, on_acquire.wait()); | |
571 | ||
572 | // Shutdown | |
573 | expect_release_lock(mock_managed_lock, 0); | |
574 | expect_unregister_watch(mock_io_ctx); | |
575 | expect_unregister_instance(mock_io_ctx, 0); | |
576 | instance_watcher->shut_down(); | |
577 | ||
578 | expect_destroy_lock(mock_managed_lock); | |
579 | delete instance_watcher; | |
580 | } | |
581 | ||
582 | TEST_F(TestMockInstanceWatcher, PeerImageReleaseWatchDNE) { | |
583 | MockManagedLock mock_managed_lock; | |
584 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
585 | ||
586 | MockInstanceReplayer mock_instance_replayer; | |
587 | auto instance_watcher = new MockInstanceWatcher( | |
588 | m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer, | |
589 | m_instance_id); | |
590 | InSequence seq; | |
591 | ||
592 | // Init | |
593 | expect_register_instance(mock_io_ctx, 0); | |
594 | expect_register_watch(mock_io_ctx); | |
595 | expect_acquire_lock(mock_managed_lock, 0); | |
596 | ASSERT_EQ(0, instance_watcher->init()); | |
597 | ||
598 | // Release image on dead (blacklisted) instance | |
599 | C_SaferCond on_acquire; | |
600 | instance_watcher->notify_image_release("dead instance", "global image id", | |
601 | &on_acquire); | |
602 | ASSERT_EQ(-ENOENT, on_acquire.wait()); | |
603 | ||
604 | // Shutdown | |
605 | expect_release_lock(mock_managed_lock, 0); | |
606 | expect_unregister_watch(mock_io_ctx); | |
607 | expect_unregister_instance(mock_io_ctx, 0); | |
608 | instance_watcher->shut_down(); | |
609 | ||
610 | expect_destroy_lock(mock_managed_lock); | |
611 | delete instance_watcher; | |
612 | } | |
613 | ||
d2e6a577 FG |
614 | TEST_F(TestMockInstanceWatcher, PeerImageRemovedCancel) { |
615 | MockManagedLock mock_managed_lock; | |
616 | librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); | |
617 | ||
618 | auto instance_watcher = new MockInstanceWatcher( | |
619 | m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); | |
620 | InSequence seq; | |
621 | ||
622 | // Init | |
623 | expect_register_instance(mock_io_ctx, 0); | |
624 | expect_register_watch(mock_io_ctx); | |
625 | expect_acquire_lock(mock_managed_lock, 0); | |
626 | ASSERT_EQ(0, instance_watcher->init()); | |
627 | ||
628 | // Send Acquire Image and cancel | |
629 | EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) | |
630 | .WillOnce(Invoke( | |
631 | [this, instance_watcher, &mock_io_ctx]( | |
632 | const std::string& o, librados::AioCompletionImpl *c, | |
633 | bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { | |
634 | c->get(); | |
635 | auto ctx = new FunctionContext( | |
636 | [instance_watcher, &mock_io_ctx, c, pbl](int r) { | |
637 | instance_watcher->cancel_notify_requests("other"); | |
11fdf7f2 | 638 | encode(librbd::watcher::NotifyResponse(), *pbl); |
d2e6a577 FG |
639 | mock_io_ctx.get_mock_rados_client()-> |
640 | finish_aio_completion(c, -ETIMEDOUT); | |
641 | }); | |
642 | m_threads->work_queue->queue(ctx, 0); | |
643 | })); | |
644 | ||
645 | C_SaferCond on_acquire; | |
646 | instance_watcher->notify_peer_image_removed("other", "gid", "uuid", | |
647 | &on_acquire); | |
648 | ASSERT_EQ(-ECANCELED, on_acquire.wait()); | |
649 | ||
650 | // Shutdown | |
651 | expect_release_lock(mock_managed_lock, 0); | |
652 | expect_unregister_watch(mock_io_ctx); | |
653 | expect_unregister_instance(mock_io_ctx, 0); | |
654 | instance_watcher->shut_down(); | |
655 | ||
656 | expect_destroy_lock(mock_managed_lock); | |
657 | delete instance_watcher; | |
658 | } | |
659 | ||
31f18b77 FG |
660 | class TestMockInstanceWatcher_NotifySync : public TestMockInstanceWatcher { |
661 | public: | |
662 | typedef ImageSyncThrottler<librbd::MockTestImageCtx> MockImageSyncThrottler; | |
663 | ||
664 | MockManagedLock mock_managed_lock; | |
665 | MockImageSyncThrottler mock_image_sync_throttler; | |
666 | std::string instance_id1; | |
667 | std::string instance_id2; | |
668 | ||
669 | librados::Rados cluster; | |
670 | librados::IoCtx io_ctx2; | |
671 | ||
672 | MockInstanceWatcher *instance_watcher1; | |
673 | MockInstanceWatcher *instance_watcher2; | |
674 | ||
675 | void SetUp() override { | |
676 | TestMockInstanceWatcher::SetUp(); | |
677 | ||
678 | instance_id1 = m_instance_id; | |
679 | librados::IoCtx& io_ctx1 = m_local_io_ctx; | |
680 | librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); | |
681 | instance_watcher1 = MockInstanceWatcher::create(io_ctx1, | |
682 | m_mock_threads->work_queue, | |
683 | nullptr); | |
684 | EXPECT_EQ("", connect_cluster_pp(cluster)); | |
685 | EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); | |
686 | instance_id2 = stringify(io_ctx2.get_instance_id()); | |
687 | librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); | |
688 | instance_watcher2 = MockInstanceWatcher::create(io_ctx2, | |
689 | m_mock_threads->work_queue, | |
690 | nullptr); | |
691 | InSequence seq; | |
692 | ||
693 | // Init instance watcher 1 (leader) | |
694 | expect_register_instance(mock_io_ctx1, 0); | |
695 | expect_register_watch(mock_io_ctx1, instance_id1); | |
696 | expect_acquire_lock(mock_managed_lock, 0); | |
697 | EXPECT_EQ(0, instance_watcher1->init()); | |
698 | instance_watcher1->handle_acquire_leader(); | |
699 | ||
700 | // Init instance watcher 2 | |
701 | expect_register_instance(mock_io_ctx2, 0); | |
702 | expect_register_watch(mock_io_ctx2, instance_id2); | |
703 | expect_acquire_lock(mock_managed_lock, 0); | |
704 | EXPECT_EQ(0, instance_watcher2->init()); | |
705 | instance_watcher2->handle_update_leader(instance_id1); | |
706 | } | |
707 | ||
708 | void TearDown() override { | |
709 | librados::IoCtx& io_ctx1 = m_local_io_ctx; | |
710 | librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); | |
711 | librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); | |
712 | ||
713 | InSequence seq; | |
714 | ||
715 | expect_throttler_destroy(); | |
716 | instance_watcher1->handle_release_leader(); | |
717 | ||
718 | // Shutdown instance watcher 1 | |
719 | expect_release_lock(mock_managed_lock, 0); | |
720 | expect_unregister_watch(mock_io_ctx1); | |
721 | expect_unregister_instance(mock_io_ctx1, 0); | |
722 | instance_watcher1->shut_down(); | |
723 | ||
724 | expect_destroy_lock(mock_managed_lock); | |
725 | delete instance_watcher1; | |
726 | ||
727 | // Shutdown instance watcher 2 | |
728 | expect_release_lock(mock_managed_lock, 0); | |
729 | expect_unregister_watch(mock_io_ctx2); | |
730 | expect_unregister_instance(mock_io_ctx2, 0); | |
731 | instance_watcher2->shut_down(); | |
732 | ||
733 | expect_destroy_lock(mock_managed_lock); | |
734 | delete instance_watcher2; | |
735 | ||
736 | TestMockInstanceWatcher::TearDown(); | |
737 | } | |
738 | ||
739 | void expect_throttler_destroy( | |
740 | std::vector<Context *> *throttler_queue = nullptr) { | |
741 | EXPECT_CALL(mock_image_sync_throttler, drain(-ESTALE)) | |
742 | .WillOnce(Invoke([throttler_queue] (int r) { | |
743 | if (throttler_queue != nullptr) { | |
744 | for (auto ctx : *throttler_queue) { | |
745 | ctx->complete(r); | |
746 | } | |
747 | } | |
748 | })); | |
749 | EXPECT_CALL(mock_image_sync_throttler, destroy()); | |
750 | } | |
751 | ||
752 | void expect_throttler_start_op(const std::string &sync_id, | |
753 | Context *on_call = nullptr, | |
754 | Context **on_start_ctx = nullptr) { | |
755 | EXPECT_CALL(mock_image_sync_throttler, start_op(sync_id, _)) | |
756 | .WillOnce(Invoke([on_call, on_start_ctx] (const std::string &, | |
757 | Context *ctx) { | |
31f18b77 FG |
758 | if (on_start_ctx != nullptr) { |
759 | *on_start_ctx = ctx; | |
760 | } else { | |
761 | ctx->complete(0); | |
762 | } | |
11fdf7f2 TL |
763 | if (on_call != nullptr) { |
764 | on_call->complete(0); | |
765 | } | |
31f18b77 FG |
766 | })); |
767 | } | |
768 | ||
769 | void expect_throttler_finish_op(const std::string &sync_id, | |
770 | Context *on_finish) { | |
771 | EXPECT_CALL(mock_image_sync_throttler, finish_op("sync_id")) | |
772 | .WillOnce(Invoke([on_finish](const std::string &) { | |
773 | on_finish->complete(0); | |
774 | })); | |
775 | } | |
776 | }; | |
777 | ||
778 | TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnLeader) { | |
779 | InSequence seq; | |
780 | ||
781 | expect_throttler_start_op("sync_id"); | |
782 | C_SaferCond on_start; | |
783 | instance_watcher1->notify_sync_request("sync_id", &on_start); | |
784 | ASSERT_EQ(0, on_start.wait()); | |
785 | ||
786 | C_SaferCond on_finish; | |
787 | expect_throttler_finish_op("sync_id", &on_finish); | |
788 | instance_watcher1->notify_sync_complete("sync_id"); | |
789 | ASSERT_EQ(0, on_finish.wait()); | |
790 | } | |
791 | ||
792 | TEST_F(TestMockInstanceWatcher_NotifySync, CancelStartedOnLeader) { | |
793 | InSequence seq; | |
794 | ||
795 | expect_throttler_start_op("sync_id"); | |
796 | C_SaferCond on_start; | |
797 | instance_watcher1->notify_sync_request("sync_id", &on_start); | |
798 | ASSERT_EQ(0, on_start.wait()); | |
799 | ||
800 | ASSERT_FALSE(instance_watcher1->cancel_sync_request("sync_id")); | |
801 | ||
802 | C_SaferCond on_finish; | |
803 | expect_throttler_finish_op("sync_id", &on_finish); | |
804 | instance_watcher1->notify_sync_complete("sync_id"); | |
805 | ASSERT_EQ(0, on_finish.wait()); | |
806 | } | |
807 | ||
808 | TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnNonLeader) { | |
809 | InSequence seq; | |
810 | ||
811 | expect_throttler_start_op("sync_id"); | |
812 | C_SaferCond on_start; | |
813 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
814 | ASSERT_EQ(0, on_start.wait()); | |
815 | ||
816 | C_SaferCond on_finish; | |
817 | expect_throttler_finish_op("sync_id", &on_finish); | |
818 | instance_watcher2->notify_sync_complete("sync_id"); | |
819 | ASSERT_EQ(0, on_finish.wait()); | |
820 | } | |
821 | ||
822 | TEST_F(TestMockInstanceWatcher_NotifySync, CancelStartedOnNonLeader) { | |
823 | InSequence seq; | |
824 | ||
825 | expect_throttler_start_op("sync_id"); | |
826 | C_SaferCond on_start; | |
827 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
828 | ASSERT_EQ(0, on_start.wait()); | |
829 | ||
830 | ASSERT_FALSE(instance_watcher2->cancel_sync_request("sync_id")); | |
831 | ||
832 | C_SaferCond on_finish; | |
833 | expect_throttler_finish_op("sync_id", &on_finish); | |
834 | instance_watcher2->notify_sync_complete("sync_id"); | |
835 | ASSERT_EQ(0, on_finish.wait()); | |
836 | } | |
837 | ||
838 | TEST_F(TestMockInstanceWatcher_NotifySync, CancelWaitingOnNonLeader) { | |
839 | InSequence seq; | |
840 | ||
841 | C_SaferCond on_start_op_called; | |
842 | Context *on_start_ctx; | |
843 | expect_throttler_start_op("sync_id", &on_start_op_called, | |
844 | &on_start_ctx); | |
845 | C_SaferCond on_start; | |
846 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
847 | ASSERT_EQ(0, on_start_op_called.wait()); | |
848 | ||
849 | ASSERT_TRUE(instance_watcher2->cancel_sync_request("sync_id")); | |
850 | // emulate watcher timeout | |
851 | on_start_ctx->complete(-ETIMEDOUT); | |
852 | ASSERT_EQ(-ECANCELED, on_start.wait()); | |
853 | } | |
854 | ||
855 | TEST_F(TestMockInstanceWatcher_NotifySync, InFlightPrevNotification) { | |
856 | // start sync when previous notification is still in flight | |
857 | ||
858 | InSequence seq; | |
859 | ||
860 | expect_throttler_start_op("sync_id"); | |
861 | C_SaferCond on_start1; | |
862 | instance_watcher2->notify_sync_request("sync_id", &on_start1); | |
863 | ASSERT_EQ(0, on_start1.wait()); | |
864 | ||
865 | C_SaferCond on_start2; | |
866 | EXPECT_CALL(mock_image_sync_throttler, finish_op("sync_id")) | |
867 | .WillOnce(Invoke([this, &on_start2](const std::string &) { | |
868 | instance_watcher2->notify_sync_request("sync_id", &on_start2); | |
869 | })); | |
870 | expect_throttler_start_op("sync_id"); | |
871 | instance_watcher2->notify_sync_complete("sync_id"); | |
872 | ||
873 | ASSERT_EQ(0, on_start2.wait()); | |
874 | C_SaferCond on_finish; | |
875 | expect_throttler_finish_op("sync_id", &on_finish); | |
876 | instance_watcher2->notify_sync_complete("sync_id"); | |
877 | ASSERT_EQ(0, on_finish.wait()); | |
878 | } | |
879 | ||
880 | TEST_F(TestMockInstanceWatcher_NotifySync, NoInFlightReleaseAcquireLeader) { | |
881 | InSequence seq; | |
882 | ||
883 | expect_throttler_destroy(); | |
884 | instance_watcher1->handle_release_leader(); | |
885 | instance_watcher1->handle_acquire_leader(); | |
886 | } | |
887 | ||
888 | TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnLeaderReleaseLeader) { | |
889 | InSequence seq; | |
890 | ||
891 | expect_throttler_destroy(); | |
892 | instance_watcher1->handle_release_leader(); | |
893 | instance_watcher2->handle_acquire_leader(); | |
894 | ||
895 | expect_throttler_start_op("sync_id"); | |
896 | C_SaferCond on_start; | |
897 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
898 | ASSERT_EQ(0, on_start.wait()); | |
899 | expect_throttler_destroy(); | |
900 | instance_watcher2->handle_release_leader(); | |
901 | instance_watcher2->notify_sync_complete("sync_id"); | |
902 | ||
903 | instance_watcher1->handle_acquire_leader(); | |
904 | } | |
905 | ||
906 | TEST_F(TestMockInstanceWatcher_NotifySync, WaitingOnLeaderReleaseLeader) { | |
907 | InSequence seq; | |
908 | ||
909 | C_SaferCond on_start_op_called; | |
910 | Context *on_start_ctx; | |
11fdf7f2 | 911 | expect_throttler_start_op("sync_id", &on_start_op_called, &on_start_ctx); |
31f18b77 FG |
912 | C_SaferCond on_start; |
913 | instance_watcher1->notify_sync_request("sync_id", &on_start); | |
914 | ASSERT_EQ(0, on_start_op_called.wait()); | |
915 | ||
916 | std::vector<Context *> throttler_queue = {on_start_ctx}; | |
917 | expect_throttler_destroy(&throttler_queue); | |
918 | instance_watcher1->handle_release_leader(); | |
11fdf7f2 | 919 | expect_throttler_start_op("sync_id"); |
31f18b77 FG |
920 | instance_watcher2->handle_acquire_leader(); |
921 | instance_watcher1->handle_update_leader(instance_id2); | |
922 | ||
31f18b77 FG |
923 | ASSERT_EQ(0, on_start.wait()); |
924 | C_SaferCond on_finish; | |
925 | expect_throttler_finish_op("sync_id", &on_finish); | |
926 | instance_watcher1->notify_sync_complete("sync_id"); | |
927 | ASSERT_EQ(0, on_finish.wait()); | |
928 | ||
929 | expect_throttler_destroy(); | |
930 | instance_watcher2->handle_release_leader(); | |
931 | instance_watcher1->handle_acquire_leader(); | |
932 | } | |
933 | ||
934 | TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnNonLeaderAcquireLeader) { | |
935 | InSequence seq; | |
936 | ||
937 | expect_throttler_destroy(); | |
938 | instance_watcher1->handle_release_leader(); | |
939 | instance_watcher2->handle_acquire_leader(); | |
940 | instance_watcher1->handle_update_leader(instance_id2); | |
941 | ||
942 | expect_throttler_start_op("sync_id"); | |
943 | C_SaferCond on_start; | |
944 | instance_watcher1->notify_sync_request("sync_id", &on_start); | |
945 | ASSERT_EQ(0, on_start.wait()); | |
946 | ||
947 | expect_throttler_destroy(); | |
948 | instance_watcher2->handle_release_leader(); | |
949 | instance_watcher1->handle_acquire_leader(); | |
11fdf7f2 | 950 | instance_watcher2->handle_update_leader(instance_id1); |
31f18b77 FG |
951 | |
952 | instance_watcher1->notify_sync_complete("sync_id"); | |
953 | } | |
954 | ||
955 | TEST_F(TestMockInstanceWatcher_NotifySync, WaitingOnNonLeaderAcquireLeader) { | |
956 | InSequence seq; | |
957 | ||
958 | C_SaferCond on_start_op_called; | |
959 | Context *on_start_ctx; | |
960 | expect_throttler_start_op("sync_id", &on_start_op_called, | |
961 | &on_start_ctx); | |
962 | C_SaferCond on_start; | |
963 | instance_watcher2->notify_sync_request("sync_id", &on_start); | |
964 | ASSERT_EQ(0, on_start_op_called.wait()); | |
965 | ||
966 | std::vector<Context *> throttler_queue = {on_start_ctx}; | |
967 | expect_throttler_destroy(&throttler_queue); | |
968 | instance_watcher1->handle_release_leader(); | |
969 | ||
970 | EXPECT_CALL(mock_image_sync_throttler, start_op("sync_id", _)) | |
971 | .WillOnce(WithArg<1>(CompleteContext(0))); | |
972 | instance_watcher2->handle_acquire_leader(); | |
973 | instance_watcher1->handle_update_leader(instance_id2); | |
974 | ||
975 | ASSERT_EQ(0, on_start.wait()); | |
976 | ||
977 | C_SaferCond on_finish; | |
978 | expect_throttler_finish_op("sync_id", &on_finish); | |
979 | instance_watcher2->notify_sync_complete("sync_id"); | |
980 | ASSERT_EQ(0, on_finish.wait()); | |
981 | ||
982 | expect_throttler_destroy(); | |
983 | instance_watcher2->handle_release_leader(); | |
984 | instance_watcher1->handle_acquire_leader(); | |
985 | } | |
986 | ||
7c673cae FG |
987 | } // namespace mirror |
988 | } // namespace rbd |