1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "librbd/Utils.h"
5 #include "test/librbd/mock/MockImageCtx.h"
6 #include "test/rbd_mirror/test_mock_fixture.h"
7 #include "tools/rbd_mirror/LeaderWatcher.h"
8 #include "tools/rbd_mirror/Threads.h"
10 using librbd::util::create_async_context_callback
;
16 struct MockTestImageCtx
: public MockImageCtx
{
17 MockTestImageCtx(librbd::ImageCtx
&image_ctx
)
18 : librbd::MockImageCtx(image_ctx
) {
22 } // anonymous namespace
24 struct MockManagedLock
{
25 static MockManagedLock
*s_instance
;
26 static MockManagedLock
&get_instance() {
27 assert(s_instance
!= nullptr);
35 bool m_release_lock_on_shutdown
= false;
37 MOCK_METHOD0(construct
, void());
38 MOCK_METHOD0(destroy
, void());
40 MOCK_CONST_METHOD0(is_lock_owner
, bool());
42 MOCK_METHOD1(shut_down
, void(Context
*));
43 MOCK_METHOD1(try_acquire_lock
, void(Context
*));
44 MOCK_METHOD1(release_lock
, void(Context
*));
45 MOCK_METHOD3(break_lock
, void(const managed_lock::Locker
&, bool, Context
*));
46 MOCK_METHOD2(get_locker
, void(managed_lock::Locker
*, Context
*));
48 MOCK_METHOD0(set_state_post_acquiring
, void());
50 MOCK_CONST_METHOD0(is_shutdown
, bool());
52 MOCK_CONST_METHOD0(is_state_post_acquiring
, bool());
53 MOCK_CONST_METHOD0(is_state_pre_releasing
, bool());
54 MOCK_CONST_METHOD0(is_state_locked
, bool());
57 MockManagedLock
*MockManagedLock::s_instance
= nullptr;
60 struct ManagedLock
<MockTestImageCtx
> {
61 ManagedLock(librados::IoCtx
& ioctx
, ContextWQ
*work_queue
,
62 const std::string
& oid
, librbd::Watcher
*watcher
,
63 managed_lock::Mode mode
, bool blacklist_on_break_lock
,
64 uint32_t blacklist_expire_seconds
)
65 : m_work_queue(work_queue
), m_lock("ManagedLock::m_lock") {
66 MockManagedLock::get_instance().construct();
69 virtual ~ManagedLock() {
70 MockManagedLock::get_instance().destroy();
73 ContextWQ
*m_work_queue
;
77 bool is_lock_owner() const {
78 return MockManagedLock::get_instance().is_lock_owner();
81 void shut_down(Context
*on_shutdown
) {
82 if (MockManagedLock::get_instance().m_release_lock_on_shutdown
) {
83 on_shutdown
= new FunctionContext(
84 [this, on_shutdown
](int r
) {
85 MockManagedLock::get_instance().m_release_lock_on_shutdown
= false;
86 shut_down(on_shutdown
);
88 release_lock(on_shutdown
);
92 MockManagedLock::get_instance().shut_down(on_shutdown
);
95 void try_acquire_lock(Context
*on_acquired
) {
96 Context
*post_acquire_ctx
= create_async_context_callback(
97 m_work_queue
, new FunctionContext(
98 [this, on_acquired
](int r
) {
99 post_acquire_lock_handler(r
, on_acquired
);
101 MockManagedLock::get_instance().try_acquire_lock(post_acquire_ctx
);
104 void release_lock(Context
*on_released
) {
105 Context
*post_release_ctx
= new FunctionContext(
106 [this, on_released
](int r
) {
107 post_release_lock_handler(false, r
, on_released
);
110 Context
*release_ctx
= new FunctionContext(
111 [this, on_released
, post_release_ctx
](int r
) {
113 on_released
->complete(r
);
115 MockManagedLock::get_instance().release_lock(post_release_ctx
);
119 Context
*pre_release_ctx
= new FunctionContext(
120 [this, release_ctx
](int r
) {
122 MockManagedLock::get_instance().m_release_lock_on_shutdown
;
123 pre_release_lock_handler(shutting_down
, release_ctx
);
126 m_work_queue
->queue(pre_release_ctx
, 0);
129 void get_locker(managed_lock::Locker
*locker
, Context
*on_finish
) {
130 MockManagedLock::get_instance().get_locker(locker
, on_finish
);
133 void break_lock(const managed_lock::Locker
&locker
, bool force_break_lock
,
134 Context
*on_finish
) {
135 MockManagedLock::get_instance().break_lock(locker
, force_break_lock
,
139 void set_state_post_acquiring() {
140 MockManagedLock::get_instance().set_state_post_acquiring();
143 bool is_shutdown() const {
144 return MockManagedLock::get_instance().is_shutdown();
147 bool is_state_post_acquiring() const {
148 return MockManagedLock::get_instance().is_state_post_acquiring();
151 bool is_state_pre_releasing() const {
152 return MockManagedLock::get_instance().is_state_pre_releasing();
155 bool is_state_locked() const {
156 return MockManagedLock::get_instance().is_state_locked();
159 virtual void post_acquire_lock_handler(int r
, Context
*on_finish
) = 0;
160 virtual void pre_release_lock_handler(bool shutting_down
,
161 Context
*on_finish
) = 0;
162 virtual void post_release_lock_handler(bool shutting_down
, int r
,
163 Context
*on_finish
) = 0;
166 } // namespace librbd
172 struct Threads
<librbd::MockTestImageCtx
> {
175 ContextWQ
*work_queue
;
177 Threads(Threads
<librbd::ImageCtx
> *threads
)
178 : timer_lock(threads
->timer_lock
), timer(threads
->timer
),
179 work_queue(threads
->work_queue
) {
184 struct MirrorStatusWatcher
<librbd::MockTestImageCtx
> {
185 static MirrorStatusWatcher
* s_instance
;
187 static MirrorStatusWatcher
*create(librados::IoCtx
&io_ctx
,
188 ContextWQ
*work_queue
) {
189 assert(s_instance
!= nullptr);
193 MirrorStatusWatcher() {
194 assert(s_instance
== nullptr);
198 ~MirrorStatusWatcher() {
199 assert(s_instance
== this);
200 s_instance
= nullptr;
203 MOCK_METHOD0(destroy
, void());
204 MOCK_METHOD1(init
, void(Context
*));
205 MOCK_METHOD1(shut_down
, void(Context
*));
208 MirrorStatusWatcher
<librbd::MockTestImageCtx
> *MirrorStatusWatcher
<librbd::MockTestImageCtx
>::s_instance
= nullptr;
211 struct Instances
<librbd::MockTestImageCtx
> {
212 static Instances
* s_instance
;
214 static Instances
*create(Threads
<librbd::MockTestImageCtx
> *threads
,
215 librados::IoCtx
&ioctx
) {
216 assert(s_instance
!= nullptr);
221 assert(s_instance
== nullptr);
226 assert(s_instance
== this);
227 s_instance
= nullptr;
230 MOCK_METHOD0(destroy
, void());
231 MOCK_METHOD1(init
, void(Context
*));
232 MOCK_METHOD1(shut_down
, void(Context
*));
233 MOCK_METHOD1(notify
, void(const std::string
&));
236 Instances
<librbd::MockTestImageCtx
> *Instances
<librbd::MockTestImageCtx
>::s_instance
= nullptr;
238 } // namespace mirror
242 // template definitions
243 #include "tools/rbd_mirror/LeaderWatcher.cc"
249 using ::testing::AtLeast
;
250 using ::testing::DoAll
;
251 using ::testing::InSequence
;
252 using ::testing::Invoke
;
253 using ::testing::Return
;
255 using librbd::MockManagedLock
;
257 struct MockListener
: public LeaderWatcher
<librbd::MockTestImageCtx
>::Listener
{
258 static MockListener
* s_instance
;
261 assert(s_instance
== nullptr);
265 ~MockListener() override
{
266 assert(s_instance
== this);
267 s_instance
= nullptr;
270 MOCK_METHOD1(post_acquire_handler
, void(Context
*));
271 MOCK_METHOD1(pre_release_handler
, void(Context
*));
273 MOCK_METHOD1(update_leader_handler
, void(const std::string
&));
276 MockListener
*MockListener::s_instance
= nullptr;
278 class TestMockLeaderWatcher
: public TestMockFixture
{
280 typedef MirrorStatusWatcher
<librbd::MockTestImageCtx
> MockMirrorStatusWatcher
;
281 typedef Instances
<librbd::MockTestImageCtx
> MockInstances
;
282 typedef LeaderWatcher
<librbd::MockTestImageCtx
> MockLeaderWatcher
;
283 typedef Threads
<librbd::MockTestImageCtx
> MockThreads
;
285 void SetUp() override
{
286 TestMockFixture::SetUp();
287 m_mock_threads
= new MockThreads(m_threads
);
290 void TearDown() override
{
291 delete m_mock_threads
;
292 TestMockFixture::TearDown();
295 void expect_construct(MockManagedLock
&mock_managed_lock
) {
296 EXPECT_CALL(mock_managed_lock
, construct());
299 void expect_destroy(MockManagedLock
&mock_managed_lock
) {
300 EXPECT_CALL(mock_managed_lock
, destroy());
303 void expect_is_lock_owner(MockManagedLock
&mock_managed_lock
, bool owner
) {
304 EXPECT_CALL(mock_managed_lock
, is_lock_owner())
305 .WillOnce(Return(owner
));
308 void expect_shut_down(MockManagedLock
&mock_managed_lock
,
309 bool release_lock_on_shutdown
, int r
) {
310 mock_managed_lock
.m_release_lock_on_shutdown
= release_lock_on_shutdown
;
311 EXPECT_CALL(mock_managed_lock
, shut_down(_
))
312 .WillOnce(CompleteContext(r
));
315 void expect_try_acquire_lock(MockManagedLock
&mock_managed_lock
, int r
) {
316 EXPECT_CALL(mock_managed_lock
, try_acquire_lock(_
))
317 .WillOnce(CompleteContext(r
));
319 expect_set_state_post_acquiring(mock_managed_lock
);
323 void expect_release_lock(MockManagedLock
&mock_managed_lock
, int r
,
324 Context
*on_finish
= nullptr) {
325 EXPECT_CALL(mock_managed_lock
, release_lock(_
))
326 .WillOnce(Invoke([on_finish
, r
](Context
*ctx
) {
328 if (on_finish
!= nullptr) {
329 on_finish
->complete(0);
334 void expect_get_locker(MockManagedLock
&mock_managed_lock
,
335 const librbd::managed_lock::Locker
&locker
, int r
) {
336 EXPECT_CALL(mock_managed_lock
, get_locker(_
, _
))
337 .WillOnce(Invoke([r
, locker
](librbd::managed_lock::Locker
*out
,
346 void expect_break_lock(MockManagedLock
&mock_managed_lock
,
347 const librbd::managed_lock::Locker
&locker
, int r
,
348 Context
*on_finish
) {
349 EXPECT_CALL(mock_managed_lock
, break_lock(locker
, true, _
))
350 .WillOnce(Invoke([on_finish
, r
](const librbd::managed_lock::Locker
&,
351 bool, Context
*ctx
) {
353 on_finish
->complete(0);
357 void expect_set_state_post_acquiring(MockManagedLock
&mock_managed_lock
) {
358 EXPECT_CALL(mock_managed_lock
, set_state_post_acquiring());
361 void expect_is_shutdown(MockManagedLock
&mock_managed_lock
) {
362 EXPECT_CALL(mock_managed_lock
, is_shutdown())
363 .Times(AtLeast(0)).WillRepeatedly(Return(false));
366 void expect_is_leader(MockManagedLock
&mock_managed_lock
, bool post_acquiring
,
368 EXPECT_CALL(mock_managed_lock
, is_state_post_acquiring())
369 .WillOnce(Return(post_acquiring
));
370 if (!post_acquiring
) {
371 EXPECT_CALL(mock_managed_lock
, is_state_locked())
372 .WillOnce(Return(locked
));
376 void expect_is_leader(MockManagedLock
&mock_managed_lock
) {
377 EXPECT_CALL(mock_managed_lock
, is_state_post_acquiring())
378 .Times(AtLeast(0)).WillRepeatedly(Return(false));
379 EXPECT_CALL(mock_managed_lock
, is_state_locked())
380 .Times(AtLeast(0)).WillRepeatedly(Return(false));
381 EXPECT_CALL(mock_managed_lock
, is_state_pre_releasing())
382 .Times(AtLeast(0)).WillRepeatedly(Return(false));
385 void expect_notify_heartbeat(MockManagedLock
&mock_managed_lock
,
386 Context
*on_finish
) {
387 // is_leader in notify_heartbeat
388 EXPECT_CALL(mock_managed_lock
, is_state_post_acquiring())
389 .WillOnce(Return(false));
390 EXPECT_CALL(mock_managed_lock
, is_state_locked())
391 .WillOnce(Return(true));
393 // is_leader in handle_notify_heartbeat
394 EXPECT_CALL(mock_managed_lock
, is_state_post_acquiring())
395 .WillOnce(Return(false));
396 EXPECT_CALL(mock_managed_lock
, is_state_locked())
397 .WillOnce(DoAll(Invoke([on_finish
]() {
398 on_finish
->complete(0);
403 void expect_destroy(MockMirrorStatusWatcher
&mock_mirror_status_watcher
) {
404 EXPECT_CALL(mock_mirror_status_watcher
, destroy());
407 void expect_init(MockMirrorStatusWatcher
&mock_mirror_status_watcher
, int r
) {
408 EXPECT_CALL(mock_mirror_status_watcher
, init(_
))
409 .WillOnce(CompleteContext(m_mock_threads
->work_queue
, r
));
412 void expect_shut_down(MockMirrorStatusWatcher
&mock_mirror_status_watcher
, int r
) {
413 EXPECT_CALL(mock_mirror_status_watcher
, shut_down(_
))
414 .WillOnce(CompleteContext(m_mock_threads
->work_queue
, r
));
415 expect_destroy(mock_mirror_status_watcher
);
418 void expect_destroy(MockInstances
&mock_instances
) {
419 EXPECT_CALL(mock_instances
, destroy());
422 void expect_init(MockInstances
&mock_instances
, int r
) {
423 EXPECT_CALL(mock_instances
, init(_
))
424 .WillOnce(CompleteContext(m_mock_threads
->work_queue
, r
));
427 void expect_shut_down(MockInstances
&mock_instances
, int r
) {
428 EXPECT_CALL(mock_instances
, shut_down(_
))
429 .WillOnce(CompleteContext(m_mock_threads
->work_queue
, r
));
430 expect_destroy(mock_instances
);
433 void expect_acquire_notify(MockManagedLock
&mock_managed_lock
,
434 MockListener
&mock_listener
, int r
) {
435 expect_is_leader(mock_managed_lock
, true, false);
436 EXPECT_CALL(mock_listener
, post_acquire_handler(_
))
437 .WillOnce(CompleteContext(r
));
438 expect_is_leader(mock_managed_lock
, true, false);
441 void expect_release_notify(MockManagedLock
&mock_managed_lock
,
442 MockListener
&mock_listener
, int r
) {
443 expect_is_leader(mock_managed_lock
, false, false);
444 EXPECT_CALL(mock_listener
, pre_release_handler(_
))
445 .WillOnce(CompleteContext(r
));
446 expect_is_leader(mock_managed_lock
, false, false);
449 MockThreads
*m_mock_threads
;
452 TEST_F(TestMockLeaderWatcher
, InitShutdown
) {
453 MockManagedLock mock_managed_lock
;
454 MockMirrorStatusWatcher mock_mirror_status_watcher
;
455 MockInstances mock_instances
;
456 MockListener listener
;
458 expect_is_shutdown(mock_managed_lock
);
459 expect_destroy(mock_managed_lock
);
463 expect_construct(mock_managed_lock
);
464 MockLeaderWatcher
leader_watcher(m_mock_threads
, m_local_io_ctx
, &listener
);
467 C_SaferCond on_heartbeat_finish
;
468 expect_is_leader(mock_managed_lock
, false, false);
469 expect_try_acquire_lock(mock_managed_lock
, 0);
470 expect_init(mock_mirror_status_watcher
, 0);
471 expect_init(mock_instances
, 0);
472 expect_acquire_notify(mock_managed_lock
, listener
, 0);
473 expect_notify_heartbeat(mock_managed_lock
, &on_heartbeat_finish
);
475 ASSERT_EQ(0, leader_watcher
.init());
476 ASSERT_EQ(0, on_heartbeat_finish
.wait());
479 expect_release_notify(mock_managed_lock
, listener
, 0);
480 expect_shut_down(mock_instances
, 0);
481 expect_shut_down(mock_mirror_status_watcher
, 0);
482 expect_is_leader(mock_managed_lock
, false, false);
483 expect_release_lock(mock_managed_lock
, 0);
484 expect_shut_down(mock_managed_lock
, true, 0);
485 expect_is_leader(mock_managed_lock
, false, false);
487 leader_watcher
.shut_down();
490 TEST_F(TestMockLeaderWatcher
, InitReleaseShutdown
) {
491 MockManagedLock mock_managed_lock
;
492 MockMirrorStatusWatcher mock_mirror_status_watcher
;
493 MockInstances mock_instances
;
494 MockListener listener
;
496 expect_is_shutdown(mock_managed_lock
);
497 expect_destroy(mock_managed_lock
);
501 expect_construct(mock_managed_lock
);
502 MockLeaderWatcher
leader_watcher(m_mock_threads
, m_local_io_ctx
, &listener
);
505 C_SaferCond on_heartbeat_finish
;
506 expect_is_leader(mock_managed_lock
, false, false);
507 expect_try_acquire_lock(mock_managed_lock
, 0);
508 expect_init(mock_mirror_status_watcher
, 0);
509 expect_init(mock_instances
, 0);
510 expect_acquire_notify(mock_managed_lock
, listener
, 0);
511 expect_notify_heartbeat(mock_managed_lock
, &on_heartbeat_finish
);
513 ASSERT_EQ(0, leader_watcher
.init());
514 ASSERT_EQ(0, on_heartbeat_finish
.wait());
517 expect_is_leader(mock_managed_lock
, false, true);
518 expect_release_notify(mock_managed_lock
, listener
, 0);
519 expect_shut_down(mock_instances
, 0);
520 expect_shut_down(mock_mirror_status_watcher
, 0);
521 expect_is_leader(mock_managed_lock
, false, false);
522 C_SaferCond on_release
;
523 expect_release_lock(mock_managed_lock
, 0, &on_release
);
525 leader_watcher
.release_leader();
526 ASSERT_EQ(0, on_release
.wait());
529 expect_shut_down(mock_managed_lock
, false, 0);
530 expect_is_leader(mock_managed_lock
, false, false);
532 leader_watcher
.shut_down();
535 TEST_F(TestMockLeaderWatcher
, AcquireError
) {
536 MockManagedLock mock_managed_lock
;
537 MockMirrorStatusWatcher mock_mirror_status_watcher
;
538 MockInstances mock_instances
;
539 MockListener listener
;
541 expect_is_shutdown(mock_managed_lock
);
542 expect_is_leader(mock_managed_lock
);
543 expect_destroy(mock_managed_lock
);
547 expect_construct(mock_managed_lock
);
548 MockLeaderWatcher
leader_watcher(m_mock_threads
, m_local_io_ctx
, &listener
);
551 C_SaferCond on_heartbeat_finish
;
552 expect_is_leader(mock_managed_lock
, false, false);
553 expect_try_acquire_lock(mock_managed_lock
, -EAGAIN
);
554 expect_get_locker(mock_managed_lock
, librbd::managed_lock::Locker(), -ENOENT
);
555 expect_try_acquire_lock(mock_managed_lock
, 0);
556 expect_init(mock_mirror_status_watcher
, 0);
557 expect_init(mock_instances
, 0);
558 expect_acquire_notify(mock_managed_lock
, listener
, 0);
559 expect_notify_heartbeat(mock_managed_lock
, &on_heartbeat_finish
);
561 ASSERT_EQ(0, leader_watcher
.init());
562 ASSERT_EQ(0, on_heartbeat_finish
.wait());
565 expect_release_notify(mock_managed_lock
, listener
, 0);
566 expect_shut_down(mock_instances
, 0);
567 expect_shut_down(mock_mirror_status_watcher
, 0);
568 expect_is_leader(mock_managed_lock
, false, false);
569 expect_release_lock(mock_managed_lock
, 0);
570 expect_shut_down(mock_managed_lock
, true, 0);
571 expect_is_leader(mock_managed_lock
, false, false);
573 leader_watcher
.shut_down();
576 TEST_F(TestMockLeaderWatcher
, ReleaseError
) {
577 MockManagedLock mock_managed_lock
;
578 MockMirrorStatusWatcher mock_mirror_status_watcher
;
579 MockInstances mock_instances
;
580 MockListener listener
;
582 expect_is_shutdown(mock_managed_lock
);
583 expect_destroy(mock_managed_lock
);
587 expect_construct(mock_managed_lock
);
588 MockLeaderWatcher
leader_watcher(m_mock_threads
, m_local_io_ctx
, &listener
);
591 C_SaferCond on_heartbeat_finish
;
592 expect_is_leader(mock_managed_lock
, false, false);
593 expect_try_acquire_lock(mock_managed_lock
, 0);
594 expect_init(mock_mirror_status_watcher
, 0);
595 expect_init(mock_instances
, 0);
596 expect_acquire_notify(mock_managed_lock
, listener
, 0);
597 expect_notify_heartbeat(mock_managed_lock
, &on_heartbeat_finish
);
599 ASSERT_EQ(0, leader_watcher
.init());
600 ASSERT_EQ(0, on_heartbeat_finish
.wait());
603 expect_is_leader(mock_managed_lock
, false, true);
604 expect_release_notify(mock_managed_lock
, listener
, -EINVAL
);
605 expect_shut_down(mock_instances
, 0);
606 expect_shut_down(mock_mirror_status_watcher
, -EINVAL
);
607 expect_is_leader(mock_managed_lock
, false, false);
608 C_SaferCond on_release
;
609 expect_release_lock(mock_managed_lock
, -EINVAL
, &on_release
);
611 leader_watcher
.release_leader();
612 ASSERT_EQ(0, on_release
.wait());
615 expect_shut_down(mock_managed_lock
, false, 0);
616 expect_is_leader(mock_managed_lock
, false, false);
618 leader_watcher
.shut_down();
621 TEST_F(TestMockLeaderWatcher
, Break
) {
622 EXPECT_EQ(0, _rados
->conf_set("rbd_mirror_leader_heartbeat_interval", "1"));
623 EXPECT_EQ(0, _rados
->conf_set("rbd_mirror_leader_max_missed_heartbeats",
625 CephContext
*cct
= reinterpret_cast<CephContext
*>(m_local_io_ctx
.cct());
626 int max_acquire_attempts
=
627 cct
->_conf
->rbd_mirror_leader_max_acquire_attempts_before_break
;
629 MockManagedLock mock_managed_lock
;
630 MockMirrorStatusWatcher mock_mirror_status_watcher
;
631 MockInstances mock_instances
;
632 MockListener listener
;
633 librbd::managed_lock::Locker
634 locker
{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
636 expect_is_shutdown(mock_managed_lock
);
637 expect_is_leader(mock_managed_lock
);
638 expect_destroy(mock_managed_lock
);
639 EXPECT_CALL(listener
, update_leader_handler(_
));
643 expect_construct(mock_managed_lock
);
644 MockLeaderWatcher
leader_watcher(m_mock_threads
, m_local_io_ctx
, &listener
);
647 expect_is_leader(mock_managed_lock
, false, false);
648 for (int i
= 0; i
< max_acquire_attempts
; i
++) {
649 expect_try_acquire_lock(mock_managed_lock
, -EAGAIN
);
650 expect_get_locker(mock_managed_lock
, locker
, 0);
652 C_SaferCond on_break
;
653 expect_break_lock(mock_managed_lock
, locker
, 0, &on_break
);
654 C_SaferCond on_heartbeat_finish
;
655 expect_try_acquire_lock(mock_managed_lock
, 0);
656 expect_init(mock_mirror_status_watcher
, 0);
657 expect_init(mock_instances
, 0);
658 expect_acquire_notify(mock_managed_lock
, listener
, 0);
659 expect_notify_heartbeat(mock_managed_lock
, &on_heartbeat_finish
);
661 ASSERT_EQ(0, leader_watcher
.init());
662 ASSERT_EQ(0, on_heartbeat_finish
.wait());
665 expect_release_notify(mock_managed_lock
, listener
, 0);
666 expect_shut_down(mock_instances
, 0);
667 expect_shut_down(mock_mirror_status_watcher
, 0);
668 expect_is_leader(mock_managed_lock
, false, false);
669 expect_release_lock(mock_managed_lock
, 0);
670 expect_shut_down(mock_managed_lock
, true, 0);
671 expect_is_leader(mock_managed_lock
, false, false);
673 leader_watcher
.shut_down();
676 } // namespace mirror