]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/cephfs/test_failover.py
7147807bf521cc4e770d5eae1c5affdbae601391
5 from random
import randint
7 from tasks
.cephfs
.cephfs_test_case
import CephFSTestCase
8 from teuthology
.exceptions
import CommandFailedError
9 from tasks
.cephfs
.fuse_mount
import FuseMount
11 log
= logging
.getLogger(__name__
)
13 class TestClusterAffinity(CephFSTestCase
):
17 def _verify_join_fs(self
, target
, status
=None):
19 status
= self
.fs
.wait_for_daemons(timeout
=30)
20 log
.debug("%s", status
)
21 target
= sorted(target
, key
=operator
.itemgetter('name'))
22 log
.info("target = %s", target
)
23 current
= list(status
.get_all())
24 current
= sorted(current
, key
=operator
.itemgetter('name'))
25 log
.info("current = %s", current
)
26 self
.assertEqual(len(current
), len(target
))
27 for i
in range(len(current
)):
28 for attr
in target
[i
]:
29 self
.assertIn(attr
, current
[i
])
30 self
.assertEqual(target
[i
][attr
], current
[i
][attr
])
32 def _change_target_state(self
, state
, name
, changes
):
34 if entity
['name'] == name
:
35 for k
, v
in changes
.items():
38 self
.fail("no entity")
40 def _verify_init(self
):
41 status
= self
.fs
.status()
42 log
.info("status = {0}".format(status
))
43 target
= [{'join_fscid': -1, 'name': info
['name']} for info
in status
.get_all()]
44 self
._verify
_join
_fs
(target
, status
=status
)
45 return (status
, target
)
47 def _reach_target(self
, target
):
50 self
._verify
_join
_fs
(target
)
52 except AssertionError as e
:
55 self
.wait_until_true(takeover
, 30)
57 def test_join_fs_runtime(self
):
59 That setting mds_join_fs at runtime affects the cluster layout.
61 status
, target
= self
._verify
_init
()
62 standbys
= list(status
.get_standbys())
63 self
.config_set('mds.'+standbys
[0]['name'], 'mds_join_fs', 'cephfs')
64 self
._change
_target
_state
(target
, standbys
[0]['name'], {'join_fscid': self
.fs
.id, 'state': 'up:active'})
65 self
._reach
_target
(target
)
67 def test_join_fs_unset(self
):
69 That unsetting mds_join_fs will cause failover if another high-affinity standby exists.
71 status
, target
= self
._verify
_init
()
72 standbys
= list(status
.get_standbys())
73 names
= (standbys
[0]['name'], standbys
[1]['name'])
74 self
.config_set('mds.'+names
[0], 'mds_join_fs', 'cephfs')
75 self
.config_set('mds.'+names
[1], 'mds_join_fs', 'cephfs')
76 self
._change
_target
_state
(target
, names
[0], {'join_fscid': self
.fs
.id})
77 self
._change
_target
_state
(target
, names
[1], {'join_fscid': self
.fs
.id})
78 self
._reach
_target
(target
)
79 status
= self
.fs
.status()
80 active
= self
.fs
.get_active_names(status
=status
)[0]
81 self
.assertIn(active
, names
)
82 self
.config_rm('mds.'+active
, 'mds_join_fs')
83 self
._change
_target
_state
(target
, active
, {'join_fscid': -1})
84 new_active
= (set(names
) - set((active
,))).pop()
85 self
._change
_target
_state
(target
, new_active
, {'state': 'up:active'})
86 self
._reach
_target
(target
)
88 def test_join_fs_drop(self
):
90 That unsetting mds_join_fs will not cause failover if no high-affinity standby exists.
92 status
, target
= self
._verify
_init
()
93 standbys
= list(status
.get_standbys())
94 active
= standbys
[0]['name']
95 self
.config_set('mds.'+active
, 'mds_join_fs', 'cephfs')
96 self
._change
_target
_state
(target
, active
, {'join_fscid': self
.fs
.id, 'state': 'up:active'})
97 self
._reach
_target
(target
)
98 self
.config_rm('mds.'+active
, 'mds_join_fs')
99 self
._change
_target
_state
(target
, active
, {'join_fscid': -1})
100 self
._reach
_target
(target
)
102 def test_join_fs_vanilla(self
):
104 That a vanilla standby is preferred over others with mds_join_fs set to another fs.
106 fs2
= self
.mds_cluster
.newfs(name
="cephfs2")
107 status
, target
= self
._verify
_init
()
108 active
= self
.fs
.get_active_names(status
=status
)[0]
109 standbys
= [info
['name'] for info
in status
.get_standbys()]
110 victim
= standbys
.pop()
111 # Set a bogus fs on the others
113 self
.config_set('mds.'+mds
, 'mds_join_fs', 'cephfs2')
114 self
._change
_target
_state
(target
, mds
, {'join_fscid': fs2
.id})
116 self
._change
_target
_state
(target
, victim
, {'state': 'up:active'})
117 self
._reach
_target
(target
)
118 status
= self
.fs
.status()
119 active
= self
.fs
.get_active_names(status
=status
)[0]
120 self
.assertEqual(active
, victim
)
122 def test_join_fs_last_resort(self
):
124 That a standby with mds_join_fs set to another fs is still used if necessary.
126 status
, target
= self
._verify
_init
()
127 standbys
= [info
['name'] for info
in status
.get_standbys()]
129 self
.config_set('mds.'+mds
, 'mds_join_fs', 'cephfs2')
130 fs2
= self
.mds_cluster
.newfs(name
="cephfs2")
132 self
._change
_target
_state
(target
, mds
, {'join_fscid': fs2
.id})
134 status
= self
.fs
.status()
135 ranks
= list(self
.fs
.get_ranks(status
=status
))
136 self
.assertEqual(len(ranks
), 1)
137 self
.assertIn(ranks
[0]['name'], standbys
)
138 # Note that we would expect the former active to reclaim its spot, but
139 # we're not testing that here.
141 def test_join_fs_steady(self
):
143 That a sole MDS with mds_join_fs set will come back as active eventually even after failover.
145 status
, target
= self
._verify
_init
()
146 active
= self
.fs
.get_active_names(status
=status
)[0]
147 self
.config_set('mds.'+active
, 'mds_join_fs', 'cephfs')
148 self
._change
_target
_state
(target
, active
, {'join_fscid': self
.fs
.id})
149 self
._reach
_target
(target
)
151 self
._reach
_target
(target
)
153 def test_join_fs_standby_replay(self
):
155 That a standby-replay daemon with weak affinity is replaced by a stronger one.
157 status
, target
= self
._verify
_init
()
158 standbys
= [info
['name'] for info
in status
.get_standbys()]
159 self
.config_set('mds.'+standbys
[0], 'mds_join_fs', 'cephfs')
160 self
._change
_target
_state
(target
, standbys
[0], {'join_fscid': self
.fs
.id, 'state': 'up:active'})
161 self
._reach
_target
(target
)
162 self
.fs
.set_allow_standby_replay(True)
163 status
= self
.fs
.status()
164 standbys
= [info
['name'] for info
in status
.get_standbys()]
165 self
.config_set('mds.'+standbys
[0], 'mds_join_fs', 'cephfs')
166 self
._change
_target
_state
(target
, standbys
[0], {'join_fscid': self
.fs
.id, 'state': 'up:standby-replay'})
167 self
._reach
_target
(target
)
169 class TestClusterResize(CephFSTestCase
):
175 That the MDS cluster grows after increasing max_mds.
178 # Need all my standbys up as well as the active daemons
179 # self.wait_for_daemon_start() necessary?
185 def test_shrink(self
):
187 That the MDS cluster shrinks automatically after decreasing max_mds.
193 def test_up_less_than_max(self
):
195 That a health warning is generated when max_mds is greater than active count.
198 status
= self
.fs
.status()
199 mdss
= [info
['gid'] for info
in status
.get_all()]
200 self
.fs
.set_max_mds(len(mdss
)+1)
201 self
.wait_for_health("MDS_UP_LESS_THAN_MAX", 30)
203 self
.wait_for_health_clear(30)
205 def test_down_health(self
):
207 That marking a FS down does not generate a health warning
212 self
.wait_for_health("", 30)
213 raise RuntimeError("got health warning?")
214 except RuntimeError as e
:
215 if "Timed out after" in str(e
):
220 def test_down_twice(self
):
222 That marking a FS down twice does not wipe old_max_mds.
227 self
.fs
.wait_for_daemons()
228 self
.fs
.set_down(False)
229 self
.assertEqual(self
.fs
.get_var("max_mds"), 2)
230 self
.fs
.wait_for_daemons(timeout
=60)
232 def test_down_grow(self
):
234 That setting max_mds undoes down.
238 self
.fs
.wait_for_daemons()
240 self
.fs
.wait_for_daemons()
244 That down setting toggles and sets max_mds appropriately.
248 self
.fs
.wait_for_daemons()
249 self
.assertEqual(self
.fs
.get_var("max_mds"), 0)
250 self
.fs
.set_down(False)
251 self
.assertEqual(self
.fs
.get_var("max_mds"), 1)
252 self
.fs
.wait_for_daemons()
253 self
.assertEqual(self
.fs
.get_var("max_mds"), 1)
257 Test that a hole cannot be created in the FS ranks.
264 # Now add a delay which should slow down how quickly rank 1 stops
265 self
.config_set('mds', 'ms_inject_delay_max', '5.0')
266 self
.config_set('mds', 'ms_inject_delay_probability', '1.0')
267 self
.fs
.set_max_mds(1)
268 log
.info("status = {0}".format(self
.fs
.status()))
270 # Don't wait for rank 1 to stop
271 self
.fs
.set_max_mds(3)
272 log
.info("status = {0}".format(self
.fs
.status()))
274 # Now check that the mons didn't try to promote a standby to rank 2
275 self
.fs
.set_max_mds(2)
276 status
= self
.fs
.status()
278 status
= self
.fs
.wait_for_daemons(timeout
=90)
279 ranks
= set([info
['rank'] for info
in status
.get_ranks(fscid
)])
280 self
.assertEqual(ranks
, set([0, 1]))
282 log
.info("status = {0}".format(status
))
284 def test_thrash(self
):
286 Test that thrashing max_mds does not fail.
290 for i
in range(0, 100):
291 self
.fs
.set_max_mds(max_mds
)
292 max_mds
= (max_mds
+1)%3+1
294 self
.fs
.wait_for_daemons(timeout
=90)
296 class TestFailover(CephFSTestCase
):
300 def test_simple(self
):
302 That when the active MDS is killed, a standby MDS is promoted into
303 its rank after the grace period.
305 This is just a simple unit test, the harder cases are covered
309 (original_active
, ) = self
.fs
.get_active_names()
310 original_standbys
= self
.mds_cluster
.get_standby_daemons()
312 # Kill the rank 0 daemon's physical process
313 self
.fs
.mds_stop(original_active
)
315 # Wait until the monitor promotes his replacement
317 ranks
= list(self
.fs
.get_ranks())
318 return len(ranks
) > 0 and ranks
[0]['name'] in original_standbys
320 log
.info("Waiting for promotion of one of the original standbys {0}".format(
322 self
.wait_until_true(promoted
, timeout
=self
.fs
.beacon_timeout
)
324 # Start the original rank 0 daemon up again, see that he becomes a standby
325 self
.fs
.mds_restart(original_active
)
326 self
.wait_until_true(
327 lambda: original_active
in self
.mds_cluster
.get_standby_daemons(),
328 timeout
=60 # Approximately long enough for MDS to start and mon to notice
331 def test_client_abort(self
):
333 That a client will respect fuse_require_active_mds and error out
334 when the cluster appears to be unavailable.
337 if not isinstance(self
.mount_a
, FuseMount
):
338 self
.skipTest("Requires FUSE client to inject client metadata")
340 require_active
= self
.fs
.get_config("fuse_require_active_mds", service_type
="mon").lower() == "true"
341 if not require_active
:
342 self
.skipTest("fuse_require_active_mds is not set")
344 # Check it's not laggy to begin with
345 (original_active
, ) = self
.fs
.get_active_names()
346 self
.assertNotIn("laggy_since", self
.fs
.status().get_mds(original_active
))
348 self
.mounts
[0].umount_wait()
350 # Control: that we can mount and unmount usually, while the cluster is healthy
351 self
.mounts
[0].mount_wait()
352 self
.mounts
[0].umount_wait()
354 # Stop the daemon processes
357 # Wait for everyone to go laggy
359 mdsmap
= self
.fs
.get_mds_map()
360 for info
in mdsmap
['info'].values():
361 if "laggy_since" not in info
:
366 self
.wait_until_true(laggy
, self
.fs
.beacon_timeout
)
367 with self
.assertRaises(CommandFailedError
):
368 self
.mounts
[0].mount_wait()
370 def test_standby_count_wanted(self
):
372 That cluster health warnings are generated by insufficient standbys available.
375 # Need all my standbys up as well as the active daemons
376 self
.wait_for_daemon_start()
378 standbys
= self
.mds_cluster
.get_standby_daemons()
379 self
.assertGreaterEqual(len(standbys
), 1)
380 self
.fs
.mon_manager
.raw_cluster_cmd('fs', 'set', self
.fs
.name
, 'standby_count_wanted', str(len(standbys
)))
382 # Kill a standby and check for warning
383 victim
= standbys
.pop()
384 self
.fs
.mds_stop(victim
)
385 self
.wait_for_health("MDS_INSUFFICIENT_STANDBY", self
.fs
.beacon_timeout
)
387 # restart the standby, see that he becomes a standby, check health clears
388 self
.fs
.mds_restart(victim
)
389 self
.wait_until_true(
390 lambda: victim
in self
.mds_cluster
.get_standby_daemons(),
391 timeout
=60 # Approximately long enough for MDS to start and mon to notice
393 self
.wait_for_health_clear(timeout
=30)
395 # Set it one greater than standbys ever seen
396 standbys
= self
.mds_cluster
.get_standby_daemons()
397 self
.assertGreaterEqual(len(standbys
), 1)
398 self
.fs
.mon_manager
.raw_cluster_cmd('fs', 'set', self
.fs
.name
, 'standby_count_wanted', str(len(standbys
)+1))
399 self
.wait_for_health("MDS_INSUFFICIENT_STANDBY", self
.fs
.beacon_timeout
)
402 self
.fs
.mon_manager
.raw_cluster_cmd('fs', 'set', self
.fs
.name
, 'standby_count_wanted', '0')
403 self
.wait_for_health_clear(timeout
=30)
405 def test_discontinuous_mdsmap(self
):
407 That discontinuous mdsmap does not affect failover.
408 See http://tracker.ceph.com/issues/24856.
410 self
.fs
.set_max_mds(2)
411 status
= self
.fs
.wait_for_daemons()
413 self
.mount_a
.umount_wait()
415 monc_timeout
= float(self
.fs
.get_config("mon_client_ping_timeout", service_type
="mds"))
417 mds_0
= self
.fs
.get_rank(rank
=0, status
=status
)
418 self
.fs
.rank_freeze(True, rank
=0) # prevent failover
419 self
.fs
.rank_signal(signal
.SIGSTOP
, rank
=0, status
=status
)
420 self
.wait_until_true(
421 lambda: "laggy_since" in self
.fs
.get_rank(),
422 timeout
=self
.fs
.beacon_timeout
425 self
.fs
.rank_fail(rank
=1)
426 self
.fs
.wait_for_state('up:resolve', rank
=1, timeout
=30)
428 # Make sure of mds_0's monitor connection gets reset
429 time
.sleep(monc_timeout
* 2)
431 # Continue rank 0, it will get discontinuous mdsmap
432 self
.fs
.rank_signal(signal
.SIGCONT
, rank
=0)
433 self
.wait_until_true(
434 lambda: "laggy_since" not in self
.fs
.get_rank(rank
=0),
435 timeout
=self
.fs
.beacon_timeout
438 # mds.b will be stuck at 'reconnect' state if snapserver gets confused
439 # by discontinuous mdsmap
440 self
.fs
.wait_for_state('up:active', rank
=1, timeout
=30)
441 self
.assertEqual(mds_0
['gid'], self
.fs
.get_rank(rank
=0)['gid'])
442 self
.fs
.rank_freeze(False, rank
=0)
444 def test_connect_bootstrapping(self
):
445 self
.config_set("mds", "mds_sleep_rank_change", 10000000.0)
446 self
.config_set("mds", "mds_connect_bootstrapping", True)
447 self
.fs
.set_max_mds(2)
448 self
.fs
.wait_for_daemons()
449 self
.fs
.rank_fail(rank
=0)
450 # rank 0 will get stuck in up:resolve, see https://tracker.ceph.com/issues/53194
451 self
.fs
.wait_for_daemons()
454 class TestStandbyReplay(CephFSTestCase
):
458 def _confirm_no_replay(self
):
459 status
= self
.fs
.status()
460 _
= len(list(status
.get_standbys()))
461 self
.assertEqual(0, len(list(self
.fs
.get_replays(status
=status
))))
464 def _confirm_single_replay(self
, full
=True, status
=None, retries
=3):
465 status
= self
.fs
.wait_for_daemons(status
=status
)
466 ranks
= sorted(self
.fs
.get_mds_map(status
=status
)['in'])
467 replays
= list(self
.fs
.get_replays(status
=status
))
468 checked_replays
= set()
471 for replay
in replays
:
472 if replay
['rank'] == rank
:
473 self
.assertFalse(has_replay
)
475 checked_replays
.add(replay
['gid'])
476 if full
and not has_replay
:
478 raise RuntimeError("rank "+str(rank
)+" has no standby-replay follower")
482 self
.assertEqual(checked_replays
, set(info
['gid'] for info
in replays
))
485 def _check_replay_takeover(self
, status
, rank
=0):
486 replay
= self
.fs
.get_replay(rank
=rank
, status
=status
)
487 new_status
= self
.fs
.wait_for_daemons()
488 new_active
= self
.fs
.get_rank(rank
=rank
, status
=new_status
)
490 self
.assertEqual(replay
['gid'], new_active
['gid'])
492 # double check takeover came from a standby (or some new daemon via restart)
494 for info
in status
.get_standbys():
495 if info
['gid'] == new_active
['gid']:
499 for info
in status
.get_all():
500 self
.assertNotEqual(info
['gid'], new_active
['gid'])
503 def test_standby_replay_singleton(self
):
505 That only one MDS becomes standby-replay.
508 self
._confirm
_no
_replay
()
509 self
.fs
.set_allow_standby_replay(True)
511 self
._confirm
_single
_replay
()
513 def test_standby_replay_damaged(self
):
515 That a standby-replay daemon can cause the rank to go damaged correctly.
518 self
._confirm
_no
_replay
()
519 self
.config_set("mds", "mds_standby_replay_damaged", True)
520 self
.fs
.set_allow_standby_replay(True)
521 self
.wait_until_true(
522 lambda: len(self
.fs
.get_damaged()) > 0,
525 status
= self
.fs
.status()
526 self
.assertListEqual([], list(self
.fs
.get_ranks(status
=status
)))
527 self
.assertListEqual([0], self
.fs
.get_damaged(status
=status
))
529 def test_standby_replay_disable(self
):
531 That turning off allow_standby_replay fails all standby-replay daemons.
534 self
._confirm
_no
_replay
()
535 self
.fs
.set_allow_standby_replay(True)
537 self
._confirm
_single
_replay
()
538 self
.fs
.set_allow_standby_replay(False)
539 self
._confirm
_no
_replay
()
541 def test_standby_replay_singleton_fail(self
):
543 That failures don't violate singleton constraint.
546 self
._confirm
_no
_replay
()
547 self
.fs
.set_allow_standby_replay(True)
548 status
= self
._confirm
_single
_replay
()
551 time
.sleep(randint(1, 5))
552 self
.fs
.rank_restart(status
=status
)
553 status
= self
._check
_replay
_takeover
(status
)
554 status
= self
._confirm
_single
_replay
(status
=status
)
557 time
.sleep(randint(1, 5))
559 status
= self
._check
_replay
_takeover
(status
)
560 status
= self
._confirm
_single
_replay
(status
=status
)
562 def test_standby_replay_singleton_fail_multimds(self
):
564 That failures don't violate singleton constraint with multiple actives.
567 status
= self
._confirm
_no
_replay
()
568 new_max_mds
= randint(2, len(list(status
.get_standbys())))
569 self
.fs
.set_max_mds(new_max_mds
)
570 self
.fs
.wait_for_daemons() # wait for actives to come online!
571 self
.fs
.set_allow_standby_replay(True)
572 status
= self
._confirm
_single
_replay
(full
=False)
575 time
.sleep(randint(1, 5))
576 victim
= randint(0, new_max_mds
-1)
577 self
.fs
.rank_restart(rank
=victim
, status
=status
)
578 status
= self
._check
_replay
_takeover
(status
, rank
=victim
)
579 status
= self
._confirm
_single
_replay
(status
=status
, full
=False)
582 time
.sleep(randint(1, 5))
583 victim
= randint(0, new_max_mds
-1)
584 self
.fs
.rank_fail(rank
=victim
)
585 status
= self
._check
_replay
_takeover
(status
, rank
=victim
)
586 status
= self
._confirm
_single
_replay
(status
=status
, full
=False)
588 def test_standby_replay_failure(self
):
590 That the failure of a standby-replay daemon happens cleanly
591 and doesn't interrupt anything else.
594 status
= self
._confirm
_no
_replay
()
595 self
.fs
.set_max_mds(1)
596 self
.fs
.set_allow_standby_replay(True)
597 status
= self
._confirm
_single
_replay
()
600 time
.sleep(randint(1, 5))
601 victim
= self
.fs
.get_replay(status
=status
)
602 self
.fs
.mds_restart(mds_id
=victim
['name'])
603 status
= self
._confirm
_single
_replay
(status
=status
)
605 def test_rank_stopped(self
):
607 That when a rank is STOPPED, standby replays for
608 that rank get torn down
611 status
= self
._confirm
_no
_replay
()
612 standby_count
= len(list(status
.get_standbys()))
613 self
.fs
.set_max_mds(2)
614 self
.fs
.set_allow_standby_replay(True)
615 status
= self
._confirm
_single
_replay
()
617 self
.fs
.set_max_mds(1) # stop rank 1
619 status
= self
._confirm
_single
_replay
()
620 self
.assertTrue(standby_count
, len(list(status
.get_standbys())))
623 class TestMultiFilesystems(CephFSTestCase
):
627 # We'll create our own filesystems and start our own daemons
628 REQUIRE_FILESYSTEM
= False
631 super(TestMultiFilesystems
, self
).setUp()
632 self
.mds_cluster
.mon_manager
.raw_cluster_cmd("fs", "flag", "set",
633 "enable_multiple", "true",
634 "--yes-i-really-mean-it")
636 def _setup_two(self
):
637 fs_a
= self
.mds_cluster
.newfs(name
="alpha")
638 fs_b
= self
.mds_cluster
.newfs(name
="bravo")
640 self
.mds_cluster
.mds_restart()
642 # Wait for both filesystems to go healthy
643 fs_a
.wait_for_daemons()
644 fs_b
.wait_for_daemons()
646 # Reconfigure client auth caps
647 for mount
in self
.mounts
:
648 self
.mds_cluster
.mon_manager
.raw_cluster_cmd_result(
649 'auth', 'caps', "client.{0}".format(mount
.client_id
),
652 'osd', 'allow rw pool={0}, allow rw pool={1}'.format(
653 fs_a
.get_data_pool_name(), fs_b
.get_data_pool_name()))
657 def test_clients(self
):
658 fs_a
, fs_b
= self
._setup
_two
()
660 # Mount a client on fs_a
661 self
.mount_a
.mount_wait(cephfs_name
=fs_a
.name
)
662 self
.mount_a
.write_n_mb("pad.bin", 1)
663 self
.mount_a
.write_n_mb("test.bin", 2)
664 a_created_ino
= self
.mount_a
.path_to_ino("test.bin")
665 self
.mount_a
.create_files()
667 # Mount a client on fs_b
668 self
.mount_b
.mount_wait(cephfs_name
=fs_b
.name
)
669 self
.mount_b
.write_n_mb("test.bin", 1)
670 b_created_ino
= self
.mount_b
.path_to_ino("test.bin")
671 self
.mount_b
.create_files()
673 # Check that a non-default filesystem mount survives an MDS
674 # failover (i.e. that map subscription is continuous, not
675 # just the first time), reproduces #16022
676 old_fs_b_mds
= fs_b
.get_active_names()[0]
677 self
.mds_cluster
.mds_stop(old_fs_b_mds
)
678 self
.mds_cluster
.mds_fail(old_fs_b_mds
)
679 fs_b
.wait_for_daemons()
680 background
= self
.mount_b
.write_background()
681 # Raise exception if the write doesn't finish (i.e. if client
682 # has not kept up with MDS failure)
684 self
.wait_until_true(lambda: background
.finished
, timeout
=30)
686 # The mount is stuck, we'll have to force it to fail cleanly
687 background
.stdin
.close()
688 self
.mount_b
.umount_wait(force
=True)
691 self
.mount_a
.umount_wait()
692 self
.mount_b
.umount_wait()
694 # See that the client's files went into the correct pool
695 self
.assertTrue(fs_a
.data_objects_present(a_created_ino
, 1024 * 1024))
696 self
.assertTrue(fs_b
.data_objects_present(b_created_ino
, 1024 * 1024))
698 def test_standby(self
):
699 fs_a
, fs_b
= self
._setup
_two
()
701 # Assert that the remaining two MDS daemons are now standbys
702 a_daemons
= fs_a
.get_active_names()
703 b_daemons
= fs_b
.get_active_names()
704 self
.assertEqual(len(a_daemons
), 1)
705 self
.assertEqual(len(b_daemons
), 1)
706 original_a
= a_daemons
[0]
707 original_b
= b_daemons
[0]
708 expect_standby_daemons
= set(self
.mds_cluster
.mds_ids
) - (set(a_daemons
) |
set(b_daemons
))
710 # Need all my standbys up as well as the active daemons
711 self
.wait_for_daemon_start()
712 self
.assertEqual(expect_standby_daemons
, self
.mds_cluster
.get_standby_daemons())
714 # Kill fs_a's active MDS, see a standby take over
715 self
.mds_cluster
.mds_stop(original_a
)
716 self
.mds_cluster
.mon_manager
.raw_cluster_cmd("mds", "fail", original_a
)
717 self
.wait_until_equal(lambda: len(fs_a
.get_active_names()), 1, 30,
718 reject_fn
=lambda v
: v
> 1)
719 # Assert that it's a *different* daemon that has now appeared in the map for fs_a
720 self
.assertNotEqual(fs_a
.get_active_names()[0], original_a
)
722 # Kill fs_b's active MDS, see a standby take over
723 self
.mds_cluster
.mds_stop(original_b
)
724 self
.mds_cluster
.mon_manager
.raw_cluster_cmd("mds", "fail", original_b
)
725 self
.wait_until_equal(lambda: len(fs_b
.get_active_names()), 1, 30,
726 reject_fn
=lambda v
: v
> 1)
727 # Assert that it's a *different* daemon that has now appeared in the map for fs_a
728 self
.assertNotEqual(fs_b
.get_active_names()[0], original_b
)
730 # Both of the original active daemons should be gone, and all standbys used up
731 self
.assertEqual(self
.mds_cluster
.get_standby_daemons(), set())
733 # Restart the ones I killed, see them reappear as standbys
734 self
.mds_cluster
.mds_restart(original_a
)
735 self
.mds_cluster
.mds_restart(original_b
)
736 self
.wait_until_true(
737 lambda: {original_a
, original_b
} == self
.mds_cluster
.get_standby_daemons(),
741 def test_grow_shrink(self
):
743 fs_a
, fs_b
= self
._setup
_two
()
745 # Increase max_mds on fs_b, see a standby take up the role
747 self
.wait_until_equal(lambda: len(fs_b
.get_active_names()), 2, 30,
748 reject_fn
=lambda v
: v
> 2 or v
< 1)
750 # Increase max_mds on fs_a, see a standby take up the role
752 self
.wait_until_equal(lambda: len(fs_a
.get_active_names()), 2, 30,
753 reject_fn
=lambda v
: v
> 2 or v
< 1)
755 # Shrink fs_b back to 1, see a daemon go back to standby
757 self
.wait_until_equal(lambda: len(fs_b
.get_active_names()), 1, 30,
758 reject_fn
=lambda v
: v
> 2 or v
< 1)
760 # Grow fs_a up to 3, see the former fs_b daemon join it.
762 self
.wait_until_equal(lambda: len(fs_a
.get_active_names()), 3, 60,
763 reject_fn
=lambda v
: v
> 3 or v
< 2)