]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | |
2 | """ | |
3 | Exercise the MDS's auto repair functions | |
4 | """ | |
5 | ||
6 | import logging | |
7 | import time | |
8 | ||
9 | from teuthology.orchestra.run import CommandFailedError | |
10 | from tasks.cephfs.cephfs_test_case import CephFSTestCase | |
11 | ||
12 | ||
13 | log = logging.getLogger(__name__) | |
14 | ||
15 | ||
16 | # Arbitrary timeouts for operations involving restarting | |
17 | # an MDS or waiting for it to come up | |
18 | MDS_RESTART_GRACE = 60 | |
19 | ||
20 | ||
21 | class TestMDSAutoRepair(CephFSTestCase): | |
22 | def test_backtrace_repair(self): | |
23 | """ | |
24 | MDS should verify/fix backtrace on fetch dirfrag | |
25 | """ | |
26 | ||
27 | self.mount_a.run_shell(["mkdir", "testdir1"]) | |
28 | self.mount_a.run_shell(["touch", "testdir1/testfile"]) | |
29 | dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino("testdir1")) | |
30 | ||
31 | # drop inodes caps | |
32 | self.mount_a.umount_wait() | |
33 | ||
34 | # flush journal entries to dirfrag objects, and expire journal | |
35 | self.fs.mds_asok(['flush', 'journal']) | |
36 | ||
37 | # Restart the MDS to drop the metadata cache (because we expired the journal, | |
38 | # nothing gets replayed into cache on restart) | |
39 | self.fs.mds_stop() | |
40 | self.fs.mds_fail_restart() | |
41 | self.fs.wait_for_daemons() | |
42 | ||
43 | # remove testdir1's backtrace | |
44 | self.fs.rados(["rmxattr", dir_objname, "parent"]) | |
45 | ||
46 | # readdir (fetch dirfrag) should fix testdir1's backtrace | |
47 | self.mount_a.mount() | |
48 | self.mount_a.wait_until_mounted() | |
49 | self.mount_a.run_shell(["ls", "testdir1"]) | |
50 | ||
51 | # flush journal entries to dirfrag objects | |
52 | self.fs.mds_asok(['flush', 'journal']) | |
53 | ||
54 | # check if backtrace exists | |
55 | self.fs.rados(["getxattr", dir_objname, "parent"]) | |
56 | ||
57 | def test_mds_readonly(self): | |
58 | """ | |
59 | test if MDS behave correct when it's readonly | |
60 | """ | |
61 | # operation should successd when MDS is not readonly | |
62 | self.mount_a.run_shell(["touch", "test_file1"]) | |
63 | writer = self.mount_a.write_background(loop=True) | |
64 | ||
65 | time.sleep(10) | |
66 | self.assertFalse(writer.finished) | |
67 | ||
68 | # force MDS to read-only mode | |
69 | self.fs.mds_asok(['force_readonly']) | |
70 | time.sleep(10) | |
71 | ||
72 | # touching test file should fail | |
73 | try: | |
74 | self.mount_a.run_shell(["touch", "test_file1"]) | |
75 | except CommandFailedError: | |
76 | pass | |
77 | else: | |
78 | self.assertTrue(False) | |
79 | ||
80 | # background writer also should fail | |
81 | self.assertTrue(writer.finished) | |
82 | ||
83 | # The MDS should report its readonly health state to the mon | |
224ce89b | 84 | self.wait_for_health("MDS_READ_ONLY", timeout=30) |
7c673cae FG |
85 | |
86 | # restart mds to make it writable | |
87 | self.fs.mds_fail_restart() | |
88 | self.fs.wait_for_daemons() | |
89 | ||
90 | self.wait_for_health_clear(timeout=30) |