]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | |
2 | """ | |
3 | Exercise the MDS's auto repair functions | |
4 | """ | |
5 | ||
6 | import logging | |
7 | import time | |
8 | ||
9 | from teuthology.orchestra.run import CommandFailedError | |
10 | from tasks.cephfs.cephfs_test_case import CephFSTestCase | |
11 | ||
12 | ||
13 | log = logging.getLogger(__name__) | |
14 | ||
15 | ||
16 | # Arbitrary timeouts for operations involving restarting | |
17 | # an MDS or waiting for it to come up | |
18 | MDS_RESTART_GRACE = 60 | |
19 | ||
20 | ||
21 | class TestMDSAutoRepair(CephFSTestCase): | |
22 | def test_backtrace_repair(self): | |
23 | """ | |
24 | MDS should verify/fix backtrace on fetch dirfrag | |
25 | """ | |
26 | ||
27 | self.mount_a.run_shell(["mkdir", "testdir1"]) | |
28 | self.mount_a.run_shell(["touch", "testdir1/testfile"]) | |
29 | dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino("testdir1")) | |
30 | ||
31 | # drop inodes caps | |
32 | self.mount_a.umount_wait() | |
33 | ||
34 | # flush journal entries to dirfrag objects, and expire journal | |
35 | self.fs.mds_asok(['flush', 'journal']) | |
36 | ||
37 | # Restart the MDS to drop the metadata cache (because we expired the journal, | |
38 | # nothing gets replayed into cache on restart) | |
39 | self.fs.mds_stop() | |
40 | self.fs.mds_fail_restart() | |
41 | self.fs.wait_for_daemons() | |
42 | ||
43 | # remove testdir1's backtrace | |
44 | self.fs.rados(["rmxattr", dir_objname, "parent"]) | |
45 | ||
46 | # readdir (fetch dirfrag) should fix testdir1's backtrace | |
e306af50 | 47 | self.mount_a.mount_wait() |
7c673cae FG |
48 | self.mount_a.run_shell(["ls", "testdir1"]) |
49 | ||
50 | # flush journal entries to dirfrag objects | |
51 | self.fs.mds_asok(['flush', 'journal']) | |
52 | ||
53 | # check if backtrace exists | |
54 | self.fs.rados(["getxattr", dir_objname, "parent"]) | |
55 | ||
56 | def test_mds_readonly(self): | |
57 | """ | |
58 | test if MDS behave correct when it's readonly | |
59 | """ | |
60 | # operation should successd when MDS is not readonly | |
61 | self.mount_a.run_shell(["touch", "test_file1"]) | |
62 | writer = self.mount_a.write_background(loop=True) | |
63 | ||
64 | time.sleep(10) | |
65 | self.assertFalse(writer.finished) | |
66 | ||
67 | # force MDS to read-only mode | |
68 | self.fs.mds_asok(['force_readonly']) | |
69 | time.sleep(10) | |
70 | ||
71 | # touching test file should fail | |
72 | try: | |
73 | self.mount_a.run_shell(["touch", "test_file1"]) | |
74 | except CommandFailedError: | |
75 | pass | |
76 | else: | |
77 | self.assertTrue(False) | |
78 | ||
79 | # background writer also should fail | |
80 | self.assertTrue(writer.finished) | |
81 | ||
82 | # The MDS should report its readonly health state to the mon | |
224ce89b | 83 | self.wait_for_health("MDS_READ_ONLY", timeout=30) |
7c673cae FG |
84 | |
85 | # restart mds to make it writable | |
86 | self.fs.mds_fail_restart() | |
87 | self.fs.wait_for_daemons() | |
88 | ||
89 | self.wait_for_health_clear(timeout=30) |