projects / ceph.git / blame
| Commit | Line | Data |
|---|---|---|
| 7c673cae FG |
1 | |
| 2 | """ | |
| 3 | Exercise the MDS's auto repair functions | |
| 4 | """ | |
| 5 | ||
| 6 | import logging | |
| 7 | import time | |
| 8 | ||
| 20effc67 | 9 | from teuthology.exceptions import CommandFailedError |
| 7c673cae FG |
10 | from tasks.cephfs.cephfs_test_case import CephFSTestCase |
| 11 | ||
| 12 | ||
| 13 | log = logging.getLogger(__name__) | |
| 14 | ||
| 15 | ||
| 16 | # Arbitrary timeouts for operations involving restarting | |
| 17 | # an MDS or waiting for it to come up | |
| 18 | MDS_RESTART_GRACE = 60 | |
| 19 | ||
| 20 | ||
| 21 | class TestMDSAutoRepair(CephFSTestCase): | |
| 22 | def test_backtrace_repair(self): | |
| 23 | """ | |
| 24 | MDS should verify/fix backtrace on fetch dirfrag | |
| 25 | """ | |
| 26 | ||
| 27 | self.mount_a.run_shell(["mkdir", "testdir1"]) | |
| 28 | self.mount_a.run_shell(["touch", "testdir1/testfile"]) | |
| 29 | dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino("testdir1")) | |
| 30 | ||
| 31 | # drop inodes caps | |
| 32 | self.mount_a.umount_wait() | |
| 33 | ||
| 34 | # flush journal entries to dirfrag objects, and expire journal | |
| 35 | self.fs.mds_asok(['flush', 'journal']) | |
| 36 | ||
| 37 | # Restart the MDS to drop the metadata cache (because we expired the journal, | |
| 38 | # nothing gets replayed into cache on restart) | |
| f67539c2 | 39 | self.fs.rank_fail() |
| 7c673cae FG |
40 | self.fs.wait_for_daemons() |
| 41 | ||
| 42 | # remove testdir1's backtrace | |
| f67539c2 | 43 | self.fs.radosm(["rmxattr", dir_objname, "parent"]) |
| 7c673cae FG |
44 | |
| 45 | # readdir (fetch dirfrag) should fix testdir1's backtrace | |
| e306af50 | 46 | self.mount_a.mount_wait() |
| 7c673cae FG |
47 | self.mount_a.run_shell(["ls", "testdir1"]) |
| 48 | ||
| 49 | # flush journal entries to dirfrag objects | |
| 50 | self.fs.mds_asok(['flush', 'journal']) | |
| 51 | ||
| 52 | # check if backtrace exists | |
| f67539c2 | 53 | self.fs.radosm(["getxattr", dir_objname, "parent"]) |
| 7c673cae FG |
54 | |
| 55 | def test_mds_readonly(self): | |
| 56 | """ | |
| 57 | test if MDS behave correct when it's readonly | |
| 58 | """ | |
| 59 | # operation should successd when MDS is not readonly | |
| 60 | self.mount_a.run_shell(["touch", "test_file1"]) | |
| 61 | writer = self.mount_a.write_background(loop=True) | |
| 62 | ||
| 63 | time.sleep(10) | |
| 64 | self.assertFalse(writer.finished) | |
| 65 | ||
| 66 | # force MDS to read-only mode | |
| 67 | self.fs.mds_asok(['force_readonly']) | |
| 68 | time.sleep(10) | |
| 69 | ||
| 70 | # touching test file should fail | |
| 71 | try: | |
| 72 | self.mount_a.run_shell(["touch", "test_file1"]) | |
| 73 | except CommandFailedError: | |
| 74 | pass | |
| 75 | else: | |
| 76 | self.assertTrue(False) | |
| 77 | ||
| 78 | # background writer also should fail | |
| 79 | self.assertTrue(writer.finished) | |
| 80 | ||
| 81 | # The MDS should report its readonly health state to the mon | |
| 224ce89b | 82 | self.wait_for_health("MDS_READ_ONLY", timeout=30) |
| 7c673cae FG |
83 | |
| 84 | # restart mds to make it writable | |
| 85 | self.fs.mds_fail_restart() | |
| 86 | self.fs.wait_for_daemons() | |
| 87 | ||
| 88 | self.wait_for_health_clear(timeout=30) |