projects / ceph.git / blame
| Commit | Line | Data |
|---|---|---|
| 7c673cae FG |
1 | |
| 2 | """ | |
| 3 | Exercise the MDS's auto repair functions | |
| 4 | """ | |
| 5 | ||
| 6 | import logging | |
| 7 | import time | |
| 8 | ||
| 9 | from teuthology.orchestra.run import CommandFailedError | |
| 10 | from tasks.cephfs.cephfs_test_case import CephFSTestCase | |
| 11 | ||
| 12 | ||
| 13 | log = logging.getLogger(__name__) | |
| 14 | ||
| 15 | ||
| 16 | # Arbitrary timeouts for operations involving restarting | |
| 17 | # an MDS or waiting for it to come up | |
| 18 | MDS_RESTART_GRACE = 60 | |
| 19 | ||
| 20 | ||
| 21 | class TestMDSAutoRepair(CephFSTestCase): | |
| 22 | def test_backtrace_repair(self): | |
| 23 | """ | |
| 24 | MDS should verify/fix backtrace on fetch dirfrag | |
| 25 | """ | |
| 26 | ||
| 27 | self.mount_a.run_shell(["mkdir", "testdir1"]) | |
| 28 | self.mount_a.run_shell(["touch", "testdir1/testfile"]) | |
| 29 | dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino("testdir1")) | |
| 30 | ||
| 31 | # drop inodes caps | |
| 32 | self.mount_a.umount_wait() | |
| 33 | ||
| 34 | # flush journal entries to dirfrag objects, and expire journal | |
| 35 | self.fs.mds_asok(['flush', 'journal']) | |
| 36 | ||
| 37 | # Restart the MDS to drop the metadata cache (because we expired the journal, | |
| 38 | # nothing gets replayed into cache on restart) | |
| 39 | self.fs.mds_stop() | |
| 40 | self.fs.mds_fail_restart() | |
| 41 | self.fs.wait_for_daemons() | |
| 42 | ||
| 43 | # remove testdir1's backtrace | |
| 44 | self.fs.rados(["rmxattr", dir_objname, "parent"]) | |
| 45 | ||
| 46 | # readdir (fetch dirfrag) should fix testdir1's backtrace | |
| 47 | self.mount_a.mount() | |
| 48 | self.mount_a.wait_until_mounted() | |
| 49 | self.mount_a.run_shell(["ls", "testdir1"]) | |
| 50 | ||
| 51 | # flush journal entries to dirfrag objects | |
| 52 | self.fs.mds_asok(['flush', 'journal']) | |
| 53 | ||
| 54 | # check if backtrace exists | |
| 55 | self.fs.rados(["getxattr", dir_objname, "parent"]) | |
| 56 | ||
| 57 | def test_mds_readonly(self): | |
| 58 | """ | |
| 59 | test if MDS behave correct when it's readonly | |
| 60 | """ | |
| 61 | # operation should successd when MDS is not readonly | |
| 62 | self.mount_a.run_shell(["touch", "test_file1"]) | |
| 63 | writer = self.mount_a.write_background(loop=True) | |
| 64 | ||
| 65 | time.sleep(10) | |
| 66 | self.assertFalse(writer.finished) | |
| 67 | ||
| 68 | # force MDS to read-only mode | |
| 69 | self.fs.mds_asok(['force_readonly']) | |
| 70 | time.sleep(10) | |
| 71 | ||
| 72 | # touching test file should fail | |
| 73 | try: | |
| 74 | self.mount_a.run_shell(["touch", "test_file1"]) | |
| 75 | except CommandFailedError: | |
| 76 | pass | |
| 77 | else: | |
| 78 | self.assertTrue(False) | |
| 79 | ||
| 80 | # background writer also should fail | |
| 81 | self.assertTrue(writer.finished) | |
| 82 | ||
| 83 | # The MDS should report its readonly health state to the mon | |
| 224ce89b | 84 | self.wait_for_health("MDS_READ_ONLY", timeout=30) |
| 7c673cae FG |
85 | |
| 86 | # restart mds to make it writable | |
| 87 | self.fs.mds_fail_restart() | |
| 88 | self.fs.wait_for_daemons() | |
| 89 | ||
| 90 | self.wait_for_health_clear(timeout=30) |