ceph/qa/tasks/cephfs/test_auto_repair.py

   1
   2 """
   3 Exercise the MDS's auto repair functions
   4 """
   5
   6 import logging
   7 import time
   8
   9 from teuthology.orchestra.run import CommandFailedError
  10 from tasks.cephfs.cephfs_test_case import CephFSTestCase
  11
  12
  13 log = logging.getLogger(__name__)
  14
  15
  16 # Arbitrary timeouts for operations involving restarting
  17 # an MDS or waiting for it to come up
  18 MDS_RESTART_GRACE = 60
  19
  20
  21 class TestMDSAutoRepair(CephFSTestCase):
  22     def test_backtrace_repair(self):
  23         """
  24         MDS should verify/fix backtrace on fetch dirfrag
  25         """
  26
  27         self.mount_a.run_shell(["mkdir", "testdir1"])
  28         self.mount_a.run_shell(["touch", "testdir1/testfile"])
  29         dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino("testdir1"))
  30
  31         # drop inodes caps
  32         self.mount_a.umount_wait()
  33
  34         # flush journal entries to dirfrag objects, and expire journal
  35         self.fs.mds_asok(['flush', 'journal'])
  36
  37         # Restart the MDS to drop the metadata cache (because we expired the journal,
  38         # nothing gets replayed into cache on restart)
  39         self.fs.mds_stop()
  40         self.fs.mds_fail_restart()
  41         self.fs.wait_for_daemons()
  42
  43         # remove testdir1's backtrace
  44         self.fs.rados(["rmxattr", dir_objname, "parent"])
  45
  46         # readdir (fetch dirfrag) should fix testdir1's backtrace
  47         self.mount_a.mount_wait()
  48         self.mount_a.run_shell(["ls", "testdir1"])
  49
  50         # flush journal entries to dirfrag objects
  51         self.fs.mds_asok(['flush', 'journal'])
  52
  53         # check if backtrace exists
  54         self.fs.rados(["getxattr", dir_objname, "parent"])
  55
  56     def test_mds_readonly(self):
  57         """
  58         test if MDS behave correct when it's readonly
  59         """
  60         # operation should successd when MDS is not readonly
  61         self.mount_a.run_shell(["touch", "test_file1"])
  62         writer = self.mount_a.write_background(loop=True)
  63
  64         time.sleep(10)
  65         self.assertFalse(writer.finished)
  66
  67         # force MDS to read-only mode
  68         self.fs.mds_asok(['force_readonly'])
  69         time.sleep(10)
  70
  71         # touching test file should fail
  72         try:
  73             self.mount_a.run_shell(["touch", "test_file1"])
  74         except CommandFailedError:
  75             pass
  76         else:
  77             self.assertTrue(False)
  78
  79         # background writer also should fail
  80         self.assertTrue(writer.finished)
  81
  82         # The MDS should report its readonly health state to the mon
  83         self.wait_for_health("MDS_READ_ONLY", timeout=30)
  84
  85         # restart mds to make it writable
  86         self.fs.mds_fail_restart()
  87         self.fs.wait_for_daemons()
  88
  89         self.wait_for_health_clear(timeout=30)