ceph/qa/tasks/cephfs/test_auto_repair.py

   1
   2 """
   3 Exercise the MDS's auto repair functions
   4 """
   5
   6 import logging
   7 import time
   8
   9 from teuthology.orchestra.run import CommandFailedError
  10 from tasks.cephfs.cephfs_test_case import CephFSTestCase
  11
  12
  13 log = logging.getLogger(__name__)
  14
  15
  16 # Arbitrary timeouts for operations involving restarting
  17 # an MDS or waiting for it to come up
  18 MDS_RESTART_GRACE = 60
  19
  20
  21 class TestMDSAutoRepair(CephFSTestCase):
  22     def test_backtrace_repair(self):
  23         """
  24         MDS should verify/fix backtrace on fetch dirfrag
  25         """
  26
  27         self.mount_a.run_shell(["mkdir", "testdir1"])
  28         self.mount_a.run_shell(["touch", "testdir1/testfile"])
  29         dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino("testdir1"))
  30
  31         # drop inodes caps
  32         self.mount_a.umount_wait()
  33
  34         # flush journal entries to dirfrag objects, and expire journal
  35         self.fs.mds_asok(['flush', 'journal'])
  36
  37         # Restart the MDS to drop the metadata cache (because we expired the journal,
  38         # nothing gets replayed into cache on restart)
  39         self.fs.mds_stop()
  40         self.fs.mds_fail_restart()
  41         self.fs.wait_for_daemons()
  42
  43         # remove testdir1's backtrace
  44         self.fs.rados(["rmxattr", dir_objname, "parent"])
  45
  46         # readdir (fetch dirfrag) should fix testdir1's backtrace
  47         self.mount_a.mount()
  48         self.mount_a.wait_until_mounted()
  49         self.mount_a.run_shell(["ls", "testdir1"])
  50
  51         # flush journal entries to dirfrag objects
  52         self.fs.mds_asok(['flush', 'journal'])
  53
  54         # check if backtrace exists
  55         self.fs.rados(["getxattr", dir_objname, "parent"])
  56
  57     def test_mds_readonly(self):
  58         """
  59         test if MDS behave correct when it's readonly
  60         """
  61         # operation should successd when MDS is not readonly
  62         self.mount_a.run_shell(["touch", "test_file1"])
  63         writer = self.mount_a.write_background(loop=True)
  64
  65         time.sleep(10)
  66         self.assertFalse(writer.finished)
  67
  68         # force MDS to read-only mode
  69         self.fs.mds_asok(['force_readonly'])
  70         time.sleep(10)
  71
  72         # touching test file should fail
  73         try:
  74             self.mount_a.run_shell(["touch", "test_file1"])
  75         except CommandFailedError:
  76             pass
  77         else:
  78             self.assertTrue(False)
  79
  80         # background writer also should fail
  81         self.assertTrue(writer.finished)
  82
  83         # The MDS should report its readonly health state to the mon
  84         self.wait_for_health("MDS in read-only mode", timeout=30)
  85
  86         # restart mds to make it writable
  87         self.fs.mds_fail_restart()
  88         self.fs.wait_for_daemons()
  89
  90         self.wait_for_health_clear(timeout=30)