[ceph.git] / ceph / qa / tasks / cephfs / test_misc.py


from unittest import SkipTest
from tasks.cephfs.fuse_mount import FuseMount
from tasks.cephfs.cephfs_test_case import CephFSTestCase
from teuthology.orchestra.run import CommandFailedError, ConnectionLostError
import errno
import time
import json
import logging
import time

log = logging.getLogger(__name__)

class TestMisc(CephFSTestCase):
    CLIENTS_REQUIRED = 2

    def test_getattr_caps(self):
        """
        Check if MDS recognizes the 'mask' parameter of open request.
        The parameter allows client to request caps when opening file
        """

        if not isinstance(self.mount_a, FuseMount):
            raise SkipTest("Require FUSE client")

        # Enable debug. Client will requests CEPH_CAP_XATTR_SHARED
        # on lookup/open
        self.mount_b.umount_wait()
        self.set_conf('client', 'client debug getattr caps', 'true')
        self.mount_b.mount()
        self.mount_b.wait_until_mounted()

        # create a file and hold it open. MDS will issue CEPH_CAP_EXCL_*
        # to mount_a
        p = self.mount_a.open_background("testfile")
        self.mount_b.wait_for_visible("testfile")

        # this triggers a lookup request and an open request. The debug
        # code will check if lookup/open reply contains xattrs
        self.mount_b.run_shell(["cat", "testfile"])

        self.mount_a.kill_background(p)

    def test_root_rctime(self):
        """
        Check that the root inode has a non-default rctime on startup.
        """

        t = time.time()
        rctime = self.mount_a.getfattr(".", "ceph.dir.rctime")
        log.info("rctime = {}".format(rctime))
        self.assertGreaterEqual(rctime, t-10)

    def test_fs_new(self):
        self.mount_a.umount_wait()
        self.mount_b.umount_wait()

        data_pool_name = self.fs.get_data_pool_name()

        self.fs.mds_stop()
        self.fs.mds_fail()

        self.fs.mon_manager.raw_cluster_cmd('fs', 'rm', self.fs.name,
                                            '--yes-i-really-mean-it')

        self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'delete',
                                            self.fs.metadata_pool_name,
                                            self.fs.metadata_pool_name,
                                            '--yes-i-really-really-mean-it')
        self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'create',
                                            self.fs.metadata_pool_name,
                                            self.fs.get_pgs_per_fs_pool().__str__())

        dummyfile = '/etc/fstab'

        self.fs.put_metadata_object_raw("key", dummyfile)

        def get_pool_df(fs, name):
            try:
                return fs.get_pool_df(name)['objects'] > 0
            except RuntimeError as e:
                return False

        self.wait_until_true(lambda: get_pool_df(self.fs, self.fs.metadata_pool_name), timeout=30)

        try:
            self.fs.mon_manager.raw_cluster_cmd('fs', 'new', self.fs.name,
                                                self.fs.metadata_pool_name,
                                                data_pool_name)
        except CommandFailedError as e:
            self.assertEqual(e.exitstatus, errno.EINVAL)
        else:
            raise AssertionError("Expected EINVAL")

        self.fs.mon_manager.raw_cluster_cmd('fs', 'new', self.fs.name,
                                            self.fs.metadata_pool_name,
                                            data_pool_name, "--force")

        self.fs.mon_manager.raw_cluster_cmd('fs', 'rm', self.fs.name,
                                            '--yes-i-really-mean-it')


        self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'delete',
                                            self.fs.metadata_pool_name,
                                            self.fs.metadata_pool_name,
                                            '--yes-i-really-really-mean-it')
        self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'create',
                                            self.fs.metadata_pool_name,
                                            self.fs.get_pgs_per_fs_pool().__str__())
        self.fs.mon_manager.raw_cluster_cmd('fs', 'new', self.fs.name,
                                            self.fs.metadata_pool_name,
                                            data_pool_name)

    def test_cap_revoke_nonresponder(self):
        """
        Check that a client is evicted if it has not responded to cap revoke
        request for configured number of seconds.
        """
        session_timeout = self.fs.get_var("session_timeout")
        eviction_timeout = session_timeout / 2.0

        self.fs.mds_asok(['config', 'set', 'mds_cap_revoke_eviction_timeout',
                          str(eviction_timeout)])

        cap_holder = self.mount_a.open_background()

        # Wait for the file to be visible from another client, indicating
        # that mount_a has completed its network ops
        self.mount_b.wait_for_visible()

        # Simulate client death
        self.mount_a.kill()

        try:
            # The waiter should get stuck waiting for the capability
            # held on the MDS by the now-dead client A
            cap_waiter = self.mount_b.write_background()

            a = time.time()
            time.sleep(eviction_timeout)
            cap_waiter.wait()
            b = time.time()
            cap_waited = b - a
            log.info("cap_waiter waited {0}s".format(cap_waited))

            # check if the cap is transferred before session timeout kicked in.
            # this is a good enough check to ensure that the client got evicted
            # by the cap auto evicter rather than transitioning to stale state
            # and then getting evicted.
            self.assertLess(cap_waited, session_timeout,
                            "Capability handover took {0}, expected less than {1}".format(
                                cap_waited, session_timeout
                            ))

            self.assertTrue(self.mount_a.is_blacklisted())
            cap_holder.stdin.close()
            try:
                cap_holder.wait()
            except (CommandFailedError, ConnectionLostError):
                # We killed it (and possibly its node), so it raises an error
                pass
        finally:
            self.mount_a.kill_cleanup()

        self.mount_a.mount()
        self.mount_a.wait_until_mounted()

    def test_filtered_df(self):
        pool_name = self.fs.get_data_pool_name()
        raw_df = self.fs.get_pool_df(pool_name)
        raw_avail = float(raw_df["max_avail"])
        out = self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'get',
                                                  pool_name, 'size',
                                                  '-f', 'json-pretty')
        j = json.loads(out)
        pool_size = int(j['size'])

        proc = self.mount_a.run_shell(['df', '.'])
        output = proc.stdout.getvalue()
        fs_avail = output.split('\n')[1].split()[3]
        fs_avail = float(fs_avail) * 1024

        ratio = raw_avail / fs_avail
        assert 0.9 < ratio < 1.1

    def test_dump_inode(self):
        info = self.fs.mds_asok(['dump', 'inode', '1'])
        assert(info['path'] == "/")

    def test_dump_inode_hexademical(self):
        self.mount_a.run_shell(["mkdir", "-p", "foo"])
        ino = self.mount_a.path_to_ino("foo")
        assert type(ino) is int
        info = self.fs.mds_asok(['dump', 'inode', hex(ino)])
        assert info['path'] == "/foo"


class TestCacheDrop(CephFSTestCase):
    CLIENTS_REQUIRED = 1

    def _run_drop_cache_cmd(self, timeout=None):
        result = None
        mds_id = self.fs.get_lone_mds_id()
        if timeout is not None:
            result = self.fs.mon_manager.raw_cluster_cmd("tell", "mds.{0}".format(mds_id),
                                                    "cache", "drop", str(timeout))
        else:
            result = self.fs.mon_manager.raw_cluster_cmd("tell", "mds.{0}".format(mds_id),
                                                    "cache", "drop")
        return json.loads(result)

    def _setup(self, max_caps=20, threshold=400):
        # create some files
        self.mount_a.create_n_files("dc-dir/dc-file", 1000, sync=True)

        # Reduce this so the MDS doesn't rkcall the maximum for simple tests
        self.fs.rank_asok(['config', 'set', 'mds_recall_max_caps', str(max_caps)])
        self.fs.rank_asok(['config', 'set', 'mds_recall_max_decay_threshold', str(threshold)])

    def test_drop_cache_command(self):
        """
        Basic test for checking drop cache command.
        Confirm it halts without a timeout.
        Note that the cache size post trimming is not checked here.
        """
        mds_min_caps_per_client = int(self.fs.get_config("mds_min_caps_per_client"))
        self._setup()
        result = self._run_drop_cache_cmd()
        self.assertEqual(result['client_recall']['return_code'], 0)
        self.assertEqual(result['flush_journal']['return_code'], 0)
        # It should take at least 1 second
        self.assertGreater(result['duration'], 1)
        self.assertGreaterEqual(result['trim_cache']['trimmed'], 1000-2*mds_min_caps_per_client)

    def test_drop_cache_command_timeout(self):
        """
        Basic test for checking drop cache command.
        Confirm recall halts early via a timeout.
        Note that the cache size post trimming is not checked here.
        """
        self._setup()
        result = self._run_drop_cache_cmd(timeout=10)
        self.assertEqual(result['client_recall']['return_code'], -errno.ETIMEDOUT)
        self.assertEqual(result['flush_journal']['return_code'], 0)
        self.assertGreater(result['duration'], 10)
        self.assertGreaterEqual(result['trim_cache']['trimmed'], 100) # we did something, right?

    def test_drop_cache_command_dead_timeout(self):
        """
        Check drop cache command with non-responding client using tell
        interface. Note that the cache size post trimming is not checked
        here.
        """
        self._setup()
        self.mount_a.kill()
        # Note: recall is subject to the timeout. The journal flush will
        # be delayed due to the client being dead.
        result = self._run_drop_cache_cmd(timeout=5)
        self.assertEqual(result['client_recall']['return_code'], -errno.ETIMEDOUT)
        self.assertEqual(result['flush_journal']['return_code'], 0)
        self.assertGreater(result['duration'], 5)
        self.assertLess(result['duration'], 120)
        # Note: result['trim_cache']['trimmed'] may be >0 because dropping the
        # cache now causes the Locker to drive eviction of stale clients (a
        # stale session will be autoclosed at mdsmap['session_timeout']). The
        # particular operation causing this is journal flush which causes the
        # MDS to wait wait for cap revoke.
        #self.assertEqual(0, result['trim_cache']['trimmed'])
        self.mount_a.kill_cleanup()
        self.mount_a.mount()
        self.mount_a.wait_until_mounted()

    def test_drop_cache_command_dead(self):
        """
        Check drop cache command with non-responding client using tell
        interface. Note that the cache size post trimming is not checked
        here.
        """
        self._setup()
        self.mount_a.kill()
        result = self._run_drop_cache_cmd()
        self.assertEqual(result['client_recall']['return_code'], 0)
        self.assertEqual(result['flush_journal']['return_code'], 0)
        self.assertGreater(result['duration'], 5)
        self.assertLess(result['duration'], 120)
        # Note: result['trim_cache']['trimmed'] may be >0 because dropping the
        # cache now causes the Locker to drive eviction of stale clients (a
        # stale session will be autoclosed at mdsmap['session_timeout']). The
        # particular operation causing this is journal flush which causes the
        # MDS to wait wait for cap revoke.
        self.mount_a.kill_cleanup()
        self.mount_a.mount()
        self.mount_a.wait_until_mounted()
Commit	Line	Data
7c673cae FG	1
	2	from unittest import SkipTest
	3	from tasks.cephfs.fuse_mount import FuseMount
	4	from tasks.cephfs.cephfs_test_case import CephFSTestCase
91327a77	5	from teuthology.orchestra.run import CommandFailedError, ConnectionLostError
7c673cae FG	6	import errno
7c673cae FG	7	import time
d2e6a577	8	import json
91327a77	9	import logging
f64942e4	10	import time
7c673cae	11
91327a77	12	log = logging.getLogger(__name__)
31f18b77	13
7c673cae FG	14	class TestMisc(CephFSTestCase):
	15	CLIENTS_REQUIRED = 2
	16
7c673cae FG	17	def test_getattr_caps(self):
	18	"""
	19	Check if MDS recognizes the 'mask' parameter of open request.
11fdf7f2	20	The parameter allows client to request caps when opening file
7c673cae FG	21	"""
	22
	23	if not isinstance(self.mount_a, FuseMount):
	24	raise SkipTest("Require FUSE client")
	25
	26	# Enable debug. Client will requests CEPH_CAP_XATTR_SHARED
	27	# on lookup/open
	28	self.mount_b.umount_wait()
	29	self.set_conf('client', 'client debug getattr caps', 'true')
	30	self.mount_b.mount()
	31	self.mount_b.wait_until_mounted()
	32
	33	# create a file and hold it open. MDS will issue CEPH_CAP_EXCL_*
	34	# to mount_a
	35	p = self.mount_a.open_background("testfile")
	36	self.mount_b.wait_for_visible("testfile")
	37
11fdf7f2	38	# this triggers a lookup request and an open request. The debug
7c673cae FG	39	# code will check if lookup/open reply contains xattrs
	40	self.mount_b.run_shell(["cat", "testfile"])
	41
	42	self.mount_a.kill_background(p)
	43
f64942e4 AA	44	def test_root_rctime(self):
	45	"""
	46	Check that the root inode has a non-default rctime on startup.
	47	"""
	48
	49	t = time.time()
	50	rctime = self.mount_a.getfattr(".", "ceph.dir.rctime")
	51	log.info("rctime = {}".format(rctime))
	52	self.assertGreaterEqual(rctime, t-10)
	53
7c673cae	54	def test_fs_new(self):
a8e16298 TL	55	self.mount_a.umount_wait()
	56	self.mount_b.umount_wait()
	57
7c673cae FG	58	data_pool_name = self.fs.get_data_pool_name()
	59
	60	self.fs.mds_stop()
	61	self.fs.mds_fail()
	62
	63	self.fs.mon_manager.raw_cluster_cmd('fs', 'rm', self.fs.name,
	64	'--yes-i-really-mean-it')
	65
	66	self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'delete',
	67	self.fs.metadata_pool_name,
	68	self.fs.metadata_pool_name,
	69	'--yes-i-really-really-mean-it')
	70	self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'create',
	71	self.fs.metadata_pool_name,
	72	self.fs.get_pgs_per_fs_pool().__str__())
	73
	74	dummyfile = '/etc/fstab'
	75
	76	self.fs.put_metadata_object_raw("key", dummyfile)
	77
224ce89b WB	78	def get_pool_df(fs, name):
	79	try:
	80	return fs.get_pool_df(name)['objects'] > 0
	81	except RuntimeError as e:
	82	return False
7c673cae	83
224ce89b	84	self.wait_until_true(lambda: get_pool_df(self.fs, self.fs.metadata_pool_name), timeout=30)
7c673cae FG	85
	86	try:
	87	self.fs.mon_manager.raw_cluster_cmd('fs', 'new', self.fs.name,
	88	self.fs.metadata_pool_name,
	89	data_pool_name)
	90	except CommandFailedError as e:
	91	self.assertEqual(e.exitstatus, errno.EINVAL)
	92	else:
	93	raise AssertionError("Expected EINVAL")
	94
	95	self.fs.mon_manager.raw_cluster_cmd('fs', 'new', self.fs.name,
	96	self.fs.metadata_pool_name,
	97	data_pool_name, "--force")
	98
	99	self.fs.mon_manager.raw_cluster_cmd('fs', 'rm', self.fs.name,
	100	'--yes-i-really-mean-it')
	101
	102
	103	self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'delete',
	104	self.fs.metadata_pool_name,
	105	self.fs.metadata_pool_name,
	106	'--yes-i-really-really-mean-it')
	107	self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'create',
	108	self.fs.metadata_pool_name,
	109	self.fs.get_pgs_per_fs_pool().__str__())
	110	self.fs.mon_manager.raw_cluster_cmd('fs', 'new', self.fs.name,
	111	self.fs.metadata_pool_name,
	112	data_pool_name)
	113
91327a77 AA	114	def test_cap_revoke_nonresponder(self):
	115	"""
	116	Check that a client is evicted if it has not responded to cap revoke
	117	request for configured number of seconds.
	118	"""
	119	session_timeout = self.fs.get_var("session_timeout")
	120	eviction_timeout = session_timeout / 2.0
	121
	122	self.fs.mds_asok(['config', 'set', 'mds_cap_revoke_eviction_timeout',
	123	str(eviction_timeout)])
	124
	125	cap_holder = self.mount_a.open_background()
	126
	127	# Wait for the file to be visible from another client, indicating
	128	# that mount_a has completed its network ops
	129	self.mount_b.wait_for_visible()
	130
	131	# Simulate client death
	132	self.mount_a.kill()
	133
	134	try:
	135	# The waiter should get stuck waiting for the capability
	136	# held on the MDS by the now-dead client A
	137	cap_waiter = self.mount_b.write_background()
	138
	139	a = time.time()
	140	time.sleep(eviction_timeout)
	141	cap_waiter.wait()
	142	b = time.time()
	143	cap_waited = b - a
	144	log.info("cap_waiter waited {0}s".format(cap_waited))
	145
	146	# check if the cap is transferred before session timeout kicked in.
	147	# this is a good enough check to ensure that the client got evicted
	148	# by the cap auto evicter rather than transitioning to stale state
	149	# and then getting evicted.
	150	self.assertLess(cap_waited, session_timeout,
	151	"Capability handover took {0}, expected less than {1}".format(
	152	cap_waited, session_timeout
	153	))
	154
11fdf7f2	155	self.assertTrue(self.mount_a.is_blacklisted())
91327a77 AA	156	cap_holder.stdin.close()
	157	try:
	158	cap_holder.wait()
	159	except (CommandFailedError, ConnectionLostError):
	160	# We killed it (and possibly its node), so it raises an error
	161	pass
	162	finally:
	163	self.mount_a.kill_cleanup()
	164
	165	self.mount_a.mount()
	166	self.mount_a.wait_until_mounted()
	167
d2e6a577 FG	168	def test_filtered_df(self):
	169	pool_name = self.fs.get_data_pool_name()
	170	raw_df = self.fs.get_pool_df(pool_name)
	171	raw_avail = float(raw_df["max_avail"])
	172	out = self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'get',
	173	pool_name, 'size',
	174	'-f', 'json-pretty')
	175	j = json.loads(out)
	176	pool_size = int(j['size'])
	177
	178	proc = self.mount_a.run_shell(['df', '.'])
	179	output = proc.stdout.getvalue()
	180	fs_avail = output.split('\n')[1].split()[3]
	181	fs_avail = float(fs_avail) * 1024
	182
181888fb	183	ratio = raw_avail / fs_avail
d2e6a577	184	assert 0.9 < ratio < 1.1
f64942e4	185
11fdf7f2 TL	186	def test_dump_inode(self):
	187	info = self.fs.mds_asok(['dump', 'inode', '1'])
	188	assert(info['path'] == "/")
f64942e4	189
11fdf7f2 TL	190	def test_dump_inode_hexademical(self):
	191	self.mount_a.run_shell(["mkdir", "-p", "foo"])
	192	ino = self.mount_a.path_to_ino("foo")
	193	assert type(ino) is int
	194	info = self.fs.mds_asok(['dump', 'inode', hex(ino)])
	195	assert info['path'] == "/foo"
f64942e4	196
f64942e4	197
11fdf7f2 TL	198	class TestCacheDrop(CephFSTestCase):
11fdf7f2 TL	199	CLIENTS_REQUIRED = 1
f64942e4	200
11fdf7f2 TL	201	def _run_drop_cache_cmd(self, timeout=None):
	202	result = None
	203	mds_id = self.fs.get_lone_mds_id()
	204	if timeout is not None:
	205	result = self.fs.mon_manager.raw_cluster_cmd("tell", "mds.{0}".format(mds_id),
	206	"cache", "drop", str(timeout))
	207	else:
	208	result = self.fs.mon_manager.raw_cluster_cmd("tell", "mds.{0}".format(mds_id),
	209	"cache", "drop")
	210	return json.loads(result)
f64942e4	211
11fdf7f2	212	def _setup(self, max_caps=20, threshold=400):
f64942e4	213	# create some files
11fdf7f2	214	self.mount_a.create_n_files("dc-dir/dc-file", 1000, sync=True)
f64942e4	215
11fdf7f2 TL	216	# Reduce this so the MDS doesn't rkcall the maximum for simple tests
	217	self.fs.rank_asok(['config', 'set', 'mds_recall_max_caps', str(max_caps)])
	218	self.fs.rank_asok(['config', 'set', 'mds_recall_max_decay_threshold', str(threshold)])
f64942e4	219
11fdf7f2	220	def test_drop_cache_command(self):
f64942e4	221	"""
11fdf7f2 TL	222	Basic test for checking drop cache command.
11fdf7f2 TL	223	Confirm it halts without a timeout.
f64942e4 AA	224	Note that the cache size post trimming is not checked here.
f64942e4 AA	225	"""
11fdf7f2 TL	226	mds_min_caps_per_client = int(self.fs.get_config("mds_min_caps_per_client"))
	227	self._setup()
	228	result = self._run_drop_cache_cmd()
92f5a8d4 TL	229	self.assertEqual(result['client_recall']['return_code'], 0)
92f5a8d4 TL	230	self.assertEqual(result['flush_journal']['return_code'], 0)
11fdf7f2	231	# It should take at least 1 second
92f5a8d4	232	self.assertGreater(result['duration'], 1)
11fdf7f2 TL	233	self.assertGreaterEqual(result['trim_cache']['trimmed'], 1000-2*mds_min_caps_per_client)
	234
	235	def test_drop_cache_command_timeout(self):
f64942e4	236	"""
11fdf7f2 TL	237	Basic test for checking drop cache command.
11fdf7f2 TL	238	Confirm recall halts early via a timeout.
f64942e4 AA	239	Note that the cache size post trimming is not checked here.
f64942e4 AA	240	"""
11fdf7f2 TL	241	self._setup()
11fdf7f2 TL	242	result = self._run_drop_cache_cmd(timeout=10)
92f5a8d4 TL	243	self.assertEqual(result['client_recall']['return_code'], -errno.ETIMEDOUT)
	244	self.assertEqual(result['flush_journal']['return_code'], 0)
	245	self.assertGreater(result['duration'], 10)
11fdf7f2 TL	246	self.assertGreaterEqual(result['trim_cache']['trimmed'], 100) # we did something, right?
	247
	248	def test_drop_cache_command_dead_timeout(self):
f64942e4	249	"""
11fdf7f2 TL	250	Check drop cache command with non-responding client using tell
	251	interface. Note that the cache size post trimming is not checked
	252	here.
f64942e4	253	"""
11fdf7f2 TL	254	self._setup()
	255	self.mount_a.kill()
	256	# Note: recall is subject to the timeout. The journal flush will
	257	# be delayed due to the client being dead.
	258	result = self._run_drop_cache_cmd(timeout=5)
92f5a8d4 TL	259	self.assertEqual(result['client_recall']['return_code'], -errno.ETIMEDOUT)
	260	self.assertEqual(result['flush_journal']['return_code'], 0)
	261	self.assertGreater(result['duration'], 5)
	262	self.assertLess(result['duration'], 120)
	263	# Note: result['trim_cache']['trimmed'] may be >0 because dropping the
	264	# cache now causes the Locker to drive eviction of stale clients (a
	265	# stale session will be autoclosed at mdsmap['session_timeout']). The
	266	# particular operation causing this is journal flush which causes the
	267	# MDS to wait wait for cap revoke.
	268	#self.assertEqual(0, result['trim_cache']['trimmed'])
11fdf7f2 TL	269	self.mount_a.kill_cleanup()
	270	self.mount_a.mount()
	271	self.mount_a.wait_until_mounted()
f64942e4	272
11fdf7f2	273	def test_drop_cache_command_dead(self):
f64942e4 AA	274	"""
	275	Check drop cache command with non-responding client using tell
	276	interface. Note that the cache size post trimming is not checked
	277	here.
	278	"""
11fdf7f2 TL	279	self._setup()
	280	self.mount_a.kill()
	281	result = self._run_drop_cache_cmd()
92f5a8d4 TL	282	self.assertEqual(result['client_recall']['return_code'], 0)
	283	self.assertEqual(result['flush_journal']['return_code'], 0)
	284	self.assertGreater(result['duration'], 5)
	285	self.assertLess(result['duration'], 120)
	286	# Note: result['trim_cache']['trimmed'] may be >0 because dropping the
	287	# cache now causes the Locker to drive eviction of stale clients (a
	288	# stale session will be autoclosed at mdsmap['session_timeout']). The
	289	# particular operation causing this is journal flush which causes the
	290	# MDS to wait wait for cap revoke.
11fdf7f2 TL	291	self.mount_a.kill_cleanup()
	292	self.mount_a.mount()
	293	self.mount_a.wait_until_mounted()