[ceph.git] / ceph / qa / tasks / cephfs / xfstests_dev.py

import six
import logging
from StringIO import StringIO
from tasks.cephfs.cephfs_test_case import CephFSTestCase

logger = logging.getLogger(__name__)


# TODO: add code to run non-ACL tests too.
# TODO: get tests running with SCRATCH_DEV and SCRATCH_DIR.
# TODO: make xfstests-dev tests running without running `make install`.
# TODO: make xfstests-dev compatible with ceph-fuse. xfstests-dev remounts
# CephFS before running tests using kernel, so ceph-fuse mounts are never
# actually testsed.
class XFSTestsDev(CephFSTestCase):

    def setUp(self):
        CephFSTestCase.setUp(self)
        self.prepare_xfstests_dev()

    def prepare_xfstests_dev(self):
        self.get_repo()
        self.get_test_and_scratch_dirs_ready()
        self.install_deps()
        self.create_reqd_users()
        self.write_local_config()

        # NOTE: On teuthology machines it's necessary to run "make" as
        # superuser since the repo is cloned somewhere in /tmp.
        self.mount_a.client_remote.run(args=['sudo', 'make'],
                                       cwd=self.repo_path, stdout=StringIO(),
                                       stderr=StringIO())
        self.mount_a.client_remote.run(args=['sudo', 'make', 'install'],
                                       cwd=self.repo_path, omit_sudo=False,
                                       stdout=StringIO(), stderr=StringIO())

    def get_repo(self):
        """
        Clone xfstests_dev repository. If already present, update it.
        """
        from teuthology.orchestra import run

        # TODO: make sure that repo is not cloned for every test. it should
        # happen only once.
        remoteurl = 'git://git.ceph.com/xfstests-dev.git'
        self.repo_path = self.mount_a.client_remote.mkdtemp(suffix=
                                                            'xfstests-dev')
        self.mount_a.run_shell(['git', 'archive', '--remote=' + remoteurl,
                                'HEAD', run.Raw('|'),
                                'tar', '-C', self.repo_path, '-x', '-f', '-'])

    def get_admin_key(self):
        import configparser
        from sys import version_info as sys_version_info

        cp = configparser.ConfigParser()
        if sys_version_info.major > 2:
            cp.read_string(self.fs.mon_manager.raw_cluster_cmd(
                'auth', 'get-or-create', 'client.admin'))
        # TODO: remove this part when we stop supporting Python 2
        elif sys_version_info.major <= 2:
            cp.read_string(six.text_type(self.fs.mon_manager.raw_cluster_cmd(
                'auth', 'get-or-create', 'client.admin')))

        return cp['client.admin']['key']

    def get_test_and_scratch_dirs_ready(self):
        """ "test" and "scratch" directories are directories inside Ceph FS.
            And, test and scratch mounts are path on the local FS where "test"
            and "scratch" directories would be mounted. Look at xfstests-dev
            local.config's template inside this file to get some context.
        """
        from os.path import join

        self.test_dirname = 'test'
        self.mount_a.run_shell(['mkdir', self.test_dirname])
        # read var name as "test dir's mount path"
        self.test_dirs_mount_path = self.mount_a.client_remote.mkdtemp(
            suffix=self.test_dirname)
        self.mount_a.run_shell(['sudo','ln','-s',join(self.mount_a.mountpoint,
                                                      self.test_dirname),
                                self.test_dirs_mount_path])

        self.scratch_dirname = 'scratch'
        self.mount_a.run_shell(['mkdir', self.scratch_dirname])
        # read var name as "scratch dir's mount path"
        self.scratch_dirs_mount_path = self.mount_a.client_remote.mkdtemp(
            suffix=self.scratch_dirname)
        self.mount_a.run_shell(['sudo','ln','-s',join(self.mount_a.mountpoint,
                                                      self.scratch_dirname),
                                self.scratch_dirs_mount_path])

    def install_deps(self):
        from teuthology.misc import get_system_type

        distro, version = get_system_type(self.mount_a.client_remote,
                                          distro=True, version=True)
        distro = distro.lower()
        major_ver_num = int(version.split('.')[0]) # only keep major release
                                                   # number

        # we keep fedora here so that right deps are installed when this test
        # is run locally by a dev.
        if distro in ('redhatenterpriseserver', 'redhatenterprise', 'fedora',
                      'centos'):
            deps = """acl attr automake bc dbench dump e2fsprogs fio \
            gawk gcc indent libtool lvm2 make psmisc quota sed \
            xfsdump xfsprogs \
            libacl-devel libattr-devel libaio-devel libuuid-devel \
            xfsprogs-devel btrfs-progs-devel python2 sqlite""".split()
            deps_old_distros = ['xfsprogs-qa-devel']

            if distro != 'fedora' and major_ver_num > 7:
                    deps.remove('btrfs-progs-devel')

            args = ['sudo', 'yum', 'install', '-y'] + deps + deps_old_distros
        elif distro == 'ubuntu':
            deps = """xfslibs-dev uuid-dev libtool-bin \
            e2fsprogs automake gcc libuuid1 quota attr libattr1-dev make \
            libacl1-dev libaio-dev xfsprogs libgdbm-dev gawk fio dbench \
            uuid-runtime python sqlite3""".split()

            if major_ver_num >= 19:
                deps[deps.index('python')] ='python2'
            args = ['sudo', 'apt-get', 'install', '-y'] + deps
        else:
            raise RuntimeError('expected a yum based or a apt based system')

        self.mount_a.client_remote.run(args=args, omit_sudo=False)

    def create_reqd_users(self):
        self.mount_a.client_remote.run(args=['sudo', 'useradd', 'fsgqa'],
                                       omit_sudo=False, check_status=False)
        self.mount_a.client_remote.run(args=['sudo', 'groupadd', 'fsgqa'],
                                       omit_sudo=False, check_status=False)
        self.mount_a.client_remote.run(args=['sudo', 'useradd',
                                             '123456-fsgqa'], omit_sudo=False,
                                       check_status=False)

    def write_local_config(self):
        from os.path import join
        from textwrap import dedent
        from teuthology.misc import sudo_write_file

        mon_sock = self.fs.mon_manager.get_msgrv1_mon_socks()[0]
        self.test_dev = mon_sock + ':/' + self.test_dirname
        self.scratch_dev = mon_sock + ':/' + self.scratch_dirname

        xfstests_config_contents = dedent('''\
            export FSTYP=ceph
            export TEST_DEV={}
            export TEST_DIR={}
            #export SCRATCH_DEV={}
            #export SCRATCH_MNT={}
            export TEST_FS_MOUNT_OPTS="-o name=admin,secret={}"
            ''').format(self.test_dev, self.test_dirs_mount_path, self.scratch_dev,
                        self.scratch_dirs_mount_path, self.get_admin_key())

        sudo_write_file(self.mount_a.client_remote, join(
            self.repo_path, 'local.config'), xfstests_config_contents)

    def tearDown(self):
        self.mount_a.client_remote.run(args=['sudo', 'userdel', '--force',
                                             '--remove', 'fsgqa'],
                                       omit_sudo=False, check_status=False)
        self.mount_a.client_remote.run(args=['sudo', 'userdel', '--force',
                                             '--remove', '123456-fsgqa'],
                                       omit_sudo=False, check_status=False)
        self.mount_a.client_remote.run(args=['sudo', 'groupdel', 'fsgqa'],
                                       omit_sudo=False, check_status=False)

        self.mount_a.client_remote.run(args=['sudo', 'rm', '-rf',
                                             self.repo_path],
                                       omit_sudo=False, check_status=False)
Commit	Line	Data
9f95a23c TL	1	import six
	2	import logging
	3	from StringIO import StringIO
	4	from tasks.cephfs.cephfs_test_case import CephFSTestCase
	5
	6	logger = logging.getLogger(__name__)
	7
	8
	9	# TODO: add code to run non-ACL tests too.
	10	# TODO: get tests running with SCRATCH_DEV and SCRATCH_DIR.
	11	# TODO: make xfstests-dev tests running without running `make install`.
	12	# TODO: make xfstests-dev compatible with ceph-fuse. xfstests-dev remounts
	13	# CephFS before running tests using kernel, so ceph-fuse mounts are never
	14	# actually testsed.
	15	class XFSTestsDev(CephFSTestCase):
	16
	17	def setUp(self):
	18	CephFSTestCase.setUp(self)
	19	self.prepare_xfstests_dev()
	20
	21	def prepare_xfstests_dev(self):
	22	self.get_repo()
	23	self.get_test_and_scratch_dirs_ready()
	24	self.install_deps()
	25	self.create_reqd_users()
	26	self.write_local_config()
	27
	28	# NOTE: On teuthology machines it's necessary to run "make" as
	29	# superuser since the repo is cloned somewhere in /tmp.
	30	self.mount_a.client_remote.run(args=['sudo', 'make'],
	31	cwd=self.repo_path, stdout=StringIO(),
	32	stderr=StringIO())
	33	self.mount_a.client_remote.run(args=['sudo', 'make', 'install'],
	34	cwd=self.repo_path, omit_sudo=False,
	35	stdout=StringIO(), stderr=StringIO())
	36
	37	def get_repo(self):
	38	"""
	39	Clone xfstests_dev repository. If already present, update it.
	40	"""
	41	from teuthology.orchestra import run
	42
	43	# TODO: make sure that repo is not cloned for every test. it should
	44	# happen only once.
	45	remoteurl = 'git://git.ceph.com/xfstests-dev.git'
	46	self.repo_path = self.mount_a.client_remote.mkdtemp(suffix=
	47	'xfstests-dev')
	48	self.mount_a.run_shell(['git', 'archive', '--remote=' + remoteurl,
	49	'HEAD', run.Raw('\|'),
	50	'tar', '-C', self.repo_path, '-x', '-f', '-'])
	51
	52	def get_admin_key(self):
	53	import configparser
	54	from sys import version_info as sys_version_info
	55
	56	cp = configparser.ConfigParser()
	57	if sys_version_info.major > 2:
	58	cp.read_string(self.fs.mon_manager.raw_cluster_cmd(
	59	'auth', 'get-or-create', 'client.admin'))
	60	# TODO: remove this part when we stop supporting Python 2
	61	elif sys_version_info.major <= 2:
	62	cp.read_string(six.text_type(self.fs.mon_manager.raw_cluster_cmd(
	63	'auth', 'get-or-create', 'client.admin')))
	64
65	return cp['client.admin']['key']
66
67	def get_test_and_scratch_dirs_ready(self):
68	""" "test" and "scratch" directories are directories inside Ceph FS.
69	And, test and scratch mounts are path on the local FS where "test"
70	and "scratch" directories would be mounted. Look at xfstests-dev
71	local.config's template inside this file to get some context.
72	"""
73	from os.path import join
74
75	self.test_dirname = 'test'
76	self.mount_a.run_shell(['mkdir', self.test_dirname])
77	# read var name as "test dir's mount path"
78	self.test_dirs_mount_path = self.mount_a.client_remote.mkdtemp(
79	suffix=self.test_dirname)
80	self.mount_a.run_shell(['sudo','ln','-s',join(self.mount_a.mountpoint,
81	self.test_dirname),
82	self.test_dirs_mount_path])
83
84	self.scratch_dirname = 'scratch'
85	self.mount_a.run_shell(['mkdir', self.scratch_dirname])
86	# read var name as "scratch dir's mount path"
87	self.scratch_dirs_mount_path = self.mount_a.client_remote.mkdtemp(
88	suffix=self.scratch_dirname)
89	self.mount_a.run_shell(['sudo','ln','-s',join(self.mount_a.mountpoint,
90	self.scratch_dirname),
91	self.scratch_dirs_mount_path])
92
93	def install_deps(self):
94	from teuthology.misc import get_system_type
95
96	distro, version = get_system_type(self.mount_a.client_remote,
97	distro=True, version=True)
98	distro = distro.lower()
99	major_ver_num = int(version.split('.')[0]) # only keep major release
100	# number
101
102	# we keep fedora here so that right deps are installed when this test
103	# is run locally by a dev.
104	if distro in ('redhatenterpriseserver', 'redhatenterprise', 'fedora',
105	'centos'):
106	deps = """acl attr automake bc dbench dump e2fsprogs fio \
107	gawk gcc indent libtool lvm2 make psmisc quota sed \
108	xfsdump xfsprogs \
109	libacl-devel libattr-devel libaio-devel libuuid-devel \
110	xfsprogs-devel btrfs-progs-devel python2 sqlite""".split()
111	deps_old_distros = ['xfsprogs-qa-devel']
112
113	if distro != 'fedora' and major_ver_num > 7:
114	deps.remove('btrfs-progs-devel')
115
116	args = ['sudo', 'yum', 'install', '-y'] + deps + deps_old_distros
117	elif distro == 'ubuntu':
118	deps = """xfslibs-dev uuid-dev libtool-bin \
119	e2fsprogs automake gcc libuuid1 quota attr libattr1-dev make \
120	libacl1-dev libaio-dev xfsprogs libgdbm-dev gawk fio dbench \
121	uuid-runtime python sqlite3""".split()
122
123	if major_ver_num >= 19:
124	deps[deps.index('python')] ='python2'
125	args = ['sudo', 'apt-get', 'install', '-y'] + deps
126	else:
127	raise RuntimeError('expected a yum based or a apt based system')
128
129	self.mount_a.client_remote.run(args=args, omit_sudo=False)
130
131	def create_reqd_users(self):
132	self.mount_a.client_remote.run(args=['sudo', 'useradd', 'fsgqa'],
133	omit_sudo=False, check_status=False)
134	self.mount_a.client_remote.run(args=['sudo', 'groupadd', 'fsgqa'],
135	omit_sudo=False, check_status=False)
136	self.mount_a.client_remote.run(args=['sudo', 'useradd',
137	'123456-fsgqa'], omit_sudo=False,
138	check_status=False)
139
140	def write_local_config(self):
141	from os.path import join
142	from textwrap import dedent
143	from teuthology.misc import sudo_write_file
144
145	mon_sock = self.fs.mon_manager.get_msgrv1_mon_socks()[0]
146	self.test_dev = mon_sock + ':/' + self.test_dirname
147	self.scratch_dev = mon_sock + ':/' + self.scratch_dirname
148
149	xfstests_config_contents = dedent('''\
150	export FSTYP=ceph
151	export TEST_DEV={}
152	export TEST_DIR={}
153	#export SCRATCH_DEV={}
154	#export SCRATCH_MNT={}
155	export TEST_FS_MOUNT_OPTS="-o name=admin,secret={}"
156	''').format(self.test_dev, self.test_dirs_mount_path, self.scratch_dev,
157	self.scratch_dirs_mount_path, self.get_admin_key())
158
159	sudo_write_file(self.mount_a.client_remote, join(
160	self.repo_path, 'local.config'), xfstests_config_contents)
161
162	def tearDown(self):
163	self.mount_a.client_remote.run(args=['sudo', 'userdel', '--force',
164	'--remove', 'fsgqa'],
165	omit_sudo=False, check_status=False)
166	self.mount_a.client_remote.run(args=['sudo', 'userdel', '--force',
167	'--remove', '123456-fsgqa'],
168	omit_sudo=False, check_status=False)
169	self.mount_a.client_remote.run(args=['sudo', 'groupdel', 'fsgqa'],
170	omit_sudo=False, check_status=False)
171
172	self.mount_a.client_remote.run(args=['sudo', 'rm', '-rf',
173	self.repo_path],
174	omit_sudo=False, check_status=False)