[ceph.git] / ceph / qa / tasks / cbt.py

import logging
import os
import yaml

from teuthology import misc
from teuthology.orchestra import run
from teuthology.task import Task

log = logging.getLogger(__name__)


class CBT(Task):
    """
    Passes through a CBT configuration yaml fragment.
    """
    def __init__(self, ctx, config):
        super(CBT, self).__init__(ctx, config)
        self.log = log

    def hosts_of_type(self, type_):
        return [r.name for r in self.ctx.cluster.only(misc.is_type(type_)).remotes.keys()]

    def generate_cbt_config(self):
        mon_hosts = self.hosts_of_type('mon')
        osd_hosts = self.hosts_of_type('osd')
        client_hosts = self.hosts_of_type('client')
        rgw_client = {}
        rgw_client[client_hosts[0]] = None
        rgw_hosts = self.config.get('cluster', {}).get('rgws', rgw_client)
        cluster_config = dict(
            user=self.config.get('cluster', {}).get('user', 'ubuntu'),
            head=mon_hosts[0],
            osds=osd_hosts,
            mons=mon_hosts,
            clients=client_hosts,
            rgws=rgw_hosts,
            osds_per_node=self.config.get('cluster', {}).get('osds_per_node', 1),
            rebuild_every_test=False,
            use_existing=True,
            is_teuthology=self.config.get('cluster', {}).get('is_teuthology', True),
            iterations=self.config.get('cluster', {}).get('iterations', 1),
            tmp_dir='/tmp/cbt',
            pool_profiles=self.config.get('cluster', {}).get('pool_profiles'),
            )

        benchmark_config = self.config.get('benchmarks')
        benchmark_type = next(iter(benchmark_config.keys()))
        if benchmark_type in ['librbdfio', 'fio']:
          testdir = misc.get_testdir(self.ctx)
          benchmark_config[benchmark_type]['cmd_path'] = os.path.join(testdir, 'fio/fio')
        if benchmark_type == 'cosbench':
            # create cosbench_dir and cosbench_xml_dir
            testdir = misc.get_testdir(self.ctx)
            benchmark_config['cosbench']['cosbench_dir'] = os.path.join(testdir, 'cos')
            benchmark_config['cosbench']['cosbench_xml_dir'] = os.path.join(testdir, 'xml')
            self.ctx.cluster.run(args=['mkdir', '-p', '-m0755', '--', benchmark_config['cosbench']['cosbench_xml_dir']])
            benchmark_config['cosbench']['controller'] = osd_hosts[0]

            # set auth details
            remotes_and_roles = self.ctx.cluster.remotes.items()
            ips = [host for (host, port) in
                   (remote.ssh.get_transport().getpeername() for (remote, role_list) in remotes_and_roles)]
            benchmark_config['cosbench']['auth'] = "username=cosbench:operator;password=intel2012;url=http://%s:80/auth/v1.0;retry=9" %(ips[0])
        client_endpoints_config = self.config.get('client_endpoints', None)

        return dict(
            cluster=cluster_config,
            benchmarks=benchmark_config,
            client_endpoints = client_endpoints_config,
            )

    def install_dependencies(self):
        system_type = misc.get_system_type(self.first_mon)

        if system_type == 'rpm':
            install_cmd = ['sudo', 'yum', '-y', 'install']
            cbt_depends = ['python3-yaml', 'python3-lxml', 'librbd-devel', 'pdsh', 'collectl']
        else:
            install_cmd = ['sudo', 'apt-get', '-y', '--force-yes', 'install']
            cbt_depends = ['python3-yaml', 'python3-lxml', 'librbd-dev', 'collectl']
        self.first_mon.run(args=install_cmd + cbt_depends)

        benchmark_type = next(iter(self.cbt_config.get('benchmarks').keys()))
        self.log.info('benchmark: %s', benchmark_type)

        if benchmark_type in ['librbdfio', 'fio']:
            # install fio
            testdir = misc.get_testdir(self.ctx)
            self.first_mon.run(
                args=[
                    'git', 'clone', '-b', 'master',
                    'https://github.com/axboe/fio.git',
                    '{tdir}/fio'.format(tdir=testdir)
                ]
            )
            self.first_mon.run(
                args=[
                    'cd', os.path.join(testdir, 'fio'), run.Raw('&&'),
                    './configure', run.Raw('&&'),
                    'make'
                ]
            )

        if benchmark_type == 'cosbench':
            # install cosbench
            self.log.info('install dependencies for cosbench')
            if system_type == 'rpm':
                cosbench_depends = ['wget', 'unzip', 'java-1.7.0-openjdk', 'curl']
            else:
                cosbench_depends = ['wget', 'unzip', 'openjdk-8-jre', 'curl']
            self.first_mon.run(args=install_cmd + cosbench_depends)
            testdir = misc.get_testdir(self.ctx)
            cosbench_version = '0.4.2.c3'
            cosbench_location = 'https://github.com/intel-cloud/cosbench/releases/download/v0.4.2.c3/0.4.2.c3.zip'
            os_version = misc.get_system_type(self.first_mon, False, True)

            # additional requirements for bionic
            if os_version == '18.04':
                self.first_mon.run(
                    args=['sudo', 'apt-get', '-y', 'purge', 'openjdk-11*'])
                # use our own version of cosbench
                cosbench_version = 'cosbench-0.4.2.c3.1'
                # contains additional parameter "-N" to nc
                cosbench_location = 'http://drop.ceph.com/qa/cosbench-0.4.2.c3.1.zip'
                cosbench_dir = os.path.join(testdir, cosbench_version)
                self.ctx.cluster.run(args=['mkdir', '-p', '-m0755', '--', cosbench_dir])
                self.first_mon.run(
                    args=[
                        'cd', testdir, run.Raw('&&'),
                        'wget',
                        cosbench_location, run.Raw('&&'),
                        'unzip', '{name}.zip'.format(name=cosbench_version), '-d', cosbench_version
                    ]
                )
            else:
                self.first_mon.run(
                    args=[
                        'cd', testdir, run.Raw('&&'),
                        'wget',
                        cosbench_location, run.Raw('&&'),
                        'unzip', '{name}.zip'.format(name=cosbench_version)
                    ]
                )
            self.first_mon.run(
                args=[
                    'cd', testdir, run.Raw('&&'),
                    'ln', '-s', cosbench_version, 'cos',
                ]
            )
            self.first_mon.run(
                args=[
                    'cd', os.path.join(testdir, 'cos'), run.Raw('&&'),
                    'chmod', '+x', run.Raw('*.sh'),
                ]
            )

            # start cosbench and check info
            self.log.info('start cosbench')
            self.first_mon.run(
                args=[
                    'cd', testdir, run.Raw('&&'),
                    'cd', 'cos', run.Raw('&&'),
                    'sh', 'start-all.sh'
                ]
            )
            self.log.info('check cosbench info')
            self.first_mon.run(
                args=[
                    'cd', testdir, run.Raw('&&'),
                    'cd', 'cos', run.Raw('&&'),
                    'sh', 'cli.sh', 'info'
                ]
            )

    def checkout_cbt(self):
        testdir = misc.get_testdir(self.ctx)
        repo = self.config.get('repo', 'https://github.com/ceph/cbt.git')
        branch = self.config.get('branch', 'master')
        branch = self.config.get('force-branch', branch)
        sha1 = self.config.get('sha1')
        if sha1 is None:
            self.first_mon.run(
                args=[
                    'git', 'clone', '--depth', '1', '-b', branch, repo,
                    '{tdir}/cbt'.format(tdir=testdir)
                ]
            )
        else:
            self.first_mon.run(
                args=[
                    'git', 'clone', '-b', branch, repo,
                    '{tdir}/cbt'.format(tdir=testdir)
                ]
            )
            self.first_mon.run(
                args=[
                    'cd', os.path.join(testdir, 'cbt'), run.Raw('&&'),
                    'git', 'reset', '--hard', sha1,
                ]
            )

    def setup(self):
        super(CBT, self).setup()
        self.first_mon = next(iter(self.ctx.cluster.only(misc.get_first_mon(self.ctx, self.config)).remotes.keys()))
        self.cbt_config = self.generate_cbt_config()
        self.log.info('cbt configuration is %s', self.cbt_config)
        self.cbt_dir = os.path.join(misc.get_archive_dir(self.ctx), 'cbt')
        self.ctx.cluster.run(args=['mkdir', '-p', '-m0755', '--', self.cbt_dir])
        misc.write_file(self.first_mon, os.path.join(self.cbt_dir, 'cbt_config.yaml'),
                        yaml.safe_dump(self.cbt_config, default_flow_style=False))
        self.checkout_cbt()
        self.install_dependencies()

    def begin(self):
        super(CBT, self).begin()
        testdir = misc.get_testdir(self.ctx)
        self.first_mon.run(
            args=[
                '{tdir}/cbt/cbt.py'.format(tdir=testdir),
                '-a', self.cbt_dir,
                '{cbtdir}/cbt_config.yaml'.format(cbtdir=self.cbt_dir),
            ],
        )
        preserve_file = os.path.join(self.ctx.archive, '.preserve')
        open(preserve_file, 'a').close()

    def end(self):
        super(CBT, self).end()
        testdir = misc.get_testdir(self.ctx)
        self.first_mon.run(
            args=[
                'rm', '--one-file-system', '-rf', '--',
                '{tdir}/cbt'.format(tdir=testdir),
            ]
        )
        benchmark_type = next(iter(self.cbt_config.get('benchmarks').keys()))
        if benchmark_type in ['librbdfio', 'fio']:
            self.first_mon.run(
                args=[
                    'rm', '--one-file-system', '-rf', '--',
                    '{tdir}/fio'.format(tdir=testdir),
                ]
            )

        if benchmark_type == 'cosbench':
            os_version = misc.get_system_type(self.first_mon, False, True)
            if os_version == '18.04':
                cosbench_version = 'cosbench-0.4.2.c3.1'
            else:
                cosbench_version = '0.4.2.c3'
            # note: stop-all requires 'nc'
            self.first_mon.run(
                args=[
                    'cd', testdir, run.Raw('&&'),
                    'cd', 'cos', run.Raw('&&'),
                    'sh', 'stop-all.sh',
                    run.Raw('||'), 'true'
                ]
            )
            self.first_mon.run(
                args=[
                    'sudo', 'killall', '-9', 'java',
                    run.Raw('||'), 'true'
                ]
            )
            self.first_mon.run(
                args=[
                    'rm', '--one-file-system', '-rf', '--',
                    '{tdir}/cos'.format(tdir=testdir),
                ]
            )
            self.first_mon.run(
                args=[
                    'rm', '--one-file-system', '-rf', '--',
                    '{tdir}/{version}'.format(tdir=testdir, version=cosbench_version),
                ]
            )
            self.first_mon.run(
                args=[
                    'rm', '--one-file-system', '-rf', '--',
                    '{tdir}/{version}.zip'.format(tdir=testdir, version=cosbench_version),
                ]
            )
            self.first_mon.run(
                args=[
                    'rm', '--one-file-system', '-rf', '--',
                    '{tdir}/xml'.format(tdir=testdir),
                ]
            )


task = CBT
Commit	Line	Data
11fdf7f2 TL	1	import logging
	2	import os
	3	import yaml
	4
	5	from teuthology import misc
11fdf7f2 TL	6	from teuthology.orchestra import run
	7	from teuthology.task import Task
	8
	9	log = logging.getLogger(__name__)
	10
	11
	12	class CBT(Task):
	13	"""
	14	Passes through a CBT configuration yaml fragment.
	15	"""
	16	def __init__(self, ctx, config):
	17	super(CBT, self).__init__(ctx, config)
	18	self.log = log
	19
	20	def hosts_of_type(self, type_):
	21	return [r.name for r in self.ctx.cluster.only(misc.is_type(type_)).remotes.keys()]
	22
	23	def generate_cbt_config(self):
	24	mon_hosts = self.hosts_of_type('mon')
	25	osd_hosts = self.hosts_of_type('osd')
	26	client_hosts = self.hosts_of_type('client')
	27	rgw_client = {}
	28	rgw_client[client_hosts[0]] = None
	29	rgw_hosts = self.config.get('cluster', {}).get('rgws', rgw_client)
	30	cluster_config = dict(
	31	user=self.config.get('cluster', {}).get('user', 'ubuntu'),
	32	head=mon_hosts[0],
	33	osds=osd_hosts,
	34	mons=mon_hosts,
	35	clients=client_hosts,
	36	rgws=rgw_hosts,
	37	osds_per_node=self.config.get('cluster', {}).get('osds_per_node', 1),
	38	rebuild_every_test=False,
	39	use_existing=True,
	40	is_teuthology=self.config.get('cluster', {}).get('is_teuthology', True),
	41	iterations=self.config.get('cluster', {}).get('iterations', 1),
	42	tmp_dir='/tmp/cbt',
	43	pool_profiles=self.config.get('cluster', {}).get('pool_profiles'),
	44	)
	45
	46	benchmark_config = self.config.get('benchmarks')
e306af50	47	benchmark_type = next(iter(benchmark_config.keys()))
9f95a23c	48	if benchmark_type in ['librbdfio', 'fio']:
11fdf7f2	49	testdir = misc.get_testdir(self.ctx)
9f95a23c	50	benchmark_config[benchmark_type]['cmd_path'] = os.path.join(testdir, 'fio/fio')
11fdf7f2 TL	51	if benchmark_type == 'cosbench':
	52	# create cosbench_dir and cosbench_xml_dir
	53	testdir = misc.get_testdir(self.ctx)
	54	benchmark_config['cosbench']['cosbench_dir'] = os.path.join(testdir, 'cos')
	55	benchmark_config['cosbench']['cosbench_xml_dir'] = os.path.join(testdir, 'xml')
	56	self.ctx.cluster.run(args=['mkdir', '-p', '-m0755', '--', benchmark_config['cosbench']['cosbench_xml_dir']])
	57	benchmark_config['cosbench']['controller'] = osd_hosts[0]
	58
	59	# set auth details
	60	remotes_and_roles = self.ctx.cluster.remotes.items()
	61	ips = [host for (host, port) in
	62	(remote.ssh.get_transport().getpeername() for (remote, role_list) in remotes_and_roles)]
494da23a	63	benchmark_config['cosbench']['auth'] = "username=cosbench:operator;password=intel2012;url=http://%s:80/auth/v1.0;retry=9" %(ips[0])
9f95a23c	64	client_endpoints_config = self.config.get('client_endpoints', None)
11fdf7f2 TL	65
	66	return dict(
	67	cluster=cluster_config,
	68	benchmarks=benchmark_config,
9f95a23c	69	client_endpoints = client_endpoints_config,
11fdf7f2 TL	70	)
	71
	72	def install_dependencies(self):
	73	system_type = misc.get_system_type(self.first_mon)
	74
	75	if system_type == 'rpm':
	76	install_cmd = ['sudo', 'yum', '-y', 'install']
9f95a23c	77	cbt_depends = ['python3-yaml', 'python3-lxml', 'librbd-devel', 'pdsh', 'collectl']
11fdf7f2 TL	78	else:
11fdf7f2 TL	79	install_cmd = ['sudo', 'apt-get', '-y', '--force-yes', 'install']
92f5a8d4	80	cbt_depends = ['python3-yaml', 'python3-lxml', 'librbd-dev', 'collectl']
11fdf7f2 TL	81	self.first_mon.run(args=install_cmd + cbt_depends)
11fdf7f2 TL	82
e306af50	83	benchmark_type = next(iter(self.cbt_config.get('benchmarks').keys()))
11fdf7f2 TL	84	self.log.info('benchmark: %s', benchmark_type)
11fdf7f2 TL	85
9f95a23c	86	if benchmark_type in ['librbdfio', 'fio']:
11fdf7f2 TL	87	# install fio
	88	testdir = misc.get_testdir(self.ctx)
	89	self.first_mon.run(
	90	args=[
	91	'git', 'clone', '-b', 'master',
	92	'https://github.com/axboe/fio.git',
	93	'{tdir}/fio'.format(tdir=testdir)
	94	]
	95	)
	96	self.first_mon.run(
	97	args=[
	98	'cd', os.path.join(testdir, 'fio'), run.Raw('&&'),
	99	'./configure', run.Raw('&&'),
	100	'make'
	101	]
	102	)
	103
	104	if benchmark_type == 'cosbench':
	105	# install cosbench
	106	self.log.info('install dependencies for cosbench')
	107	if system_type == 'rpm':
	108	cosbench_depends = ['wget', 'unzip', 'java-1.7.0-openjdk', 'curl']
	109	else:
	110	cosbench_depends = ['wget', 'unzip', 'openjdk-8-jre', 'curl']
	111	self.first_mon.run(args=install_cmd + cosbench_depends)
	112	testdir = misc.get_testdir(self.ctx)
	113	cosbench_version = '0.4.2.c3'
	114	cosbench_location = 'https://github.com/intel-cloud/cosbench/releases/download/v0.4.2.c3/0.4.2.c3.zip'
	115	os_version = misc.get_system_type(self.first_mon, False, True)
	116
	117	# additional requirements for bionic
	118	if os_version == '18.04':
	119	self.first_mon.run(
	120	args=['sudo', 'apt-get', '-y', 'purge', 'openjdk-11*'])
	121	# use our own version of cosbench
	122	cosbench_version = 'cosbench-0.4.2.c3.1'
	123	# contains additional parameter "-N" to nc
	124	cosbench_location = 'http://drop.ceph.com/qa/cosbench-0.4.2.c3.1.zip'
	125	cosbench_dir = os.path.join(testdir, cosbench_version)
	126	self.ctx.cluster.run(args=['mkdir', '-p', '-m0755', '--', cosbench_dir])
	127	self.first_mon.run(
	128	args=[
	129	'cd', testdir, run.Raw('&&'),
	130	'wget',
	131	cosbench_location, run.Raw('&&'),
	132	'unzip', '{name}.zip'.format(name=cosbench_version), '-d', cosbench_version
	133	]
	134	)
	135	else:
	136	self.first_mon.run(
	137	args=[
	138	'cd', testdir, run.Raw('&&'),
	139	'wget',
	140	cosbench_location, run.Raw('&&'),
	141	'unzip', '{name}.zip'.format(name=cosbench_version)
	142	]
	143	)
	144	self.first_mon.run(
	145	args=[
	146	'cd', testdir, run.Raw('&&'),
	147	'ln', '-s', cosbench_version, 'cos',
	148	]
	149	)
	150	self.first_mon.run(
151	args=[
152	'cd', os.path.join(testdir, 'cos'), run.Raw('&&'),
153	'chmod', '+x', run.Raw('*.sh'),
154	]
155	)
156
157	# start cosbench and check info
158	self.log.info('start cosbench')
159	self.first_mon.run(
160	args=[
161	'cd', testdir, run.Raw('&&'),
162	'cd', 'cos', run.Raw('&&'),
163	'sh', 'start-all.sh'
164	]
165	)
166	self.log.info('check cosbench info')
167	self.first_mon.run(
168	args=[
169	'cd', testdir, run.Raw('&&'),
170	'cd', 'cos', run.Raw('&&'),
171	'sh', 'cli.sh', 'info'
172	]
173	)
174
175	def checkout_cbt(self):
176	testdir = misc.get_testdir(self.ctx)
177	repo = self.config.get('repo', 'https://github.com/ceph/cbt.git')
178	branch = self.config.get('branch', 'master')
179	branch = self.config.get('force-branch', branch)
180	sha1 = self.config.get('sha1')
9f95a23c TL	181	if sha1 is None:
	182	self.first_mon.run(
	183	args=[
	184	'git', 'clone', '--depth', '1', '-b', branch, repo,
	185	'{tdir}/cbt'.format(tdir=testdir)
	186	]
	187	)
	188	else:
	189	self.first_mon.run(
	190	args=[
	191	'git', 'clone', '-b', branch, repo,
	192	'{tdir}/cbt'.format(tdir=testdir)
	193	]
	194	)
11fdf7f2 TL	195	self.first_mon.run(
	196	args=[
	197	'cd', os.path.join(testdir, 'cbt'), run.Raw('&&'),
	198	'git', 'reset', '--hard', sha1,
	199	]
	200	)
	201
	202	def setup(self):
	203	super(CBT, self).setup()
e306af50	204	self.first_mon = next(iter(self.ctx.cluster.only(misc.get_first_mon(self.ctx, self.config)).remotes.keys()))
11fdf7f2 TL	205	self.cbt_config = self.generate_cbt_config()
	206	self.log.info('cbt configuration is %s', self.cbt_config)
	207	self.cbt_dir = os.path.join(misc.get_archive_dir(self.ctx), 'cbt')
	208	self.ctx.cluster.run(args=['mkdir', '-p', '-m0755', '--', self.cbt_dir])
	209	misc.write_file(self.first_mon, os.path.join(self.cbt_dir, 'cbt_config.yaml'),
	210	yaml.safe_dump(self.cbt_config, default_flow_style=False))
	211	self.checkout_cbt()
	212	self.install_dependencies()
	213
	214	def begin(self):
	215	super(CBT, self).begin()
	216	testdir = misc.get_testdir(self.ctx)
	217	self.first_mon.run(
	218	args=[
	219	'{tdir}/cbt/cbt.py'.format(tdir=testdir),
	220	'-a', self.cbt_dir,
	221	'{cbtdir}/cbt_config.yaml'.format(cbtdir=self.cbt_dir),
	222	],
	223	)
	224	preserve_file = os.path.join(self.ctx.archive, '.preserve')
	225	open(preserve_file, 'a').close()
	226
	227	def end(self):
	228	super(CBT, self).end()
	229	testdir = misc.get_testdir(self.ctx)
	230	self.first_mon.run(
	231	args=[
	232	'rm', '--one-file-system', '-rf', '--',
	233	'{tdir}/cbt'.format(tdir=testdir),
	234	]
	235	)
e306af50	236	benchmark_type = next(iter(self.cbt_config.get('benchmarks').keys()))
9f95a23c	237	if benchmark_type in ['librbdfio', 'fio']:
11fdf7f2 TL	238	self.first_mon.run(
	239	args=[
	240	'rm', '--one-file-system', '-rf', '--',
	241	'{tdir}/fio'.format(tdir=testdir),
	242	]
	243	)
	244
	245	if benchmark_type == 'cosbench':
	246	os_version = misc.get_system_type(self.first_mon, False, True)
	247	if os_version == '18.04':
	248	cosbench_version = 'cosbench-0.4.2.c3.1'
	249	else:
	250	cosbench_version = '0.4.2.c3'
eafe8130 TL	251	# note: stop-all requires 'nc'
	252	self.first_mon.run(
	253	args=[
	254	'cd', testdir, run.Raw('&&'),
	255	'cd', 'cos', run.Raw('&&'),
	256	'sh', 'stop-all.sh',
	257	run.Raw('\|\|'), 'true'
	258	]
	259	)
	260	self.first_mon.run(
	261	args=[
	262	'sudo', 'killall', '-9', 'java',
	263	run.Raw('\|\|'), 'true'
	264	]
	265	)
11fdf7f2 TL	266	self.first_mon.run(
	267	args=[
	268	'rm', '--one-file-system', '-rf', '--',
	269	'{tdir}/cos'.format(tdir=testdir),
	270	]
	271	)
	272	self.first_mon.run(
	273	args=[
	274	'rm', '--one-file-system', '-rf', '--',
	275	'{tdir}/{version}'.format(tdir=testdir, version=cosbench_version),
	276	]
	277	)
	278	self.first_mon.run(
	279	args=[
	280	'rm', '--one-file-system', '-rf', '--',
	281	'{tdir}/{version}.zip'.format(tdir=testdir, version=cosbench_version),
	282	]
	283	)
	284	self.first_mon.run(
	285	args=[
	286	'rm', '--one-file-system', '-rf', '--',
	287	'{tdir}/xml'.format(tdir=testdir),
	288	]
	289	)
	290
	291
	292	task = CBT