]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/cbt.py
5 from teuthology
import misc
6 from teuthology
.config
import config
as teuth_config
7 from teuthology
.orchestra
import run
8 from teuthology
.task
import Task
10 log
= logging
.getLogger(__name__
)
15 Passes through a CBT configuration yaml fragment.
17 def __init__(self
, ctx
, config
):
18 super(CBT
, self
).__init
__(ctx
, config
)
21 def hosts_of_type(self
, type_
):
22 return [r
.name
for r
in self
.ctx
.cluster
.only(misc
.is_type(type_
)).remotes
.keys()]
24 def generate_cbt_config(self
):
25 mon_hosts
= self
.hosts_of_type('mon')
26 osd_hosts
= self
.hosts_of_type('osd')
27 client_hosts
= self
.hosts_of_type('client')
29 rgw_client
[client_hosts
[0]] = None
30 rgw_hosts
= self
.config
.get('cluster', {}).get('rgws', rgw_client
)
31 cluster_config
= dict(
32 user
=self
.config
.get('cluster', {}).get('user', 'ubuntu'),
38 osds_per_node
=self
.config
.get('cluster', {}).get('osds_per_node', 1),
39 rebuild_every_test
=False,
41 is_teuthology
=self
.config
.get('cluster', {}).get('is_teuthology', True),
42 iterations
=self
.config
.get('cluster', {}).get('iterations', 1),
44 pool_profiles
=self
.config
.get('cluster', {}).get('pool_profiles'),
47 benchmark_config
= self
.config
.get('benchmarks')
48 benchmark_type
= benchmark_config
.keys()[0]
49 if benchmark_type
== 'librbdfio':
50 testdir
= misc
.get_testdir(self
.ctx
)
51 benchmark_config
['librbdfio']['cmd_path'] = os
.path
.join(testdir
, 'fio/fio')
52 if benchmark_type
== 'cosbench':
53 # create cosbench_dir and cosbench_xml_dir
54 testdir
= misc
.get_testdir(self
.ctx
)
55 benchmark_config
['cosbench']['cosbench_dir'] = os
.path
.join(testdir
, 'cos')
56 benchmark_config
['cosbench']['cosbench_xml_dir'] = os
.path
.join(testdir
, 'xml')
57 self
.ctx
.cluster
.run(args
=['mkdir', '-p', '-m0755', '--', benchmark_config
['cosbench']['cosbench_xml_dir']])
58 benchmark_config
['cosbench']['controller'] = osd_hosts
[0]
61 remotes_and_roles
= self
.ctx
.cluster
.remotes
.items()
62 ips
= [host
for (host
, port
) in
63 (remote
.ssh
.get_transport().getpeername() for (remote
, role_list
) in remotes_and_roles
)]
64 benchmark_config
['cosbench']['auth'] = "username=cosbench:operator;password=intel2012;url=http://%s:80/auth/v1.0;retry=9" %(ips
[0])
67 cluster
=cluster_config
,
68 benchmarks
=benchmark_config
,
71 def install_dependencies(self
):
72 system_type
= misc
.get_system_type(self
.first_mon
)
74 if system_type
== 'rpm':
75 install_cmd
= ['sudo', 'yum', '-y', 'install']
76 cbt_depends
= ['python36-PyYAML', 'python36-lxml', 'librbd-devel', 'pdsh', 'collectl']
78 install_cmd
= ['sudo', 'apt-get', '-y', '--force-yes', 'install']
79 cbt_depends
= ['python3-yaml', 'python3-lxml', 'librbd-dev', 'collectl']
80 self
.first_mon
.run(args
=install_cmd
+ cbt_depends
)
82 benchmark_type
= self
.cbt_config
.get('benchmarks').keys()[0]
83 self
.log
.info('benchmark: %s', benchmark_type
)
85 if benchmark_type
== 'librbdfio':
87 testdir
= misc
.get_testdir(self
.ctx
)
90 'git', 'clone', '-b', 'master',
91 'https://github.com/axboe/fio.git',
92 '{tdir}/fio'.format(tdir
=testdir
)
97 'cd', os
.path
.join(testdir
, 'fio'), run
.Raw('&&'),
98 './configure', run
.Raw('&&'),
103 if benchmark_type
== 'cosbench':
105 self
.log
.info('install dependencies for cosbench')
106 if system_type
== 'rpm':
107 cosbench_depends
= ['wget', 'unzip', 'java-1.7.0-openjdk', 'curl']
109 cosbench_depends
= ['wget', 'unzip', 'openjdk-8-jre', 'curl']
110 self
.first_mon
.run(args
=install_cmd
+ cosbench_depends
)
111 testdir
= misc
.get_testdir(self
.ctx
)
112 cosbench_version
= '0.4.2.c3'
113 cosbench_location
= 'https://github.com/intel-cloud/cosbench/releases/download/v0.4.2.c3/0.4.2.c3.zip'
114 os_version
= misc
.get_system_type(self
.first_mon
, False, True)
116 # additional requirements for bionic
117 if os_version
== '18.04':
119 args
=['sudo', 'apt-get', '-y', 'purge', 'openjdk-11*'])
120 # use our own version of cosbench
121 cosbench_version
= 'cosbench-0.4.2.c3.1'
122 # contains additional parameter "-N" to nc
123 cosbench_location
= 'http://drop.ceph.com/qa/cosbench-0.4.2.c3.1.zip'
124 cosbench_dir
= os
.path
.join(testdir
, cosbench_version
)
125 self
.ctx
.cluster
.run(args
=['mkdir', '-p', '-m0755', '--', cosbench_dir
])
128 'cd', testdir
, run
.Raw('&&'),
130 cosbench_location
, run
.Raw('&&'),
131 'unzip', '{name}.zip'.format(name
=cosbench_version
), '-d', cosbench_version
137 'cd', testdir
, run
.Raw('&&'),
139 cosbench_location
, run
.Raw('&&'),
140 'unzip', '{name}.zip'.format(name
=cosbench_version
)
145 'cd', testdir
, run
.Raw('&&'),
146 'ln', '-s', cosbench_version
, 'cos',
151 'cd', os
.path
.join(testdir
, 'cos'), run
.Raw('&&'),
152 'chmod', '+x', run
.Raw('*.sh'),
156 # start cosbench and check info
157 self
.log
.info('start cosbench')
160 'cd', testdir
, run
.Raw('&&'),
161 'cd', 'cos', run
.Raw('&&'),
165 self
.log
.info('check cosbench info')
168 'cd', testdir
, run
.Raw('&&'),
169 'cd', 'cos', run
.Raw('&&'),
170 'sh', 'cli.sh', 'info'
174 def checkout_cbt(self
):
175 testdir
= misc
.get_testdir(self
.ctx
)
176 repo
= self
.config
.get('repo', 'https://github.com/ceph/cbt.git')
177 branch
= self
.config
.get('branch', 'master')
178 branch
= self
.config
.get('force-branch', branch
)
179 sha1
= self
.config
.get('sha1')
182 'git', 'clone', '-b', branch
, repo
,
183 '{tdir}/cbt'.format(tdir
=testdir
)
189 'cd', os
.path
.join(testdir
, 'cbt'), run
.Raw('&&'),
190 'git', 'reset', '--hard', sha1
,
195 super(CBT
, self
).setup()
196 self
.first_mon
= self
.ctx
.cluster
.only(misc
.get_first_mon(self
.ctx
, self
.config
)).remotes
.keys()[0]
197 self
.cbt_config
= self
.generate_cbt_config()
198 self
.log
.info('cbt configuration is %s', self
.cbt_config
)
199 self
.cbt_dir
= os
.path
.join(misc
.get_archive_dir(self
.ctx
), 'cbt')
200 self
.ctx
.cluster
.run(args
=['mkdir', '-p', '-m0755', '--', self
.cbt_dir
])
201 misc
.write_file(self
.first_mon
, os
.path
.join(self
.cbt_dir
, 'cbt_config.yaml'),
202 yaml
.safe_dump(self
.cbt_config
, default_flow_style
=False))
204 self
.install_dependencies()
207 super(CBT
, self
).begin()
208 testdir
= misc
.get_testdir(self
.ctx
)
211 '{tdir}/cbt/cbt.py'.format(tdir
=testdir
),
213 '{cbtdir}/cbt_config.yaml'.format(cbtdir
=self
.cbt_dir
),
216 preserve_file
= os
.path
.join(self
.ctx
.archive
, '.preserve')
217 open(preserve_file
, 'a').close()
220 super(CBT
, self
).end()
221 testdir
= misc
.get_testdir(self
.ctx
)
224 'rm', '--one-file-system', '-rf', '--',
225 '{tdir}/cbt'.format(tdir
=testdir
),
228 benchmark_type
= self
.cbt_config
.get('benchmarks').keys()[0]
229 if benchmark_type
== 'librbdfio':
232 'rm', '--one-file-system', '-rf', '--',
233 '{tdir}/fio'.format(tdir
=testdir
),
237 if benchmark_type
== 'cosbench':
238 os_version
= misc
.get_system_type(self
.first_mon
, False, True)
239 if os_version
== '18.04':
240 cosbench_version
= 'cosbench-0.4.2.c3.1'
242 cosbench_version
= '0.4.2.c3'
243 # note: stop-all requires 'nc'
246 'cd', testdir
, run
.Raw('&&'),
247 'cd', 'cos', run
.Raw('&&'),
249 run
.Raw('||'), 'true'
254 'sudo', 'killall', '-9', 'java',
255 run
.Raw('||'), 'true'
260 'rm', '--one-file-system', '-rf', '--',
261 '{tdir}/cos'.format(tdir
=testdir
),
266 'rm', '--one-file-system', '-rf', '--',
267 '{tdir}/{version}'.format(tdir
=testdir
, version
=cosbench_version
),
272 'rm', '--one-file-system', '-rf', '--',
273 '{tdir}/{version}.zip'.format(tdir
=testdir
, version
=cosbench_version
),
278 'rm', '--one-file-system', '-rf', '--',
279 '{tdir}/xml'.format(tdir
=testdir
),