]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/rbd_fio.py
2 Long running fio tests on rbd mapped devices for format/features provided in config
3 Many fio parameters can be configured so that this task can be used along with thrash/power-cut tests
4 and exercise IO on full disk for all format/features
5 - This test should not be run on VM due to heavy use of resource
13 from teuthology
.parallel
import parallel
14 from teuthology
import misc
as teuthology
15 from tempfile
import NamedTemporaryFile
16 from teuthology
.orchestra
import run
17 from teuthology
.packaging
import install_package
, remove_package
19 log
= logging
.getLogger(__name__
)
21 @contextlib.contextmanager
22 def task(ctx
, config
):
25 fio-io-size: 100g or 80% or 100m
28 features: [[layering],[striping],[layering,exclusive-lock,object-map]]
29 test-clone-io: 1 #remove this option to not run create rbd clone and not run io on clone
30 io-engine: "sync or rbd or any io-engine"
43 features: [[layering],[striping]]
46 Create rbd image + device and exercise IO for format/features provided in config file
47 Config can be per client or one config can be used for all clients, fio jobs are run in parallel for client provided
51 client_config
= config
['all']
52 clients
= ctx
.cluster
.only(teuthology
.is_type('client'))
53 rbd_test_dir
= teuthology
.get_testdir(ctx
) + "/rbd_fio_test"
54 for remote
,role
in clients
.remotes
.items():
55 if 'client_config' in locals():
57 p
.spawn(run_fio
, remote
, client_config
, rbd_test_dir
)
59 for client_config
in config
:
60 if client_config
in role
:
62 p
.spawn(run_fio
, remote
, config
[client_config
], rbd_test_dir
)
67 def get_ioengine_package_name(ioengine
, remote
):
68 system_type
= teuthology
.get_system_type(remote
)
70 return 'librbd1-devel' if system_type
== 'rpm' else 'librbd-dev'
71 elif ioengine
== 'libaio':
72 return 'libaio-devel' if system_type
== 'rpm' else 'libaio-dev'
77 def run_rbd_map(remote
, image
, iodepth
):
78 iodepth
= max(iodepth
, 128) # RBD_QUEUE_DEPTH_DEFAULT
79 dev
= remote
.sh(['sudo', 'rbd', 'device', 'map', '-o',
80 'queue_depth={}'.format(iodepth
), image
]).rstrip('\n')
81 remote
.sudo_write_file(
82 '/sys/block/{}/queue/nr_requests'.format(os
.path
.basename(dev
)),
87 def run_fio(remote
, config
, rbd_test_dir
):
89 create fio config file with options based on above config
90 get the fio from github, generate binary, and use it to run on
91 the generated fio config file
93 fio_config
=NamedTemporaryFile(mode
='w', prefix
='fio_rbd_', dir='/tmp/', delete
=False)
94 fio_config
.write('[global]\n')
95 if config
.get('io-engine'):
96 ioengine
=config
['io-engine']
97 fio_config
.write('ioengine={ioe}\n'.format(ioe
=ioengine
))
99 fio_config
.write('ioengine=sync\n')
102 fio_config
.write('bs={bs}\n'.format(bs
=bs
))
104 fio_config
.write('bs=4k\n')
105 iodepth
= config
.get('io-depth', 2)
106 fio_config
.write('iodepth={iod}\n'.format(iod
=iodepth
))
107 if config
.get('fio-io-size'):
108 size
=config
['fio-io-size']
109 fio_config
.write('size={size}\n'.format(size
=size
))
111 fio_config
.write('size=100m\n')
113 fio_config
.write('time_based\n')
114 if config
.get('runtime'):
115 runtime
=config
['runtime']
116 fio_config
.write('runtime={runtime}\n'.format(runtime
=runtime
))
118 fio_config
.write('runtime=1800\n')
119 fio_config
.write('allow_file_create=0\n')
121 if config
.get('image_size'):
122 image_size
=config
['image_size']
125 features
=[['layering'],['striping'],['exclusive-lock','object-map']]
127 if config
.get('formats'):
128 formats
=config
['formats']
129 if config
.get('features'):
130 features
=config
['features']
131 if config
.get('fio-version'):
132 fio_version
=config
['fio-version']
134 # handle package required for ioengine, if any
136 ioengine_pkg
= get_ioengine_package_name(ioengine
, remote
)
138 install_package(ioengine_pkg
, remote
)
140 fio_config
.write('norandommap\n')
141 if ioengine
== 'rbd':
142 fio_config
.write('clientname=admin\n')
143 fio_config
.write('pool=rbd\n')
144 fio_config
.write('invalidate=0\n')
145 elif ioengine
== 'libaio':
146 fio_config
.write('direct=1\n')
148 for feature
in features
:
149 log
.info("Creating rbd images on {sn}".format(sn
=sn
))
150 feature_name
= '-'.join(feature
)
151 rbd_name
= 'i{i}f{f}{sn}'.format(i
=frmt
,f
=feature_name
,sn
=sn
)
152 rbd_snap_name
= 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i
=frmt
,f
=feature_name
,sn
=sn
)
153 rbd_clone_name
= 'i{i}f{f}{sn}Clone'.format(i
=frmt
,f
=feature_name
,sn
=sn
)
154 create_args
=['rbd', 'create',
155 '--size', '{size}'.format(size
=image_size
),
157 '--image-format', '{f}'.format(f
=frmt
)]
158 map(lambda x
: create_args
.extend(['--image-feature', x
]), feature
)
159 if config
.get('thick-provision'):
160 create_args
.append('--thick-provision')
161 remote
.run(args
=create_args
)
162 remote
.run(args
=['rbd', 'info', rbd_name
])
163 if ioengine
!= 'rbd':
164 rbd_dev
= run_rbd_map(remote
, rbd_name
, iodepth
)
165 if config
.get('test-clone-io'):
166 log
.info("Testing clones using fio")
167 remote
.run(args
=['rbd', 'snap', 'create', rbd_snap_name
])
168 remote
.run(args
=['rbd', 'snap', 'protect', rbd_snap_name
])
169 remote
.run(args
=['rbd', 'clone', rbd_snap_name
, rbd_clone_name
])
170 rbd_clone_dev
= run_rbd_map(remote
, rbd_clone_name
, iodepth
)
171 fio_config
.write('[{rbd_dev}]\n'.format(rbd_dev
=rbd_dev
))
174 fio_config
.write('rw={rw}\n'.format(rw
=rw
))
176 fio_config
.write('rw=randrw\n')
177 fio_config
.write('filename={rbd_dev}\n'.format(rbd_dev
=rbd_dev
))
178 if config
.get('test-clone-io'):
179 fio_config
.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev
=rbd_clone_dev
))
180 fio_config
.write('rw={rw}\n'.format(rw
=rw
))
181 fio_config
.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev
=rbd_clone_dev
))
183 if config
.get('test-clone-io'):
184 log
.info("Testing clones using fio")
185 remote
.run(args
=['rbd', 'snap', 'create', rbd_snap_name
])
186 remote
.run(args
=['rbd', 'snap', 'protect', rbd_snap_name
])
187 remote
.run(args
=['rbd', 'clone', rbd_snap_name
, rbd_clone_name
])
188 fio_config
.write('[{img_name}]\n'.format(img_name
=rbd_name
))
191 fio_config
.write('rw={rw}\n'.format(rw
=rw
))
193 fio_config
.write('rw=randrw\n')
194 fio_config
.write('rbdname={img_name}\n'.format(img_name
=rbd_name
))
195 if config
.get('test-clone-io'):
196 fio_config
.write('[{clone_img_name}]\n'.format(clone_img_name
=rbd_clone_name
))
197 fio_config
.write('rw={rw}\n'.format(rw
=rw
))
198 fio_config
.write('rbdname={clone_img_name}\n'.format(clone_img_name
=rbd_clone_name
))
202 remote
.put_file(fio_config
.name
,fio_config
.name
)
204 log
.info("Running rbd feature - fio test on {sn}".format(sn
=sn
))
205 fio
= "https://github.com/axboe/fio/archive/fio-" + fio_version
+ ".tar.gz"
206 remote
.run(args
=['mkdir', run
.Raw(rbd_test_dir
),])
207 remote
.run(args
=['cd' , run
.Raw(rbd_test_dir
),
208 run
.Raw(';'), 'wget', fio
, run
.Raw(';'), run
.Raw('tar -xvf fio*tar.gz'), run
.Raw(';'),
209 run
.Raw('cd fio-fio*'), run
.Raw(';'), './configure', run
.Raw(';'), 'make'])
210 remote
.run(args
=['ceph', '-s'])
211 remote
.run(args
=[run
.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir
=rbd_test_dir
,v
=fio_version
,f
=fio_config
.name
))])
212 remote
.run(args
=['sudo', run
.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir
=rbd_test_dir
,v
=fio_version
,f
=fio_config
.name
))])
213 remote
.run(args
=['ceph', '-s'])
215 out
= remote
.sh('rbd device list --format=json')
216 mapped_images
= json
.loads(out
)
218 log
.info("Unmapping rbd images on {sn}".format(sn
=sn
))
219 for image
in mapped_images
:
220 remote
.run(args
=['sudo', 'rbd', 'device', 'unmap',
221 str(image
['device'])])
222 log
.info("Cleaning up fio install")
223 remote
.run(args
=['rm','-rf', run
.Raw(rbd_test_dir
)])
225 remove_package(ioengine_pkg
, remote
)