]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | """ |
2 | Long running fio tests on rbd mapped devices for format/features provided in config | |
3 | Many fio parameters can be configured so that this task can be used along with thrash/power-cut tests | |
4 | and exercise IO on full disk for all format/features | |
5 | - This test should not be run on VM due to heavy use of resource | |
6 | ||
7 | """ | |
8 | import contextlib | |
9 | import json | |
10 | import logging | |
c07f9fc5 | 11 | import os |
7c673cae | 12 | import StringIO |
7c673cae FG |
13 | |
14 | from teuthology.parallel import parallel | |
15 | from teuthology import misc as teuthology | |
16 | from tempfile import NamedTemporaryFile | |
17 | from teuthology.orchestra import run | |
c07f9fc5 | 18 | from teuthology.packaging import install_package, remove_package |
7c673cae FG |
19 | |
20 | log = logging.getLogger(__name__) | |
21 | ||
22 | @contextlib.contextmanager | |
23 | def task(ctx, config): | |
24 | """ | |
25 | client.0: | |
26 | fio-io-size: 100g or 80% or 100m | |
27 | fio-version: 2.2.9 | |
28 | formats: [2] | |
29 | features: [[layering],[striping],[layering,exclusive-lock,object-map]] | |
30 | test-clone-io: 1 #remove this option to not run create rbd clone and not run io on clone | |
31 | io-engine: "sync or rbd or any io-engine" | |
32 | rw: randrw | |
33 | client.1: | |
34 | fio-io-size: 100g | |
35 | fio-version: 2.2.9 | |
36 | rw: read | |
37 | image-size:20480 | |
38 | ||
39 | or | |
40 | all: | |
41 | fio-io-size: 400g | |
42 | rw: randrw | |
43 | formats: [2] | |
44 | features: [[layering],[striping]] | |
45 | io-engine: libaio | |
46 | ||
47 | Create rbd image + device and exercise IO for format/features provided in config file | |
48 | Config can be per client or one config can be used for all clients, fio jobs are run in parallel for client provided | |
49 | ||
50 | """ | |
51 | if config.get('all'): | |
52 | client_config = config['all'] | |
53 | clients = ctx.cluster.only(teuthology.is_type('client')) | |
54 | rbd_test_dir = teuthology.get_testdir(ctx) + "/rbd_fio_test" | |
55 | for remote,role in clients.remotes.iteritems(): | |
56 | if 'client_config' in locals(): | |
57 | with parallel() as p: | |
58 | p.spawn(run_fio, remote, client_config, rbd_test_dir) | |
59 | else: | |
60 | for client_config in config: | |
61 | if client_config in role: | |
62 | with parallel() as p: | |
63 | p.spawn(run_fio, remote, config[client_config], rbd_test_dir) | |
64 | ||
65 | yield | |
66 | ||
67 | ||
c07f9fc5 FG |
68 | def get_ioengine_package_name(ioengine, remote): |
69 | system_type = teuthology.get_system_type(remote) | |
70 | if ioengine == 'rbd': | |
71 | return 'librbd1-devel' if system_type == 'rpm' else 'librbd-dev' | |
72 | elif ioengine == 'libaio': | |
73 | return 'libaio-devel' if system_type == 'rpm' else 'libaio-dev' | |
74 | else: | |
75 | return None | |
76 | ||
77 | ||
78 | def run_rbd_map(remote, image, iodepth): | |
79 | iodepth = max(iodepth, 128) # RBD_QUEUE_DEPTH_DEFAULT | |
80 | out = StringIO.StringIO() | |
11fdf7f2 TL |
81 | remote.run(args=['sudo', 'rbd', 'device', 'map', '-o', |
82 | 'queue_depth={}'.format(iodepth), image], stdout=out) | |
c07f9fc5 FG |
83 | dev = out.getvalue().rstrip('\n') |
84 | teuthology.sudo_write_file( | |
85 | remote, | |
86 | '/sys/block/{}/queue/nr_requests'.format(os.path.basename(dev)), | |
87 | str(iodepth)) | |
88 | return dev | |
89 | ||
90 | ||
7c673cae FG |
91 | def run_fio(remote, config, rbd_test_dir): |
92 | """ | |
93 | create fio config file with options based on above config | |
94 | get the fio from github, generate binary, and use it to run on | |
95 | the generated fio config file | |
96 | """ | |
97 | fio_config=NamedTemporaryFile(prefix='fio_rbd_', dir='/tmp/', delete=False) | |
98 | fio_config.write('[global]\n') | |
99 | if config.get('io-engine'): | |
100 | ioengine=config['io-engine'] | |
101 | fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine)) | |
102 | else: | |
103 | fio_config.write('ioengine=sync\n') | |
104 | if config.get('bs'): | |
105 | bs=config['bs'] | |
106 | fio_config.write('bs={bs}\n'.format(bs=bs)) | |
107 | else: | |
108 | fio_config.write('bs=4k\n') | |
c07f9fc5 FG |
109 | iodepth = config.get('io-depth', 2) |
110 | fio_config.write('iodepth={iod}\n'.format(iod=iodepth)) | |
7c673cae FG |
111 | if config.get('fio-io-size'): |
112 | size=config['fio-io-size'] | |
113 | fio_config.write('size={size}\n'.format(size=size)) | |
114 | else: | |
115 | fio_config.write('size=100m\n') | |
116 | ||
117 | fio_config.write('time_based\n') | |
118 | if config.get('runtime'): | |
119 | runtime=config['runtime'] | |
120 | fio_config.write('runtime={runtime}\n'.format(runtime=runtime)) | |
121 | else: | |
122 | fio_config.write('runtime=1800\n') | |
123 | fio_config.write('allow_file_create=0\n') | |
124 | image_size=10240 | |
125 | if config.get('image_size'): | |
126 | image_size=config['image_size'] | |
127 | ||
128 | formats=[1,2] | |
129 | features=[['layering'],['striping'],['exclusive-lock','object-map']] | |
c07f9fc5 | 130 | fio_version='2.21' |
7c673cae FG |
131 | if config.get('formats'): |
132 | formats=config['formats'] | |
133 | if config.get('features'): | |
134 | features=config['features'] | |
135 | if config.get('fio-version'): | |
136 | fio_version=config['fio-version'] | |
137 | ||
c07f9fc5 | 138 | # handle package required for ioengine, if any |
7c673cae | 139 | sn=remote.shortname |
c07f9fc5 FG |
140 | ioengine_pkg = get_ioengine_package_name(ioengine, remote) |
141 | if ioengine_pkg: | |
142 | install_package(ioengine_pkg, remote) | |
143 | ||
144 | fio_config.write('norandommap\n') | |
7c673cae FG |
145 | if ioengine == 'rbd': |
146 | fio_config.write('clientname=admin\n') | |
147 | fio_config.write('pool=rbd\n') | |
c07f9fc5 FG |
148 | fio_config.write('invalidate=0\n') |
149 | elif ioengine == 'libaio': | |
150 | fio_config.write('direct=1\n') | |
7c673cae FG |
151 | for frmt in formats: |
152 | for feature in features: | |
153 | log.info("Creating rbd images on {sn}".format(sn=sn)) | |
154 | feature_name = '-'.join(feature) | |
155 | rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn) | |
156 | rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn) | |
157 | rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn) | |
158 | create_args=['rbd', 'create', | |
159 | '--size', '{size}'.format(size=image_size), | |
160 | '--image', rbd_name, | |
161 | '--image-format', '{f}'.format(f=frmt)] | |
162 | map(lambda x: create_args.extend(['--image-feature', x]), feature) | |
163 | remote.run(args=create_args) | |
164 | remote.run(args=['rbd', 'info', rbd_name]) | |
165 | if ioengine != 'rbd': | |
c07f9fc5 | 166 | rbd_dev = run_rbd_map(remote, rbd_name, iodepth) |
7c673cae FG |
167 | if config.get('test-clone-io'): |
168 | log.info("Testing clones using fio") | |
169 | remote.run(args=['rbd', 'snap', 'create', rbd_snap_name]) | |
170 | remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name]) | |
171 | remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name]) | |
c07f9fc5 | 172 | rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth) |
7c673cae FG |
173 | fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev)) |
174 | if config.get('rw'): | |
175 | rw=config['rw'] | |
176 | fio_config.write('rw={rw}\n'.format(rw=rw)) | |
177 | else: | |
178 | fio_config .write('rw=randrw\n') | |
179 | fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev)) | |
180 | if config.get('test-clone-io'): | |
181 | fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev)) | |
182 | fio_config.write('rw={rw}\n'.format(rw=rw)) | |
183 | fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev)) | |
184 | else: | |
185 | if config.get('test-clone-io'): | |
186 | log.info("Testing clones using fio") | |
187 | remote.run(args=['rbd', 'snap', 'create', rbd_snap_name]) | |
188 | remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name]) | |
189 | remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name]) | |
190 | fio_config.write('[{img_name}]\n'.format(img_name=rbd_name)) | |
191 | if config.get('rw'): | |
192 | rw=config['rw'] | |
193 | fio_config.write('rw={rw}\n'.format(rw=rw)) | |
194 | else: | |
195 | fio_config.write('rw=randrw\n') | |
196 | fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name)) | |
197 | if config.get('test-clone-io'): | |
198 | fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name)) | |
199 | fio_config.write('rw={rw}\n'.format(rw=rw)) | |
200 | fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name)) | |
201 | ||
202 | ||
203 | fio_config.close() | |
204 | remote.put_file(fio_config.name,fio_config.name) | |
205 | try: | |
206 | log.info("Running rbd feature - fio test on {sn}".format(sn=sn)) | |
207 | fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz" | |
208 | remote.run(args=['mkdir', run.Raw(rbd_test_dir),]) | |
209 | remote.run(args=['cd' , run.Raw(rbd_test_dir), | |
81eedcae TL |
210 | run.Raw(';'), 'wget', fio, run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'), |
211 | run.Raw('cd fio-fio*'), run.Raw(';'), './configure', run.Raw(';'), 'make']) | |
7c673cae | 212 | remote.run(args=['ceph', '-s']) |
c07f9fc5 | 213 | remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))]) |
7c673cae FG |
214 | remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))]) |
215 | remote.run(args=['ceph', '-s']) | |
216 | finally: | |
217 | out=StringIO.StringIO() | |
11fdf7f2 | 218 | remote.run(args=['rbd', 'device', 'list', '--format=json'], stdout=out) |
7c673cae FG |
219 | mapped_images = json.loads(out.getvalue()) |
220 | if mapped_images: | |
221 | log.info("Unmapping rbd images on {sn}".format(sn=sn)) | |
11fdf7f2 TL |
222 | for image in mapped_images: |
223 | remote.run(args=['sudo', 'rbd', 'device', 'unmap', | |
224 | str(image['device'])]) | |
7c673cae FG |
225 | log.info("Cleaning up fio install") |
226 | remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)]) | |
c07f9fc5 FG |
227 | if ioengine_pkg: |
228 | remove_package(ioengine_pkg, remote) |