ceph/qa/tasks/rbd_fio.py

   1 """
   2  Long running fio tests on rbd mapped devices for format/features provided in config
   3  Many fio parameters can be configured so that this task can be used along with thrash/power-cut tests
   4  and exercise IO on full disk for all format/features
   5   - This test should not be run on VM due to heavy use of resource
   6
   7 """
   8 import contextlib
   9 import json
  10 import logging
  11 import os
  12
  13 from teuthology.parallel import parallel
  14 from teuthology import misc as teuthology
  15 from tempfile import NamedTemporaryFile
  16 from teuthology.orchestra import run
  17 from teuthology.packaging import install_package, remove_package
  18
  19 log = logging.getLogger(__name__)
  20
  21 @contextlib.contextmanager
  22 def task(ctx, config):
  23     """
  24     client.0:
  25        fio-io-size: 100g or 80% or 100m
  26        fio-version: 2.2.9
  27        formats: [2]
  28        features: [[layering],[striping],[layering,exclusive-lock,object-map]]
  29        test-clone-io: 1  #remove this option to not run create rbd clone and not run io on clone
  30        io-engine: "sync or rbd or any io-engine"
  31        rw: randrw
  32     client.1:
  33        fio-io-size: 100g
  34        fio-version: 2.2.9
  35        rw: read
  36        image-size:20480
  37
  38 or
  39     all:
  40        fio-io-size: 400g
  41        rw: randrw
  42        formats: [2]
  43        features: [[layering],[striping]]
  44        io-engine: libaio
  45
  46     Create rbd image + device and exercise IO for format/features provided in config file
  47     Config can be per client or one config can be used for all clients, fio jobs are run in parallel for client provided
  48
  49     """
  50     if config.get('all'):
  51         client_config = config['all']
  52     clients = ctx.cluster.only(teuthology.is_type('client'))
  53     rbd_test_dir = teuthology.get_testdir(ctx) + "/rbd_fio_test"
  54     for remote,role in clients.remotes.items():
  55         if 'client_config' in locals():
  56            with parallel() as p:
  57                p.spawn(run_fio, remote, client_config, rbd_test_dir)
  58         else:
  59            for client_config in config:
  60               if client_config in role:
  61                  with parallel() as p:
  62                      p.spawn(run_fio, remote, config[client_config], rbd_test_dir)
  63
  64     yield
  65
  66
  67 def get_ioengine_package_name(ioengine, remote):
  68     system_type = teuthology.get_system_type(remote)
  69     if ioengine == 'rbd':
  70         return 'librbd1-devel' if system_type == 'rpm' else 'librbd-dev'
  71     elif ioengine == 'libaio':
  72         return 'libaio-devel' if system_type == 'rpm' else 'libaio-dev'
  73     else:
  74         return None
  75
  76
  77 def run_rbd_map(remote, image, iodepth):
  78     iodepth = max(iodepth, 128)  # RBD_QUEUE_DEPTH_DEFAULT
  79     dev = remote.sh(['sudo', 'rbd', 'device', 'map', '-o',
  80                      'queue_depth={}'.format(iodepth), image]).rstrip('\n')
  81     teuthology.sudo_write_file(
  82         remote,
  83         '/sys/block/{}/queue/nr_requests'.format(os.path.basename(dev)),
  84         str(iodepth))
  85     return dev
  86
  87
  88 def run_fio(remote, config, rbd_test_dir):
  89     """
  90     create fio config file with options based on above config
  91     get the fio from github, generate binary, and use it to run on
  92     the generated fio config file
  93     """
  94     fio_config=NamedTemporaryFile(mode='w', prefix='fio_rbd_', dir='/tmp/', delete=False)
  95     fio_config.write('[global]\n')
  96     if config.get('io-engine'):
  97         ioengine=config['io-engine']
  98         fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine))
  99     else:
 100         fio_config.write('ioengine=sync\n')
 101     if config.get('bs'):
 102         bs=config['bs']
 103         fio_config.write('bs={bs}\n'.format(bs=bs))
 104     else:
 105         fio_config.write('bs=4k\n')
 106     iodepth = config.get('io-depth', 2)
 107     fio_config.write('iodepth={iod}\n'.format(iod=iodepth))
 108     if config.get('fio-io-size'):
 109         size=config['fio-io-size']
 110         fio_config.write('size={size}\n'.format(size=size))
 111     else:
 112         fio_config.write('size=100m\n')
 113
 114     fio_config.write('time_based\n')
 115     if config.get('runtime'):
 116         runtime=config['runtime']
 117         fio_config.write('runtime={runtime}\n'.format(runtime=runtime))
 118     else:
 119         fio_config.write('runtime=1800\n')
 120     fio_config.write('allow_file_create=0\n')
 121     image_size=10240
 122     if config.get('image_size'):
 123         image_size=config['image_size']
 124
 125     formats=[1,2]
 126     features=[['layering'],['striping'],['exclusive-lock','object-map']]
 127     fio_version='3.16'
 128     if config.get('formats'):
 129         formats=config['formats']
 130     if config.get('features'):
 131         features=config['features']
 132     if config.get('fio-version'):
 133         fio_version=config['fio-version']
 134
 135     # handle package required for ioengine, if any
 136     sn=remote.shortname
 137     ioengine_pkg = get_ioengine_package_name(ioengine, remote)
 138     if ioengine_pkg:
 139         install_package(ioengine_pkg, remote)
 140
 141     fio_config.write('norandommap\n')
 142     if ioengine == 'rbd':
 143         fio_config.write('clientname=admin\n')
 144         fio_config.write('pool=rbd\n')
 145         fio_config.write('invalidate=0\n')
 146     elif ioengine == 'libaio':
 147         fio_config.write('direct=1\n')
 148     for frmt in formats:
 149         for feature in features:
 150            log.info("Creating rbd images on {sn}".format(sn=sn))
 151            feature_name = '-'.join(feature)
 152            rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn)
 153            rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn)
 154            rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn)
 155            create_args=['rbd', 'create',
 156                         '--size', '{size}'.format(size=image_size),
 157                         '--image', rbd_name,
 158                         '--image-format', '{f}'.format(f=frmt)]
 159            map(lambda x: create_args.extend(['--image-feature', x]), feature)
 160            if config.get('thick-provision'):
 161                create_args.append('--thick-provision')
 162            remote.run(args=create_args)
 163            remote.run(args=['rbd', 'info', rbd_name])
 164            if ioengine != 'rbd':
 165                rbd_dev = run_rbd_map(remote, rbd_name, iodepth)
 166                if config.get('test-clone-io'):
 167                     log.info("Testing clones using fio")
 168                     remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
 169                     remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
 170                     remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
 171                     rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth)
 172                fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev))
 173                if config.get('rw'):
 174                    rw=config['rw']
 175                    fio_config.write('rw={rw}\n'.format(rw=rw))
 176                else:
 177                    fio_config .write('rw=randrw\n')
 178                fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev))
 179                if config.get('test-clone-io'):
 180                    fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev))
 181                    fio_config.write('rw={rw}\n'.format(rw=rw))
 182                    fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev))
 183            else:
 184                if config.get('test-clone-io'):
 185                     log.info("Testing clones using fio")
 186                     remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
 187                     remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
 188                     remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
 189                fio_config.write('[{img_name}]\n'.format(img_name=rbd_name))
 190                if config.get('rw'):
 191                    rw=config['rw']
 192                    fio_config.write('rw={rw}\n'.format(rw=rw))
 193                else:
 194                    fio_config.write('rw=randrw\n')
 195                fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name))
 196                if config.get('test-clone-io'):
 197                    fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name))
 198                    fio_config.write('rw={rw}\n'.format(rw=rw))
 199                    fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name))
 200
 201
 202     fio_config.close()
 203     remote.put_file(fio_config.name,fio_config.name)
 204     try:
 205         log.info("Running rbd feature - fio test on {sn}".format(sn=sn))
 206         fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz"
 207         remote.run(args=['mkdir', run.Raw(rbd_test_dir),])
 208         remote.run(args=['cd' , run.Raw(rbd_test_dir),
 209                          run.Raw(';'), 'wget', fio, run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'),
 210                          run.Raw('cd fio-fio*'), run.Raw(';'), './configure', run.Raw(';'), 'make'])
 211         remote.run(args=['ceph', '-s'])
 212         remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
 213         remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
 214         remote.run(args=['ceph', '-s'])
 215     finally:
 216         out = remote.sh('rbd device list --format=json')
 217         mapped_images = json.loads(out)
 218         if mapped_images:
 219             log.info("Unmapping rbd images on {sn}".format(sn=sn))
 220             for image in mapped_images:
 221                 remote.run(args=['sudo', 'rbd', 'device', 'unmap',
 222                                  str(image['device'])])
 223         log.info("Cleaning up fio install")
 224         remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)])
 225         if ioengine_pkg:
 226             remove_package(ioengine_pkg, remote)