ceph/qa/tasks/rbd_fio.py

   1 """
   2  Long running fio tests on rbd mapped devices for format/features provided in config
   3  Many fio parameters can be configured so that this task can be used along with thrash/power-cut tests
   4  and exercise IO on full disk for all format/features
   5   - This test should not be run on VM due to heavy use of resource
   6
   7 """
   8 import contextlib
   9 import json
  10 import logging
  11 import os
  12
  13 from teuthology.parallel import parallel
  14 from teuthology import misc as teuthology
  15 from tempfile import NamedTemporaryFile
  16 from teuthology.orchestra import run
  17 from teuthology.packaging import install_package, remove_package
  18
  19 log = logging.getLogger(__name__)
  20
  21 @contextlib.contextmanager
  22 def task(ctx, config):
  23     """
  24     client.0:
  25        fio-io-size: 100g or 80% or 100m
  26        fio-version: 2.2.9
  27        formats: [2]
  28        features: [[layering],[striping],[layering,exclusive-lock,object-map]]
  29        test-clone-io: 1  #remove this option to not run create rbd clone and not run io on clone
  30        io-engine: "sync or rbd or any io-engine"
  31        rw: randrw
  32     client.1:
  33        fio-io-size: 100g
  34        fio-version: 2.2.9
  35        rw: read
  36        image-size:20480
  37
  38 or
  39     all:
  40        fio-io-size: 400g
  41        rw: randrw
  42        formats: [2]
  43        features: [[layering],[striping]]
  44        io-engine: libaio
  45
  46     Create rbd image + device and exercise IO for format/features provided in config file
  47     Config can be per client or one config can be used for all clients, fio jobs are run in parallel for client provided
  48
  49     """
  50     if config.get('all'):
  51         client_config = config['all']
  52     clients = ctx.cluster.only(teuthology.is_type('client'))
  53     rbd_test_dir = teuthology.get_testdir(ctx) + "/rbd_fio_test"
  54     for remote,role in clients.remotes.items():
  55         if 'client_config' in locals():
  56            with parallel() as p:
  57                p.spawn(run_fio, remote, client_config, rbd_test_dir)
  58         else:
  59            for client_config in config:
  60               if client_config in role:
  61                  with parallel() as p:
  62                      p.spawn(run_fio, remote, config[client_config], rbd_test_dir)
  63
  64     yield
  65
  66
  67 def get_ioengine_package_name(ioengine, remote):
  68     system_type = teuthology.get_system_type(remote)
  69     if ioengine == 'rbd':
  70         return 'librbd1-devel' if system_type == 'rpm' else 'librbd-dev'
  71     elif ioengine == 'libaio':
  72         return 'libaio-devel' if system_type == 'rpm' else 'libaio-dev'
  73     else:
  74         return None
  75
  76
  77 def run_rbd_map(remote, image, iodepth):
  78     iodepth = max(iodepth, 128)  # RBD_QUEUE_DEPTH_DEFAULT
  79     dev = remote.sh(['sudo', 'rbd', 'device', 'map', '-o',
  80                      'queue_depth={}'.format(iodepth), image]).rstrip('\n')
  81     remote.sudo_write_file(
  82         '/sys/block/{}/queue/nr_requests'.format(os.path.basename(dev)),
  83         str(iodepth))
  84     return dev
  85
  86
  87 def run_fio(remote, config, rbd_test_dir):
  88     """
  89     create fio config file with options based on above config
  90     get the fio from github, generate binary, and use it to run on
  91     the generated fio config file
  92     """
  93     fio_config=NamedTemporaryFile(mode='w', prefix='fio_rbd_', dir='/tmp/', delete=False)
  94     fio_config.write('[global]\n')
  95     if config.get('io-engine'):
  96         ioengine=config['io-engine']
  97         fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine))
  98     else:
  99         fio_config.write('ioengine=sync\n')
 100     if config.get('bs'):
 101         bs=config['bs']
 102         fio_config.write('bs={bs}\n'.format(bs=bs))
 103     else:
 104         fio_config.write('bs=4k\n')
 105     iodepth = config.get('io-depth', 2)
 106     fio_config.write('iodepth={iod}\n'.format(iod=iodepth))
 107     if config.get('fio-io-size'):
 108         size=config['fio-io-size']
 109         fio_config.write('size={size}\n'.format(size=size))
 110     else:
 111         fio_config.write('size=100m\n')
 112
 113     fio_config.write('time_based\n')
 114     if config.get('runtime'):
 115         runtime=config['runtime']
 116         fio_config.write('runtime={runtime}\n'.format(runtime=runtime))
 117     else:
 118         fio_config.write('runtime=1800\n')
 119     fio_config.write('allow_file_create=0\n')
 120     image_size=10240
 121     if config.get('image_size'):
 122         image_size=config['image_size']
 123
 124     formats=[1,2]
 125     features=[['layering'],['striping'],['exclusive-lock','object-map']]
 126     fio_version='3.16'
 127     if config.get('formats'):
 128         formats=config['formats']
 129     if config.get('features'):
 130         features=config['features']
 131     if config.get('fio-version'):
 132         fio_version=config['fio-version']
 133
 134     # handle package required for ioengine, if any
 135     sn=remote.shortname
 136     ioengine_pkg = get_ioengine_package_name(ioengine, remote)
 137     if ioengine_pkg:
 138         install_package(ioengine_pkg, remote)
 139
 140     fio_config.write('norandommap\n')
 141     if ioengine == 'rbd':
 142         fio_config.write('clientname=admin\n')
 143         fio_config.write('pool=rbd\n')
 144         fio_config.write('invalidate=0\n')
 145     elif ioengine == 'libaio':
 146         fio_config.write('direct=1\n')
 147     for frmt in formats:
 148         for feature in features:
 149            log.info("Creating rbd images on {sn}".format(sn=sn))
 150            feature_name = '-'.join(feature)
 151            rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn)
 152            rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn)
 153            rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn)
 154            create_args=['rbd', 'create',
 155                         '--size', '{size}'.format(size=image_size),
 156                         '--image', rbd_name,
 157                         '--image-format', '{f}'.format(f=frmt)]
 158            map(lambda x: create_args.extend(['--image-feature', x]), feature)
 159            if config.get('thick-provision'):
 160                create_args.append('--thick-provision')
 161            remote.run(args=create_args)
 162            remote.run(args=['rbd', 'info', rbd_name])
 163            if ioengine != 'rbd':
 164                rbd_dev = run_rbd_map(remote, rbd_name, iodepth)
 165                if config.get('test-clone-io'):
 166                     log.info("Testing clones using fio")
 167                     remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
 168                     remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
 169                     remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
 170                     rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth)
 171                fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev))
 172                if config.get('rw'):
 173                    rw=config['rw']
 174                    fio_config.write('rw={rw}\n'.format(rw=rw))
 175                else:
 176                    fio_config .write('rw=randrw\n')
 177                fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev))
 178                if config.get('test-clone-io'):
 179                    fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev))
 180                    fio_config.write('rw={rw}\n'.format(rw=rw))
 181                    fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev))
 182            else:
 183                if config.get('test-clone-io'):
 184                     log.info("Testing clones using fio")
 185                     remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
 186                     remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
 187                     remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
 188                fio_config.write('[{img_name}]\n'.format(img_name=rbd_name))
 189                if config.get('rw'):
 190                    rw=config['rw']
 191                    fio_config.write('rw={rw}\n'.format(rw=rw))
 192                else:
 193                    fio_config.write('rw=randrw\n')
 194                fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name))
 195                if config.get('test-clone-io'):
 196                    fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name))
 197                    fio_config.write('rw={rw}\n'.format(rw=rw))
 198                    fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name))
 199
 200
 201     fio_config.close()
 202     remote.put_file(fio_config.name,fio_config.name)
 203     try:
 204         log.info("Running rbd feature - fio test on {sn}".format(sn=sn))
 205         fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz"
 206         remote.run(args=['mkdir', run.Raw(rbd_test_dir),])
 207         remote.run(args=['cd' , run.Raw(rbd_test_dir),
 208                          run.Raw(';'), 'wget', fio, run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'),
 209                          run.Raw('cd fio-fio*'), run.Raw(';'), './configure', run.Raw(';'), 'make'])
 210         remote.run(args=['ceph', '-s'])
 211         remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
 212         remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
 213         remote.run(args=['ceph', '-s'])
 214     finally:
 215         out = remote.sh('rbd device list --format=json')
 216         mapped_images = json.loads(out)
 217         if mapped_images:
 218             log.info("Unmapping rbd images on {sn}".format(sn=sn))
 219             for image in mapped_images:
 220                 remote.run(args=['sudo', 'rbd', 'device', 'unmap',
 221                                  str(image['device'])])
 222         log.info("Cleaning up fio install")
 223         remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)])
 224         if ioengine_pkg:
 225             remove_package(ioengine_pkg, remote)