ceph/qa/tasks/rbd_fio.py

   1 """
   2  Long running fio tests on rbd mapped devices for format/features provided in config
   3  Many fio parameters can be configured so that this task can be used along with thrash/power-cut tests
   4  and exercise IO on full disk for all format/features
   5   - This test should not be run on VM due to heavy use of resource
   6
   7 """
   8 import contextlib
   9 import json
  10 import logging
  11 import os
  12 import StringIO
  13
  14 from teuthology.parallel import parallel
  15 from teuthology import misc as teuthology
  16 from tempfile import NamedTemporaryFile
  17 from teuthology.orchestra import run
  18 from teuthology.packaging import install_package, remove_package
  19
  20 log = logging.getLogger(__name__)
  21
  22 @contextlib.contextmanager
  23 def task(ctx, config):
  24     """
  25     client.0:
  26        fio-io-size: 100g or 80% or 100m
  27        fio-version: 2.2.9
  28        formats: [2]
  29        features: [[layering],[striping],[layering,exclusive-lock,object-map]]
  30        test-clone-io: 1  #remove this option to not run create rbd clone and not run io on clone
  31        io-engine: "sync or rbd or any io-engine"
  32        rw: randrw
  33     client.1:
  34        fio-io-size: 100g
  35        fio-version: 2.2.9
  36        rw: read
  37        image-size:20480
  38
  39 or
  40     all:
  41        fio-io-size: 400g
  42        rw: randrw
  43        formats: [2]
  44        features: [[layering],[striping]]
  45        io-engine: libaio
  46
  47     Create rbd image + device and exercise IO for format/features provided in config file
  48     Config can be per client or one config can be used for all clients, fio jobs are run in parallel for client provided
  49
  50     """
  51     if config.get('all'):
  52         client_config = config['all']
  53     clients = ctx.cluster.only(teuthology.is_type('client'))
  54     rbd_test_dir = teuthology.get_testdir(ctx) + "/rbd_fio_test"
  55     for remote,role in clients.remotes.items():
  56         if 'client_config' in locals():
  57            with parallel() as p:
  58                p.spawn(run_fio, remote, client_config, rbd_test_dir)
  59         else:
  60            for client_config in config:
  61               if client_config in role:
  62                  with parallel() as p:
  63                      p.spawn(run_fio, remote, config[client_config], rbd_test_dir)
  64
  65     yield
  66
  67
  68 def get_ioengine_package_name(ioengine, remote):
  69     system_type = teuthology.get_system_type(remote)
  70     if ioengine == 'rbd':
  71         return 'librbd1-devel' if system_type == 'rpm' else 'librbd-dev'
  72     elif ioengine == 'libaio':
  73         return 'libaio-devel' if system_type == 'rpm' else 'libaio-dev'
  74     else:
  75         return None
  76
  77
  78 def run_rbd_map(remote, image, iodepth):
  79     iodepth = max(iodepth, 128)  # RBD_QUEUE_DEPTH_DEFAULT
  80     out = StringIO.StringIO()
  81     remote.run(args=['sudo', 'rbd', 'device', 'map', '-o',
  82                      'queue_depth={}'.format(iodepth), image], stdout=out)
  83     dev = out.getvalue().rstrip('\n')
  84     teuthology.sudo_write_file(
  85         remote,
  86         '/sys/block/{}/queue/nr_requests'.format(os.path.basename(dev)),
  87         str(iodepth))
  88     return dev
  89
  90
  91 def run_fio(remote, config, rbd_test_dir):
  92     """
  93     create fio config file with options based on above config
  94     get the fio from github, generate binary, and use it to run on
  95     the generated fio config file
  96     """
  97     fio_config=NamedTemporaryFile(prefix='fio_rbd_', dir='/tmp/', delete=False)
  98     fio_config.write('[global]\n')
  99     if config.get('io-engine'):
 100         ioengine=config['io-engine']
 101         fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine))
 102     else:
 103         fio_config.write('ioengine=sync\n')
 104     if config.get('bs'):
 105         bs=config['bs']
 106         fio_config.write('bs={bs}\n'.format(bs=bs))
 107     else:
 108         fio_config.write('bs=4k\n')
 109     iodepth = config.get('io-depth', 2)
 110     fio_config.write('iodepth={iod}\n'.format(iod=iodepth))
 111     if config.get('fio-io-size'):
 112         size=config['fio-io-size']
 113         fio_config.write('size={size}\n'.format(size=size))
 114     else:
 115         fio_config.write('size=100m\n')
 116
 117     fio_config.write('time_based\n')
 118     if config.get('runtime'):
 119         runtime=config['runtime']
 120         fio_config.write('runtime={runtime}\n'.format(runtime=runtime))
 121     else:
 122         fio_config.write('runtime=1800\n')
 123     fio_config.write('allow_file_create=0\n')
 124     image_size=10240
 125     if config.get('image_size'):
 126         image_size=config['image_size']
 127
 128     formats=[1,2]
 129     features=[['layering'],['striping'],['exclusive-lock','object-map']]
 130     fio_version='3.16'
 131     if config.get('formats'):
 132         formats=config['formats']
 133     if config.get('features'):
 134         features=config['features']
 135     if config.get('fio-version'):
 136         fio_version=config['fio-version']
 137
 138     # handle package required for ioengine, if any
 139     sn=remote.shortname
 140     ioengine_pkg = get_ioengine_package_name(ioengine, remote)
 141     if ioengine_pkg:
 142         install_package(ioengine_pkg, remote)
 143
 144     fio_config.write('norandommap\n')
 145     if ioengine == 'rbd':
 146         fio_config.write('clientname=admin\n')
 147         fio_config.write('pool=rbd\n')
 148         fio_config.write('invalidate=0\n')
 149     elif ioengine == 'libaio':
 150         fio_config.write('direct=1\n')
 151     for frmt in formats:
 152         for feature in features:
 153            log.info("Creating rbd images on {sn}".format(sn=sn))
 154            feature_name = '-'.join(feature)
 155            rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn)
 156            rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn)
 157            rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn)
 158            create_args=['rbd', 'create',
 159                         '--size', '{size}'.format(size=image_size),
 160                         '--image', rbd_name,
 161                         '--image-format', '{f}'.format(f=frmt)]
 162            map(lambda x: create_args.extend(['--image-feature', x]), feature)
 163            remote.run(args=create_args)
 164            remote.run(args=['rbd', 'info', rbd_name])
 165            if ioengine != 'rbd':
 166                rbd_dev = run_rbd_map(remote, rbd_name, iodepth)
 167                if config.get('test-clone-io'):
 168                     log.info("Testing clones using fio")
 169                     remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
 170                     remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
 171                     remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
 172                     rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth)
 173                fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev))
 174                if config.get('rw'):
 175                    rw=config['rw']
 176                    fio_config.write('rw={rw}\n'.format(rw=rw))
 177                else:
 178                    fio_config .write('rw=randrw\n')
 179                fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev))
 180                if config.get('test-clone-io'):
 181                    fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev))
 182                    fio_config.write('rw={rw}\n'.format(rw=rw))
 183                    fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev))
 184            else:
 185                if config.get('test-clone-io'):
 186                     log.info("Testing clones using fio")
 187                     remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
 188                     remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
 189                     remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
 190                fio_config.write('[{img_name}]\n'.format(img_name=rbd_name))
 191                if config.get('rw'):
 192                    rw=config['rw']
 193                    fio_config.write('rw={rw}\n'.format(rw=rw))
 194                else:
 195                    fio_config.write('rw=randrw\n')
 196                fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name))
 197                if config.get('test-clone-io'):
 198                    fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name))
 199                    fio_config.write('rw={rw}\n'.format(rw=rw))
 200                    fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name))
 201
 202
 203     fio_config.close()
 204     remote.put_file(fio_config.name,fio_config.name)
 205     try:
 206         log.info("Running rbd feature - fio test on {sn}".format(sn=sn))
 207         fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz"
 208         remote.run(args=['mkdir', run.Raw(rbd_test_dir),])
 209         remote.run(args=['cd' , run.Raw(rbd_test_dir),
 210                          run.Raw(';'), 'wget', fio, run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'),
 211                          run.Raw('cd fio-fio*'), run.Raw(';'), './configure', run.Raw(';'), 'make'])
 212         remote.run(args=['ceph', '-s'])
 213         remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
 214         remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
 215         remote.run(args=['ceph', '-s'])
 216     finally:
 217         out=StringIO.StringIO()
 218         remote.run(args=['rbd', 'device', 'list', '--format=json'], stdout=out)
 219         mapped_images = json.loads(out.getvalue())
 220         if mapped_images:
 221             log.info("Unmapping rbd images on {sn}".format(sn=sn))
 222             for image in mapped_images:
 223                 remote.run(args=['sudo', 'rbd', 'device', 'unmap',
 224                                  str(image['device'])])
 225         log.info("Cleaning up fio install")
 226         remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)])
 227         if ioengine_pkg:
 228             remove_package(ioengine_pkg, remote)