ceph/qa/tasks/rbd.py

   1 """
   2 Rbd testing task
   3 """
   4 import contextlib
   5 import logging
   6 import os
   7 import tempfile
   8 import sys
   9
  10 from io import BytesIO
  11 from teuthology.orchestra import run
  12 from teuthology import misc as teuthology
  13 from teuthology import contextutil
  14 from teuthology.parallel import parallel
  15 from teuthology.task.common_fs_utils import generic_mkfs
  16 from teuthology.task.common_fs_utils import generic_mount
  17 from teuthology.task.common_fs_utils import default_image_name
  18
  19 import six
  20
  21 #V1 image unsupported but required for testing purposes
  22 os.environ["RBD_FORCE_ALLOW_V1"] = "1"
  23
  24 log = logging.getLogger(__name__)
  25
  26 @contextlib.contextmanager
  27 def create_image(ctx, config):
  28     """
  29     Create an rbd image.
  30
  31     For example::
  32
  33         tasks:
  34         - ceph:
  35         - rbd.create_image:
  36             client.0:
  37                 image_name: testimage
  38                 image_size: 100
  39                 image_format: 1
  40             client.1:
  41
  42     Image size is expressed as a number of megabytes; default value
  43     is 10240.
  44
  45     Image format value must be either 1 or 2; default value is 1.
  46
  47     """
  48     assert isinstance(config, dict) or isinstance(config, list), \
  49         "task create_image only supports a list or dictionary for configuration"
  50
  51     if isinstance(config, dict):
  52         images = config.items()
  53     else:
  54         images = [(role, None) for role in config]
  55
  56     testdir = teuthology.get_testdir(ctx)
  57     for role, properties in images:
  58         if properties is None:
  59             properties = {}
  60         name = properties.get('image_name', default_image_name(role))
  61         size = properties.get('image_size', 10240)
  62         fmt = properties.get('image_format', 1)
  63         (remote,) = ctx.cluster.only(role).remotes.keys()
  64         log.info('Creating image {name} with size {size}'.format(name=name,
  65                                                                  size=size))
  66         args = [
  67                 'adjust-ulimits',
  68                 'ceph-coverage',
  69                 '{tdir}/archive/coverage'.format(tdir=testdir),
  70                 'rbd',
  71                 '-p', 'rbd',
  72                 'create',
  73                 '--size', str(size),
  74                 name,
  75             ]
  76         # omit format option if using the default (format 1)
  77         # since old versions of don't support it
  78         if int(fmt) != 1:
  79             args += ['--image-format', str(fmt)]
  80         remote.run(args=args)
  81     try:
  82         yield
  83     finally:
  84         log.info('Deleting rbd images...')
  85         for role, properties in images:
  86             if properties is None:
  87                 properties = {}
  88             name = properties.get('image_name', default_image_name(role))
  89             (remote,) = ctx.cluster.only(role).remotes.keys()
  90             remote.run(
  91                 args=[
  92                     'adjust-ulimits',
  93                     'ceph-coverage',
  94                     '{tdir}/archive/coverage'.format(tdir=testdir),
  95                     'rbd',
  96                     '-p', 'rbd',
  97                     'rm',
  98                     name,
  99                     ],
 100                 )
 101
 102 @contextlib.contextmanager
 103 def clone_image(ctx, config):
 104     """
 105     Clones a parent imag
 106
 107     For example::
 108
 109         tasks:
 110         - ceph:
 111         - rbd.clone_image:
 112             client.0:
 113                 parent_name: testimage
 114                 image_name: cloneimage
 115     """
 116     assert isinstance(config, dict) or isinstance(config, list), \
 117         "task clone_image only supports a list or dictionary for configuration"
 118
 119     if isinstance(config, dict):
 120         images = config.items()
 121     else:
 122         images = [(role, None) for role in config]
 123
 124     testdir = teuthology.get_testdir(ctx)
 125     for role, properties in images:
 126         if properties is None:
 127             properties = {}
 128
 129         name = properties.get('image_name', default_image_name(role))
 130         parent_name = properties.get('parent_name')
 131         assert parent_name is not None, \
 132             "parent_name is required"
 133         parent_spec = '{name}@{snap}'.format(name=parent_name, snap=name)
 134
 135         (remote,) = ctx.cluster.only(role).remotes.keys()
 136         log.info('Clone image {parent} to {child}'.format(parent=parent_name,
 137                                                           child=name))
 138         for cmd in [('snap', 'create', parent_spec),
 139                     ('snap', 'protect', parent_spec),
 140                     ('clone', parent_spec, name)]:
 141             args = [
 142                     'adjust-ulimits',
 143                     'ceph-coverage',
 144                     '{tdir}/archive/coverage'.format(tdir=testdir),
 145                     'rbd', '-p', 'rbd'
 146                     ]
 147             args.extend(cmd)
 148             remote.run(args=args)
 149
 150     try:
 151         yield
 152     finally:
 153         log.info('Deleting rbd clones...')
 154         for role, properties in images:
 155             if properties is None:
 156                 properties = {}
 157             name = properties.get('image_name', default_image_name(role))
 158             parent_name = properties.get('parent_name')
 159             parent_spec = '{name}@{snap}'.format(name=parent_name, snap=name)
 160
 161             (remote,) = ctx.cluster.only(role).remotes.keys()
 162
 163             for cmd in [('rm', name),
 164                         ('snap', 'unprotect', parent_spec),
 165                         ('snap', 'rm', parent_spec)]:
 166                 args = [
 167                         'adjust-ulimits',
 168                         'ceph-coverage',
 169                         '{tdir}/archive/coverage'.format(tdir=testdir),
 170                         'rbd', '-p', 'rbd'
 171                         ]
 172                 args.extend(cmd)
 173                 remote.run(args=args)
 174
 175 @contextlib.contextmanager
 176 def modprobe(ctx, config):
 177     """
 178     Load the rbd kernel module..
 179
 180     For example::
 181
 182         tasks:
 183         - ceph:
 184         - rbd.create_image: [client.0]
 185         - rbd.modprobe: [client.0]
 186     """
 187     log.info('Loading rbd kernel module...')
 188     for role in config:
 189         (remote,) = ctx.cluster.only(role).remotes.keys()
 190         remote.run(
 191             args=[
 192                 'sudo',
 193                 'modprobe',
 194                 'rbd',
 195                 ],
 196             )
 197     try:
 198         yield
 199     finally:
 200         log.info('Unloading rbd kernel module...')
 201         for role in config:
 202             (remote,) = ctx.cluster.only(role).remotes.keys()
 203             remote.run(
 204                 args=[
 205                     'sudo',
 206                     'modprobe',
 207                     '-r',
 208                     'rbd',
 209                     # force errors to be ignored; necessary if more
 210                     # than one device was created, which may mean
 211                     # the module isn't quite ready to go the first
 212                     # time through.
 213                     run.Raw('||'),
 214                     'true',
 215                     ],
 216                 )
 217
 218 @contextlib.contextmanager
 219 def dev_create(ctx, config):
 220     """
 221     Map block devices to rbd images.
 222
 223     For example::
 224
 225         tasks:
 226         - ceph:
 227         - rbd.create_image: [client.0]
 228         - rbd.modprobe: [client.0]
 229         - rbd.dev_create:
 230             client.0: testimage.client.0
 231     """
 232     assert isinstance(config, dict) or isinstance(config, list), \
 233         "task dev_create only supports a list or dictionary for configuration"
 234
 235     if isinstance(config, dict):
 236         role_images = config.items()
 237     else:
 238         role_images = [(role, None) for role in config]
 239
 240     log.info('Creating rbd block devices...')
 241
 242     testdir = teuthology.get_testdir(ctx)
 243
 244     for role, image in role_images:
 245         if image is None:
 246             image = default_image_name(role)
 247         (remote,) = ctx.cluster.only(role).remotes.keys()
 248
 249         remote.run(
 250             args=[
 251                 'sudo',
 252                 'adjust-ulimits',
 253                 'ceph-coverage',
 254                 '{tdir}/archive/coverage'.format(tdir=testdir),
 255                 'rbd',
 256                 '--user', role.rsplit('.')[-1],
 257                 '-p', 'rbd',
 258                 'map',
 259                 image,
 260                 run.Raw('&&'),
 261                 # wait for the symlink to be created by udev
 262                 'while', 'test', '!', '-e', '/dev/rbd/rbd/{image}'.format(image=image), run.Raw(';'), 'do',
 263                 'sleep', '1', run.Raw(';'),
 264                 'done',
 265                 ],
 266             )
 267     try:
 268         yield
 269     finally:
 270         log.info('Unmapping rbd devices...')
 271         for role, image in role_images:
 272             if image is None:
 273                 image = default_image_name(role)
 274             (remote,) = ctx.cluster.only(role).remotes.keys()
 275             remote.run(
 276                 args=[
 277                     'LD_LIBRARY_PATH={tdir}/binary/usr/local/lib'.format(tdir=testdir),
 278                     'sudo',
 279                     'adjust-ulimits',
 280                     'ceph-coverage',
 281                     '{tdir}/archive/coverage'.format(tdir=testdir),
 282                     'rbd',
 283                     '-p', 'rbd',
 284                     'unmap',
 285                     '/dev/rbd/rbd/{imgname}'.format(imgname=image),
 286                     run.Raw('&&'),
 287                     # wait for the symlink to be deleted by udev
 288                     'while', 'test', '-e', '/dev/rbd/rbd/{image}'.format(image=image),
 289                     run.Raw(';'),
 290                     'do',
 291                     'sleep', '1', run.Raw(';'),
 292                     'done',
 293                     ],
 294                 )
 295
 296
 297 def rbd_devname_rtn(ctx, image):
 298     return '/dev/rbd/rbd/{image}'.format(image=image)
 299
 300 def canonical_path(ctx, role, path):
 301     """
 302     Determine the canonical path for a given path on the host
 303     representing the given role.  A canonical path contains no
 304     . or .. components, and includes no symbolic links.
 305     """
 306     version_fp = BytesIO()
 307     ctx.cluster.only(role).run(
 308         args=[ 'readlink', '-f', path ],
 309         stdout=version_fp,
 310         )
 311     canonical_path = six.ensure_str(version_fp.getvalue()).rstrip('\n')
 312     version_fp.close()
 313     return canonical_path
 314
 315 @contextlib.contextmanager
 316 def run_xfstests(ctx, config):
 317     """
 318     Run xfstests over specified devices.
 319
 320     Warning: both the test and scratch devices specified will be
 321     overwritten.  Normally xfstests modifies (but does not destroy)
 322     the test device, but for now the run script used here re-makes
 323     both filesystems.
 324
 325     Note: Only one instance of xfstests can run on a single host at
 326     a time, although this is not enforced.
 327
 328     This task in its current form needs some improvement.  For
 329     example, it assumes all roles provided in the config are
 330     clients, and that the config provided is a list of key/value
 331     pairs.  For now please use the xfstests() interface, below.
 332
 333     For example::
 334
 335         tasks:
 336         - ceph:
 337         - rbd.run_xfstests:
 338             client.0:
 339                 count: 2
 340                 test_dev: 'test_dev'
 341                 scratch_dev: 'scratch_dev'
 342                 fs_type: 'xfs'
 343                 tests: 'generic/100 xfs/003 xfs/005 xfs/006 generic/015'
 344                 exclude:
 345                 - generic/42
 346                 randomize: true
 347     """
 348     with parallel() as p:
 349         for role, properties in config.items():
 350             p.spawn(run_xfstests_one_client, ctx, role, properties)
 351         exc_info = None
 352         while True:
 353             try:
 354                 p.next()
 355             except StopIteration:
 356                 break
 357             except:
 358                 exc_info = sys.exc_info()
 359         if exc_info:
 360             six.reraise(exc_info[0], exc_info[1], exc_info[2])
 361     yield
 362
 363 def run_xfstests_one_client(ctx, role, properties):
 364     """
 365     Spawned routine to handle xfs tests for a single client
 366     """
 367     testdir = teuthology.get_testdir(ctx)
 368     try:
 369         count = properties.get('count')
 370         test_dev = properties.get('test_dev')
 371         assert test_dev is not None, \
 372             "task run_xfstests requires test_dev to be defined"
 373         test_dev = canonical_path(ctx, role, test_dev)
 374
 375         scratch_dev = properties.get('scratch_dev')
 376         assert scratch_dev is not None, \
 377             "task run_xfstests requires scratch_dev to be defined"
 378         scratch_dev = canonical_path(ctx, role, scratch_dev)
 379
 380         fs_type = properties.get('fs_type')
 381         tests = properties.get('tests')
 382         exclude_list = properties.get('exclude')
 383         randomize = properties.get('randomize')
 384
 385         (remote,) = ctx.cluster.only(role).remotes.keys()
 386
 387         # Fetch the test script
 388         test_root = teuthology.get_testdir(ctx)
 389         test_script = 'run_xfstests.sh'
 390         test_path = os.path.join(test_root, test_script)
 391
 392         xfstests_url = properties.get('xfstests_url')
 393         assert xfstests_url is not None, \
 394             "task run_xfstests requires xfstests_url to be defined"
 395
 396         xfstests_krbd_url = xfstests_url + '/' + test_script
 397
 398         log.info('Fetching {script} for {role} from {url}'.format(
 399             script=test_script,
 400             role=role,
 401             url=xfstests_krbd_url))
 402
 403         args = [ 'wget', '-O', test_path, '--', xfstests_krbd_url ]
 404         remote.run(args=args)
 405
 406         log.info('Running xfstests on {role}:'.format(role=role))
 407         log.info('   iteration count: {count}:'.format(count=count))
 408         log.info('       test device: {dev}'.format(dev=test_dev))
 409         log.info('    scratch device: {dev}'.format(dev=scratch_dev))
 410         log.info('     using fs_type: {fs_type}'.format(fs_type=fs_type))
 411         log.info('      tests to run: {tests}'.format(tests=tests))
 412         log.info('      exclude list: {}'.format(' '.join(exclude_list)))
 413         log.info('         randomize: {randomize}'.format(randomize=randomize))
 414
 415         if exclude_list:
 416             with tempfile.NamedTemporaryFile(mode='w', prefix='exclude') as exclude_file:
 417                 for test in exclude_list:
 418                     exclude_file.write("{}\n".format(test))
 419                 exclude_file.flush()
 420                 remote.put_file(exclude_file.name, exclude_file.name)
 421
 422         # Note that the device paths are interpreted using
 423         # readlink -f <path> in order to get their canonical
 424         # pathname (so it matches what the kernel remembers).
 425         args = [
 426             '/usr/bin/sudo',
 427             'TESTDIR={tdir}'.format(tdir=testdir),
 428             'adjust-ulimits',
 429             'ceph-coverage',
 430             '{tdir}/archive/coverage'.format(tdir=testdir),
 431             '/bin/bash',
 432             test_path,
 433             '-c', str(count),
 434             '-f', fs_type,
 435             '-t', test_dev,
 436             '-s', scratch_dev,
 437             ]
 438         if exclude_list:
 439             args.extend(['-x', exclude_file.name])
 440         if randomize:
 441             args.append('-r')
 442         if tests:
 443             args.extend(['--', tests])
 444         remote.run(args=args, logger=log.getChild(role))
 445     finally:
 446         log.info('Removing {script} on {role}'.format(script=test_script,
 447                                                       role=role))
 448         remote.run(args=['rm', '-f', test_path])
 449
 450 @contextlib.contextmanager
 451 def xfstests(ctx, config):
 452     """
 453     Run xfstests over rbd devices.  This interface sets up all
 454     required configuration automatically if not otherwise specified.
 455     Note that only one instance of xfstests can run on a single host
 456     at a time.  By default, the set of tests specified is run once.
 457     If a (non-zero) count value is supplied, the complete set of
 458     tests will be run that number of times.
 459
 460     For example::
 461
 462         tasks:
 463         - ceph:
 464         # Image sizes are in MB
 465         - rbd.xfstests:
 466             client.0:
 467                 count: 3
 468                 test_image: 'test_image'
 469                 test_size: 250
 470                 test_format: 2
 471                 scratch_image: 'scratch_image'
 472                 scratch_size: 250
 473                 scratch_format: 1
 474                 fs_type: 'xfs'
 475                 tests: 'generic/100 xfs/003 xfs/005 xfs/006 generic/015'
 476                 exclude:
 477                 - generic/42
 478                 randomize: true
 479                 xfstests_branch: master
 480                 xfstests_url: 'https://raw.github.com/ceph/branch/master/qa'
 481     """
 482     if config is None:
 483         config = { 'all': None }
 484     assert isinstance(config, dict) or isinstance(config, list), \
 485         "task xfstests only supports a list or dictionary for configuration"
 486     if isinstance(config, dict):
 487         config = teuthology.replace_all_with_clients(ctx.cluster, config)
 488         runs = config.items()
 489     else:
 490         runs = [(role, None) for role in config]
 491
 492     running_xfstests = {}
 493     for role, properties in runs:
 494         assert role.startswith('client.'), \
 495             "task xfstests can only run on client nodes"
 496         for host, roles_for_host in ctx.cluster.remotes.items():
 497             if role in roles_for_host:
 498                 assert host not in running_xfstests, \
 499                     "task xfstests allows only one instance at a time per host"
 500                 running_xfstests[host] = True
 501
 502     images_config = {}
 503     scratch_config = {}
 504     modprobe_config = {}
 505     image_map_config = {}
 506     scratch_map_config = {}
 507     xfstests_config = {}
 508     for role, properties in runs:
 509         if properties is None:
 510             properties = {}
 511
 512         test_image = properties.get('test_image', 'test_image.{role}'.format(role=role))
 513         test_size = properties.get('test_size', 10000) # 10G
 514         test_fmt = properties.get('test_format', 1)
 515         scratch_image = properties.get('scratch_image', 'scratch_image.{role}'.format(role=role))
 516         scratch_size = properties.get('scratch_size', 10000) # 10G
 517         scratch_fmt = properties.get('scratch_format', 1)
 518
 519         images_config[role] = dict(
 520             image_name=test_image,
 521             image_size=test_size,
 522             image_format=test_fmt,
 523             )
 524
 525         scratch_config[role] = dict(
 526             image_name=scratch_image,
 527             image_size=scratch_size,
 528             image_format=scratch_fmt,
 529             )
 530
 531         xfstests_branch = properties.get('xfstests_branch', 'master')
 532         xfstests_url = properties.get('xfstests_url', 'https://raw.github.com/ceph/ceph/{branch}/qa'.format(branch=xfstests_branch))
 533
 534         xfstests_config[role] = dict(
 535             count=properties.get('count', 1),
 536             test_dev='/dev/rbd/rbd/{image}'.format(image=test_image),
 537             scratch_dev='/dev/rbd/rbd/{image}'.format(image=scratch_image),
 538             fs_type=properties.get('fs_type', 'xfs'),
 539             randomize=properties.get('randomize', False),
 540             tests=properties.get('tests'),
 541             exclude=properties.get('exclude', []),
 542             xfstests_url=xfstests_url,
 543             )
 544
 545         log.info('Setting up xfstests using RBD images:')
 546         log.info('      test ({size} MB): {image}'.format(size=test_size,
 547                                                         image=test_image))
 548         log.info('   scratch ({size} MB): {image}'.format(size=scratch_size,
 549                                                         image=scratch_image))
 550         modprobe_config[role] = None
 551         image_map_config[role] = test_image
 552         scratch_map_config[role] = scratch_image
 553
 554     with contextutil.nested(
 555         lambda: create_image(ctx=ctx, config=images_config),
 556         lambda: create_image(ctx=ctx, config=scratch_config),
 557         lambda: modprobe(ctx=ctx, config=modprobe_config),
 558         lambda: dev_create(ctx=ctx, config=image_map_config),
 559         lambda: dev_create(ctx=ctx, config=scratch_map_config),
 560         lambda: run_xfstests(ctx=ctx, config=xfstests_config),
 561         ):
 562         yield
 563
 564
 565 @contextlib.contextmanager
 566 def task(ctx, config):
 567     """
 568     Create and mount an rbd image.
 569
 570     For example, you can specify which clients to run on::
 571
 572         tasks:
 573         - ceph:
 574         - rbd: [client.0, client.1]
 575
 576     There are a few image options::
 577
 578         tasks:
 579         - ceph:
 580         - rbd:
 581             client.0: # uses defaults
 582             client.1:
 583                 image_name: foo
 584                 image_size: 2048
 585                 image_format: 2
 586                 fs_type: xfs
 587
 588     To use default options on all clients::
 589
 590         tasks:
 591         - ceph:
 592         - rbd:
 593             all:
 594
 595     To create 20GiB images and format them with xfs on all clients::
 596
 597         tasks:
 598         - ceph:
 599         - rbd:
 600             all:
 601               image_size: 20480
 602               fs_type: xfs
 603     """
 604     if config is None:
 605         config = { 'all': None }
 606     norm_config = config
 607     if isinstance(config, dict):
 608         norm_config = teuthology.replace_all_with_clients(ctx.cluster, config)
 609     if isinstance(norm_config, dict):
 610         role_images = {}
 611         for role, properties in norm_config.items():
 612             if properties is None:
 613                 properties = {}
 614             role_images[role] = properties.get('image_name')
 615     else:
 616         role_images = norm_config
 617
 618     log.debug('rbd config is: %s', norm_config)
 619
 620     with contextutil.nested(
 621         lambda: create_image(ctx=ctx, config=norm_config),
 622         lambda: modprobe(ctx=ctx, config=norm_config),
 623         lambda: dev_create(ctx=ctx, config=role_images),
 624         lambda: generic_mkfs(ctx=ctx, config=norm_config,
 625                 devname_rtn=rbd_devname_rtn),
 626         lambda: generic_mount(ctx=ctx, config=role_images,
 627                 devname_rtn=rbd_devname_rtn),
 628         ):
 629         yield