ceph/qa/tasks/rbd.py

   1 """
   2 Rbd testing task
   3 """
   4 import contextlib
   5 import logging
   6 import os
   7 import tempfile
   8 import sys
   9
  10 from io import StringIO
  11 from teuthology.orchestra import run
  12 from teuthology import misc as teuthology
  13 from teuthology import contextutil
  14 from teuthology.parallel import parallel
  15 from teuthology.task.common_fs_utils import generic_mkfs
  16 from teuthology.task.common_fs_utils import generic_mount
  17 from teuthology.task.common_fs_utils import default_image_name
  18
  19
  20 #V1 image unsupported but required for testing purposes
  21 os.environ["RBD_FORCE_ALLOW_V1"] = "1"
  22
  23 log = logging.getLogger(__name__)
  24
  25 ENCRYPTION_PASSPHRASE = "password"
  26
  27 @contextlib.contextmanager
  28 def create_image(ctx, config):
  29     """
  30     Create an rbd image.
  31
  32     For example::
  33
  34         tasks:
  35         - ceph:
  36         - rbd.create_image:
  37             client.0:
  38                 image_name: testimage
  39                 image_size: 100
  40                 image_format: 1
  41                 encryption_format: luks2
  42             client.1:
  43
  44     Image size is expressed as a number of megabytes; default value
  45     is 10240.
  46
  47     Image format value must be either 1 or 2; default value is 1.
  48
  49     """
  50     assert isinstance(config, dict) or isinstance(config, list), \
  51         "task create_image only supports a list or dictionary for configuration"
  52
  53     if isinstance(config, dict):
  54         images = config.items()
  55     else:
  56         images = [(role, None) for role in config]
  57
  58     testdir = teuthology.get_testdir(ctx)
  59     passphrase_file = '{tdir}/passphrase'.format(tdir=testdir)
  60     for role, properties in images:
  61         if properties is None:
  62             properties = {}
  63         name = properties.get('image_name', default_image_name(role))
  64         size = properties.get('image_size', 10240)
  65         fmt = properties.get('image_format', 1)
  66         encryption_format = properties.get('encryption_format', 'none')
  67         (remote,) = ctx.cluster.only(role).remotes.keys()
  68         log.info('Creating image {name} with size {size}'.format(name=name,
  69                                                                  size=size))
  70         args = [
  71                 'adjust-ulimits',
  72                 'ceph-coverage',
  73                 '{tdir}/archive/coverage'.format(tdir=testdir),
  74                 'rbd',
  75                 '-p', 'rbd',
  76                 'create',
  77                 '--size', str(size),
  78                 name,
  79             ]
  80         # omit format option if using the default (format 1)
  81         # since old versions of don't support it
  82         if int(fmt) != 1:
  83             args += ['--image-format', str(fmt)]
  84         remote.run(args=args)
  85
  86         if encryption_format != 'none':
  87             remote.run(
  88                 args=[
  89                     'echo',
  90                     ENCRYPTION_PASSPHRASE,
  91                     run.Raw('>'),
  92                     passphrase_file
  93                     ]
  94                 )
  95             remote.run(
  96                 args=[
  97                     'adjust-ulimits',
  98                     'ceph-coverage',
  99                     '{tdir}/archive/coverage'.format(tdir=testdir),
 100                     'rbd',
 101                     'encryption',
 102                     'format',
 103                     name,
 104                     encryption_format,
 105                     passphrase_file,
 106                     '-p',
 107                     'rbd'
 108                     ]
 109                 )
 110     try:
 111         yield
 112     finally:
 113         log.info('Deleting rbd images...')
 114         remote.run(args=['rm', '-f', passphrase_file])
 115         for role, properties in images:
 116             if properties is None:
 117                 properties = {}
 118             name = properties.get('image_name', default_image_name(role))
 119             (remote,) = ctx.cluster.only(role).remotes.keys()
 120             remote.run(
 121                 args=[
 122                     'adjust-ulimits',
 123                     'ceph-coverage',
 124                     '{tdir}/archive/coverage'.format(tdir=testdir),
 125                     'rbd',
 126                     '-p', 'rbd',
 127                     'rm',
 128                     name,
 129                     ],
 130                 )
 131
 132 @contextlib.contextmanager
 133 def clone_image(ctx, config):
 134     """
 135     Clones a parent imag
 136
 137     For example::
 138
 139         tasks:
 140         - ceph:
 141         - rbd.clone_image:
 142             client.0:
 143                 parent_name: testimage
 144                 image_name: cloneimage
 145     """
 146     assert isinstance(config, dict) or isinstance(config, list), \
 147         "task clone_image only supports a list or dictionary for configuration"
 148
 149     if isinstance(config, dict):
 150         images = config.items()
 151     else:
 152         images = [(role, None) for role in config]
 153
 154     testdir = teuthology.get_testdir(ctx)
 155     for role, properties in images:
 156         if properties is None:
 157             properties = {}
 158
 159         name = properties.get('image_name', default_image_name(role))
 160         parent_name = properties.get('parent_name')
 161         assert parent_name is not None, \
 162             "parent_name is required"
 163         parent_spec = '{name}@{snap}'.format(name=parent_name, snap=name)
 164
 165         (remote,) = ctx.cluster.only(role).remotes.keys()
 166         log.info('Clone image {parent} to {child}'.format(parent=parent_name,
 167                                                           child=name))
 168         for cmd in [('snap', 'create', parent_spec),
 169                     ('snap', 'protect', parent_spec),
 170                     ('clone', parent_spec, name)]:
 171             args = [
 172                     'adjust-ulimits',
 173                     'ceph-coverage',
 174                     '{tdir}/archive/coverage'.format(tdir=testdir),
 175                     'rbd', '-p', 'rbd'
 176                     ]
 177             args.extend(cmd)
 178             remote.run(args=args)
 179
 180     try:
 181         yield
 182     finally:
 183         log.info('Deleting rbd clones...')
 184         for role, properties in images:
 185             if properties is None:
 186                 properties = {}
 187             name = properties.get('image_name', default_image_name(role))
 188             parent_name = properties.get('parent_name')
 189             parent_spec = '{name}@{snap}'.format(name=parent_name, snap=name)
 190
 191             (remote,) = ctx.cluster.only(role).remotes.keys()
 192
 193             for cmd in [('rm', name),
 194                         ('snap', 'unprotect', parent_spec),
 195                         ('snap', 'rm', parent_spec)]:
 196                 args = [
 197                         'adjust-ulimits',
 198                         'ceph-coverage',
 199                         '{tdir}/archive/coverage'.format(tdir=testdir),
 200                         'rbd', '-p', 'rbd'
 201                         ]
 202                 args.extend(cmd)
 203                 remote.run(args=args)
 204
 205 @contextlib.contextmanager
 206 def modprobe(ctx, config):
 207     """
 208     Load the rbd kernel module..
 209
 210     For example::
 211
 212         tasks:
 213         - ceph:
 214         - rbd.create_image: [client.0]
 215         - rbd.modprobe: [client.0]
 216     """
 217     log.info('Loading rbd kernel module...')
 218     for role in config:
 219         (remote,) = ctx.cluster.only(role).remotes.keys()
 220         remote.run(
 221             args=[
 222                 'sudo',
 223                 'modprobe',
 224                 'rbd',
 225                 ],
 226             )
 227     try:
 228         yield
 229     finally:
 230         log.info('Unloading rbd kernel module...')
 231         for role in config:
 232             (remote,) = ctx.cluster.only(role).remotes.keys()
 233             remote.run(
 234                 args=[
 235                     'sudo',
 236                     'modprobe',
 237                     '-r',
 238                     'rbd',
 239                     # force errors to be ignored; necessary if more
 240                     # than one device was created, which may mean
 241                     # the module isn't quite ready to go the first
 242                     # time through.
 243                     run.Raw('||'),
 244                     'true',
 245                     ],
 246                 )
 247
 248 @contextlib.contextmanager
 249 def dev_create(ctx, config):
 250     """
 251     Map block devices to rbd images.
 252
 253     For example::
 254
 255         tasks:
 256         - ceph:
 257         - rbd.create_image: [client.0]
 258         - rbd.modprobe: [client.0]
 259         - rbd.dev_create:
 260             client.0:
 261                 image_name: testimage.client.0
 262                 encryption_format: luks2
 263     """
 264     assert isinstance(config, dict) or isinstance(config, list), \
 265         "task dev_create only supports a list or dictionary for configuration"
 266
 267     if isinstance(config, dict):
 268         images = config.items()
 269     else:
 270         images = [(role, None) for role in config]
 271
 272     log.info('Creating rbd block devices...')
 273
 274     testdir = teuthology.get_testdir(ctx)
 275     passphrase_file = '{tdir}/passphrase'.format(tdir=testdir)
 276     device_path = {}
 277
 278     for role, properties in images:
 279         if properties is None:
 280             properties = {}
 281         name = properties.get('image_name', default_image_name(role))
 282         encryption_format = properties.get('encryption_format', 'none')
 283         (remote,) = ctx.cluster.only(role).remotes.keys()
 284
 285         if encryption_format == 'none':
 286             device_path[role] = '/dev/rbd/rbd/{image}'.format(image=name)
 287             device_specific_args = []
 288         else:
 289             remote.run(
 290                 args=[
 291                     'echo',
 292                     ENCRYPTION_PASSPHRASE,
 293                     run.Raw('>'),
 294                     passphrase_file
 295                     ]
 296                 )
 297             device_specific_args = [
 298                 '-t', 'nbd', '-o',
 299                 'encryption-format=%s,encryption-passphrase-file=%s' % (
 300                     encryption_format, passphrase_file)]
 301
 302         map_fp = StringIO()
 303         remote.run(
 304             args=[
 305                 'sudo',
 306                 'adjust-ulimits',
 307                 'ceph-coverage',
 308                 '{tdir}/archive/coverage'.format(tdir=testdir),
 309                 'rbd',
 310                 '--id', role.rsplit('.')[-1],
 311                 '-p', 'rbd',
 312                 'map',
 313                 name] + device_specific_args,
 314             stdout=map_fp,
 315             )
 316
 317         if encryption_format != 'none':
 318             device_path[role] = map_fp.getvalue().rstrip()
 319             properties['device_path'] = device_path[role]
 320             remote.run(args=['sudo', 'chmod', '666', device_path[role]])
 321     try:
 322         yield
 323     finally:
 324         log.info('Unmapping rbd devices...')
 325         remote.run(args=['rm', '-f', passphrase_file])
 326         for role, properties in images:
 327             if not device_path.get(role):
 328                 continue
 329
 330             if properties is None:
 331                 properties = {}
 332             encryption_format = properties.get('encryption_format', 'none')
 333             (remote,) = ctx.cluster.only(role).remotes.keys()
 334
 335             if encryption_format == 'none':
 336                 device_specific_args = []
 337             else:
 338                 device_specific_args = ['-t', 'nbd']
 339
 340             remote.run(
 341                 args=[
 342                     'LD_LIBRARY_PATH={tdir}/binary/usr/local/lib'.format(tdir=testdir),
 343                     'sudo',
 344                     'adjust-ulimits',
 345                     'ceph-coverage',
 346                     '{tdir}/archive/coverage'.format(tdir=testdir),
 347                     'rbd',
 348                     '-p', 'rbd',
 349                     'unmap',
 350                     device_path[role],
 351                     ] + device_specific_args,
 352                 )
 353
 354
 355 def rbd_devname_rtn(ctx, image):
 356     return '/dev/rbd/rbd/{image}'.format(image=image)
 357
 358 def canonical_path(ctx, role, path):
 359     """
 360     Determine the canonical path for a given path on the host
 361     representing the given role.  A canonical path contains no
 362     . or .. components, and includes no symbolic links.
 363     """
 364     version_fp = StringIO()
 365     ctx.cluster.only(role).run(
 366         args=[ 'readlink', '-f', path ],
 367         stdout=version_fp,
 368         )
 369     canonical_path = version_fp.getvalue().rstrip('\n')
 370     version_fp.close()
 371     return canonical_path
 372
 373 @contextlib.contextmanager
 374 def run_xfstests(ctx, config):
 375     """
 376     Run xfstests over specified devices.
 377
 378     Warning: both the test and scratch devices specified will be
 379     overwritten.  Normally xfstests modifies (but does not destroy)
 380     the test device, but for now the run script used here re-makes
 381     both filesystems.
 382
 383     Note: Only one instance of xfstests can run on a single host at
 384     a time, although this is not enforced.
 385
 386     This task in its current form needs some improvement.  For
 387     example, it assumes all roles provided in the config are
 388     clients, and that the config provided is a list of key/value
 389     pairs.  For now please use the xfstests() interface, below.
 390
 391     For example::
 392
 393         tasks:
 394         - ceph:
 395         - rbd.run_xfstests:
 396             client.0:
 397                 count: 2
 398                 test_dev: 'test_dev'
 399                 scratch_dev: 'scratch_dev'
 400                 fs_type: 'xfs'
 401                 tests: 'generic/100 xfs/003 xfs/005 xfs/006 generic/015'
 402                 exclude:
 403                 - generic/42
 404                 randomize: true
 405     """
 406     with parallel() as p:
 407         for role, properties in config.items():
 408             p.spawn(run_xfstests_one_client, ctx, role, properties)
 409         exc = None
 410         while True:
 411             try:
 412                 p.next()
 413             except StopIteration:
 414                 break
 415             except:
 416                 exc = sys.exc_info()[1]
 417         if exc is not None:
 418             raise exc
 419     yield
 420
 421 def run_xfstests_one_client(ctx, role, properties):
 422     """
 423     Spawned routine to handle xfs tests for a single client
 424     """
 425     testdir = teuthology.get_testdir(ctx)
 426     try:
 427         count = properties.get('count')
 428         test_dev = properties.get('test_dev')
 429         assert test_dev is not None, \
 430             "task run_xfstests requires test_dev to be defined"
 431         test_dev = canonical_path(ctx, role, test_dev)
 432
 433         scratch_dev = properties.get('scratch_dev')
 434         assert scratch_dev is not None, \
 435             "task run_xfstests requires scratch_dev to be defined"
 436         scratch_dev = canonical_path(ctx, role, scratch_dev)
 437
 438         fs_type = properties.get('fs_type')
 439         tests = properties.get('tests')
 440         exclude_list = properties.get('exclude')
 441         randomize = properties.get('randomize')
 442
 443         (remote,) = ctx.cluster.only(role).remotes.keys()
 444
 445         # Fetch the test script
 446         test_root = teuthology.get_testdir(ctx)
 447         test_script = 'run_xfstests.sh'
 448         test_path = os.path.join(test_root, test_script)
 449
 450         xfstests_url = properties.get('xfstests_url')
 451         assert xfstests_url is not None, \
 452             "task run_xfstests requires xfstests_url to be defined"
 453
 454         xfstests_krbd_url = xfstests_url + '/' + test_script
 455
 456         log.info('Fetching {script} for {role} from {url}'.format(
 457             script=test_script,
 458             role=role,
 459             url=xfstests_krbd_url))
 460
 461         args = [ 'wget', '-O', test_path, '--', xfstests_krbd_url ]
 462         remote.run(args=args)
 463
 464         log.info('Running xfstests on {role}:'.format(role=role))
 465         log.info('   iteration count: {count}:'.format(count=count))
 466         log.info('       test device: {dev}'.format(dev=test_dev))
 467         log.info('    scratch device: {dev}'.format(dev=scratch_dev))
 468         log.info('     using fs_type: {fs_type}'.format(fs_type=fs_type))
 469         log.info('      tests to run: {tests}'.format(tests=tests))
 470         log.info('      exclude list: {}'.format(' '.join(exclude_list)))
 471         log.info('         randomize: {randomize}'.format(randomize=randomize))
 472
 473         if exclude_list:
 474             with tempfile.NamedTemporaryFile(mode='w', prefix='exclude') as exclude_file:
 475                 for test in exclude_list:
 476                     exclude_file.write("{}\n".format(test))
 477                 exclude_file.flush()
 478                 remote.put_file(exclude_file.name, exclude_file.name)
 479
 480         # Note that the device paths are interpreted using
 481         # readlink -f <path> in order to get their canonical
 482         # pathname (so it matches what the kernel remembers).
 483         args = [
 484             '/usr/bin/sudo',
 485             'TESTDIR={tdir}'.format(tdir=testdir),
 486             'adjust-ulimits',
 487             'ceph-coverage',
 488             '{tdir}/archive/coverage'.format(tdir=testdir),
 489             '/bin/bash',
 490             test_path,
 491             '-c', str(count),
 492             '-f', fs_type,
 493             '-t', test_dev,
 494             '-s', scratch_dev,
 495             ]
 496         if exclude_list:
 497             args.extend(['-x', exclude_file.name])
 498         if randomize:
 499             args.append('-r')
 500         if tests:
 501             args.extend(['--', tests])
 502         remote.run(args=args, logger=log.getChild(role))
 503     finally:
 504         log.info('Removing {script} on {role}'.format(script=test_script,
 505                                                       role=role))
 506         remote.run(args=['rm', '-f', test_path])
 507
 508 @contextlib.contextmanager
 509 def xfstests(ctx, config):
 510     """
 511     Run xfstests over rbd devices.  This interface sets up all
 512     required configuration automatically if not otherwise specified.
 513     Note that only one instance of xfstests can run on a single host
 514     at a time.  By default, the set of tests specified is run once.
 515     If a (non-zero) count value is supplied, the complete set of
 516     tests will be run that number of times.
 517
 518     For example::
 519
 520         tasks:
 521         - ceph:
 522         # Image sizes are in MB
 523         - rbd.xfstests:
 524             client.0:
 525                 count: 3
 526                 test_image: 'test_image'
 527                 test_size: 250
 528                 test_format: 2
 529                 scratch_image: 'scratch_image'
 530                 scratch_size: 250
 531                 scratch_format: 1
 532                 fs_type: 'xfs'
 533                 tests: 'generic/100 xfs/003 xfs/005 xfs/006 generic/015'
 534                 exclude:
 535                 - generic/42
 536                 randomize: true
 537                 xfstests_url: 'https://raw.github.com/ceph/ceph-ci/wip-55555/qa'
 538     """
 539     if config is None:
 540         config = { 'all': None }
 541     assert isinstance(config, dict) or isinstance(config, list), \
 542         "task xfstests only supports a list or dictionary for configuration"
 543     if isinstance(config, dict):
 544         config = teuthology.replace_all_with_clients(ctx.cluster, config)
 545         runs = config.items()
 546     else:
 547         runs = [(role, None) for role in config]
 548
 549     running_xfstests = {}
 550     for role, properties in runs:
 551         assert role.startswith('client.'), \
 552             "task xfstests can only run on client nodes"
 553         for host, roles_for_host in ctx.cluster.remotes.items():
 554             if role in roles_for_host:
 555                 assert host not in running_xfstests, \
 556                     "task xfstests allows only one instance at a time per host"
 557                 running_xfstests[host] = True
 558
 559     images_config = {}
 560     scratch_config = {}
 561     modprobe_config = {}
 562     image_map_config = {}
 563     scratch_map_config = {}
 564     xfstests_config = {}
 565     for role, properties in runs:
 566         if properties is None:
 567             properties = {}
 568
 569         test_image = properties.get('test_image', 'test_image.{role}'.format(role=role))
 570         test_size = properties.get('test_size', 10000) # 10G
 571         test_fmt = properties.get('test_format', 1)
 572         scratch_image = properties.get('scratch_image', 'scratch_image.{role}'.format(role=role))
 573         scratch_size = properties.get('scratch_size', 10000) # 10G
 574         scratch_fmt = properties.get('scratch_format', 1)
 575
 576         images_config[role] = dict(
 577             image_name=test_image,
 578             image_size=test_size,
 579             image_format=test_fmt,
 580             )
 581
 582         scratch_config[role] = dict(
 583             image_name=scratch_image,
 584             image_size=scratch_size,
 585             image_format=scratch_fmt,
 586             )
 587
 588         xfstests_branch = properties.get('xfstests_branch', 'master')
 589         xfstests_url = properties.get('xfstests_url', 'https://raw.github.com/ceph/ceph/{branch}/qa'.format(branch=xfstests_branch))
 590
 591         xfstests_config[role] = dict(
 592             count=properties.get('count', 1),
 593             test_dev='/dev/rbd/rbd/{image}'.format(image=test_image),
 594             scratch_dev='/dev/rbd/rbd/{image}'.format(image=scratch_image),
 595             fs_type=properties.get('fs_type', 'xfs'),
 596             randomize=properties.get('randomize', False),
 597             tests=properties.get('tests'),
 598             exclude=properties.get('exclude', []),
 599             xfstests_url=xfstests_url,
 600             )
 601
 602         log.info('Setting up xfstests using RBD images:')
 603         log.info('      test ({size} MB): {image}'.format(size=test_size,
 604                                                         image=test_image))
 605         log.info('   scratch ({size} MB): {image}'.format(size=scratch_size,
 606                                                         image=scratch_image))
 607         modprobe_config[role] = None
 608         image_map_config[role] = {'image_name': test_image}
 609         scratch_map_config[role] = {'image_name': scratch_image}
 610
 611     with contextutil.nested(
 612         lambda: create_image(ctx=ctx, config=images_config),
 613         lambda: create_image(ctx=ctx, config=scratch_config),
 614         lambda: modprobe(ctx=ctx, config=modprobe_config),
 615         lambda: dev_create(ctx=ctx, config=image_map_config),
 616         lambda: dev_create(ctx=ctx, config=scratch_map_config),
 617         lambda: run_xfstests(ctx=ctx, config=xfstests_config),
 618         ):
 619         yield
 620
 621
 622 @contextlib.contextmanager
 623 def task(ctx, config):
 624     """
 625     Create and mount an rbd image.
 626
 627     For example, you can specify which clients to run on::
 628
 629         tasks:
 630         - ceph:
 631         - rbd: [client.0, client.1]
 632
 633     There are a few image options::
 634
 635         tasks:
 636         - ceph:
 637         - rbd:
 638             client.0: # uses defaults
 639             client.1:
 640                 image_name: foo
 641                 image_size: 2048
 642                 image_format: 2
 643                 fs_type: xfs
 644
 645     To use default options on all clients::
 646
 647         tasks:
 648         - ceph:
 649         - rbd:
 650             all:
 651
 652     To create 20GiB images and format them with xfs on all clients::
 653
 654         tasks:
 655         - ceph:
 656         - rbd:
 657             all:
 658               image_size: 20480
 659               fs_type: xfs
 660     """
 661     if config is None:
 662         config = { 'all': None }
 663     norm_config = config
 664     if isinstance(config, dict):
 665         norm_config = teuthology.replace_all_with_clients(ctx.cluster, config)
 666     if isinstance(norm_config, dict):
 667         role_images = {}
 668         for role, properties in norm_config.items():
 669             if properties is None:
 670                 properties = {}
 671             role_images[role] = properties.get('image_name')
 672     else:
 673         role_images = norm_config
 674
 675     log.debug('rbd config is: %s', norm_config)
 676
 677     with contextutil.nested(
 678         lambda: create_image(ctx=ctx, config=norm_config),
 679         lambda: modprobe(ctx=ctx, config=norm_config),
 680         lambda: dev_create(ctx=ctx, config=norm_config),
 681         lambda: generic_mkfs(ctx=ctx, config=norm_config,
 682                 devname_rtn=rbd_devname_rtn),
 683         lambda: generic_mount(ctx=ctx, config=role_images,
 684                 devname_rtn=rbd_devname_rtn),
 685         ):
 686         yield