ceph/qa/tasks/qemu.py

   1 """
   2 Qemu task
   3 """
   4
   5 import contextlib
   6 import logging
   7 import os
   8 import yaml
   9 import time
  10
  11 from tasks import rbd
  12 from tasks.util.workunit import get_refspec_after_overrides
  13 from teuthology import contextutil
  14 from teuthology import misc as teuthology
  15 from teuthology.config import config as teuth_config
  16 from teuthology.orchestra import run
  17
  18 log = logging.getLogger(__name__)
  19
  20 DEFAULT_NUM_DISKS = 2
  21 DEFAULT_IMAGE_URL = 'http://download.ceph.com/qa/ubuntu-12.04.qcow2'
  22 DEFAULT_IMAGE_SIZE = 10240 # in megabytes
  23 DEFAULT_CPUS = 1
  24 DEFAULT_MEM = 4096 # in megabytes
  25
  26 def create_images(ctx, config, managers):
  27     for client, client_config in config.items():
  28         disks = client_config.get('disks', DEFAULT_NUM_DISKS)
  29         if not isinstance(disks, list):
  30             disks = [{} for n in range(int(disks))]
  31         clone = client_config.get('clone', False)
  32         assert disks, 'at least one rbd device must be used'
  33         for i, disk in enumerate(disks[1:]):
  34             create_config = {
  35                 client: {
  36                     'image_name': '{client}.{num}'.format(client=client,
  37                                                           num=i + 1),
  38                     'image_format': 2 if clone else 1,
  39                     'image_size': (disk or {}).get('image_size',
  40                                                    DEFAULT_IMAGE_SIZE),
  41                     }
  42                 }
  43             managers.append(
  44                 lambda create_config=create_config:
  45                 rbd.create_image(ctx=ctx, config=create_config)
  46                 )
  47
  48 def create_clones(ctx, config, managers):
  49     for client, client_config in config.items():
  50         clone = client_config.get('clone', False)
  51         if clone:
  52             num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
  53             if isinstance(num_disks, list):
  54                 num_disks = len(num_disks)
  55             for i in range(num_disks):
  56                 create_config = {
  57                     client: {
  58                         'image_name':
  59                         '{client}.{num}-clone'.format(client=client, num=i),
  60                         'parent_name':
  61                         '{client}.{num}'.format(client=client, num=i),
  62                         }
  63                     }
  64                 managers.append(
  65                     lambda create_config=create_config:
  66                     rbd.clone_image(ctx=ctx, config=create_config)
  67                     )
  68
  69 @contextlib.contextmanager
  70 def create_dirs(ctx, config):
  71     """
  72     Handle directory creation and cleanup
  73     """
  74     testdir = teuthology.get_testdir(ctx)
  75     for client, client_config in config.items():
  76         assert 'test' in client_config, 'You must specify a test to run'
  77         (remote,) = ctx.cluster.only(client).remotes.keys()
  78         remote.run(
  79             args=[
  80                 'install', '-d', '-m0755', '--',
  81                 '{tdir}/qemu'.format(tdir=testdir),
  82                 '{tdir}/archive/qemu'.format(tdir=testdir),
  83                 ]
  84             )
  85     try:
  86         yield
  87     finally:
  88         for client, client_config in config.items():
  89             assert 'test' in client_config, 'You must specify a test to run'
  90             (remote,) = ctx.cluster.only(client).remotes.keys()
  91             remote.run(
  92                 args=[
  93                     'rmdir', '{tdir}/qemu'.format(tdir=testdir), run.Raw('||'), 'true',
  94                     ]
  95                 )
  96
  97 @contextlib.contextmanager
  98 def generate_iso(ctx, config):
  99     """Execute system commands to generate iso"""
 100     log.info('generating iso...')
 101     testdir = teuthology.get_testdir(ctx)
 102
 103     # use ctx.config instead of config, because config has been
 104     # through teuthology.replace_all_with_clients()
 105     refspec = get_refspec_after_overrides(ctx.config, {})
 106
 107     git_url = teuth_config.get_ceph_qa_suite_git_url()
 108     log.info('Pulling tests from %s ref %s', git_url, refspec)
 109
 110     for client, client_config in config.items():
 111         assert 'test' in client_config, 'You must specify a test to run'
 112         test = client_config['test']
 113
 114         (remote,) = ctx.cluster.only(client).remotes.keys()
 115
 116         clone_dir = '{tdir}/qemu_clone.{role}'.format(tdir=testdir, role=client)
 117         remote.run(args=refspec.clone(git_url, clone_dir))
 118
 119         src_dir = os.path.dirname(__file__)
 120         userdata_path = os.path.join(testdir, 'qemu', 'userdata.' + client)
 121         metadata_path = os.path.join(testdir, 'qemu', 'metadata.' + client)
 122
 123         with open(os.path.join(src_dir, 'userdata_setup.yaml')) as f:
 124             test_setup = ''.join(f.readlines())
 125             # configuring the commands to setup the nfs mount
 126             mnt_dir = "/export/{client}".format(client=client)
 127             test_setup = test_setup.format(
 128                 mnt_dir=mnt_dir
 129             )
 130
 131         with open(os.path.join(src_dir, 'userdata_teardown.yaml')) as f:
 132             test_teardown = ''.join(f.readlines())
 133
 134         user_data = test_setup
 135         if client_config.get('type', 'filesystem') == 'filesystem':
 136             num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
 137             if isinstance(num_disks, list):
 138                 num_disks = len(num_disks)
 139             for i in range(1, num_disks):
 140                 dev_letter = chr(ord('a') + i)
 141                 user_data += """
 142 - |
 143   #!/bin/bash
 144   mkdir /mnt/test_{dev_letter}
 145   mkfs -t xfs /dev/vd{dev_letter}
 146   mount -t xfs /dev/vd{dev_letter} /mnt/test_{dev_letter}
 147 """.format(dev_letter=dev_letter)
 148
 149         user_data += """
 150 - |
 151   #!/bin/bash
 152   test -d /etc/ceph || mkdir /etc/ceph
 153   cp /mnt/cdrom/ceph.* /etc/ceph/
 154 """
 155
 156         cloud_config_archive = client_config.get('cloud_config_archive', [])
 157         if cloud_config_archive:
 158           user_data += yaml.safe_dump(cloud_config_archive, default_style='|',
 159                                       default_flow_style=False)
 160
 161         # this may change later to pass the directories as args to the
 162         # script or something. xfstests needs that.
 163         user_data += """
 164 - |
 165   #!/bin/bash
 166   test -d /mnt/test_b && cd /mnt/test_b
 167   /mnt/cdrom/test.sh > /mnt/log/test.log 2>&1 && touch /mnt/log/success
 168 """ + test_teardown
 169
 170         user_data = user_data.format(
 171             ceph_branch=ctx.config.get('branch'),
 172             ceph_sha1=ctx.config.get('sha1'))
 173         teuthology.write_file(remote, userdata_path, user_data)
 174
 175         with open(os.path.join(src_dir, 'metadata.yaml'), 'rb') as f:
 176             teuthology.write_file(remote, metadata_path, f)
 177
 178         test_file = '{tdir}/qemu/{client}.test.sh'.format(tdir=testdir, client=client)
 179
 180         log.info('fetching test %s for %s', test, client)
 181         remote.run(
 182             args=[
 183                 'cp', '--', os.path.join(clone_dir, test), test_file,
 184                 run.Raw('&&'),
 185                 'chmod', '755', test_file,
 186                 ],
 187             )
 188         remote.run(
 189             args=[
 190                 'genisoimage', '-quiet', '-input-charset', 'utf-8',
 191                 '-volid', 'cidata', '-joliet', '-rock',
 192                 '-o', '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
 193                 '-graft-points',
 194                 'user-data={userdata}'.format(userdata=userdata_path),
 195                 'meta-data={metadata}'.format(metadata=metadata_path),
 196                 'ceph.conf=/etc/ceph/ceph.conf',
 197                 'ceph.keyring=/etc/ceph/ceph.keyring',
 198                 'test.sh={file}'.format(file=test_file),
 199                 ],
 200             )
 201     try:
 202         yield
 203     finally:
 204         for client in config.keys():
 205             (remote,) = ctx.cluster.only(client).remotes.keys()
 206             remote.run(
 207                 args=[
 208                     'rm', '-rf',
 209                     '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
 210                     os.path.join(testdir, 'qemu', 'userdata.' + client),
 211                     os.path.join(testdir, 'qemu', 'metadata.' + client),
 212                     '{tdir}/qemu/{client}.test.sh'.format(tdir=testdir, client=client),
 213                     '{tdir}/qemu_clone.{client}'.format(tdir=testdir, client=client),
 214                     ],
 215                 )
 216
 217 @contextlib.contextmanager
 218 def download_image(ctx, config):
 219     """Downland base image, remove image file when done"""
 220     log.info('downloading base image')
 221     testdir = teuthology.get_testdir(ctx)
 222     for client, client_config in config.items():
 223         (remote,) = ctx.cluster.only(client).remotes.keys()
 224         base_file = '{tdir}/qemu/base.{client}.qcow2'.format(tdir=testdir, client=client)
 225         image_url = client_config.get('image_url', DEFAULT_IMAGE_URL)
 226         remote.run(
 227             args=[
 228                 'wget', '-nv', '-O', base_file, image_url,
 229                 ]
 230             )
 231
 232         disks = client_config.get('disks', None)
 233         if not isinstance(disks, list):
 234             disks = [{}]
 235         image_name = '{client}.0'.format(client=client)
 236         image_size = (disks[0] or {}).get('image_size', DEFAULT_IMAGE_SIZE)
 237         remote.run(
 238             args=[
 239                 'qemu-img', 'convert', '-f', 'qcow2', '-O', 'raw',
 240                 base_file, 'rbd:rbd/{image_name}'.format(image_name=image_name)
 241                 ]
 242             )
 243         remote.run(
 244             args=[
 245                 'rbd', 'resize',
 246                 '--size={image_size}M'.format(image_size=image_size),
 247                 image_name,
 248                 ]
 249             )
 250     try:
 251         yield
 252     finally:
 253         log.debug('cleaning up base image files')
 254         for client in config.keys():
 255             base_file = '{tdir}/qemu/base.{client}.qcow2'.format(
 256                 tdir=testdir,
 257                 client=client,
 258                 )
 259             (remote,) = ctx.cluster.only(client).remotes.keys()
 260             remote.run(
 261                 args=[
 262                     'rm', '-f', base_file,
 263                     ],
 264                 )
 265
 266
 267 def _setup_nfs_mount(remote, client, service_name, mount_dir):
 268     """
 269     Sets up an nfs mount on the remote that the guest can use to
 270     store logs. This nfs mount is also used to touch a file
 271     at the end of the test to indicate if the test was successful
 272     or not.
 273     """
 274     export_dir = "/export/{client}".format(client=client)
 275     log.info("Creating the nfs export directory...")
 276     remote.run(args=[
 277         'sudo', 'mkdir', '-p', export_dir,
 278     ])
 279     log.info("Mounting the test directory...")
 280     remote.run(args=[
 281         'sudo', 'mount', '--bind', mount_dir, export_dir,
 282     ])
 283     log.info("Adding mount to /etc/exports...")
 284     export = "{dir} *(rw,no_root_squash,no_subtree_check,insecure)".format(
 285         dir=export_dir
 286     )
 287     remote.run(args=[
 288         'sudo', 'sed', '-i', '/^\/export\//d', "/etc/exports",
 289     ])
 290     remote.run(args=[
 291         'echo', export, run.Raw("|"),
 292         'sudo', 'tee', '-a', "/etc/exports",
 293     ])
 294     log.info("Restarting NFS...")
 295     if remote.os.package_type == "deb":
 296         remote.run(args=['sudo', 'service', 'nfs-kernel-server', 'restart'])
 297     else:
 298         remote.run(args=['sudo', 'systemctl', 'restart', service_name])
 299
 300
 301 def _teardown_nfs_mount(remote, client, service_name):
 302     """
 303     Tears down the nfs mount on the remote used for logging and reporting the
 304     status of the tests being ran in the guest.
 305     """
 306     log.info("Tearing down the nfs mount for {remote}".format(remote=remote))
 307     export_dir = "/export/{client}".format(client=client)
 308     log.info("Stopping NFS...")
 309     if remote.os.package_type == "deb":
 310         remote.run(args=[
 311             'sudo', 'service', 'nfs-kernel-server', 'stop'
 312         ])
 313     else:
 314         remote.run(args=[
 315             'sudo', 'systemctl', 'stop', service_name
 316         ])
 317     log.info("Unmounting exported directory...")
 318     remote.run(args=[
 319         'sudo', 'umount', export_dir
 320     ])
 321     log.info("Deleting exported directory...")
 322     remote.run(args=[
 323         'sudo', 'rm', '-r', '/export'
 324     ])
 325     log.info("Deleting export from /etc/exports...")
 326     remote.run(args=[
 327         'sudo', 'sed', '-i', '$ d', '/etc/exports'
 328     ])
 329     log.info("Starting NFS...")
 330     if remote.os.package_type == "deb":
 331         remote.run(args=[
 332             'sudo', 'service', 'nfs-kernel-server', 'start'
 333         ])
 334     else:
 335         remote.run(args=[
 336             'sudo', 'systemctl', 'start', service_name
 337         ])
 338
 339
 340 @contextlib.contextmanager
 341 def run_qemu(ctx, config):
 342     """Setup kvm environment and start qemu"""
 343     procs = []
 344     testdir = teuthology.get_testdir(ctx)
 345     for client, client_config in config.items():
 346         (remote,) = ctx.cluster.only(client).remotes.keys()
 347         log_dir = '{tdir}/archive/qemu/{client}'.format(tdir=testdir, client=client)
 348         remote.run(
 349             args=[
 350                 'mkdir', log_dir, run.Raw('&&'),
 351                 'sudo', 'modprobe', 'kvm',
 352                 ]
 353             )
 354
 355         nfs_service_name = 'nfs'
 356         if remote.os.name in ['rhel', 'centos'] and float(remote.os.version) >= 8:
 357             nfs_service_name = 'nfs-server'
 358
 359         # make an nfs mount to use for logging and to
 360         # allow to test to tell teuthology the tests outcome
 361         _setup_nfs_mount(remote, client, nfs_service_name, log_dir)
 362
 363         # Hack to make sure /dev/kvm permissions are set correctly
 364         # See http://tracker.ceph.com/issues/17977 and
 365         # https://bugzilla.redhat.com/show_bug.cgi?id=1333159
 366         remote.run(args='sudo udevadm control --reload')
 367         remote.run(args='sudo udevadm trigger /dev/kvm')
 368         remote.run(args='ls -l /dev/kvm')
 369
 370         qemu_cmd = 'qemu-system-x86_64'
 371         if remote.os.package_type == "rpm":
 372             qemu_cmd = "/usr/libexec/qemu-kvm"
 373         args=[
 374             'adjust-ulimits',
 375             'ceph-coverage',
 376             '{tdir}/archive/coverage'.format(tdir=testdir),
 377             'daemon-helper',
 378             'term',
 379             qemu_cmd, '-enable-kvm', '-nographic', '-cpu', 'host',
 380             '-smp', str(client_config.get('cpus', DEFAULT_CPUS)),
 381             '-m', str(client_config.get('memory', DEFAULT_MEM)),
 382             # cd holding metadata for cloud-init
 383             '-cdrom', '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
 384             ]
 385
 386         cachemode = 'none'
 387         ceph_config = ctx.ceph['ceph'].conf.get('global', {})
 388         ceph_config.update(ctx.ceph['ceph'].conf.get('client', {}))
 389         ceph_config.update(ctx.ceph['ceph'].conf.get(client, {}))
 390         if ceph_config.get('rbd cache', True):
 391             if ceph_config.get('rbd cache max dirty', 1) > 0:
 392                 cachemode = 'writeback'
 393             else:
 394                 cachemode = 'writethrough'
 395
 396         clone = client_config.get('clone', False)
 397         num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
 398         if isinstance(num_disks, list):
 399             num_disks = len(num_disks)
 400         for i in range(num_disks):
 401             suffix = '-clone' if clone else ''
 402             args.extend([
 403                 '-drive',
 404                 'file=rbd:rbd/{img}:id={id},format=raw,if=virtio,cache={cachemode}'.format(
 405                     img='{client}.{num}{suffix}'.format(client=client, num=i,
 406                                                         suffix=suffix),
 407                     id=client[len('client.'):],
 408                     cachemode=cachemode,
 409                     ),
 410                 ])
 411         time_wait = client_config.get('time_wait', 0)
 412
 413         log.info('starting qemu...')
 414         procs.append(
 415             remote.run(
 416                 args=args,
 417                 logger=log.getChild(client),
 418                 stdin=run.PIPE,
 419                 wait=False,
 420                 )
 421             )
 422
 423     try:
 424         yield
 425     finally:
 426         log.info('waiting for qemu tests to finish...')
 427         run.wait(procs)
 428
 429         if time_wait > 0:
 430             log.debug('waiting {time_wait} sec for workloads detect finish...'.format(
 431                 time_wait=time_wait));
 432             time.sleep(time_wait)
 433
 434         log.debug('checking that qemu tests succeeded...')
 435         for client in config.keys():
 436             (remote,) = ctx.cluster.only(client).remotes.keys()
 437
 438             # ensure we have permissions to all the logs
 439             log_dir = '{tdir}/archive/qemu/{client}'.format(tdir=testdir,
 440                                                             client=client)
 441             remote.run(
 442                 args=[
 443                     'sudo', 'chmod', 'a+rw', '-R', log_dir
 444                     ]
 445                 )
 446
 447             # teardown nfs mount
 448             _teardown_nfs_mount(remote, client, nfs_service_name)
 449             # check for test status
 450             remote.run(
 451                 args=[
 452                     'test', '-f',
 453                     '{tdir}/archive/qemu/{client}/success'.format(
 454                         tdir=testdir,
 455                         client=client
 456                         ),
 457                     ],
 458                 )
 459
 460
 461 @contextlib.contextmanager
 462 def task(ctx, config):
 463     """
 464     Run a test inside of QEMU on top of rbd. Only one test
 465     is supported per client.
 466
 467     For example, you can specify which clients to run on::
 468
 469         tasks:
 470         - ceph:
 471         - qemu:
 472             client.0:
 473               test: http://download.ceph.com/qa/test.sh
 474             client.1:
 475               test: http://download.ceph.com/qa/test2.sh
 476
 477     Or use the same settings on all clients:
 478
 479         tasks:
 480         - ceph:
 481         - qemu:
 482             all:
 483               test: http://download.ceph.com/qa/test.sh
 484
 485     For tests that don't need a filesystem, set type to block::
 486
 487         tasks:
 488         - ceph:
 489         - qemu:
 490             client.0:
 491               test: http://download.ceph.com/qa/test.sh
 492               type: block
 493
 494     The test should be configured to run on /dev/vdb and later
 495     devices.
 496
 497     If you want to run a test that uses more than one rbd image,
 498     specify how many images to use::
 499
 500         tasks:
 501         - ceph:
 502         - qemu:
 503             client.0:
 504               test: http://download.ceph.com/qa/test.sh
 505               type: block
 506               disks: 2
 507
 508     - or -
 509
 510         tasks:
 511         - ceph:
 512         - qemu:
 513             client.0:
 514               test: http://ceph.com/qa/test.sh
 515               type: block
 516               disks:
 517                 - image_size: 1024
 518                 - image_size: 2048
 519
 520     You can set the amount of CPUs and memory the VM has (default is 1 CPU and
 521     4096 MB)::
 522
 523         tasks:
 524         - ceph:
 525         - qemu:
 526             client.0:
 527               test: http://download.ceph.com/qa/test.sh
 528               cpus: 4
 529               memory: 512 # megabytes
 530
 531     If you want to run a test against a cloned rbd image, set clone to true::
 532
 533         tasks:
 534         - ceph:
 535         - qemu:
 536             client.0:
 537               test: http://download.ceph.com/qa/test.sh
 538               clone: true
 539
 540     If you need to configure additional cloud-config options, set cloud_config
 541     to the required data set::
 542
 543         tasks:
 544         - ceph
 545         - qemu:
 546             client.0:
 547                 test: http://ceph.com/qa/test.sh
 548                 cloud_config_archive:
 549                     - |
 550                       #/bin/bash
 551                       touch foo1
 552                     - content: |
 553                         test data
 554                       type: text/plain
 555                       filename: /tmp/data
 556
 557     If you need to override the default cloud image, set image_url:
 558
 559         tasks:
 560         - ceph
 561         - qemu:
 562             client.0:
 563                 test: http://ceph.com/qa/test.sh
 564                 image_url: https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
 565     """
 566     assert isinstance(config, dict), \
 567            "task qemu only supports a dictionary for configuration"
 568
 569     config = teuthology.replace_all_with_clients(ctx.cluster, config)
 570
 571     managers = []
 572     create_images(ctx=ctx, config=config, managers=managers)
 573     managers.extend([
 574         lambda: create_dirs(ctx=ctx, config=config),
 575         lambda: generate_iso(ctx=ctx, config=config),
 576         lambda: download_image(ctx=ctx, config=config),
 577         ])
 578     create_clones(ctx=ctx, config=config, managers=managers)
 579     managers.append(
 580         lambda: run_qemu(ctx=ctx, config=config),
 581         )
 582
 583     with contextutil.nested(*managers):
 584         yield