ceph/qa/tasks/qemu.py

   1 """
   2 Qemu task
   3 """
   4 from cStringIO import StringIO
   5
   6 import contextlib
   7 import logging
   8 import os
   9 import yaml
  10 import time
  11
  12 from teuthology import misc as teuthology
  13 from teuthology import contextutil
  14 from tasks import rbd
  15 from teuthology.orchestra import run
  16 from teuthology.config import config as teuth_config
  17
  18 from util.workunit import get_refspec_after_overrides
  19
  20 log = logging.getLogger(__name__)
  21
  22 DEFAULT_NUM_DISKS = 2
  23 DEFAULT_IMAGE_URL = 'http://download.ceph.com/qa/ubuntu-12.04.qcow2'
  24 DEFAULT_IMAGE_SIZE = 10240 # in megabytes
  25 DEFAULT_CPUS = 1
  26 DEFAULT_MEM = 4096 # in megabytes
  27
  28 def create_images(ctx, config, managers):
  29     for client, client_config in config.iteritems():
  30         disks = client_config.get('disks', DEFAULT_NUM_DISKS)
  31         if not isinstance(disks, list):
  32             disks = [{} for n in range(int(disks))]
  33         clone = client_config.get('clone', False)
  34         assert disks, 'at least one rbd device must be used'
  35         for i, disk in enumerate(disks[1:]):
  36             create_config = {
  37                 client: {
  38                     'image_name': '{client}.{num}'.format(client=client,
  39                                                           num=i + 1),
  40                     'image_format': 2 if clone else 1,
  41                     'image_size': (disk or {}).get('image_size',
  42                                                    DEFAULT_IMAGE_SIZE),
  43                     }
  44                 }
  45             managers.append(
  46                 lambda create_config=create_config:
  47                 rbd.create_image(ctx=ctx, config=create_config)
  48                 )
  49
  50 def create_clones(ctx, config, managers):
  51     for client, client_config in config.iteritems():
  52         clone = client_config.get('clone', False)
  53         if clone:
  54             num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
  55             if isinstance(num_disks, list):
  56                 num_disks = len(num_disks)
  57             for i in xrange(num_disks):
  58                 create_config = {
  59                     client: {
  60                         'image_name':
  61                         '{client}.{num}-clone'.format(client=client, num=i),
  62                         'parent_name':
  63                         '{client}.{num}'.format(client=client, num=i),
  64                         }
  65                     }
  66                 managers.append(
  67                     lambda create_config=create_config:
  68                     rbd.clone_image(ctx=ctx, config=create_config)
  69                     )
  70
  71 @contextlib.contextmanager
  72 def create_dirs(ctx, config):
  73     """
  74     Handle directory creation and cleanup
  75     """
  76     testdir = teuthology.get_testdir(ctx)
  77     for client, client_config in config.iteritems():
  78         assert 'test' in client_config, 'You must specify a test to run'
  79         (remote,) = ctx.cluster.only(client).remotes.keys()
  80         remote.run(
  81             args=[
  82                 'install', '-d', '-m0755', '--',
  83                 '{tdir}/qemu'.format(tdir=testdir),
  84                 '{tdir}/archive/qemu'.format(tdir=testdir),
  85                 ]
  86             )
  87     try:
  88         yield
  89     finally:
  90         for client, client_config in config.iteritems():
  91             assert 'test' in client_config, 'You must specify a test to run'
  92             (remote,) = ctx.cluster.only(client).remotes.keys()
  93             remote.run(
  94                 args=[
  95                     'rmdir', '{tdir}/qemu'.format(tdir=testdir), run.Raw('||'), 'true',
  96                     ]
  97                 )
  98
  99 @contextlib.contextmanager
 100 def generate_iso(ctx, config):
 101     """Execute system commands to generate iso"""
 102     log.info('generating iso...')
 103     testdir = teuthology.get_testdir(ctx)
 104
 105     # use ctx.config instead of config, because config has been
 106     # through teuthology.replace_all_with_clients()
 107     refspec = get_refspec_after_overrides(ctx.config, {})
 108
 109     git_url = teuth_config.get_ceph_qa_suite_git_url()
 110     log.info('Pulling tests from %s ref %s', git_url, refspec)
 111
 112     for client, client_config in config.iteritems():
 113         assert 'test' in client_config, 'You must specify a test to run'
 114         test = client_config['test']
 115
 116         (remote,) = ctx.cluster.only(client).remotes.keys()
 117
 118         clone_dir = '{tdir}/clone.{role}'.format(tdir=testdir, role=client)
 119         remote.run(args=refspec.clone(git_url, clone_dir))
 120
 121         src_dir = os.path.dirname(__file__)
 122         userdata_path = os.path.join(testdir, 'qemu', 'userdata.' + client)
 123         metadata_path = os.path.join(testdir, 'qemu', 'metadata.' + client)
 124
 125         with file(os.path.join(src_dir, 'userdata_setup.yaml'), 'rb') as f:
 126             test_setup = ''.join(f.readlines())
 127             # configuring the commands to setup the nfs mount
 128             mnt_dir = "/export/{client}".format(client=client)
 129             test_setup = test_setup.format(
 130                 mnt_dir=mnt_dir
 131             )
 132
 133         with file(os.path.join(src_dir, 'userdata_teardown.yaml'), 'rb') as f:
 134             test_teardown = ''.join(f.readlines())
 135
 136         user_data = test_setup
 137         if client_config.get('type', 'filesystem') == 'filesystem':
 138             num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
 139             if isinstance(num_disks, list):
 140                 num_disks = len(num_disks)
 141             for i in xrange(1, num_disks):
 142                 dev_letter = chr(ord('a') + i)
 143                 user_data += """
 144 - |
 145   #!/bin/bash
 146   mkdir /mnt/test_{dev_letter}
 147   mkfs -t xfs /dev/vd{dev_letter}
 148   mount -t xfs /dev/vd{dev_letter} /mnt/test_{dev_letter}
 149 """.format(dev_letter=dev_letter)
 150
 151         user_data += """
 152 - |
 153   #!/bin/bash
 154   test -d /etc/ceph || mkdir /etc/ceph
 155   cp /mnt/cdrom/ceph.* /etc/ceph/
 156 """
 157
 158         cloud_config_archive = client_config.get('cloud_config_archive', [])
 159         if cloud_config_archive:
 160           user_data += yaml.safe_dump(cloud_config_archive, default_style='|',
 161                                       default_flow_style=False)
 162
 163         # this may change later to pass the directories as args to the
 164         # script or something. xfstests needs that.
 165         user_data += """
 166 - |
 167   #!/bin/bash
 168   test -d /mnt/test_b && cd /mnt/test_b
 169   /mnt/cdrom/test.sh > /mnt/log/test.log 2>&1 && touch /mnt/log/success
 170 """ + test_teardown
 171
 172         user_data = user_data.format(
 173             ceph_branch=ctx.config.get('branch'),
 174             ceph_sha1=ctx.config.get('sha1'))
 175         teuthology.write_file(remote, userdata_path, StringIO(user_data))
 176
 177         with file(os.path.join(src_dir, 'metadata.yaml'), 'rb') as f:
 178             teuthology.write_file(remote, metadata_path, f)
 179
 180         test_file = '{tdir}/qemu/{client}.test.sh'.format(tdir=testdir, client=client)
 181
 182         log.info('fetching test %s for %s', test, client)
 183         remote.run(
 184             args=[
 185                 'cp', '--', os.path.join(clone_dir, test), test_file,
 186                 run.Raw('&&'),
 187                 'chmod', '755', test_file,
 188                 ],
 189             )
 190         remote.run(
 191             args=[
 192                 'genisoimage', '-quiet', '-input-charset', 'utf-8',
 193                 '-volid', 'cidata', '-joliet', '-rock',
 194                 '-o', '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
 195                 '-graft-points',
 196                 'user-data={userdata}'.format(userdata=userdata_path),
 197                 'meta-data={metadata}'.format(metadata=metadata_path),
 198                 'ceph.conf=/etc/ceph/ceph.conf',
 199                 'ceph.keyring=/etc/ceph/ceph.keyring',
 200                 'test.sh={file}'.format(file=test_file),
 201                 ],
 202             )
 203     try:
 204         yield
 205     finally:
 206         for client in config.iterkeys():
 207             (remote,) = ctx.cluster.only(client).remotes.keys()
 208             remote.run(
 209                 args=[
 210                     'rm', '-rf',
 211                     '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
 212                     os.path.join(testdir, 'qemu', 'userdata.' + client),
 213                     os.path.join(testdir, 'qemu', 'metadata.' + client),
 214                     '{tdir}/qemu/{client}.test.sh'.format(tdir=testdir, client=client),
 215                     '{tdir}/clone.{client}'.format(tdir=testdir, client=client),
 216                     ],
 217                 )
 218
 219 @contextlib.contextmanager
 220 def download_image(ctx, config):
 221     """Downland base image, remove image file when done"""
 222     log.info('downloading base image')
 223     testdir = teuthology.get_testdir(ctx)
 224     for client, client_config in config.iteritems():
 225         (remote,) = ctx.cluster.only(client).remotes.keys()
 226         base_file = '{tdir}/qemu/base.{client}.qcow2'.format(tdir=testdir, client=client)
 227         image_url = client_config.get('image_url', DEFAULT_IMAGE_URL)
 228         remote.run(
 229             args=[
 230                 'wget', '-nv', '-O', base_file, image_url,
 231                 ]
 232             )
 233
 234         disks = client_config.get('disks', None)
 235         if not isinstance(disks, list):
 236             disks = [{}]
 237         image_name = '{client}.0'.format(client=client)
 238         image_size = (disks[0] or {}).get('image_size', DEFAULT_IMAGE_SIZE)
 239         remote.run(
 240             args=[
 241                 'qemu-img', 'convert', '-f', 'qcow2', '-O', 'raw',
 242                 base_file, 'rbd:rbd/{image_name}'.format(image_name=image_name)
 243                 ]
 244             )
 245         remote.run(
 246             args=[
 247                 'rbd', 'resize',
 248                 '--size={image_size}M'.format(image_size=image_size),
 249                 image_name,
 250                 ]
 251             )
 252     try:
 253         yield
 254     finally:
 255         log.debug('cleaning up base image files')
 256         for client in config.iterkeys():
 257             base_file = '{tdir}/qemu/base.{client}.qcow2'.format(
 258                 tdir=testdir,
 259                 client=client,
 260                 )
 261             (remote,) = ctx.cluster.only(client).remotes.keys()
 262             remote.run(
 263                 args=[
 264                     'rm', '-f', base_file,
 265                     ],
 266                 )
 267
 268
 269 def _setup_nfs_mount(remote, client, mount_dir):
 270     """
 271     Sets up an nfs mount on the remote that the guest can use to
 272     store logs. This nfs mount is also used to touch a file
 273     at the end of the test to indiciate if the test was successful
 274     or not.
 275     """
 276     export_dir = "/export/{client}".format(client=client)
 277     log.info("Creating the nfs export directory...")
 278     remote.run(args=[
 279         'sudo', 'mkdir', '-p', export_dir,
 280     ])
 281     log.info("Mounting the test directory...")
 282     remote.run(args=[
 283         'sudo', 'mount', '--bind', mount_dir, export_dir,
 284     ])
 285     log.info("Adding mount to /etc/exports...")
 286     export = "{dir} *(rw,no_root_squash,no_subtree_check,insecure)".format(
 287         dir=export_dir
 288     )
 289     remote.run(args=[
 290         'sudo', 'sed', '-i', '/^\/export\//d', "/etc/exports",
 291     ])
 292     remote.run(args=[
 293         'echo', export, run.Raw("|"),
 294         'sudo', 'tee', '-a', "/etc/exports",
 295     ])
 296     log.info("Restarting NFS...")
 297     if remote.os.package_type == "deb":
 298         remote.run(args=['sudo', 'service', 'nfs-kernel-server', 'restart'])
 299     else:
 300         remote.run(args=['sudo', 'systemctl', 'restart', 'nfs'])
 301
 302
 303 def _teardown_nfs_mount(remote, client):
 304     """
 305     Tears down the nfs mount on the remote used for logging and reporting the
 306     status of the tests being ran in the guest.
 307     """
 308     log.info("Tearing down the nfs mount for {remote}".format(remote=remote))
 309     export_dir = "/export/{client}".format(client=client)
 310     log.info("Stopping NFS...")
 311     if remote.os.package_type == "deb":
 312         remote.run(args=[
 313             'sudo', 'service', 'nfs-kernel-server', 'stop'
 314         ])
 315     else:
 316         remote.run(args=[
 317             'sudo', 'systemctl', 'stop', 'nfs'
 318         ])
 319     log.info("Unmounting exported directory...")
 320     remote.run(args=[
 321         'sudo', 'umount', export_dir
 322     ])
 323     log.info("Deleting exported directory...")
 324     remote.run(args=[
 325         'sudo', 'rm', '-r', '/export'
 326     ])
 327     log.info("Deleting export from /etc/exports...")
 328     remote.run(args=[
 329         'sudo', 'sed', '-i', '$ d', '/etc/exports'
 330     ])
 331     log.info("Starting NFS...")
 332     if remote.os.package_type == "deb":
 333         remote.run(args=[
 334             'sudo', 'service', 'nfs-kernel-server', 'start'
 335         ])
 336     else:
 337         remote.run(args=[
 338             'sudo', 'systemctl', 'start', 'nfs'
 339         ])
 340
 341
 342 @contextlib.contextmanager
 343 def run_qemu(ctx, config):
 344     """Setup kvm environment and start qemu"""
 345     procs = []
 346     testdir = teuthology.get_testdir(ctx)
 347     for client, client_config in config.iteritems():
 348         (remote,) = ctx.cluster.only(client).remotes.keys()
 349         log_dir = '{tdir}/archive/qemu/{client}'.format(tdir=testdir, client=client)
 350         remote.run(
 351             args=[
 352                 'mkdir', log_dir, run.Raw('&&'),
 353                 'sudo', 'modprobe', 'kvm',
 354                 ]
 355             )
 356
 357         # make an nfs mount to use for logging and to
 358         # allow to test to tell teuthology the tests outcome
 359         _setup_nfs_mount(remote, client, log_dir)
 360
 361         # Hack to make sure /dev/kvm permissions are set correctly
 362         # See http://tracker.ceph.com/issues/17977 and
 363         # https://bugzilla.redhat.com/show_bug.cgi?id=1333159
 364         remote.run(args='sudo udevadm control --reload')
 365         remote.run(args='sudo udevadm trigger /dev/kvm')
 366         remote.run(args='ls -l /dev/kvm')
 367
 368         qemu_cmd = 'qemu-system-x86_64'
 369         if remote.os.package_type == "rpm":
 370             qemu_cmd = "/usr/libexec/qemu-kvm"
 371         args=[
 372             'adjust-ulimits',
 373             'ceph-coverage',
 374             '{tdir}/archive/coverage'.format(tdir=testdir),
 375             'daemon-helper',
 376             'term',
 377             qemu_cmd, '-enable-kvm', '-nographic', '-cpu', 'host',
 378             '-smp', str(client_config.get('cpus', DEFAULT_CPUS)),
 379             '-m', str(client_config.get('memory', DEFAULT_MEM)),
 380             # cd holding metadata for cloud-init
 381             '-cdrom', '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
 382             ]
 383
 384         cachemode = 'none'
 385         ceph_config = ctx.ceph['ceph'].conf.get('global', {})
 386         ceph_config.update(ctx.ceph['ceph'].conf.get('client', {}))
 387         ceph_config.update(ctx.ceph['ceph'].conf.get(client, {}))
 388         if ceph_config.get('rbd cache', True):
 389             if ceph_config.get('rbd cache max dirty', 1) > 0:
 390                 cachemode = 'writeback'
 391             else:
 392                 cachemode = 'writethrough'
 393
 394         clone = client_config.get('clone', False)
 395         num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
 396         if isinstance(num_disks, list):
 397             num_disks = len(num_disks)
 398         for i in xrange(num_disks):
 399             suffix = '-clone' if clone else ''
 400             args.extend([
 401                 '-drive',
 402                 'file=rbd:rbd/{img}:id={id},format=raw,if=virtio,cache={cachemode}'.format(
 403                     img='{client}.{num}{suffix}'.format(client=client, num=i,
 404                                                         suffix=suffix),
 405                     id=client[len('client.'):],
 406                     cachemode=cachemode,
 407                     ),
 408                 ])
 409         time_wait = client_config.get('time_wait', 0)
 410
 411         log.info('starting qemu...')
 412         procs.append(
 413             remote.run(
 414                 args=args,
 415                 logger=log.getChild(client),
 416                 stdin=run.PIPE,
 417                 wait=False,
 418                 )
 419             )
 420
 421     try:
 422         yield
 423     finally:
 424         log.info('waiting for qemu tests to finish...')
 425         run.wait(procs)
 426
 427         if time_wait > 0:
 428             log.debug('waiting {time_wait} sec for workloads detect finish...'.format(
 429                 time_wait=time_wait));
 430             time.sleep(time_wait)
 431
 432         log.debug('checking that qemu tests succeeded...')
 433         for client in config.iterkeys():
 434             (remote,) = ctx.cluster.only(client).remotes.keys()
 435
 436             # ensure we have permissions to all the logs
 437             log_dir = '{tdir}/archive/qemu/{client}'.format(tdir=testdir,
 438                                                             client=client)
 439             remote.run(
 440                 args=[
 441                     'sudo', 'chmod', 'a+rw', '-R', log_dir
 442                     ]
 443                 )
 444
 445             # teardown nfs mount
 446             _teardown_nfs_mount(remote, client)
 447             # check for test status
 448             remote.run(
 449                 args=[
 450                     'test', '-f',
 451                     '{tdir}/archive/qemu/{client}/success'.format(
 452                         tdir=testdir,
 453                         client=client
 454                         ),
 455                     ],
 456                 )
 457
 458
 459 @contextlib.contextmanager
 460 def task(ctx, config):
 461     """
 462     Run a test inside of QEMU on top of rbd. Only one test
 463     is supported per client.
 464
 465     For example, you can specify which clients to run on::
 466
 467         tasks:
 468         - ceph:
 469         - qemu:
 470             client.0:
 471               test: http://download.ceph.com/qa/test.sh
 472             client.1:
 473               test: http://download.ceph.com/qa/test2.sh
 474
 475     Or use the same settings on all clients:
 476
 477         tasks:
 478         - ceph:
 479         - qemu:
 480             all:
 481               test: http://download.ceph.com/qa/test.sh
 482
 483     For tests that don't need a filesystem, set type to block::
 484
 485         tasks:
 486         - ceph:
 487         - qemu:
 488             client.0:
 489               test: http://download.ceph.com/qa/test.sh
 490               type: block
 491
 492     The test should be configured to run on /dev/vdb and later
 493     devices.
 494
 495     If you want to run a test that uses more than one rbd image,
 496     specify how many images to use::
 497
 498         tasks:
 499         - ceph:
 500         - qemu:
 501             client.0:
 502               test: http://download.ceph.com/qa/test.sh
 503               type: block
 504               disks: 2
 505
 506     - or -
 507
 508         tasks:
 509         - ceph:
 510         - qemu:
 511             client.0:
 512               test: http://ceph.com/qa/test.sh
 513               type: block
 514               disks:
 515                 - image_size: 1024
 516                 - image_size: 2048
 517
 518     You can set the amount of CPUs and memory the VM has (default is 1 CPU and
 519     4096 MB)::
 520
 521         tasks:
 522         - ceph:
 523         - qemu:
 524             client.0:
 525               test: http://download.ceph.com/qa/test.sh
 526               cpus: 4
 527               memory: 512 # megabytes
 528
 529     If you want to run a test against a cloned rbd image, set clone to true::
 530
 531         tasks:
 532         - ceph:
 533         - qemu:
 534             client.0:
 535               test: http://download.ceph.com/qa/test.sh
 536               clone: true
 537
 538     If you need to configure additional cloud-config options, set cloud_config
 539     to the required data set::
 540
 541         tasks:
 542         - ceph
 543         - qemu:
 544             client.0:
 545                 test: http://ceph.com/qa/test.sh
 546                 cloud_config_archive:
 547                     - |
 548                       #/bin/bash
 549                       touch foo1
 550                     - content: |
 551                         test data
 552                       type: text/plain
 553                       filename: /tmp/data
 554
 555     If you need to override the default cloud image, set image_url:
 556
 557         tasks:
 558         - ceph
 559         - qemu:
 560             client.0:
 561                 test: http://ceph.com/qa/test.sh
 562                 image_url: https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
 563     """
 564     assert isinstance(config, dict), \
 565            "task qemu only supports a dictionary for configuration"
 566
 567     config = teuthology.replace_all_with_clients(ctx.cluster, config)
 568
 569     managers = []
 570     create_images(ctx=ctx, config=config, managers=managers)
 571     managers.extend([
 572         lambda: create_dirs(ctx=ctx, config=config),
 573         lambda: generate_iso(ctx=ctx, config=config),
 574         lambda: download_image(ctx=ctx, config=config),
 575         ])
 576     create_clones(ctx=ctx, config=config, managers=managers)
 577     managers.append(
 578         lambda: run_qemu(ctx=ctx, config=config),
 579         )
 580
 581     with contextutil.nested(*managers):
 582         yield