12 from tasks
.util
.workunit
import get_refspec_after_overrides
13 from teuthology
import contextutil
14 from teuthology
import misc
as teuthology
15 from teuthology
.config
import config
as teuth_config
16 from teuthology
.orchestra
import run
17 from teuthology
.packaging
import install_package
, remove_package
19 log
= logging
.getLogger(__name__
)
22 DEFAULT_IMAGE_URL
= 'http://download.ceph.com/qa/ubuntu-12.04.qcow2'
23 DEFAULT_IMAGE_SIZE
= 10240 # in megabytes
24 ENCRYPTION_HEADER_SIZE
= 16 # in megabytes
26 DEFAULT_MEM
= 4096 # in megabytes
28 def normalize_disks(config
):
29 # normalize the 'disks' parameter into a list of dictionaries
30 for client
, client_config
in config
.items():
31 clone
= client_config
.get('clone', False)
32 image_url
= client_config
.get('image_url', DEFAULT_IMAGE_URL
)
33 device_type
= client_config
.get('type', 'filesystem')
34 encryption_format
= client_config
.get('encryption_format', 'none')
36 disks
= client_config
.get('disks', DEFAULT_NUM_DISKS
)
37 if not isinstance(disks
, list):
38 disks
= [{'image_name': '{client}.{num}'.format(client
=client
,
40 for i
in range(int(disks
))]
41 client_config
['disks'] = disks
43 for i
, disk
in enumerate(disks
):
44 if 'action' not in disk
:
45 disk
['action'] = 'create'
46 assert disk
['action'] in ['none', 'create', 'clone'], 'invalid disk action'
47 assert disk
['action'] != 'clone' or 'parent_name' in disk
, 'parent_name required for clone'
49 if 'image_size' not in disk
:
50 disk
['image_size'] = DEFAULT_IMAGE_SIZE
51 disk
['image_size'] = int(disk
['image_size'])
53 if 'image_url' not in disk
and i
== 0:
54 disk
['image_url'] = image_url
56 if 'device_type' not in disk
:
57 disk
['device_type'] = device_type
59 disk
['device_letter'] = chr(ord('a') + i
)
61 if 'encryption_format' not in disk
:
62 disk
['encryption_format'] = encryption_format
63 assert disk
['encryption_format'] in ['none', 'luks1', 'luks2'], 'invalid encryption format'
65 assert disks
, 'at least one rbd device must be used'
69 if disk
['action'] != 'create':
72 clone
['action'] = 'clone'
73 clone
['parent_name'] = clone
['image_name']
74 clone
['image_name'] += '-clone'
75 del disk
['device_letter']
78 def create_images(ctx
, config
, managers
):
79 for client
, client_config
in config
.items():
80 disks
= client_config
['disks']
82 if disk
.get('action') != 'create' or (
83 'image_url' in disk
and
84 disk
['encryption_format'] == 'none'):
86 image_size
= disk
['image_size']
87 if disk
['encryption_format'] != 'none':
88 image_size
+= ENCRYPTION_HEADER_SIZE
91 'image_name': disk
['image_name'],
93 'image_size': image_size
,
94 'encryption_format': disk
['encryption_format'],
98 lambda create_config
=create_config
:
99 rbd
.create_image(ctx
=ctx
, config
=create_config
)
102 def create_clones(ctx
, config
, managers
):
103 for client
, client_config
in config
.items():
104 disks
= client_config
['disks']
106 if disk
['action'] != 'clone':
111 'image_name': disk
['image_name'],
112 'parent_name': disk
['parent_name']
116 lambda create_config
=create_config
:
117 rbd
.clone_image(ctx
=ctx
, config
=create_config
)
120 def create_encrypted_devices(ctx
, config
, managers
):
121 for client
, client_config
in config
.items():
122 disks
= client_config
['disks']
124 if disk
['encryption_format'] == 'none' or \
125 'device_letter' not in disk
:
128 dev_config
= {client
: disk
}
130 lambda dev_config
=dev_config
:
131 rbd
.dev_create(ctx
=ctx
, config
=dev_config
)
134 @contextlib.contextmanager
135 def create_dirs(ctx
, config
):
137 Handle directory creation and cleanup
139 testdir
= teuthology
.get_testdir(ctx
)
140 for client
, client_config
in config
.items():
141 assert 'test' in client_config
, 'You must specify a test to run'
142 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
145 'install', '-d', '-m0755', '--',
146 '{tdir}/qemu'.format(tdir
=testdir
),
147 '{tdir}/archive/qemu'.format(tdir
=testdir
),
153 for client
, client_config
in config
.items():
154 assert 'test' in client_config
, 'You must specify a test to run'
155 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
158 'rmdir', '{tdir}/qemu'.format(tdir
=testdir
), run
.Raw('||'), 'true',
162 @contextlib.contextmanager
163 def install_block_rbd_driver(ctx
, config
):
165 Make sure qemu rbd block driver (block-rbd.so) is installed
167 for client
, client_config
in config
.items():
168 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
169 if remote
.os
.package_type
== 'rpm':
170 block_rbd_pkg
= 'qemu-kvm-block-rbd'
172 block_rbd_pkg
= 'qemu-block-extra'
173 install_package(block_rbd_pkg
, remote
)
177 for client
, client_config
in config
.items():
178 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
179 remove_package(block_rbd_pkg
, remote
)
181 @contextlib.contextmanager
182 def generate_iso(ctx
, config
):
183 """Execute system commands to generate iso"""
184 log
.info('generating iso...')
185 testdir
= teuthology
.get_testdir(ctx
)
187 # use ctx.config instead of config, because config has been
188 # through teuthology.replace_all_with_clients()
189 refspec
= get_refspec_after_overrides(ctx
.config
, {})
191 git_url
= teuth_config
.get_ceph_qa_suite_git_url()
192 log
.info('Pulling tests from %s ref %s', git_url
, refspec
)
194 for client
, client_config
in config
.items():
195 assert 'test' in client_config
, 'You must specify a test to run'
196 test
= client_config
['test']
198 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
200 clone_dir
= '{tdir}/qemu_clone.{role}'.format(tdir
=testdir
, role
=client
)
201 remote
.run(args
=refspec
.clone(git_url
, clone_dir
))
203 src_dir
= os
.path
.dirname(__file__
)
204 userdata_path
= os
.path
.join(testdir
, 'qemu', 'userdata.' + client
)
205 metadata_path
= os
.path
.join(testdir
, 'qemu', 'metadata.' + client
)
207 with
open(os
.path
.join(src_dir
, 'userdata_setup.yaml')) as f
:
208 test_setup
= ''.join(f
.readlines())
209 # configuring the commands to setup the nfs mount
210 mnt_dir
= "/export/{client}".format(client
=client
)
211 test_setup
= test_setup
.format(
215 with
open(os
.path
.join(src_dir
, 'userdata_teardown.yaml')) as f
:
216 test_teardown
= ''.join(f
.readlines())
218 user_data
= test_setup
220 disks
= client_config
['disks']
222 if disk
['device_type'] != 'filesystem' or \
223 'device_letter' not in disk
or \
226 if disk
['encryption_format'] == 'none':
227 dev_name
= 'vd' + disk
['device_letter']
229 # encrypted disks use if=ide interface, instead of if=virtio
230 dev_name
= 'sd' + disk
['device_letter']
234 mkdir /mnt/test_{dev_name}
235 mkfs -t xfs /dev/{dev_name}
236 mount -t xfs /dev/{dev_name} /mnt/test_{dev_name}
237 """.format(dev_name
=dev_name
)
242 test -d /etc/ceph || mkdir /etc/ceph
243 cp /mnt/cdrom/ceph.* /etc/ceph/
246 cloud_config_archive
= client_config
.get('cloud_config_archive', [])
247 if cloud_config_archive
:
248 user_data
+= yaml
.safe_dump(cloud_config_archive
, default_style
='|',
249 default_flow_style
=False)
251 # this may change later to pass the directories as args to the
252 # script or something. xfstests needs that.
256 test -d /mnt/test_b && cd /mnt/test_b
257 /mnt/cdrom/test.sh > /mnt/log/test.log 2>&1 && touch /mnt/log/success
260 user_data
= user_data
.format(
261 ceph_branch
=ctx
.config
.get('branch'),
262 ceph_sha1
=ctx
.config
.get('sha1'))
263 remote
.write_file(userdata_path
, user_data
)
265 with
open(os
.path
.join(src_dir
, 'metadata.yaml'), 'rb') as f
:
266 remote
.write_file(metadata_path
, f
)
268 test_file
= '{tdir}/qemu/{client}.test.sh'.format(tdir
=testdir
, client
=client
)
270 log
.info('fetching test %s for %s', test
, client
)
273 'cp', '--', os
.path
.join(clone_dir
, test
), test_file
,
275 'chmod', '755', test_file
,
280 'genisoimage', '-quiet', '-input-charset', 'utf-8',
281 '-volid', 'cidata', '-joliet', '-rock',
282 '-o', '{tdir}/qemu/{client}.iso'.format(tdir
=testdir
, client
=client
),
284 'user-data={userdata}'.format(userdata
=userdata_path
),
285 'meta-data={metadata}'.format(metadata
=metadata_path
),
286 'ceph.conf=/etc/ceph/ceph.conf',
287 'ceph.keyring=/etc/ceph/ceph.keyring',
288 'test.sh={file}'.format(file=test_file
),
294 for client
in config
.keys():
295 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
299 '{tdir}/qemu/{client}.iso'.format(tdir
=testdir
, client
=client
),
300 os
.path
.join(testdir
, 'qemu', 'userdata.' + client
),
301 os
.path
.join(testdir
, 'qemu', 'metadata.' + client
),
302 '{tdir}/qemu/{client}.test.sh'.format(tdir
=testdir
, client
=client
),
303 '{tdir}/qemu_clone.{client}'.format(tdir
=testdir
, client
=client
),
307 @contextlib.contextmanager
308 def download_image(ctx
, config
):
309 """Downland base image, remove image file when done"""
310 log
.info('downloading base image')
311 testdir
= teuthology
.get_testdir(ctx
)
313 client_base_files
= {}
314 for client
, client_config
in config
.items():
315 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
317 client_base_files
[client
] = []
318 disks
= client_config
['disks']
320 if disk
['action'] != 'create' or 'image_url' not in disk
:
323 base_file
= '{tdir}/qemu/base.{name}.qcow2'.format(tdir
=testdir
,
324 name
=disk
['image_name'])
325 client_base_files
[client
].append(base_file
)
329 'wget', '-nv', '-O', base_file
, disk
['image_url'],
333 if disk
['encryption_format'] == 'none':
336 'qemu-img', 'convert', '-f', 'qcow2', '-O', 'raw',
337 base_file
, 'rbd:rbd/{image_name}'.format(image_name
=disk
['image_name'])
341 dev_config
= {client
: {'image_name': disk
['image_name'],
342 'encryption_format': disk
['encryption_format']}}
343 raw_file
= '{tdir}/qemu/base.{name}.raw'.format(
344 tdir
=testdir
, name
=disk
['image_name'])
345 client_base_files
[client
].append(raw_file
)
348 'qemu-img', 'convert', '-f', 'qcow2', '-O', 'raw',
352 with rbd
.dev_create(ctx
, dev_config
):
355 'dd', 'if={name}'.format(name
=raw_file
),
356 'of={name}'.format(name
=dev_config
[client
]['device_path']),
357 'bs=4M', 'conv=fdatasync'
362 if disk
['action'] == 'clone' or \
363 disk
['encryption_format'] != 'none' or \
364 (disk
['action'] == 'create' and 'image_url' not in disk
):
370 '--size={image_size}M'.format(image_size
=disk
['image_size']),
371 disk
['image_name'], run
.Raw('||'), 'true'
378 log
.debug('cleaning up base image files')
379 for client
, base_files
in client_base_files
.items():
380 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
381 for base_file
in base_files
:
384 'rm', '-f', base_file
,
389 def _setup_nfs_mount(remote
, client
, service_name
, mount_dir
):
391 Sets up an nfs mount on the remote that the guest can use to
392 store logs. This nfs mount is also used to touch a file
393 at the end of the test to indicate if the test was successful
396 export_dir
= "/export/{client}".format(client
=client
)
397 log
.info("Creating the nfs export directory...")
399 'sudo', 'mkdir', '-p', export_dir
,
401 log
.info("Mounting the test directory...")
403 'sudo', 'mount', '--bind', mount_dir
, export_dir
,
405 log
.info("Adding mount to /etc/exports...")
406 export
= "{dir} *(rw,no_root_squash,no_subtree_check,insecure)".format(
409 log
.info("Deleting export from /etc/exports...")
411 'sudo', 'sed', '-i', "\|{export_dir}|d".format(export_dir
=export_dir
),
415 'echo', export
, run
.Raw("|"),
416 'sudo', 'tee', '-a', "/etc/exports",
418 log
.info("Restarting NFS...")
419 if remote
.os
.package_type
== "deb":
420 remote
.run(args
=['sudo', 'service', 'nfs-kernel-server', 'restart'])
422 remote
.run(args
=['sudo', 'systemctl', 'restart', service_name
])
425 def _teardown_nfs_mount(remote
, client
, service_name
):
427 Tears down the nfs mount on the remote used for logging and reporting the
428 status of the tests being ran in the guest.
430 log
.info("Tearing down the nfs mount for {remote}".format(remote
=remote
))
431 export_dir
= "/export/{client}".format(client
=client
)
432 log
.info("Stopping NFS...")
433 if remote
.os
.package_type
== "deb":
435 'sudo', 'service', 'nfs-kernel-server', 'stop'
439 'sudo', 'systemctl', 'stop', service_name
441 log
.info("Unmounting exported directory...")
443 'sudo', 'umount', export_dir
445 log
.info("Deleting export from /etc/exports...")
447 'sudo', 'sed', '-i', "\|{export_dir}|d".format(export_dir
=export_dir
),
450 log
.info("Starting NFS...")
451 if remote
.os
.package_type
== "deb":
453 'sudo', 'service', 'nfs-kernel-server', 'start'
457 'sudo', 'systemctl', 'start', service_name
461 @contextlib.contextmanager
462 def run_qemu(ctx
, config
):
463 """Setup kvm environment and start qemu"""
465 testdir
= teuthology
.get_testdir(ctx
)
466 for client
, client_config
in config
.items():
467 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
468 log_dir
= '{tdir}/archive/qemu/{client}'.format(tdir
=testdir
, client
=client
)
471 'mkdir', log_dir
, run
.Raw('&&'),
472 'sudo', 'modprobe', 'kvm',
476 nfs_service_name
= 'nfs'
477 if remote
.os
.name
in ['rhel', 'centos'] and float(remote
.os
.version
) >= 8:
478 nfs_service_name
= 'nfs-server'
480 # make an nfs mount to use for logging and to
481 # allow to test to tell teuthology the tests outcome
482 _setup_nfs_mount(remote
, client
, nfs_service_name
, log_dir
)
484 # Hack to make sure /dev/kvm permissions are set correctly
485 # See http://tracker.ceph.com/issues/17977 and
486 # https://bugzilla.redhat.com/show_bug.cgi?id=1333159
487 remote
.run(args
='sudo udevadm control --reload')
488 remote
.run(args
='sudo udevadm trigger /dev/kvm')
489 remote
.run(args
='ls -l /dev/kvm')
491 qemu_cmd
= 'qemu-system-x86_64'
492 if remote
.os
.package_type
== "rpm":
493 qemu_cmd
= "/usr/libexec/qemu-kvm"
497 '{tdir}/archive/coverage'.format(tdir
=testdir
),
500 qemu_cmd
, '-enable-kvm', '-nographic', '-cpu', 'host',
501 '-smp', str(client_config
.get('cpus', DEFAULT_CPUS
)),
502 '-m', str(client_config
.get('memory', DEFAULT_MEM
)),
503 # cd holding metadata for cloud-init
504 '-cdrom', '{tdir}/qemu/{client}.iso'.format(tdir
=testdir
, client
=client
),
508 ceph_config
= ctx
.ceph
['ceph'].conf
.get('global', {})
509 ceph_config
.update(ctx
.ceph
['ceph'].conf
.get('client', {}))
510 ceph_config
.update(ctx
.ceph
['ceph'].conf
.get(client
, {}))
511 if ceph_config
.get('rbd cache', True):
512 if ceph_config
.get('rbd cache max dirty', 1) > 0:
513 cachemode
= 'writeback'
515 cachemode
= 'writethrough'
517 disks
= client_config
['disks']
519 if 'device_letter' not in disk
:
522 if disk
['encryption_format'] == 'none':
524 disk_spec
= 'rbd:rbd/{img}:id={id}'.format(
525 img
=disk
['image_name'],
526 id=client
[len('client.'):]
529 # encrypted disks use ide as a temporary workaround for
530 # a bug in qemu when using virtio over nbd
531 # TODO: use librbd encryption directly via qemu (not via nbd)
533 disk_spec
= disk
['device_path']
537 'file={disk_spec},format=raw,if={interface},cache={cachemode}'.format(
543 time_wait
= client_config
.get('time_wait', 0)
545 log
.info('starting qemu...')
549 logger
=log
.getChild(client
),
558 log
.info('waiting for qemu tests to finish...')
562 log
.debug('waiting {time_wait} sec for workloads detect finish...'.format(
563 time_wait
=time_wait
));
564 time
.sleep(time_wait
)
566 log
.debug('checking that qemu tests succeeded...')
567 for client
in config
.keys():
568 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
570 # ensure we have permissions to all the logs
571 log_dir
= '{tdir}/archive/qemu/{client}'.format(tdir
=testdir
,
575 'sudo', 'chmod', 'a+rw', '-R', log_dir
580 _teardown_nfs_mount(remote
, client
, nfs_service_name
)
581 # check for test status
585 '{tdir}/archive/qemu/{client}/success'.format(
591 log
.info("Deleting exported directory...")
592 for client
in config
.keys():
593 (remote
,) = ctx
.cluster
.only(client
).remotes
.keys()
595 'sudo', 'rm', '-r', '/export'
599 @contextlib.contextmanager
600 def task(ctx
, config
):
602 Run a test inside of QEMU on top of rbd. Only one test
603 is supported per client.
605 For example, you can specify which clients to run on::
611 test: http://download.ceph.com/qa/test.sh
613 test: http://download.ceph.com/qa/test2.sh
615 Or use the same settings on all clients:
621 test: http://download.ceph.com/qa/test.sh
623 For tests that want to explicitly describe the RBD images to connect:
629 test: http://download.ceph.com/qa/test.sh
630 clone: True/False (optionally clone all created disks),
631 image_url: <URL> (optional default image URL)
632 type: filesystem / block (optional default device type)
635 action: create / clone / none (optional, defaults to create)
636 image_name: <image name> (optional)
637 parent_name: <parent_name> (if action == clone),
638 type: filesystem / block (optional, defaults to fileystem)
639 image_url: <URL> (optional),
640 image_size: <MiB> (optional)
641 encryption_format: luks1 / luks2 / none (optional, defaults to none)
645 You can set the amount of CPUs and memory the VM has (default is 1 CPU and
652 test: http://download.ceph.com/qa/test.sh
654 memory: 512 # megabytes
656 If you need to configure additional cloud-config options, set cloud_config
657 to the required data set::
663 test: http://ceph.com/qa/test.sh
664 cloud_config_archive:
673 assert isinstance(config
, dict), \
674 "task qemu only supports a dictionary for configuration"
676 config
= teuthology
.replace_all_with_clients(ctx
.cluster
, config
)
677 normalize_disks(config
)
680 create_images(ctx
=ctx
, config
=config
, managers
=managers
)
682 lambda: create_dirs(ctx
=ctx
, config
=config
),
683 lambda: install_block_rbd_driver(ctx
=ctx
, config
=config
),
684 lambda: generate_iso(ctx
=ctx
, config
=config
),
685 lambda: download_image(ctx
=ctx
, config
=config
),
687 create_clones(ctx
=ctx
, config
=config
, managers
=managers
)
688 create_encrypted_devices(ctx
=ctx
, config
=config
, managers
=managers
)
690 lambda: run_qemu(ctx
=ctx
, config
=config
),
693 with contextutil
.nested(*managers
):