[mirror_qemu.git] / tests / qemu-iotests / check

#!/usr/bin/env python3
#
# Configure environment and run group of tests in it.
#
# Copyright (c) 2020-2021 Virtuozzo International GmbH
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import argparse
import shutil
from pathlib import Path

from findtests import TestFinder
from testenv import TestEnv
from testrunner import TestRunner

def get_default_path(follow_link=False):
    """
    Try to automagically figure out the path we are running from.
    """
    # called from the build tree?
    if os.path.islink(sys.argv[0]):
        if follow_link:
            return os.path.dirname(os.readlink(sys.argv[0]))
        else:
            return os.path.dirname(os.path.abspath(sys.argv[0]))
    else:  # or source tree?
        return os.getcwd()

def make_argparser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(
        description="Test run options",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    p.add_argument('-n', '--dry-run', action='store_true',
                   help='show me, do not run tests')
    p.add_argument('-j', dest='jobs', type=int, default=1,
                   help='run tests in multiple parallel jobs')

    p.add_argument('-d', dest='debug', action='store_true', help='debug')
    p.add_argument('-p', dest='print', action='store_true',
                   help='redirects qemu\'s stdout and stderr to '
                        'the test output')
    p.add_argument('-gdb', action='store_true',
                   help="start gdbserver with $GDB_OPTIONS options "
                        "('localhost:12345' if $GDB_OPTIONS is empty)")
    p.add_argument('-valgrind', action='store_true',
                   help='use valgrind, sets VALGRIND_QEMU environment '
                        'variable')

    p.add_argument('-misalign', action='store_true',
                   help='misalign memory allocations')
    p.add_argument('--color', choices=['on', 'off', 'auto'],
                   default='auto', help="use terminal colors. The default "
                   "'auto' value means use colors if terminal stdout detected")
    p.add_argument('-tap', action='store_true',
                   help='produce TAP output')

    g_env = p.add_argument_group('test environment options')
    mg = g_env.add_mutually_exclusive_group()
    # We don't set default for cachemode, as we need to distinguish default
    # from user input later.
    mg.add_argument('-nocache', dest='cachemode', action='store_const',
                    const='none', help='set cache mode "none" (O_DIRECT), '
                    'sets CACHEMODE environment variable')
    mg.add_argument('-c', dest='cachemode',
                    help='sets CACHEMODE environment variable')

    g_env.add_argument('-i', dest='aiomode', default='threads',
                       help='sets AIOMODE environment variable')

    p.set_defaults(imgfmt='raw', imgproto='file')

    format_list = ['raw', 'bochs', 'cloop', 'parallels', 'qcow', 'qcow2',
                   'qed', 'vdi', 'vpc', 'vhdx', 'vmdk', 'luks', 'dmg']
    g_fmt = p.add_argument_group(
        '  image format options',
        'The following options set the IMGFMT environment variable. '
        'At most one choice is allowed, default is "raw"')
    mg = g_fmt.add_mutually_exclusive_group()
    for fmt in format_list:
        mg.add_argument('-' + fmt, dest='imgfmt', action='store_const',
                        const=fmt, help=f'test {fmt}')

    protocol_list = ['file', 'rbd', 'nbd', 'ssh', 'nfs', 'fuse']
    g_prt = p.add_argument_group(
        '  image protocol options',
        'The following options set the IMGPROTO environment variable. '
        'At most one choice is allowed, default is "file"')
    mg = g_prt.add_mutually_exclusive_group()
    for prt in protocol_list:
        mg.add_argument('-' + prt, dest='imgproto', action='store_const',
                        const=prt, help=f'test {prt}')

    g_bash = p.add_argument_group('bash tests options',
                                  'The following options are ignored by '
                                  'python tests.')
    # TODO: make support for the following options in iotests.py
    g_bash.add_argument('-o', dest='imgopts',
                        help='options to pass to qemu-img create/convert, '
                        'sets IMGOPTS environment variable')

    g_sel = p.add_argument_group('test selecting options',
                                 'The following options specify test set '
                                 'to run.')
    g_sel.add_argument('-g', '--groups', metavar='group1,...',
                       help='include tests from these groups')
    g_sel.add_argument('-x', '--exclude-groups', metavar='group1,...',
                       help='exclude tests from these groups')
    g_sel.add_argument('--start-from', metavar='TEST',
                       help='Start from specified test: make sorted sequence '
                       'of tests as usual and then drop tests from the first '
                       'one to TEST (not inclusive). This may be used to '
                       'rerun failed ./check command, starting from the '
                       'middle of the process.')
    g_sel.add_argument('tests', metavar='TEST_FILES', nargs='*',
                       help='tests to run, or "--" followed by a command')
    g_sel.add_argument('--build-dir', default=get_default_path(),
                       help='Path to iotests build directory')
    g_sel.add_argument('--source-dir',
                       default=get_default_path(follow_link=True),
                       help='Path to iotests build directory')

    return p


if __name__ == '__main__':
    args = make_argparser().parse_args()

    env = TestEnv(source_dir=args.source_dir,
                  build_dir=args.build_dir,
                  imgfmt=args.imgfmt, imgproto=args.imgproto,
                  aiomode=args.aiomode, cachemode=args.cachemode,
                  imgopts=args.imgopts, misalign=args.misalign,
                  debug=args.debug, valgrind=args.valgrind,
                  gdb=args.gdb, qprint=args.print,
                  dry_run=args.dry_run)

    if len(sys.argv) > 1 and sys.argv[-len(args.tests)-1] == '--':
        if not args.tests:
            sys.exit("missing command after '--'")
        cmd = args.tests
        env.print_env()
        exec_pathstr = shutil.which(cmd[0])
        if exec_pathstr is None:
            sys.exit('command not found: ' + cmd[0])
        exec_path = Path(exec_pathstr).resolve()
        cmd[0] = str(exec_path)
        full_env = env.prepare_subprocess(cmd)
        os.chdir(exec_path.parent)
        os.execve(cmd[0], cmd, full_env)

    testfinder = TestFinder(test_dir=env.source_iotests)

    groups = args.groups.split(',') if args.groups else None
    x_groups = args.exclude_groups.split(',') if args.exclude_groups else None

    group_local = os.path.join(env.source_iotests, 'group.local')
    if os.path.isfile(group_local):
        try:
            testfinder.add_group_file(group_local)
        except ValueError as e:
            sys.exit(f"Failed to parse group file '{group_local}': {e}")

    try:
        tests = testfinder.find_tests(groups=groups, exclude_groups=x_groups,
                                      tests=args.tests,
                                      start_from=args.start_from)
        if not tests:
            raise ValueError('No tests selected')
    except ValueError as e:
        sys.exit(str(e))

    if args.dry_run:
        with env:
            print('\n'.join([os.path.basename(t) for t in tests]))
    else:
        with TestRunner(env, tap=args.tap,
                        color=args.color) as tr:
            paths = [os.path.join(env.source_iotests, t) for t in tests]
            ok = tr.run_tests(paths, args.jobs)
            if not ok:
                sys.exit(1)
Commit	Line	Data
f203080b	1	#!/usr/bin/env python3
6bf19c94	2	#
f203080b VSO	3	# Configure environment and run group of tests in it.
	4	#
	5	# Copyright (c) 2020-2021 Virtuozzo International GmbH
6bf19c94 CH	6	#
	7	# This program is free software; you can redistribute it and/or
	8	# modify it under the terms of the GNU General Public License as
	9	# published by the Free Software Foundation.
	10	#
	11	# This program is distributed in the hope that it would be useful,
	12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	# GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License
e8c212d6	17	# along with this program. If not, see <http://www.gnu.org/licenses/>.
6bf19c94	18
f203080b VSO	19	import os
	20	import sys
	21	import argparse
480b75ee PB	22	import shutil
	23	from pathlib import Path
	24
f203080b VSO	25	from findtests import TestFinder
	26	from testenv import TestEnv
	27	from testrunner import TestRunner
	28
0c8076b0 DB	29	def get_default_path(follow_link=False):
	30	"""
	31	Try to automagically figure out the path we are running from.
	32	"""
	33	# called from the build tree?
	34	if os.path.islink(sys.argv[0]):
	35	if follow_link:
	36	return os.path.dirname(os.readlink(sys.argv[0]))
	37	else:
	38	return os.path.dirname(os.path.abspath(sys.argv[0]))
	39	else: # or source tree?
	40	return os.getcwd()
f203080b VSO	41
f203080b VSO	42	def make_argparser() -> argparse.ArgumentParser:
0c8076b0 DB	43	p = argparse.ArgumentParser(
	44	description="Test run options",
	45	formatter_class=argparse.ArgumentDefaultsHelpFormatter)
f203080b VSO	46
	47	p.add_argument('-n', '--dry-run', action='store_true',
	48	help='show me, do not run tests')
722f87df VSO	49	p.add_argument('-j', dest='jobs', type=int, default=1,
722f87df VSO	50	help='run tests in multiple parallel jobs')
f203080b VSO	51
f203080b VSO	52	p.add_argument('-d', dest='debug', action='store_true', help='debug')
eb7a91d0	53	p.add_argument('-p', dest='print', action='store_true',
87e4d4a2 EGE	54	help='redirects qemu\'s stdout and stderr to '
87e4d4a2 EGE	55	'the test output')
cfb9b0b7	56	p.add_argument('-gdb', action='store_true',
87e4d4a2 EGE	57	help="start gdbserver with $GDB_OPTIONS options "
87e4d4a2 EGE	58	"('localhost:12345' if $GDB_OPTIONS is empty)")
a9b4c6bb	59	p.add_argument('-valgrind', action='store_true',
87e4d4a2 EGE	60	help='use valgrind, sets VALGRIND_QEMU environment '
87e4d4a2 EGE	61	'variable')
a9b4c6bb	62
f203080b VSO	63	p.add_argument('-misalign', action='store_true',
	64	help='misalign memory allocations')
	65	p.add_argument('--color', choices=['on', 'off', 'auto'],
	66	default='auto', help="use terminal colors. The default "
	67	"'auto' value means use colors if terminal stdout detected")
d316859f PB	68	p.add_argument('-tap', action='store_true',
d316859f PB	69	help='produce TAP output')
f203080b VSO	70
	71	g_env = p.add_argument_group('test environment options')
	72	mg = g_env.add_mutually_exclusive_group()
	73	# We don't set default for cachemode, as we need to distinguish default
	74	# from user input later.
	75	mg.add_argument('-nocache', dest='cachemode', action='store_const',
	76	const='none', help='set cache mode "none" (O_DIRECT), '
	77	'sets CACHEMODE environment variable')
	78	mg.add_argument('-c', dest='cachemode',
	79	help='sets CACHEMODE environment variable')
	80
	81	g_env.add_argument('-i', dest='aiomode', default='threads',
	82	help='sets AIOMODE environment variable')
	83
	84	p.set_defaults(imgfmt='raw', imgproto='file')
	85
	86	format_list = ['raw', 'bochs', 'cloop', 'parallels', 'qcow', 'qcow2',
	87	'qed', 'vdi', 'vpc', 'vhdx', 'vmdk', 'luks', 'dmg']
	88	g_fmt = p.add_argument_group(
	89	' image format options',
	90	'The following options set the IMGFMT environment variable. '
	91	'At most one choice is allowed, default is "raw"')
	92	mg = g_fmt.add_mutually_exclusive_group()
	93	for fmt in format_list:
	94	mg.add_argument('-' + fmt, dest='imgfmt', action='store_const',
	95	const=fmt, help=f'test {fmt}')
	96
09ec8517	97	protocol_list = ['file', 'rbd', 'nbd', 'ssh', 'nfs', 'fuse']
f203080b VSO	98	g_prt = p.add_argument_group(
	99	' image protocol options',
	100	'The following options set the IMGPROTO environment variable. '
	101	'At most one choice is allowed, default is "file"')
	102	mg = g_prt.add_mutually_exclusive_group()
	103	for prt in protocol_list:
	104	mg.add_argument('-' + prt, dest='imgproto', action='store_const',
	105	const=prt, help=f'test {prt}')
	106
	107	g_bash = p.add_argument_group('bash tests options',
	108	'The following options are ignored by '
	109	'python tests.')
	110	# TODO: make support for the following options in iotests.py
	111	g_bash.add_argument('-o', dest='imgopts',
	112	help='options to pass to qemu-img create/convert, '
	113	'sets IMGOPTS environment variable')
f203080b VSO	114
	115	g_sel = p.add_argument_group('test selecting options',
	116	'The following options specify test set '
	117	'to run.')
	118	g_sel.add_argument('-g', '--groups', metavar='group1,...',
	119	help='include tests from these groups')
	120	g_sel.add_argument('-x', '--exclude-groups', metavar='group1,...',
	121	help='exclude tests from these groups')
	122	g_sel.add_argument('--start-from', metavar='TEST',
	123	help='Start from specified test: make sorted sequence '
	124	'of tests as usual and then drop tests from the first '
	125	'one to TEST (not inclusive). This may be used to '
	126	'rerun failed ./check command, starting from the '
	127	'middle of the process.')
	128	g_sel.add_argument('tests', metavar='TEST_FILES', nargs='*',
480b75ee	129	help='tests to run, or "--" followed by a command')
0c8076b0 DB	130	g_sel.add_argument('--build-dir', default=get_default_path(),
	131	help='Path to iotests build directory')
	132	g_sel.add_argument('--source-dir',
	133	default=get_default_path(follow_link=True),
	134	help='Path to iotests build directory')
f203080b VSO	135
	136	return p
	137
	138
	139	if __name__ == '__main__':
	140	args = make_argparser().parse_args()
	141
0c8076b0 DB	142	env = TestEnv(source_dir=args.source_dir,
	143	build_dir=args.build_dir,
	144	imgfmt=args.imgfmt, imgproto=args.imgproto,
f203080b VSO	145	aiomode=args.aiomode, cachemode=args.cachemode,
f203080b VSO	146	imgopts=args.imgopts, misalign=args.misalign,
cfb9b0b7	147	debug=args.debug, valgrind=args.valgrind,
a9e21786 DB	148	gdb=args.gdb, qprint=args.print,
a9e21786 DB	149	dry_run=args.dry_run)
f203080b	150
480b75ee PB	151	if len(sys.argv) > 1 and sys.argv[-len(args.tests)-1] == '--':
	152	if not args.tests:
	153	sys.exit("missing command after '--'")
	154	cmd = args.tests
	155	env.print_env()
	156	exec_pathstr = shutil.which(cmd[0])
	157	if exec_pathstr is None:
	158	sys.exit('command not found: ' + cmd[0])
	159	exec_path = Path(exec_pathstr).resolve()
	160	cmd[0] = str(exec_path)
	161	full_env = env.prepare_subprocess(cmd)
	162	os.chdir(exec_path.parent)
	163	os.execve(cmd[0], cmd, full_env)
	164
f203080b VSO	165	testfinder = TestFinder(test_dir=env.source_iotests)
	166
	167	groups = args.groups.split(',') if args.groups else None
	168	x_groups = args.exclude_groups.split(',') if args.exclude_groups else None
	169
	170	group_local = os.path.join(env.source_iotests, 'group.local')
	171	if os.path.isfile(group_local):
	172	try:
	173	testfinder.add_group_file(group_local)
	174	except ValueError as e:
	175	sys.exit(f"Failed to parse group file '{group_local}': {e}")
	176
	177	try:
	178	tests = testfinder.find_tests(groups=groups, exclude_groups=x_groups,
	179	tests=args.tests,
	180	start_from=args.start_from)
	181	if not tests:
	182	raise ValueError('No tests selected')
	183	except ValueError as e:
5bcf18b0	184	sys.exit(str(e))
f203080b VSO	185
f203080b VSO	186	if args.dry_run:
c645bac4 DB	187	with env:
c645bac4 DB	188	print('\n'.join([os.path.basename(t) for t in tests]))
f203080b	189	else:
d316859f	190	with TestRunner(env, tap=args.tap,
f203080b	191	color=args.color) as tr:
3ae50942	192	paths = [os.path.join(env.source_iotests, t) for t in tests]
722f87df	193	ok = tr.run_tests(paths, args.jobs)
3ae50942 VSO	194	if not ok:
3ae50942 VSO	195	sys.exit(1)