[mirror_zfs.git] / scripts / zfs-tests.sh

#!/bin/bash
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License, Version 1.0 only
# (the "License").  You may not use this file except in compliance
# with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

BASE_DIR=$(dirname "$0")
SCRIPT_COMMON=common.sh
if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
. "${BASE_DIR}/${SCRIPT_COMMON}"
else
echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
fi

PROG=zfs-tests.sh
VERBOSE="no"
QUIET=
CLEANUP="yes"
CLEANUPALL="no"
LOOPBACK="yes"
STACK_TRACER="no"
FILESIZE="4G"
RUNFILE=${RUNFILE:-"linux.run"}
FILEDIR=${FILEDIR:-/var/tmp}
DISKS=${DISKS:-""}
SINGLETEST=()
SINGLETESTUSER="root"
TAGS=""
ITERATIONS=1
ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
LOSETUP=${LOSETUP:-/sbin/losetup}
DMSETUP=${DMSETUP:-/sbin/dmsetup}

#
# Log an informational message when additional verbosity is enabled.
#
msg() {
	if [ "$VERBOSE" = "yes" ]; then
		echo "$@"
	fi
}

#
# Log a failure message, cleanup, and return an error.
#
fail() {
        echo -e "$PROG: $1" >&2
	cleanup
	exit 1
}

#
# Attempt to remove loopback devices and files which where created earlier
# by this script to run the test framework.  The '-k' option may be passed
# to the script to suppress cleanup for debugging purposes.
#
cleanup() {
	if [ "$CLEANUP" = "no" ]; then
		return 0
	fi

	if [ "$LOOPBACK" = "yes" ]; then
		for TEST_LOOPBACK in ${LOOPBACKS}; do
			LOOP_DEV=$(basename "$TEST_LOOPBACK")
			DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
			    grep "${LOOP_DEV}" | cut -f1)

			if [ -n "$DM_DEV" ]; then
				sudo "${DMSETUP}" remove "${DM_DEV}" ||
				    echo "Failed to remove: ${DM_DEV}"
			fi

			if [ -n "${TEST_LOOPBACK}" ]; then
				sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
				    echo "Failed to remove: ${TEST_LOOPBACK}"
			fi
		done
	fi

	for TEST_FILE in ${FILES}; do
		rm -f "${TEST_FILE}" &>/dev/null
	done

	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
		rm -Rf "$STF_PATH"
	fi
}
trap cleanup EXIT

#
# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
# loopback devices, and files.  This is a useful way to cleanup a previous
# test run failure which has left the system in an unknown state.  This can
# be dangerous and should only be used in a dedicated test environment.
#
cleanup_all() {
	local TEST_POOLS
	TEST_POOLS=$(sudo "$ZPOOL" list -H -o name | grep testpool)
	local TEST_LOOPBACKS
	TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
	local TEST_FILES
	TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)

	msg
	msg "--- Cleanup ---"
	msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
	for TEST_POOL in $TEST_POOLS; do
		sudo "$ZPOOL" destroy "${TEST_POOL}"
	done

	msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
	    grep loop | tr '\n' ' ')"
	sudo "${DMSETUP}" remove_all

	msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
		sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
	done

	msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
	for TEST_FILE in $TEST_FILES; do
		sudo rm -f "${TEST_FILE}"
	done
}

#
# Takes a name as the only arguments and looks for the following variations
# on that name.  If one is found it is returned.
#
# $RUNFILE_DIR/<name>
# $RUNFILE_DIR/<name>.run
# <name>
# <name>.run
#
find_runfile() {
	local NAME=$1
	local RESULT=""

	if [ -f "$RUNFILE_DIR/$NAME" ]; then
		RESULT="$RUNFILE_DIR/$NAME"
	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
		RESULT="$RUNFILE_DIR/$NAME.run"
	elif [ -f "$NAME" ]; then
		RESULT="$NAME"
	elif [ -f "$NAME.run" ]; then
		RESULT="$NAME.run"
	fi

	echo "$RESULT"
}

#
# Symlink file if it appears under any of the given paths.
#
create_links() {
	local dir_list="$1"
	local file_list="$2"

	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"

	for i in $file_list; do
		for j in $dir_list; do
			[ ! -e "$STF_PATH/$i" ] || continue

			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
				ln -s "$j/$i" "$STF_PATH/$i" || \
				    fail "Couldn't link $i"
				break
			fi
		done

		[ ! -e "$STF_PATH/$i" ] && STF_MISSING_BIN="$STF_MISSING_BIN$i "
	done
}

#
# Constrain the path to limit the available binaries to a known set.
# When running in-tree a top level ./bin/ directory is created for
# convenience, otherwise a temporary directory is used.
#
constrain_path() {
	. "$STF_SUITE/include/commands.cfg"

	if [ "$INTREE" = "yes" ]; then
		# Constrained path set to ./zfs/bin/
		STF_PATH="$BIN_DIR"
		STF_PATH_REMOVE="no"
		STF_MISSING_BIN=""
		if [ ! -d "$STF_PATH" ]; then
			mkdir "$STF_PATH"
			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
		fi

		# Special case links for standard zfs utilities
		DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
		    ! -name .libs \) -print | tr '\n' ' ')"
		create_links "$DIRS" "$ZFS_FILES"

		# Special case links for zfs test suite utilities
		DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
		    ! -name .libs \) -print | tr '\n' ' ')"
		create_links "$DIRS" "$ZFSTEST_FILES"
	else
		# Constrained path set to /var/tmp/constrained_path.*
		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXX}
		STF_PATH=$(/bin/mktemp -d "$SYSTEMDIR")
		STF_PATH_REMOVE="yes"
		STF_MISSING_BIN=""

		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"

		# Special case links for standard zfs utilities
		create_links "/bin /usr/bin /sbin /usr/sbin" "$ZFS_FILES"

		# Special case links for zfs test suite utilities
		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
	fi

	# Standard system utilities
	create_links "/bin /usr/bin /sbin /usr/sbin" "$SYSTEM_FILES"

	# Exceptions
	ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
	ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
	ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
	ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
	ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
	ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
	ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"

	if [ -L "$STF_PATH/arc_summary3" ]; then
		ln -fs "$STF_PATH/arc_summary3" "$STF_PATH/arc_summary"
	fi
}

#
# Output a useful usage message.
#
usage() {
cat << EOF
USAGE:
$0 [hvqxkfS] [-s SIZE] [-r RUNFILE] [-t PATH] [-u USER]

DESCRIPTION:
	ZFS Test Suite launch script

OPTIONS:
	-h          Show this message
	-v          Verbose zfs-tests.sh output
	-q          Quiet test-runner output
	-x          Remove all testpools, dm, lo, and files (unsafe)
	-k          Disable cleanup after test failure
	-f          Use files only, disables block device tests
	-S          Enable stack tracer (negative performance impact)
	-c          Only create and populate constrained path
	-n NFSFILE  Use the nfsfile to determine the NFS configuration
	-I NUM      Number of iterations
	-d DIR      Use DIR for files and loopback devices
	-s SIZE     Use vdevs of SIZE (default: 4G)
	-r RUNFILE  Run tests in RUNFILE (default: linux.run)
	-t PATH     Run single test at PATH relative to test suite
	-T TAGS     Comma separated list of tags (default: 'functional')
	-u USER     Run single test as USER (default: root)

EXAMPLES:
# Run the default (linux) suite of tests and output the configuration used.
$0 -v

# Run a smaller suite of tests designed to run more quickly.
$0 -r linux-fast

# Cleanup a previous run of the test suite prior to testing, run the
# default (linux) suite of tests and perform no cleanup on exit.
$0 -x

EOF
}

while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do
	case $OPTION in
	h)
		usage
		exit 1
		;;
	v)
		# shellcheck disable=SC2034
		VERBOSE="yes"
		;;
	q)
		QUIET="-q"
		;;
	x)
		CLEANUPALL="yes"
		;;
	k)
		CLEANUP="no"
		;;
	f)
		LOOPBACK="no"
		;;
	S)
		STACK_TRACER="yes"
		;;
	c)
		constrain_path
		exit
		;;
	n)
		nfsfile=$OPTARG
		[[ -f $nfsfile ]] || fail "Cannot read file: $nfsfile"
		export NFS=1
		. "$nfsfile"
		;;
	d)
		FILEDIR="$OPTARG"
		;;
	I)
		ITERATIONS="$OPTARG"
		if [ "$ITERATIONS" -le 0 ]; then
			fail "Iterations must be greater than 0."
		fi
		;;
	s)
		FILESIZE="$OPTARG"
		;;
	r)
		RUNFILE="$OPTARG"
		;;
	t)
		if [ ${#SINGLETEST[@]} -ne 0 ]; then
			fail "-t can only be provided once."
		fi
		SINGLETEST+=("$OPTARG")
		;;
	T)
		TAGS="$OPTARG"
		;;
	u)
		SINGLETESTUSER="$OPTARG"
		;;
	?)
		usage
		exit
		;;
	esac
done

shift $((OPTIND-1))

FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
LOOPBACKS=${LOOPBACKS:-""}

if [ ${#SINGLETEST[@]} -ne 0 ]; then
	if [ -n "$TAGS" ]; then
		fail "-t and -T are mutually exclusive."
	fi
	RUNFILE_DIR="/var/tmp"
	RUNFILE="zfs-tests.$$.run"
	SINGLEQUIET="False"

	if [ -n "$QUIET" ]; then
		SINGLEQUIET="True"
	fi

	cat >$RUNFILE_DIR/$RUNFILE << EOF
[DEFAULT]
pre =
quiet = $SINGLEQUIET
pre_user = root
user = $SINGLETESTUSER
timeout = 600
post_user = root
post =
outputdir = /var/tmp/test_results
EOF
	for t in "${SINGLETEST[@]}"
	do
		SINGLETESTDIR=$(dirname "$t")
		SINGLETESTFILE=$(basename "$t")
		SETUPSCRIPT=
		CLEANUPSCRIPT=

		if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
			SETUPSCRIPT="setup"
		fi

		if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
			CLEANUPSCRIPT="cleanup"
		fi

		cat >>$RUNFILE_DIR/$RUNFILE << EOF

[$SINGLETESTDIR]
tests = ['$SINGLETESTFILE']
pre = $SETUPSCRIPT
post = $CLEANUPSCRIPT
tags = ['functional']
EOF
	done
fi

#
# Use default tag if none was specified
#
TAGS=${TAGS:='functional'}

#
# Attempt to locate the runfile describing the test workload.
#
if [ -n "$RUNFILE" ]; then
	SAVED_RUNFILE="$RUNFILE"
	RUNFILE=$(find_runfile "$RUNFILE")
	[ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
fi

if [ ! -r "$RUNFILE" ]; then
	fail "Cannot read runfile: $RUNFILE"
fi

#
# This script should not be run as root.  Instead the test user, which may
# be a normal user account, needs to be configured such that it can
# run commands via sudo passwordlessly.
#
if [ "$(id -u)" = "0" ]; then
	fail "This script must not be run as root."
fi

if [ "$(sudo whoami)" != "root" ]; then
	fail "Passwordless sudo access required."
fi

#
# Constrain the available binaries to a known set.
#
constrain_path

#
# Check if ksh exists
#
[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
[ -e "$STF_SUITE/include/default.cfg" ] || fail \
    "Missing $STF_SUITE/include/default.cfg file."

#
# Verify the ZFS module stack is loaded.
#
if [ "$STACK_TRACER" = "yes" ]; then
	sudo "${ZFS_SH}" -S &>/dev/null
else
	sudo "${ZFS_SH}" &>/dev/null
fi

#
# Attempt to cleanup all previous state for a new test run.
#
if [ "$CLEANUPALL" = "yes" ]; then
	cleanup_all
fi

#
# By default preserve any existing pools
# NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
# space-delimited to newline-delimited.
#
if [ -z "${KEEP}" ]; then
	KEEP="$(sudo "$ZPOOL" list -H -o name)"
	if [ -z "${KEEP}" ]; then
		KEEP="rpool"
	fi
else
	KEEP="$(echo -e "${KEEP//[[:blank:]]/\n}")"
fi

#
# NOTE: The following environment variables are undocumented
# and should be used for testing purposes only:
#
# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
#
# See libzfs/libzfs_config.c for more information.
#
__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"

. "$STF_SUITE/include/default.cfg"

msg
msg "--- Configuration ---"
msg "Runfile:         $RUNFILE"
msg "STF_TOOLS:       $STF_TOOLS"
msg "STF_SUITE:       $STF_SUITE"
msg "STF_PATH:        $STF_PATH"

#
# No DISKS have been provided so a basic file or loopback based devices
# must be created for the test suite to use.
#
if [ -z "${DISKS}" ]; then
	#
	# Create sparse files for the test suite.  These may be used
	# directory or have loopback devices layered on them.
	#
	for TEST_FILE in ${FILES}; do
		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
		    fail "Failed creating: ${TEST_FILE} ($?)"
		if [[ "$DISKS" ]]; then
			DISKS="$DISKS $TEST_FILE"
		else
			DISKS="$TEST_FILE"
		fi
	done

	#
	# If requested setup loopback devices backed by the sparse files.
	#
	if [ "$LOOPBACK" = "yes" ]; then
		DISKS=""

		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"

		for TEST_FILE in ${FILES}; do
			TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
			sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
			    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
			LOOPBACKS="${LOOPBACKS}${TEST_LOOPBACK} "
			BASELOOPBACKS=$(basename "$TEST_LOOPBACK")
			if [[ "$DISKS" ]]; then
				DISKS="$DISKS $BASELOOPBACKS"
			else
				DISKS="$BASELOOPBACKS"
			fi
		done
	fi
fi

NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"

#
# Disable SELinux until the ZFS Test Suite has been updated accordingly.
#
if [ -x "$STF_PATH/setenforce" ]; then
	sudo setenforce permissive &>/dev/null
fi

#
# Enable internal ZFS debug log and clear it.
#
if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
	sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
	sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
fi

msg "FILEDIR:         $FILEDIR"
msg "FILES:           $FILES"
msg "LOOPBACKS:       $LOOPBACKS"
msg "DISKS:           $DISKS"
msg "NUM_DISKS:       $NUM_DISKS"
msg "FILESIZE:        $FILESIZE"
msg "ITERATIONS:      $ITERATIONS"
msg "TAGS:            $TAGS"
msg "STACK_TRACER:    $STACK_TRACER"
msg "Keep pool(s):    $KEEP"
msg "Missing util(s): $STF_MISSING_BIN"
msg ""

export STF_TOOLS
export STF_SUITE
export STF_PATH
export DISKS
export FILEDIR
export KEEP
export __ZFS_POOL_EXCLUDE
export TESTFAIL_CALLBACKS
export PATH=$STF_PATH

RESULTS_FILE=$(mktemp -u -t zts-results.XXXX -p "$FILEDIR")
REPORT_FILE=$(mktemp -u -t zts-report.XXXX -p "$FILEDIR")

#
# Run all the tests as specified.
#
msg "${TEST_RUNNER} ${QUIET} -c ${RUNFILE} -T ${TAGS} -i ${STF_SUITE}" \
    "-I ${ITERATIONS}"
${TEST_RUNNER} ${QUIET} -c "${RUNFILE}" -T "${TAGS}" -i "${STF_SUITE}" \
    -I "${ITERATIONS}" 2>&1 | tee "$RESULTS_FILE"

#
# Analyze the results.
#
set -o pipefail
${ZTS_REPORT} "$RESULTS_FILE" | tee "$REPORT_FILE"
RESULT=$?
set +o pipefail

RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
if [ -d "$RESULTS_DIR" ]; then
	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
fi

rm -f "$RESULTS_FILE" "$REPORT_FILE"

if [ ${#SINGLETEST[@]} -ne 0 ]; then
	rm -f "$RUNFILE" &>/dev/null
fi

exit ${RESULT}
Commit	Line	Data
6bb24f4d BB	1	#!/bin/bash
	2	#
	3	# CDDL HEADER START
	4	#
	5	# The contents of this file are subject to the terms of the
	6	# Common Development and Distribution License, Version 1.0 only
	7	# (the "License"). You may not use this file except in compliance
	8	# with the License.
	9	#
	10	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	11	# or http://www.opensolaris.org/os/licensing.
	12	# See the License for the specific language governing permissions
	13	# and limitations under the License.
	14	#
	15	# When distributing Covered Code, include this CDDL HEADER in each
	16	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	17	# If applicable, add the following below this CDDL HEADER, with the
	18	# fields enclosed by brackets "[]" replaced with your own identifying
	19	# information: Portions Copyright [yyyy] [name of copyright owner]
	20	#
	21	# CDDL HEADER END
	22	#
6bb24f4d	23
c8f9061f	24	BASE_DIR=$(dirname "$0")
6bb24f4d	25	SCRIPT_COMMON=common.sh
c8f9061f BB	26	if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
c8f9061f BB	27	. "${BASE_DIR}/${SCRIPT_COMMON}"
6bb24f4d BB	28	else
	29	echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
	30	fi
	31
6bb24f4d	32	PROG=zfs-tests.sh
c8f9061f	33	VERBOSE="no"
6bb24f4d	34	QUIET=
c8f9061f BB	35	CLEANUP="yes"
	36	CLEANUPALL="no"
	37	LOOPBACK="yes"
ef57371a	38	STACK_TRACER="no"
d7958b4c	39	FILESIZE="4G"
6bb24f4d BB	40	RUNFILE=${RUNFILE:-"linux.run"}
	41	FILEDIR=${FILEDIR:-/var/tmp}
	42	DISKS=${DISKS:-""}
d8fa599f GDN	43	SINGLETEST=()
d8fa599f GDN	44	SINGLETESTUSER="root"
271955da	45	TAGS=""
9a810efb	46	ITERATIONS=1
00481e7d BB	47	ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
00481e7d BB	48	ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
379ca9cf OF	49	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
379ca9cf OF	50	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
c8f9061f BB	51	LOSETUP=${LOSETUP:-/sbin/losetup}
	52	DMSETUP=${DMSETUP:-/sbin/dmsetup}
	53
	54	#
	55	# Log an informational message when additional verbosity is enabled.
	56	#
	57	msg() {
	58	if [ "$VERBOSE" = "yes" ]; then
	59	echo "$@"
	60	fi
	61	}
	62
	63	#
	64	# Log a failure message, cleanup, and return an error.
	65	#
	66	fail() {
	67	echo -e "$PROG: $1" >&2
	68	cleanup
	69	exit 1
	70	}
6bb24f4d BB	71
	72	#
	73	# Attempt to remove loopback devices and files which where created earlier
	74	# by this script to run the test framework. The '-k' option may be passed
	75	# to the script to suppress cleanup for debugging purposes.
	76	#
	77	cleanup() {
c8f9061f	78	if [ "$CLEANUP" = "no" ]; then
6bb24f4d BB	79	return 0
	80	fi
	81
c8f9061f	82	if [ "$LOOPBACK" = "yes" ]; then
6bb24f4d	83	for TEST_LOOPBACK in ${LOOPBACKS}; do
c552fbc5	84	LOOP_DEV=$(basename "$TEST_LOOPBACK")
c1d9abf9	85	DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null \| \
c552fbc5	86	grep "${LOOP_DEV}" \| cut -f1)
6bb24f4d BB	87
6bb24f4d BB	88	if [ -n "$DM_DEV" ]; then
c1d9abf9	89	sudo "${DMSETUP}" remove "${DM_DEV}" \|\|
6bb24f4d BB	90	echo "Failed to remove: ${DM_DEV}"
	91	fi
	92
	93	if [ -n "${TEST_LOOPBACK}" ]; then
c1d9abf9	94	sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" \|\|
6bb24f4d BB	95	echo "Failed to remove: ${TEST_LOOPBACK}"
	96	fi
	97	done
	98	fi
	99
	100	for TEST_FILE in ${FILES}; do
c552fbc5	101	rm -f "${TEST_FILE}" &>/dev/null
6bb24f4d	102	done
c1d9abf9	103
c8f9061f	104	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
c1d9abf9 JWK	105	rm -Rf "$STF_PATH"
c1d9abf9 JWK	106	fi
6bb24f4d BB	107	}
	108	trap cleanup EXIT
	109
	110	#
	111	# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
	112	# loopback devices, and files. This is a useful way to cleanup a previous
	113	# test run failure which has left the system in an unknown state. This can
	114	# be dangerous and should only be used in a dedicated test environment.
	115	#
	116	cleanup_all() {
c552fbc5	117	local TEST_POOLS
74698631	118	TEST_POOLS=$(sudo "$ZPOOL" list -H -o name \| grep testpool)
c552fbc5	119	local TEST_LOOPBACKS
c1d9abf9	120	TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a\|grep file-vdev\|cut -f1 -d:)
c552fbc5 GDN	121	local TEST_FILES
c552fbc5 GDN	122	TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
6bb24f4d BB	123
	124	msg
	125	msg "--- Cleanup ---"
c552fbc5	126	msg "Removing pool(s): $(echo "${TEST_POOLS}" \| tr '\n' ' ')"
6bb24f4d	127	for TEST_POOL in $TEST_POOLS; do
74698631	128	sudo "$ZPOOL" destroy "${TEST_POOL}"
6bb24f4d BB	129	done
6bb24f4d BB	130
c1d9abf9	131	msg "Removing dm(s): $(sudo "${DMSETUP}" ls \|
6bb24f4d	132	grep loop \| tr '\n' ' ')"
c1d9abf9	133	sudo "${DMSETUP}" remove_all
6bb24f4d	134
c552fbc5	135	msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" \| tr '\n' ' ')"
6bb24f4d	136	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
c1d9abf9	137	sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
6bb24f4d BB	138	done
6bb24f4d BB	139
c552fbc5	140	msg "Removing files(s): $(echo "${TEST_FILES}" \| tr '\n' ' ')"
6bb24f4d	141	for TEST_FILE in $TEST_FILES; do
c1d9abf9	142	sudo rm -f "${TEST_FILE}"
6bb24f4d BB	143	done
	144	}
	145
6bb24f4d BB	146	#
	147	# Takes a name as the only arguments and looks for the following variations
	148	# on that name. If one is found it is returned.
	149	#
c8f9061f BB	150	# $RUNFILE_DIR/<name>
c8f9061f BB	151	# $RUNFILE_DIR/<name>.run
6bb24f4d BB	152	# <name>
	153	# <name>.run
	154	#
	155	find_runfile() {
	156	local NAME=$1
	157	local RESULT=""
	158
c8f9061f BB	159	if [ -f "$RUNFILE_DIR/$NAME" ]; then
	160	RESULT="$RUNFILE_DIR/$NAME"
	161	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
	162	RESULT="$RUNFILE_DIR/$NAME.run"
6bb24f4d BB	163	elif [ -f "$NAME" ]; then
	164	RESULT="$NAME"
	165	elif [ -f "$NAME.run" ]; then
	166	RESULT="$NAME.run"
	167	fi
	168
	169	echo "$RESULT"
	170	}
	171
c1d9abf9 JWK	172	#
	173	# Symlink file if it appears under any of the given paths.
	174	#
	175	create_links() {
	176	local dir_list="$1"
	177	local file_list="$2"
	178
74698631	179	[ -n "$STF_PATH" ] \|\| fail "STF_PATH wasn't correctly set"
c1d9abf9 JWK	180
	181	for i in $file_list; do
	182	for j in $dir_list; do
	183	[ ! -e "$STF_PATH/$i" ] \|\| continue
	184
74698631 GDN	185	if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
74698631 GDN	186	ln -s "$j/$i" "$STF_PATH/$i" \|\| \
c1d9abf9 JWK	187	fail "Couldn't link $i"
	188	break
	189	fi
	190	done
	191
74698631	192	[ ! -e "$STF_PATH/$i" ] && STF_MISSING_BIN="$STF_MISSING_BIN$i "
c1d9abf9 JWK	193	done
	194	}
	195
	196	#
	197	# Constrain the path to limit the available binaries to a known set.
	198	# When running in-tree a top level ./bin/ directory is created for
	199	# convenience, otherwise a temporary directory is used.
	200	#
	201	constrain_path() {
74698631	202	. "$STF_SUITE/include/commands.cfg"
c1d9abf9	203
c8f9061f BB	204	if [ "$INTREE" = "yes" ]; then
	205	# Constrained path set to ./zfs/bin/
	206	STF_PATH="$BIN_DIR"
	207	STF_PATH_REMOVE="no"
	208	STF_MISSING_BIN=""
c1d9abf9 JWK	209	if [ ! -d "$STF_PATH" ]; then
c1d9abf9 JWK	210	mkdir "$STF_PATH"
c8f9061f	211	chmod 755 "$STF_PATH" \|\| fail "Couldn't chmod $STF_PATH"
c1d9abf9	212	fi
c8f9061f BB	213
	214	# Special case links for standard zfs utilities
	215	DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
	216	! -name .libs \) -print \| tr '\n' ' ')"
	217	create_links "$DIRS" "$ZFS_FILES"
	218
	219	# Special case links for zfs test suite utilities
	220	DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
	221	! -name .libs \) -print \| tr '\n' ' ')"
	222	create_links "$DIRS" "$ZFSTEST_FILES"
c1d9abf9	223	else
c8f9061f	224	# Constrained path set to /var/tmp/constrained_path.*
c1d9abf9 JWK	225	SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXX}
c1d9abf9 JWK	226	STF_PATH=$(/bin/mktemp -d "$SYSTEMDIR")
c8f9061f BB	227	STF_PATH_REMOVE="yes"
c8f9061f BB	228	STF_MISSING_BIN=""
c1d9abf9	229
c8f9061f	230	chmod 755 "$STF_PATH" \|\| fail "Couldn't chmod $STF_PATH"
c1d9abf9	231
c1d9abf9 JWK	232	# Special case links for standard zfs utilities
	233	create_links "/bin /usr/bin /sbin /usr/sbin" "$ZFS_FILES"
	234
c8f9061f BB	235	# Special case links for zfs test suite utilities
c8f9061f BB	236	create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
c1d9abf9 JWK	237	fi
c1d9abf9 JWK	238
c8f9061f BB	239	# Standard system utilities
	240	create_links "/bin /usr/bin /sbin /usr/sbin" "$SYSTEM_FILES"
	241
c1d9abf9	242	# Exceptions
74698631	243	ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
5c214ae3 BB	244	ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
5c214ae3 BB	245	ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
74698631 GDN	246	ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
	247	ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
	248	ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
	249	ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
530248d1 BB	250
	251	if [ -L "$STF_PATH/arc_summary3" ]; then
	252	ln -fs "$STF_PATH/arc_summary3" "$STF_PATH/arc_summary"
	253	fi
c1d9abf9 JWK	254	}
c1d9abf9 JWK	255
6bb24f4d BB	256	#
	257	# Output a useful usage message.
	258	#
	259	usage() {
	260	cat << EOF
	261	USAGE:
ef57371a	262	$0 [hvqxkfS] [-s SIZE] [-r RUNFILE] [-t PATH] [-u USER]
6bb24f4d BB	263
	264	DESCRIPTION:
	265	ZFS Test Suite launch script
	266
	267	OPTIONS:
	268	-h Show this message
	269	-v Verbose zfs-tests.sh output
	270	-q Quiet test-runner output
	271	-x Remove all testpools, dm, lo, and files (unsafe)
	272	-k Disable cleanup after test failure
	273	-f Use files only, disables block device tests
ef57371a	274	-S Enable stack tracer (negative performance impact)
c8f9061f	275	-c Only create and populate constrained path
96342996	276	-n NFSFILE Use the nfsfile to determine the NFS configuration
9a810efb	277	-I NUM Number of iterations
6bb24f4d	278	-d DIR Use DIR for files and loopback devices
d7958b4c	279	-s SIZE Use vdevs of SIZE (default: 4G)
6bb24f4d	280	-r RUNFILE Run tests in RUNFILE (default: linux.run)
d25534f8	281	-t PATH Run single test at PATH relative to test suite
271955da	282	-T TAGS Comma separated list of tags (default: 'functional')
d8fa599f	283	-u USER Run single test as USER (default: root)
6bb24f4d BB	284
	285	EXAMPLES:
	286	# Run the default (linux) suite of tests and output the configuration used.
	287	$0 -v
	288
	289	# Run a smaller suite of tests designed to run more quickly.
	290	$0 -r linux-fast
	291
	292	# Cleanup a previous run of the test suite prior to testing, run the
	293	# default (linux) suite of tests and perform no cleanup on exit.
e6123796	294	$0 -x
6bb24f4d BB	295
	296	EOF
	297	}
	298
96342996	299	while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do
6bb24f4d BB	300	case $OPTION in
	301	h)
	302	usage
	303	exit 1
	304	;;
	305	v)
c552fbc5	306	# shellcheck disable=SC2034
c8f9061f	307	VERBOSE="yes"
6bb24f4d BB	308	;;
	309	q)
	310	QUIET="-q"
	311	;;
	312	x)
c8f9061f	313	CLEANUPALL="yes"
6bb24f4d BB	314	;;
6bb24f4d BB	315	k)
c8f9061f	316	CLEANUP="no"
6bb24f4d BB	317	;;
6bb24f4d BB	318	f)
c8f9061f BB	319	LOOPBACK="no"
c8f9061f BB	320	;;
ef57371a TN	321	S)
	322	STACK_TRACER="yes"
	323	;;
c8f9061f BB	324	c)
	325	constrain_path
	326	exit
6bb24f4d	327	;;
96342996 AG	328	n)
	329	nfsfile=$OPTARG
	330	[[ -f $nfsfile ]] \|\| fail "Cannot read file: $nfsfile"
	331	export NFS=1
	332	. "$nfsfile"
	333	;;
6bb24f4d BB	334	d)
	335	FILEDIR="$OPTARG"
	336	;;
9a810efb GDN	337	I)
	338	ITERATIONS="$OPTARG"
	339	if [ "$ITERATIONS" -le 0 ]; then
	340	fail "Iterations must be greater than 0."
	341	fi
	342	;;
6bb24f4d BB	343	s)
	344	FILESIZE="$OPTARG"
	345	;;
	346	r)
	347	RUNFILE="$OPTARG"
	348	;;
d8fa599f GDN	349	t)
	350	if [ ${#SINGLETEST[@]} -ne 0 ]; then
	351	fail "-t can only be provided once."
	352	fi
	353	SINGLETEST+=("$OPTARG")
	354	;;
9a810efb GDN	355	T)
	356	TAGS="$OPTARG"
	357	;;
d8fa599f GDN	358	u)
	359	SINGLETESTUSER="$OPTARG"
	360	;;
6bb24f4d BB	361	?)
	362	usage
	363	exit
	364	;;
	365	esac
	366	done
	367
	368	shift $((OPTIND-1))
	369
	370	FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
	371	LOOPBACKS=${LOOPBACKS:-""}
	372
d8fa599f	373	if [ ${#SINGLETEST[@]} -ne 0 ]; then
271955da	374	if [ -n "$TAGS" ]; then
	375	fail "-t and -T are mutually exclusive."
	376	fi
c8f9061f	377	RUNFILE_DIR="/var/tmp"
d8fa599f GDN	378	RUNFILE="zfs-tests.$$.run"
	379	SINGLEQUIET="False"
	380
	381	if [ -n "$QUIET" ]; then
	382	SINGLEQUIET="True"
	383	fi
	384
c8f9061f	385	cat >$RUNFILE_DIR/$RUNFILE << EOF
d8fa599f GDN	386	[DEFAULT]
	387	pre =
	388	quiet = $SINGLEQUIET
	389	pre_user = root
	390	user = $SINGLETESTUSER
	391	timeout = 600
	392	post_user = root
	393	post =
	394	outputdir = /var/tmp/test_results
	395	EOF
	396	for t in "${SINGLETEST[@]}"
	397	do
	398	SINGLETESTDIR=$(dirname "$t")
	399	SINGLETESTFILE=$(basename "$t")
	400	SETUPSCRIPT=
	401	CLEANUPSCRIPT=
	402
d25534f8	403	if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
d8fa599f GDN	404	SETUPSCRIPT="setup"
	405	fi
	406
d25534f8	407	if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
d8fa599f GDN	408	CLEANUPSCRIPT="cleanup"
	409	fi
	410
c8f9061f	411	cat >>$RUNFILE_DIR/$RUNFILE << EOF
d8fa599f GDN	412
	413	[$SINGLETESTDIR]
	414	tests = ['$SINGLETESTFILE']
	415	pre = $SETUPSCRIPT
	416	post = $CLEANUPSCRIPT
271955da	417	tags = ['functional']
d8fa599f GDN	418	EOF
	419	done
	420	fi
	421
271955da	422	#
	423	# Use default tag if none was specified
	424	#
	425	TAGS=${TAGS:='functional'}
	426
6bb24f4d BB	427	#
	428	# Attempt to locate the runfile describing the test workload.
	429	#
	430	if [ -n "$RUNFILE" ]; then
	431	SAVED_RUNFILE="$RUNFILE"
	432	RUNFILE=$(find_runfile "$RUNFILE")
	433	[ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
	434	fi
	435
	436	if [ ! -r "$RUNFILE" ]; then
	437	fail "Cannot read runfile: $RUNFILE"
	438	fi
	439
	440	#
	441	# This script should not be run as root. Instead the test user, which may
	442	# be a normal user account, needs to be configured such that it can
	443	# run commands via sudo passwordlessly.
	444	#
c552fbc5	445	if [ "$(id -u)" = "0" ]; then
6bb24f4d BB	446	fail "This script must not be run as root."
	447	fi
	448
c552fbc5	449	if [ "$(sudo whoami)" != "root" ]; then
6bb24f4d BB	450	fail "Passwordless sudo access required."
	451	fi
	452
c1d9abf9	453	#
c8f9061f	454	# Constrain the available binaries to a known set.
c1d9abf9 JWK	455	#
	456	constrain_path
	457
e6123796 JX	458	#
	459	# Check if ksh exists
	460	#
74698631	461	[ -e "$STF_PATH/ksh" ] \|\| fail "This test suite requires ksh."
3f03fc8d BB	462	[ -e "$STF_SUITE/include/default.cfg" ] \|\| fail \
3f03fc8d BB	463	"Missing $STF_SUITE/include/default.cfg file."
e6123796	464
6bb24f4d	465	#
c8f9061f	466	# Verify the ZFS module stack is loaded.
6bb24f4d	467	#
ef57371a TN	468	if [ "$STACK_TRACER" = "yes" ]; then
	469	sudo "${ZFS_SH}" -S &>/dev/null
	470	else
	471	sudo "${ZFS_SH}" &>/dev/null
	472	fi
6bb24f4d BB	473
	474	#
	475	# Attempt to cleanup all previous state for a new test run.
	476	#
c8f9061f	477	if [ "$CLEANUPALL" = "yes" ]; then
6bb24f4d BB	478	cleanup_all
	479	fi
	480
	481	#
	482	# By default preserve any existing pools
3fd3e56c	483	# NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
3fd3e56c	484	# space-delimited to newline-delimited.
6bb24f4d BB	485	#
6bb24f4d BB	486	if [ -z "${KEEP}" ]; then
3fd3e56c	487	KEEP="$(sudo "$ZPOOL" list -H -o name)"
6bb24f4d BB	488	if [ -z "${KEEP}" ]; then
	489	KEEP="rpool"
	490	fi
3fd3e56c	491	else
3fd3e56c	492	KEEP="$(echo -e "${KEEP//[[:blank:]]/\n}")"
6bb24f4d BB	493	fi
6bb24f4d BB	494
3fd3e56c	495	#
	496	# NOTE: The following environment variables are undocumented
	497	# and should be used for testing purposes only:
	498	#
	499	# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
	500	# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
	501	#
	502	# See libzfs/libzfs_config.c for more information.
	503	#
	504	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" \| sed ':a;N;s/\n/ /g;ba')"
d21d5b82	505
74698631	506	. "$STF_SUITE/include/default.cfg"
c1d9abf9	507
6bb24f4d BB	508	msg
	509	msg "--- Configuration ---"
	510	msg "Runfile: $RUNFILE"
	511	msg "STF_TOOLS: $STF_TOOLS"
	512	msg "STF_SUITE: $STF_SUITE"
c1d9abf9	513	msg "STF_PATH: $STF_PATH"
6bb24f4d BB	514
	515	#
	516	# No DISKS have been provided so a basic file or loopback based devices
	517	# must be created for the test suite to use.
	518	#
	519	if [ -z "${DISKS}" ]; then
	520	#
	521	# Create sparse files for the test suite. These may be used
	522	# directory or have loopback devices layered on them.
	523	#
	524	for TEST_FILE in ${FILES}; do
	525	[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
c552fbc5	526	truncate -s "${FILESIZE}" "${TEST_FILE}" \|\|
6bb24f4d	527	fail "Failed creating: ${TEST_FILE} ($?)"
5c596ba7 TC	528	if [[ "$DISKS" ]]; then
	529	DISKS="$DISKS $TEST_FILE"
	530	else
	531	DISKS="$TEST_FILE"
	532	fi
6bb24f4d BB	533	done
	534
	535	#
	536	# If requested setup loopback devices backed by the sparse files.
	537	#
c8f9061f	538	if [ "$LOOPBACK" = "yes" ]; then
6bb24f4d	539	DISKS=""
c8f9061f BB	540
c8f9061f BB	541	test -x "$LOSETUP" \|\| fail "$LOSETUP utility must be installed"
6bb24f4d BB	542
6bb24f4d BB	543	for TEST_FILE in ${FILES}; do
c1d9abf9 JWK	544	TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
c1d9abf9 JWK	545	sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" \|\|
6bb24f4d BB	546	fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
6bb24f4d BB	547	LOOPBACKS="${LOOPBACKS}${TEST_LOOPBACK} "
c552fbc5	548	BASELOOPBACKS=$(basename "$TEST_LOOPBACK")
5c596ba7 TC	549	if [[ "$DISKS" ]]; then
	550	DISKS="$DISKS $BASELOOPBACKS"
	551	else
	552	DISKS="$BASELOOPBACKS"
	553	fi
6bb24f4d BB	554	done
	555	fi
	556	fi
	557
c8f9061f	558	NUM_DISKS=$(echo "${DISKS}" \| awk '{print NF}')
c552fbc5	559	[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
6bb24f4d BB	560
	561	#
	562	# Disable SELinux until the ZFS Test Suite has been updated accordingly.
	563	#
c1d9abf9 JWK	564	if [ -x "$STF_PATH/setenforce" ]; then
c1d9abf9 JWK	565	sudo setenforce permissive &>/dev/null
6bb24f4d BB	566	fi
6bb24f4d BB	567
00481e7d	568	#
27ef66ef	569	# Enable internal ZFS debug log and clear it.
00481e7d BB	570	#
	571	if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
	572	sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
	573	sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
	574	fi
	575
6bb24f4d BB	576	msg "FILEDIR: $FILEDIR"
	577	msg "FILES: $FILES"
	578	msg "LOOPBACKS: $LOOPBACKS"
	579	msg "DISKS: $DISKS"
	580	msg "NUM_DISKS: $NUM_DISKS"
	581	msg "FILESIZE: $FILESIZE"
9a810efb GDN	582	msg "ITERATIONS: $ITERATIONS"
9a810efb GDN	583	msg "TAGS: $TAGS"
ef57371a	584	msg "STACK_TRACER: $STACK_TRACER"
6bb24f4d	585	msg "Keep pool(s): $KEEP"
c1d9abf9	586	msg "Missing util(s): $STF_MISSING_BIN"
6bb24f4d BB	587	msg ""
	588
	589	export STF_TOOLS
	590	export STF_SUITE
c1d9abf9	591	export STF_PATH
6bb24f4d	592	export DISKS
3fd3e56c	593	export FILEDIR
6bb24f4d	594	export KEEP
d21d5b82	595	export __ZFS_POOL_EXCLUDE
00481e7d	596	export TESTFAIL_CALLBACKS
c1d9abf9	597	export PATH=$STF_PATH
6bb24f4d	598
e4a3297a BB	599	RESULTS_FILE=$(mktemp -u -t zts-results.XXXX -p "$FILEDIR")
	600	REPORT_FILE=$(mktemp -u -t zts-report.XXXX -p "$FILEDIR")
	601
	602	#
	603	# Run all the tests as specified.
	604	#
9a810efb GDN	605	msg "${TEST_RUNNER} ${QUIET} -c ${RUNFILE} -T ${TAGS} -i ${STF_SUITE}" \
	606	"-I ${ITERATIONS}"
	607	${TEST_RUNNER} ${QUIET} -c "${RUNFILE}" -T "${TAGS}" -i "${STF_SUITE}" \
e4a3297a BB	608	-I "${ITERATIONS}" 2>&1 \| tee "$RESULTS_FILE"
	609
	610	#
	611	# Analyze the results.
	612	#
	613	set -o pipefail
	614	${ZTS_REPORT} "$RESULTS_FILE" \| tee "$REPORT_FILE"
6bb24f4d	615	RESULT=$?
e4a3297a BB	616	set +o pipefail
	617
	618	RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
	619	if [ -d "$RESULTS_DIR" ]; then
	620	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
	621	fi
	622
	623	rm -f "$RESULTS_FILE" "$REPORT_FILE"
6bb24f4d	624
d8fa599f	625	if [ ${#SINGLETEST[@]} -ne 0 ]; then
d25534f8	626	rm -f "$RUNFILE" &>/dev/null
d8fa599f GDN	627	fi
d8fa599f GDN	628
6bb24f4d	629	exit ${RESULT}