scripts/zfs-tests.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2154
   3 #
   4 # CDDL HEADER START
   5 #
   6 # The contents of this file are subject to the terms of the
   7 # Common Development and Distribution License, Version 1.0 only
   8 # (the "License").  You may not use this file except in compliance
   9 # with the License.
  10 #
  11 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  12 # or http://www.opensolaris.org/os/licensing.
  13 # See the License for the specific language governing permissions
  14 # and limitations under the License.
  15 #
  16 # When distributing Covered Code, include this CDDL HEADER in each
  17 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  18 # If applicable, add the following below this CDDL HEADER, with the
  19 # fields enclosed by brackets "[]" replaced with your own identifying
  20 # information: Portions Copyright [yyyy] [name of copyright owner]
  21 #
  22 # CDDL HEADER END
  23 #
  24
  25 #
  26 # Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  27 #
  28
  29 BASE_DIR=$(dirname "$0")
  30 SCRIPT_COMMON=common.sh
  31 if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
  32 . "${BASE_DIR}/${SCRIPT_COMMON}"
  33 else
  34 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  35 fi
  36
  37 PROG=zfs-tests.sh
  38 VERBOSE="no"
  39 QUIET=""
  40 CLEANUP="yes"
  41 CLEANUPALL="no"
  42 KMSG=""
  43 LOOPBACK="yes"
  44 STACK_TRACER="no"
  45 FILESIZE="4G"
  46 DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
  47 RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
  48 FILEDIR=${FILEDIR:-/var/tmp}
  49 DISKS=${DISKS:-""}
  50 SINGLETEST=""
  51 SINGLETESTUSER="root"
  52 TAGS=""
  53 ITERATIONS=1
  54 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
  55 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
  56 UNAME=$(uname -s)
  57 RERUN=""
  58 KMEMLEAK=""
  59
  60 # Override some defaults if on FreeBSD
  61 if [ "$UNAME" = "FreeBSD" ] ; then
  62         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
  63         LOSETUP=/sbin/mdconfig
  64         DMSETUP=/sbin/gpart
  65 else
  66         ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
  67         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
  68         LOSETUP=${LOSETUP:-/sbin/losetup}
  69         DMSETUP=${DMSETUP:-/sbin/dmsetup}
  70 fi
  71
  72 #
  73 # Log an informational message when additional verbosity is enabled.
  74 #
  75 msg() {
  76         if [ "$VERBOSE" = "yes" ]; then
  77                 echo "$@"
  78         fi
  79 }
  80
  81 #
  82 # Log a failure message, cleanup, and return an error.
  83 #
  84 fail() {
  85         echo "$PROG: $1" >&2
  86         cleanup
  87         exit 1
  88 }
  89
  90 cleanup_freebsd_loopback() {
  91         for TEST_LOOPBACK in ${LOOPBACKS}; do
  92                 if [ -c "/dev/${TEST_LOOPBACK}" ]; then
  93                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
  94                             echo "Failed to destroy: ${TEST_LOOPBACK}"
  95                 fi
  96         done
  97 }
  98
  99 cleanup_linux_loopback() {
 100         for TEST_LOOPBACK in ${LOOPBACKS}; do
 101                 LOOP_DEV="${TEST_LOOPBACK##*/}"
 102                 DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
 103                     grep "${LOOP_DEV}" | cut -f1)
 104
 105                 if [ -n "$DM_DEV" ]; then
 106                         sudo "${DMSETUP}" remove "${DM_DEV}" ||
 107                             echo "Failed to remove: ${DM_DEV}"
 108                 fi
 109
 110                 if [ -n "${TEST_LOOPBACK}" ]; then
 111                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
 112                             echo "Failed to remove: ${TEST_LOOPBACK}"
 113                 fi
 114         done
 115 }
 116
 117 #
 118 # Attempt to remove loopback devices and files which where created earlier
 119 # by this script to run the test framework.  The '-k' option may be passed
 120 # to the script to suppress cleanup for debugging purposes.
 121 #
 122 cleanup() {
 123         if [ "$CLEANUP" = "no" ]; then
 124                 return 0
 125         fi
 126
 127
 128         if [ "$LOOPBACK" = "yes" ]; then
 129                 if [ "$UNAME" = "FreeBSD" ] ; then
 130                         cleanup_freebsd_loopback
 131                 else
 132                         cleanup_linux_loopback
 133                 fi
 134         fi
 135
 136         for TEST_FILE in ${FILES}; do
 137                 rm -f "${TEST_FILE}" >/dev/null 2>&1
 138         done
 139
 140         if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
 141                 rm -Rf "$STF_PATH"
 142         fi
 143 }
 144 trap cleanup EXIT
 145
 146 #
 147 # Attempt to remove all testpools (testpool.XXX), unopened dm devices,
 148 # loopback devices, and files.  This is a useful way to cleanup a previous
 149 # test run failure which has left the system in an unknown state.  This can
 150 # be dangerous and should only be used in a dedicated test environment.
 151 #
 152 cleanup_all() {
 153         TEST_POOLS=$(sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -H -o name | grep testpool)
 154         if [ "$UNAME" = "FreeBSD" ] ; then
 155                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
 156         else
 157                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
 158         fi
 159         TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
 160
 161         msg
 162         msg "--- Cleanup ---"
 163         msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
 164         for TEST_POOL in $TEST_POOLS; do
 165                 sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
 166         done
 167
 168         if [ "$UNAME" != "FreeBSD" ] ; then
 169                 msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
 170                     grep loop | tr '\n' ' ')"
 171                 sudo "${DMSETUP}" remove_all
 172         fi
 173
 174         msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
 175         for TEST_LOOPBACK in $TEST_LOOPBACKS; do
 176                 if [ "$UNAME" = "FreeBSD" ] ; then
 177                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
 178                 else
 179                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
 180                 fi
 181         done
 182
 183         msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
 184         for TEST_FILE in $TEST_FILES; do
 185                 sudo rm -f "${TEST_FILE}"
 186         done
 187 }
 188
 189 #
 190 # Takes a name as the only arguments and looks for the following variations
 191 # on that name.  If one is found it is returned.
 192 #
 193 # $RUNFILE_DIR/<name>
 194 # $RUNFILE_DIR/<name>.run
 195 # <name>
 196 # <name>.run
 197 #
 198 find_runfile() {
 199         NAME=$1
 200         RESULT=""
 201
 202         if [ -f "$RUNFILE_DIR/$NAME" ]; then
 203                 RESULT="$RUNFILE_DIR/$NAME"
 204         elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
 205                 RESULT="$RUNFILE_DIR/$NAME.run"
 206         elif [ -f "$NAME" ]; then
 207                 RESULT="$NAME"
 208         elif [ -f "$NAME.run" ]; then
 209                 RESULT="$NAME.run"
 210         fi
 211
 212         echo "$RESULT"
 213 }
 214
 215 #
 216 # Symlink file if it appears under any of the given paths.
 217 #
 218 create_links() {
 219         dir_list="$1"
 220         file_list="$2"
 221
 222         [ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
 223
 224         for i in $file_list; do
 225                 for j in $dir_list; do
 226                         [ ! -e "$STF_PATH/$i" ] || continue
 227
 228                         if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
 229                                 ln -sf "$j/$i" "$STF_PATH/$i" || \
 230                                     fail "Couldn't link $i"
 231                                 break
 232                         fi
 233                 done
 234
 235                 [ ! -e "$STF_PATH/$i" ] && \
 236                     STF_MISSING_BIN="$STF_MISSING_BIN $i"
 237         done
 238         STF_MISSING_BIN=${STF_MISSING_BIN# }
 239 }
 240
 241 #
 242 # Constrain the path to limit the available binaries to a known set.
 243 # When running in-tree a top level ./bin/ directory is created for
 244 # convenience, otherwise a temporary directory is used.
 245 #
 246 constrain_path() {
 247         . "$STF_SUITE/include/commands.cfg"
 248
 249         # On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
 250         # install to /usr/local/sbin. To avoid testing the wrong utils we
 251         # need /usr/local to come before / in the path search order.
 252         SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
 253         SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
 254
 255         if [ "$INTREE" = "yes" ]; then
 256                 # Constrained path set to ./zfs/bin/
 257                 STF_PATH="$BIN_DIR"
 258                 STF_PATH_REMOVE="no"
 259                 STF_MISSING_BIN=""
 260                 if [ ! -d "$STF_PATH" ]; then
 261                         mkdir "$STF_PATH"
 262                         chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 263                 fi
 264
 265                 # Special case links for standard zfs utilities
 266                 DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
 267                     ! -name .libs \) -print | tr '\n' ' ')"
 268                 create_links "$DIRS" "$ZFS_FILES"
 269
 270                 # Special case links for zfs test suite utilities
 271                 DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
 272                     ! -name .libs \) -print | tr '\n' ' ')"
 273                 create_links "$DIRS" "$ZFSTEST_FILES"
 274         else
 275                 # Constrained path set to /var/tmp/constrained_path.*
 276                 SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
 277                 STF_PATH=$(mktemp -d "$SYSTEMDIR")
 278                 STF_PATH_REMOVE="yes"
 279                 STF_MISSING_BIN=""
 280
 281                 chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 282
 283                 # Special case links for standard zfs utilities
 284                 create_links "$SYSTEM_DIRS" "$ZFS_FILES"
 285
 286                 # Special case links for zfs test suite utilities
 287                 create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
 288         fi
 289
 290         # Standard system utilities
 291         SYSTEM_FILES="$SYSTEM_FILES_COMMON"
 292         if [ "$UNAME" = "FreeBSD" ] ; then
 293                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
 294         else
 295                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
 296         fi
 297         create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
 298
 299         # Exceptions
 300         ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
 301         if [ "$UNAME" = "Linux" ] ; then
 302                 ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
 303                 ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
 304                 ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
 305                 ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
 306                 ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
 307                 ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
 308         elif [ "$UNAME" = "FreeBSD" ] ; then
 309                 ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
 310         fi
 311 }
 312
 313 #
 314 # Output a useful usage message.
 315 #
 316 usage() {
 317 cat << EOF
 318 USAGE:
 319 $0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
 320
 321 DESCRIPTION:
 322         ZFS Test Suite launch script
 323
 324 OPTIONS:
 325         -h          Show this message
 326         -v          Verbose zfs-tests.sh output
 327         -q          Quiet test-runner output
 328         -x          Remove all testpools, dm, lo, and files (unsafe)
 329         -k          Disable cleanup after test failure
 330         -K          Log test names to /dev/kmsg
 331         -f          Use files only, disables block device tests
 332         -S          Enable stack tracer (negative performance impact)
 333         -c          Only create and populate constrained path
 334         -R          Automatically rerun failing tests
 335         -m          Enable kmemleak reporting (Linux only)
 336         -n NFSFILE  Use the nfsfile to determine the NFS configuration
 337         -I NUM      Number of iterations
 338         -d DIR      Use DIR for files and loopback devices
 339         -s SIZE     Use vdevs of SIZE (default: 4G)
 340         -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
 341         -t PATH     Run single test at PATH relative to test suite
 342         -T TAGS     Comma separated list of tags (default: 'functional')
 343         -u USER     Run single test as USER (default: root)
 344
 345 EXAMPLES:
 346 # Run the default (linux) suite of tests and output the configuration used.
 347 $0 -v
 348
 349 # Run a smaller suite of tests designed to run more quickly.
 350 $0 -r linux-fast
 351
 352 # Run a single test
 353 $0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
 354
 355 # Cleanup a previous run of the test suite prior to testing, run the
 356 # default (linux) suite of tests and perform no cleanup on exit.
 357 $0 -x
 358
 359 EOF
 360 }
 361
 362 while getopts 'hvqxkKfScRmn:d:s:r:?t:T:u:I:' OPTION; do
 363         case $OPTION in
 364         h)
 365                 usage
 366                 exit 1
 367                 ;;
 368         v)
 369                 VERBOSE="yes"
 370                 ;;
 371         q)
 372                 QUIET="yes"
 373                 ;;
 374         x)
 375                 CLEANUPALL="yes"
 376                 ;;
 377         k)
 378                 CLEANUP="no"
 379                 ;;
 380         K)
 381                 KMSG="yes"
 382                 ;;
 383         f)
 384                 LOOPBACK="no"
 385                 ;;
 386         S)
 387                 STACK_TRACER="yes"
 388                 ;;
 389         c)
 390                 constrain_path
 391                 exit
 392                 ;;
 393         R)
 394                 RERUN="yes"
 395                 ;;
 396         m)
 397                 KMEMLEAK="yes"
 398                 ;;
 399         n)
 400                 nfsfile=$OPTARG
 401                 [ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
 402                 export NFS=1
 403                 . "$nfsfile"
 404                 ;;
 405         d)
 406                 FILEDIR="$OPTARG"
 407                 ;;
 408         I)
 409                 ITERATIONS="$OPTARG"
 410                 if [ "$ITERATIONS" -le 0 ]; then
 411                         fail "Iterations must be greater than 0."
 412                 fi
 413                 ;;
 414         s)
 415                 FILESIZE="$OPTARG"
 416                 ;;
 417         r)
 418                 RUNFILES="$OPTARG"
 419                 ;;
 420         t)
 421                 if [ -n "$SINGLETEST" ]; then
 422                         fail "-t can only be provided once."
 423                 fi
 424                 SINGLETEST="$OPTARG"
 425                 ;;
 426         T)
 427                 TAGS="$OPTARG"
 428                 ;;
 429         u)
 430                 SINGLETESTUSER="$OPTARG"
 431                 ;;
 432         ?)
 433                 usage
 434                 exit
 435                 ;;
 436         *)
 437                 ;;
 438         esac
 439 done
 440
 441 shift $((OPTIND-1))
 442
 443 FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
 444 LOOPBACKS=${LOOPBACKS:-""}
 445
 446 if [ -n "$SINGLETEST" ]; then
 447         if [ -n "$TAGS" ]; then
 448                 fail "-t and -T are mutually exclusive."
 449         fi
 450         RUNFILE_DIR="/var/tmp"
 451         RUNFILES="zfs-tests.$$.run"
 452         SINGLEQUIET="False"
 453
 454         if [ -n "$QUIET" ]; then
 455                 SINGLEQUIET="True"
 456         fi
 457
 458         cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
 459 [DEFAULT]
 460 pre =
 461 quiet = $SINGLEQUIET
 462 pre_user = root
 463 user = $SINGLETESTUSER
 464 timeout = 600
 465 post_user = root
 466 post =
 467 outputdir = /var/tmp/test_results
 468 EOF
 469         SINGLETESTDIR=$(dirname "$SINGLETEST")
 470         SINGLETESTFILE=$(basename "$SINGLETEST")
 471         SETUPSCRIPT=
 472         CLEANUPSCRIPT=
 473
 474         if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
 475                 SETUPSCRIPT="setup"
 476         fi
 477
 478         if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
 479                 CLEANUPSCRIPT="cleanup"
 480         fi
 481
 482         cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
 483
 484 [$SINGLETESTDIR]
 485 tests = ['$SINGLETESTFILE']
 486 pre = $SETUPSCRIPT
 487 post = $CLEANUPSCRIPT
 488 tags = ['functional']
 489 EOF
 490 fi
 491
 492 #
 493 # Use default tag if none was specified
 494 #
 495 TAGS=${TAGS:='functional'}
 496
 497 #
 498 # Attempt to locate the runfiles describing the test workload.
 499 #
 500 R=""
 501 IFS=,
 502 for RUNFILE in $RUNFILES; do
 503         if [ -n "$RUNFILE" ]; then
 504                 SAVED_RUNFILE="$RUNFILE"
 505                 RUNFILE=$(find_runfile "$RUNFILE")
 506                 [ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
 507                 R="$R,$RUNFILE"
 508         fi
 509
 510         if [ ! -r "$RUNFILE" ]; then
 511                 fail "Cannot read runfile: $RUNFILE"
 512         fi
 513 done
 514 unset IFS
 515 RUNFILES=${R#,}
 516
 517 #
 518 # This script should not be run as root.  Instead the test user, which may
 519 # be a normal user account, needs to be configured such that it can
 520 # run commands via sudo passwordlessly.
 521 #
 522 if [ "$(id -u)" = "0" ]; then
 523         fail "This script must not be run as root."
 524 fi
 525
 526 if [ "$(sudo whoami)" != "root" ]; then
 527         fail "Passwordless sudo access required."
 528 fi
 529
 530 #
 531 # Constrain the available binaries to a known set.
 532 #
 533 constrain_path
 534
 535 #
 536 # Check if ksh exists
 537 #
 538 if [ "$UNAME" = "FreeBSD" ]; then
 539         sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
 540 fi
 541 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
 542 [ -e "$STF_SUITE/include/default.cfg" ] || fail \
 543     "Missing $STF_SUITE/include/default.cfg file."
 544
 545 #
 546 # Verify the ZFS module stack is loaded.
 547 #
 548 if [ "$STACK_TRACER" = "yes" ]; then
 549         sudo "${ZFS_SH}" -S >/dev/null 2>&1
 550 else
 551         sudo "${ZFS_SH}" >/dev/null 2>&1
 552 fi
 553
 554 #
 555 # Attempt to cleanup all previous state for a new test run.
 556 #
 557 if [ "$CLEANUPALL" = "yes" ]; then
 558         cleanup_all
 559 fi
 560
 561 #
 562 # By default preserve any existing pools
 563 # NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
 564 # space-delimited to newline-delimited.
 565 #
 566 if [ -z "${KEEP}" ]; then
 567         KEEP="$(sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -H -o name)"
 568         if [ -z "${KEEP}" ]; then
 569                 KEEP="rpool"
 570         fi
 571 else
 572         KEEP="$(echo "$KEEP" | tr '[:blank:]' '\n')"
 573 fi
 574
 575 #
 576 # NOTE: The following environment variables are undocumented
 577 # and should be used for testing purposes only:
 578 #
 579 # __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
 580 # __ZFS_POOL_RESTRICT - iterate only over the pools it lists
 581 #
 582 # See libzfs/libzfs_config.c for more information.
 583 #
 584 if [ "$UNAME" = "FreeBSD" ] ; then
 585         __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
 586 else
 587         __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
 588 fi
 589
 590 . "$STF_SUITE/include/default.cfg"
 591
 592 #
 593 # No DISKS have been provided so a basic file or loopback based devices
 594 # must be created for the test suite to use.
 595 #
 596 if [ -z "${DISKS}" ]; then
 597         #
 598         # If this is a performance run, prevent accidental use of
 599         # loopback devices.
 600         #
 601         [ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
 602
 603         #
 604         # Create sparse files for the test suite.  These may be used
 605         # directory or have loopback devices layered on them.
 606         #
 607         for TEST_FILE in ${FILES}; do
 608                 [ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
 609                 truncate -s "${FILESIZE}" "${TEST_FILE}" ||
 610                     fail "Failed creating: ${TEST_FILE} ($?)"
 611         done
 612
 613         #
 614         # If requested setup loopback devices backed by the sparse files.
 615         #
 616         if [ "$LOOPBACK" = "yes" ]; then
 617                 test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
 618
 619                 for TEST_FILE in ${FILES}; do
 620                         if [ "$UNAME" = "FreeBSD" ] ; then
 621                                 MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
 622                                 if [ -z "$MDDEVICE" ] ; then
 623                                         fail "Failed: ${TEST_FILE} -> loopback"
 624                                 fi
 625                                 DISKS="$DISKS $MDDEVICE"
 626                                 LOOPBACKS="$LOOPBACKS $MDDEVICE"
 627                         else
 628                                 TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
 629                                 sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
 630                                     fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
 631                                 BASELOOPBACK="${TEST_LOOPBACK##*/}"
 632                                 DISKS="$DISKS $BASELOOPBACK"
 633                                 LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
 634                         fi
 635                 done
 636                 DISKS=${DISKS# }
 637                 LOOPBACKS=${LOOPBACKS# }
 638         else
 639                 DISKS="$FILES"
 640         fi
 641 fi
 642
 643 #
 644 # It may be desirable to test with fewer disks than the default when running
 645 # the performance tests, but the functional tests require at least three.
 646 #
 647 NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
 648 if [ "$TAGS" != "perf" ]; then
 649         [ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
 650 fi
 651
 652 #
 653 # Disable SELinux until the ZFS Test Suite has been updated accordingly.
 654 #
 655 if [ -x "$STF_PATH/setenforce" ]; then
 656         sudo setenforce permissive >/dev/null 2>&1
 657 fi
 658
 659 #
 660 # Enable internal ZFS debug log and clear it.
 661 #
 662 if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
 663         sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
 664         sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
 665 fi
 666
 667 msg
 668 msg "--- Configuration ---"
 669 msg "Runfiles:        $RUNFILES"
 670 msg "STF_TOOLS:       $STF_TOOLS"
 671 msg "STF_SUITE:       $STF_SUITE"
 672 msg "STF_PATH:        $STF_PATH"
 673 msg "FILEDIR:         $FILEDIR"
 674 msg "FILES:           $FILES"
 675 msg "LOOPBACKS:       $LOOPBACKS"
 676 msg "DISKS:           $DISKS"
 677 msg "NUM_DISKS:       $NUM_DISKS"
 678 msg "FILESIZE:        $FILESIZE"
 679 msg "ITERATIONS:      $ITERATIONS"
 680 msg "TAGS:            $TAGS"
 681 msg "STACK_TRACER:    $STACK_TRACER"
 682 msg "Keep pool(s):    $KEEP"
 683 msg "Missing util(s): $STF_MISSING_BIN"
 684 msg ""
 685
 686 export STF_TOOLS
 687 export STF_SUITE
 688 export STF_PATH
 689 export DISKS
 690 export FILEDIR
 691 export KEEP
 692 export __ZFS_POOL_EXCLUDE
 693 export TESTFAIL_CALLBACKS
 694 export PATH=$STF_PATH
 695
 696 mktemp_file() {
 697         if [ "$UNAME" = "FreeBSD" ]; then
 698                 mktemp -u "${FILEDIR}/$1.XXXXXX"
 699         else
 700                 mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
 701         fi
 702 }
 703 mkdir -p "$FILEDIR" || :
 704 RESULTS_FILE=$(mktemp_file zts-results)
 705 REPORT_FILE=$(mktemp_file zts-report)
 706
 707 #
 708 # Run all the tests as specified.
 709 #
 710 msg "${TEST_RUNNER}" \
 711     "${QUIET:+-q}" \
 712     "${KMEMLEAK:+-m}" \
 713     "${KMSG:+-K}" \
 714     "-c \"${RUNFILES}\"" \
 715     "-T \"${TAGS}\"" \
 716     "-i \"${STF_SUITE}\"" \
 717     "-I \"${ITERATIONS}\""
 718 { ${TEST_RUNNER} \
 719     ${QUIET:+-q} \
 720     ${KMEMLEAK:+-m} \
 721     ${KMSG:+-K} \
 722     -c "${RUNFILES}" \
 723     -T "${TAGS}" \
 724     -i "${STF_SUITE}" \
 725     -I "${ITERATIONS}" \
 726     2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 727 read -r RUNRESULT <"$REPORT_FILE"
 728
 729 #
 730 # Analyze the results.
 731 #
 732 ${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
 733 RESULT=$?
 734
 735 if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
 736         MAYBES="$($ZTS_REPORT --list-maybes)"
 737         TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
 738         TEST_LIST=$(mktemp_file test-list)
 739         grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
 740         for test_name in $MAYBES; do
 741                 grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
 742         done
 743         { ${TEST_RUNNER} \
 744             ${QUIET:+-q} \
 745             ${KMEMLEAK:+-m} \
 746             -c "${RUNFILES}" \
 747             -T "${TAGS}" \
 748             -i "${STF_SUITE}" \
 749             -I "${ITERATIONS}" \
 750             -l "${TEST_LIST}" \
 751             2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 752         read -r RUNRESULT <"$REPORT_FILE"
 753         #
 754         # Analyze the results.
 755         #
 756         ${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
 757         RESULT=$?
 758 fi
 759
 760
 761 cat "$REPORT_FILE"
 762
 763 RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
 764 if [ -d "$RESULTS_DIR" ]; then
 765         cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
 766 fi
 767
 768 rm -f "$RESULTS_FILE" "$REPORT_FILE"
 769
 770 if [ -n "$SINGLETEST" ]; then
 771         rm -f "$RUNFILES" >/dev/null 2>&1
 772 fi
 773
 774 [ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"