scripts/zfs-tests.sh

   1 #!/bin/bash
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License, Version 1.0 only
   7 # (the "License").  You may not use this file except in compliance
   8 # with the License.
   9 #
  10 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  11 # or http://www.opensolaris.org/os/licensing.
  12 # See the License for the specific language governing permissions
  13 # and limitations under the License.
  14 #
  15 # When distributing Covered Code, include this CDDL HEADER in each
  16 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  17 # If applicable, add the following below this CDDL HEADER, with the
  18 # fields enclosed by brackets "[]" replaced with your own identifying
  19 # information: Portions Copyright [yyyy] [name of copyright owner]
  20 #
  21 # CDDL HEADER END
  22 #
  23
  24 BASE_DIR=$(dirname "$0")
  25 SCRIPT_COMMON=common.sh
  26 if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
  27 . "${BASE_DIR}/${SCRIPT_COMMON}"
  28 else
  29 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  30 fi
  31
  32 PROG=zfs-tests.sh
  33 VERBOSE="no"
  34 QUIET=
  35 CLEANUP="yes"
  36 CLEANUPALL="no"
  37 LOOPBACK="yes"
  38 STACK_TRACER="no"
  39 FILESIZE="4G"
  40 RUNFILE=${RUNFILE:-"linux.run"}
  41 FILEDIR=${FILEDIR:-/var/tmp}
  42 DISKS=${DISKS:-""}
  43 SINGLETEST=()
  44 SINGLETESTUSER="root"
  45 TAGS=""
  46 ITERATIONS=1
  47 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
  48 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
  49 ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
  50 TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
  51 LOSETUP=${LOSETUP:-/sbin/losetup}
  52 DMSETUP=${DMSETUP:-/sbin/dmsetup}
  53
  54 #
  55 # Log an informational message when additional verbosity is enabled.
  56 #
  57 msg() {
  58         if [ "$VERBOSE" = "yes" ]; then
  59                 echo "$@"
  60         fi
  61 }
  62
  63 #
  64 # Log a failure message, cleanup, and return an error.
  65 #
  66 fail() {
  67         echo -e "$PROG: $1" >&2
  68         cleanup
  69         exit 1
  70 }
  71
  72 #
  73 # Attempt to remove loopback devices and files which where created earlier
  74 # by this script to run the test framework.  The '-k' option may be passed
  75 # to the script to suppress cleanup for debugging purposes.
  76 #
  77 cleanup() {
  78         if [ "$CLEANUP" = "no" ]; then
  79                 return 0
  80         fi
  81
  82         if [ "$LOOPBACK" = "yes" ]; then
  83                 for TEST_LOOPBACK in ${LOOPBACKS}; do
  84                         LOOP_DEV=$(basename "$TEST_LOOPBACK")
  85                         DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
  86                             grep "${LOOP_DEV}" | cut -f1)
  87
  88                         if [ -n "$DM_DEV" ]; then
  89                                 sudo "${DMSETUP}" remove "${DM_DEV}" ||
  90                                     echo "Failed to remove: ${DM_DEV}"
  91                         fi
  92
  93                         if [ -n "${TEST_LOOPBACK}" ]; then
  94                                 sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
  95                                     echo "Failed to remove: ${TEST_LOOPBACK}"
  96                         fi
  97                 done
  98         fi
  99
 100         for TEST_FILE in ${FILES}; do
 101                 rm -f "${TEST_FILE}" &>/dev/null
 102         done
 103
 104         if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
 105                 rm -Rf "$STF_PATH"
 106         fi
 107 }
 108 trap cleanup EXIT
 109
 110 #
 111 # Attempt to remove all testpools (testpool.XXX), unopened dm devices,
 112 # loopback devices, and files.  This is a useful way to cleanup a previous
 113 # test run failure which has left the system in an unknown state.  This can
 114 # be dangerous and should only be used in a dedicated test environment.
 115 #
 116 cleanup_all() {
 117         local TEST_POOLS
 118         TEST_POOLS=$(sudo "$ZPOOL" list -H -o name | grep testpool)
 119         local TEST_LOOPBACKS
 120         TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
 121         local TEST_FILES
 122         TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
 123
 124         msg
 125         msg "--- Cleanup ---"
 126         msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
 127         for TEST_POOL in $TEST_POOLS; do
 128                 sudo "$ZPOOL" destroy "${TEST_POOL}"
 129         done
 130
 131         msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
 132             grep loop | tr '\n' ' ')"
 133         sudo "${DMSETUP}" remove_all
 134
 135         msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
 136         for TEST_LOOPBACK in $TEST_LOOPBACKS; do
 137                 sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
 138         done
 139
 140         msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
 141         for TEST_FILE in $TEST_FILES; do
 142                 sudo rm -f "${TEST_FILE}"
 143         done
 144 }
 145
 146 #
 147 # Takes a name as the only arguments and looks for the following variations
 148 # on that name.  If one is found it is returned.
 149 #
 150 # $RUNFILE_DIR/<name>
 151 # $RUNFILE_DIR/<name>.run
 152 # <name>
 153 # <name>.run
 154 #
 155 find_runfile() {
 156         local NAME=$1
 157         local RESULT=""
 158
 159         if [ -f "$RUNFILE_DIR/$NAME" ]; then
 160                 RESULT="$RUNFILE_DIR/$NAME"
 161         elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
 162                 RESULT="$RUNFILE_DIR/$NAME.run"
 163         elif [ -f "$NAME" ]; then
 164                 RESULT="$NAME"
 165         elif [ -f "$NAME.run" ]; then
 166                 RESULT="$NAME.run"
 167         fi
 168
 169         echo "$RESULT"
 170 }
 171
 172 #
 173 # Symlink file if it appears under any of the given paths.
 174 #
 175 create_links() {
 176         local dir_list="$1"
 177         local file_list="$2"
 178
 179         [ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
 180
 181         for i in $file_list; do
 182                 for j in $dir_list; do
 183                         [ ! -e "$STF_PATH/$i" ] || continue
 184
 185                         if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
 186                                 ln -s "$j/$i" "$STF_PATH/$i" || \
 187                                     fail "Couldn't link $i"
 188                                 break
 189                         fi
 190                 done
 191
 192                 [ ! -e "$STF_PATH/$i" ] && STF_MISSING_BIN="$STF_MISSING_BIN$i "
 193         done
 194 }
 195
 196 #
 197 # Constrain the path to limit the available binaries to a known set.
 198 # When running in-tree a top level ./bin/ directory is created for
 199 # convenience, otherwise a temporary directory is used.
 200 #
 201 constrain_path() {
 202         . "$STF_SUITE/include/commands.cfg"
 203
 204         if [ "$INTREE" = "yes" ]; then
 205                 # Constrained path set to ./zfs/bin/
 206                 STF_PATH="$BIN_DIR"
 207                 STF_PATH_REMOVE="no"
 208                 STF_MISSING_BIN=""
 209                 if [ ! -d "$STF_PATH" ]; then
 210                         mkdir "$STF_PATH"
 211                         chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 212                 fi
 213
 214                 # Special case links for standard zfs utilities
 215                 DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
 216                     ! -name .libs \) -print | tr '\n' ' ')"
 217                 create_links "$DIRS" "$ZFS_FILES"
 218
 219                 # Special case links for zfs test suite utilities
 220                 DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
 221                     ! -name .libs \) -print | tr '\n' ' ')"
 222                 create_links "$DIRS" "$ZFSTEST_FILES"
 223         else
 224                 # Constrained path set to /var/tmp/constrained_path.*
 225                 SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXX}
 226                 STF_PATH=$(/bin/mktemp -d "$SYSTEMDIR")
 227                 STF_PATH_REMOVE="yes"
 228                 STF_MISSING_BIN=""
 229
 230                 chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 231
 232                 # Special case links for standard zfs utilities
 233                 create_links "/bin /usr/bin /sbin /usr/sbin" "$ZFS_FILES"
 234
 235                 # Special case links for zfs test suite utilities
 236                 create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
 237         fi
 238
 239         # Standard system utilities
 240         create_links "/bin /usr/bin /sbin /usr/sbin" "$SYSTEM_FILES"
 241
 242         # Exceptions
 243         ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
 244         ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
 245         ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
 246         ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
 247         ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
 248         ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
 249         ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
 250
 251         if [ -L "$STF_PATH/arc_summary3" ]; then
 252                 ln -fs "$STF_PATH/arc_summary3" "$STF_PATH/arc_summary"
 253         fi
 254 }
 255
 256 #
 257 # Output a useful usage message.
 258 #
 259 usage() {
 260 cat << EOF
 261 USAGE:
 262 $0 [hvqxkfS] [-s SIZE] [-r RUNFILE] [-t PATH] [-u USER]
 263
 264 DESCRIPTION:
 265         ZFS Test Suite launch script
 266
 267 OPTIONS:
 268         -h          Show this message
 269         -v          Verbose zfs-tests.sh output
 270         -q          Quiet test-runner output
 271         -x          Remove all testpools, dm, lo, and files (unsafe)
 272         -k          Disable cleanup after test failure
 273         -f          Use files only, disables block device tests
 274         -S          Enable stack tracer (negative performance impact)
 275         -c          Only create and populate constrained path
 276         -n NFSFILE  Use the nfsfile to determine the NFS configuration
 277         -I NUM      Number of iterations
 278         -d DIR      Use DIR for files and loopback devices
 279         -s SIZE     Use vdevs of SIZE (default: 4G)
 280         -r RUNFILE  Run tests in RUNFILE (default: linux.run)
 281         -t PATH     Run single test at PATH relative to test suite
 282         -T TAGS     Comma separated list of tags (default: 'functional')
 283         -u USER     Run single test as USER (default: root)
 284
 285 EXAMPLES:
 286 # Run the default (linux) suite of tests and output the configuration used.
 287 $0 -v
 288
 289 # Run a smaller suite of tests designed to run more quickly.
 290 $0 -r linux-fast
 291
 292 # Cleanup a previous run of the test suite prior to testing, run the
 293 # default (linux) suite of tests and perform no cleanup on exit.
 294 $0 -x
 295
 296 EOF
 297 }
 298
 299 while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do
 300         case $OPTION in
 301         h)
 302                 usage
 303                 exit 1
 304                 ;;
 305         v)
 306                 # shellcheck disable=SC2034
 307                 VERBOSE="yes"
 308                 ;;
 309         q)
 310                 QUIET="-q"
 311                 ;;
 312         x)
 313                 CLEANUPALL="yes"
 314                 ;;
 315         k)
 316                 CLEANUP="no"
 317                 ;;
 318         f)
 319                 LOOPBACK="no"
 320                 ;;
 321         S)
 322                 STACK_TRACER="yes"
 323                 ;;
 324         c)
 325                 constrain_path
 326                 exit
 327                 ;;
 328         n)
 329                 nfsfile=$OPTARG
 330                 [[ -f $nfsfile ]] || fail "Cannot read file: $nfsfile"
 331                 export NFS=1
 332                 . "$nfsfile"
 333                 ;;
 334         d)
 335                 FILEDIR="$OPTARG"
 336                 ;;
 337         I)
 338                 ITERATIONS="$OPTARG"
 339                 if [ "$ITERATIONS" -le 0 ]; then
 340                         fail "Iterations must be greater than 0."
 341                 fi
 342                 ;;
 343         s)
 344                 FILESIZE="$OPTARG"
 345                 ;;
 346         r)
 347                 RUNFILE="$OPTARG"
 348                 ;;
 349         t)
 350                 if [ ${#SINGLETEST[@]} -ne 0 ]; then
 351                         fail "-t can only be provided once."
 352                 fi
 353                 SINGLETEST+=("$OPTARG")
 354                 ;;
 355         T)
 356                 TAGS="$OPTARG"
 357                 ;;
 358         u)
 359                 SINGLETESTUSER="$OPTARG"
 360                 ;;
 361         ?)
 362                 usage
 363                 exit
 364                 ;;
 365         esac
 366 done
 367
 368 shift $((OPTIND-1))
 369
 370 FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
 371 LOOPBACKS=${LOOPBACKS:-""}
 372
 373 if [ ${#SINGLETEST[@]} -ne 0 ]; then
 374         if [ -n "$TAGS" ]; then
 375                 fail "-t and -T are mutually exclusive."
 376         fi
 377         RUNFILE_DIR="/var/tmp"
 378         RUNFILE="zfs-tests.$$.run"
 379         SINGLEQUIET="False"
 380
 381         if [ -n "$QUIET" ]; then
 382                 SINGLEQUIET="True"
 383         fi
 384
 385         cat >$RUNFILE_DIR/$RUNFILE << EOF
 386 [DEFAULT]
 387 pre =
 388 quiet = $SINGLEQUIET
 389 pre_user = root
 390 user = $SINGLETESTUSER
 391 timeout = 600
 392 post_user = root
 393 post =
 394 outputdir = /var/tmp/test_results
 395 EOF
 396         for t in "${SINGLETEST[@]}"
 397         do
 398                 SINGLETESTDIR=$(dirname "$t")
 399                 SINGLETESTFILE=$(basename "$t")
 400                 SETUPSCRIPT=
 401                 CLEANUPSCRIPT=
 402
 403                 if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
 404                         SETUPSCRIPT="setup"
 405                 fi
 406
 407                 if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
 408                         CLEANUPSCRIPT="cleanup"
 409                 fi
 410
 411                 cat >>$RUNFILE_DIR/$RUNFILE << EOF
 412
 413 [$SINGLETESTDIR]
 414 tests = ['$SINGLETESTFILE']
 415 pre = $SETUPSCRIPT
 416 post = $CLEANUPSCRIPT
 417 tags = ['functional']
 418 EOF
 419         done
 420 fi
 421
 422 #
 423 # Use default tag if none was specified
 424 #
 425 TAGS=${TAGS:='functional'}
 426
 427 #
 428 # Attempt to locate the runfile describing the test workload.
 429 #
 430 if [ -n "$RUNFILE" ]; then
 431         SAVED_RUNFILE="$RUNFILE"
 432         RUNFILE=$(find_runfile "$RUNFILE")
 433         [ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
 434 fi
 435
 436 if [ ! -r "$RUNFILE" ]; then
 437         fail "Cannot read runfile: $RUNFILE"
 438 fi
 439
 440 #
 441 # This script should not be run as root.  Instead the test user, which may
 442 # be a normal user account, needs to be configured such that it can
 443 # run commands via sudo passwordlessly.
 444 #
 445 if [ "$(id -u)" = "0" ]; then
 446         fail "This script must not be run as root."
 447 fi
 448
 449 if [ "$(sudo whoami)" != "root" ]; then
 450         fail "Passwordless sudo access required."
 451 fi
 452
 453 #
 454 # Constrain the available binaries to a known set.
 455 #
 456 constrain_path
 457
 458 #
 459 # Check if ksh exists
 460 #
 461 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
 462 [ -e "$STF_SUITE/include/default.cfg" ] || fail \
 463     "Missing $STF_SUITE/include/default.cfg file."
 464
 465 #
 466 # Verify the ZFS module stack is loaded.
 467 #
 468 if [ "$STACK_TRACER" = "yes" ]; then
 469         sudo "${ZFS_SH}" -S &>/dev/null
 470 else
 471         sudo "${ZFS_SH}" &>/dev/null
 472 fi
 473
 474 #
 475 # Attempt to cleanup all previous state for a new test run.
 476 #
 477 if [ "$CLEANUPALL" = "yes" ]; then
 478         cleanup_all
 479 fi
 480
 481 #
 482 # By default preserve any existing pools
 483 # NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
 484 # space-delimited to newline-delimited.
 485 #
 486 if [ -z "${KEEP}" ]; then
 487         KEEP="$(sudo "$ZPOOL" list -H -o name)"
 488         if [ -z "${KEEP}" ]; then
 489                 KEEP="rpool"
 490         fi
 491 else
 492         KEEP="$(echo -e "${KEEP//[[:blank:]]/\n}")"
 493 fi
 494
 495 #
 496 # NOTE: The following environment variables are undocumented
 497 # and should be used for testing purposes only:
 498 #
 499 # __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
 500 # __ZFS_POOL_RESTRICT - iterate only over the pools it lists
 501 #
 502 # See libzfs/libzfs_config.c for more information.
 503 #
 504 __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
 505
 506 . "$STF_SUITE/include/default.cfg"
 507
 508 msg
 509 msg "--- Configuration ---"
 510 msg "Runfile:         $RUNFILE"
 511 msg "STF_TOOLS:       $STF_TOOLS"
 512 msg "STF_SUITE:       $STF_SUITE"
 513 msg "STF_PATH:        $STF_PATH"
 514
 515 #
 516 # No DISKS have been provided so a basic file or loopback based devices
 517 # must be created for the test suite to use.
 518 #
 519 if [ -z "${DISKS}" ]; then
 520         #
 521         # Create sparse files for the test suite.  These may be used
 522         # directory or have loopback devices layered on them.
 523         #
 524         for TEST_FILE in ${FILES}; do
 525                 [ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
 526                 truncate -s "${FILESIZE}" "${TEST_FILE}" ||
 527                     fail "Failed creating: ${TEST_FILE} ($?)"
 528                 if [[ "$DISKS" ]]; then
 529                         DISKS="$DISKS $TEST_FILE"
 530                 else
 531                         DISKS="$TEST_FILE"
 532                 fi
 533         done
 534
 535         #
 536         # If requested setup loopback devices backed by the sparse files.
 537         #
 538         if [ "$LOOPBACK" = "yes" ]; then
 539                 DISKS=""
 540
 541                 test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
 542
 543                 for TEST_FILE in ${FILES}; do
 544                         TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
 545                         sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
 546                             fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
 547                         LOOPBACKS="${LOOPBACKS}${TEST_LOOPBACK} "
 548                         BASELOOPBACKS=$(basename "$TEST_LOOPBACK")
 549                         if [[ "$DISKS" ]]; then
 550                                 DISKS="$DISKS $BASELOOPBACKS"
 551                         else
 552                                 DISKS="$BASELOOPBACKS"
 553                         fi
 554                 done
 555         fi
 556 fi
 557
 558 NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
 559 [ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
 560
 561 #
 562 # Disable SELinux until the ZFS Test Suite has been updated accordingly.
 563 #
 564 if [ -x "$STF_PATH/setenforce" ]; then
 565         sudo setenforce permissive &>/dev/null
 566 fi
 567
 568 #
 569 # Enable internal ZFS debug log and clear it.
 570 #
 571 if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
 572         sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
 573         sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
 574 fi
 575
 576 msg "FILEDIR:         $FILEDIR"
 577 msg "FILES:           $FILES"
 578 msg "LOOPBACKS:       $LOOPBACKS"
 579 msg "DISKS:           $DISKS"
 580 msg "NUM_DISKS:       $NUM_DISKS"
 581 msg "FILESIZE:        $FILESIZE"
 582 msg "ITERATIONS:      $ITERATIONS"
 583 msg "TAGS:            $TAGS"
 584 msg "STACK_TRACER:    $STACK_TRACER"
 585 msg "Keep pool(s):    $KEEP"
 586 msg "Missing util(s): $STF_MISSING_BIN"
 587 msg ""
 588
 589 export STF_TOOLS
 590 export STF_SUITE
 591 export STF_PATH
 592 export DISKS
 593 export FILEDIR
 594 export KEEP
 595 export __ZFS_POOL_EXCLUDE
 596 export TESTFAIL_CALLBACKS
 597 export PATH=$STF_PATH
 598
 599 RESULTS_FILE=$(mktemp -u -t zts-results.XXXX -p "$FILEDIR")
 600 REPORT_FILE=$(mktemp -u -t zts-report.XXXX -p "$FILEDIR")
 601
 602 #
 603 # Run all the tests as specified.
 604 #
 605 msg "${TEST_RUNNER} ${QUIET} -c ${RUNFILE} -T ${TAGS} -i ${STF_SUITE}" \
 606     "-I ${ITERATIONS}"
 607 ${TEST_RUNNER} ${QUIET} -c "${RUNFILE}" -T "${TAGS}" -i "${STF_SUITE}" \
 608     -I "${ITERATIONS}" 2>&1 | tee "$RESULTS_FILE"
 609
 610 #
 611 # Analyze the results.
 612 #
 613 set -o pipefail
 614 ${ZTS_REPORT} "$RESULTS_FILE" | tee "$REPORT_FILE"
 615 RESULT=$?
 616 set +o pipefail
 617
 618 RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
 619 if [ -d "$RESULTS_DIR" ]; then
 620         cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
 621 fi
 622
 623 rm -f "$RESULTS_FILE" "$REPORT_FILE"
 624
 625 if [ ${#SINGLETEST[@]} -ne 0 ]; then
 626         rm -f "$RUNFILE" &>/dev/null
 627 fi
 628
 629 exit ${RESULT}