]> git.proxmox.com Git - mirror_zfs.git/blobdiff - tests/zfs-tests/include/libtest.shlib
Do not iterate through filesystems unnecessarily
[mirror_zfs.git] / tests / zfs-tests / include / libtest.shlib
index 5d8500ddfce1a3a8fcdedc6b9095740edb9adc3c..ea9448353b68fc0732b9542015ae70310fa63386 100644 (file)
@@ -1,4 +1,3 @@
-#!/bin/ksh -p
 #
 # CDDL HEADER START
 #
 #
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
-# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
-# Copyright 2016 Nexenta Systems, Inc.
+# Copyright (c) 2012, 2017 by Delphix. All rights reserved.
+# Copyright (c) 2017 by Tim Chase. All rights reserved.
+# Copyright (c) 2017 by Nexenta Systems, Inc. All rights reserved.
 # Copyright (c) 2017 Lawrence Livermore National Security, LLC.
 # Copyright (c) 2017 Datto Inc.
+# Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
+# Use is subject to license terms.
 #
 
 . ${STF_TOOLS}/include/logapi.shlib
 . ${STF_SUITE}/include/math.shlib
+. ${STF_SUITE}/include/blkdev.shlib
 
 #
 # Apply constrained path when available.  This is required since the
@@ -40,6 +43,43 @@ if [ -n "$STF_PATH" ]; then
        PATH="$STF_PATH"
 fi
 
+#
+# Generic dot version comparison function
+#
+# Returns success when version $1 is greater than or equal to $2.
+#
+function compare_version_gte
+{
+       if [[ "$(printf "$1\n$2" | sort -V | tail -n1)" == "$1" ]]; then
+               return 0
+       else
+               return 1
+       fi
+}
+
+# Linux kernel version comparison function
+#
+# $1 Linux version ("4.10", "2.6.32") or blank for installed Linux version
+#
+# Used for comparison: if [ $(linux_version) -ge $(linux_version "2.6.32") ]
+#
+function linux_version
+{
+       typeset ver="$1"
+
+       [[ -z "$ver" ]] && ver=$(uname -r | grep -Eo "^[0-9]+\.[0-9]+\.[0-9]+")
+
+       typeset version=$(echo $ver | cut -d '.' -f 1)
+       typeset major=$(echo $ver | cut -d '.' -f 2)
+       typeset minor=$(echo $ver | cut -d '.' -f 3)
+
+       [[ -z "$version" ]] && version=0
+       [[ -z "$major" ]] && major=0
+       [[ -z "$minor" ]] && minor=0
+
+       echo $((version * 10000 + major * 100 + minor))
+}
+
 # Determine if this is a Linux test system
 #
 # Return 0 if platform Linux, 1 if otherwise
@@ -117,7 +157,7 @@ function ismounted
 
                        [[ "$1" == "$dir" || "$1" == "$name" ]] && return 0
                ;;
-               ext2)
+               ext*)
                        out=$(df -t $fstype $1 2>/dev/null)
                        return $?
                ;;
@@ -260,12 +300,12 @@ function default_container_volume_setup
 # Create a snapshot on a filesystem or volume. Defaultly create a snapshot on
 # filesystem
 #
-# $1 Existing filesystem or volume name. Default, $TESTFS
+# $1 Existing filesystem or volume name. Default, $TESTPOOL/$TESTFS
 # $2 snapshot name. Default, $TESTSNAP
 #
 function create_snapshot
 {
-       typeset fs_vol=${1:-$TESTFS}
+       typeset fs_vol=${1:-$TESTPOOL/$TESTFS}
        typeset snap=${2:-$TESTSNAP}
 
        [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined."
@@ -328,6 +368,41 @@ function create_bookmark
        log_must zfs bookmark $fs_vol@$snap $fs_vol#$bkmark
 }
 
+#
+# Create a temporary clone result of an interrupted resumable 'zfs receive'
+# $1 Destination filesystem name. Must not exist, will be created as the result
+#    of this function along with its %recv temporary clone
+# $2 Source filesystem name. Must not exist, will be created and destroyed
+#
+function create_recv_clone
+{
+       typeset recvfs="$1"
+       typeset sendfs="${2:-$TESTPOOL/create_recv_clone}"
+       typeset snap="$sendfs@snap1"
+       typeset incr="$sendfs@snap2"
+       typeset mountpoint="$TESTDIR/create_recv_clone"
+       typeset sendfile="$TESTDIR/create_recv_clone.zsnap"
+
+       [[ -z $recvfs ]] && log_fail "Recv filesystem's name is undefined."
+
+       datasetexists $recvfs && log_fail "Recv filesystem must not exist."
+       datasetexists $sendfs && log_fail "Send filesystem must not exist."
+
+       log_must zfs create -o mountpoint="$mountpoint" $sendfs
+       log_must zfs snapshot $snap
+       log_must eval "zfs send $snap | zfs recv -u $recvfs"
+       log_must mkfile 1m "$mountpoint/data"
+       log_must zfs snapshot $incr
+       log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 > $sendfile"
+       log_mustnot eval "zfs recv -su $recvfs < $sendfile"
+       destroy_dataset "$sendfs" "-r"
+       log_must rm -f "$sendfile"
+
+       if [[ $(get_prop 'inconsistent' "$recvfs/%recv") -ne 1 ]]; then
+               log_fail "Error creating temporary $recvfs/%recv clone"
+       fi
+}
+
 function default_mirror_setup
 {
        default_mirror_setup_noexit $1 $2 $3
@@ -423,7 +498,7 @@ function default_raidz_setup
        fi
 
        [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL
-       log_must zpool create -f $TESTPOOL raidz $1 $2 $3
+       log_must zpool create -f $TESTPOOL raidz $disklist
        log_must zfs create $TESTPOOL/$TESTFS
        log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
 
@@ -445,9 +520,19 @@ function default_cleanup
        log_pass
 }
 
+#
+# Utility function used to list all available pool names.
+#
+# NOTE: $KEEP is a variable containing pool names, separated by a newline
+# character, that must be excluded from the returned list.
+#
+function get_all_pools
+{
+       zpool list -H -o name | grep -Fvx "$KEEP" | grep -v "$NO_POOLS"
+}
+
 function default_cleanup_noexit
 {
-       typeset exclude=""
        typeset pool=""
        #
        # Destroying the pool will also destroy any
@@ -455,9 +540,7 @@ function default_cleanup_noexit
        #
        if is_global_zone; then
                zfs unmount -a > /dev/null 2>&1
-               exclude=`eval echo \"'(${KEEP})'\"`
-               ALL_POOLS=$(zpool list -H -o name \
-                   | grep -v "$NO_POOLS" | egrep -v "$exclude")
+               ALL_POOLS=$(get_all_pools)
                # Here, we loop through the pools we're allowed to
                # destroy, only destroying them if it's safe to do
                # so.
@@ -469,9 +552,7 @@ function default_cleanup_noexit
                                then
                                        destroy_pool $pool
                                fi
-                               ALL_POOLS=$(zpool list -H -o name \
-                                   | grep -v "$NO_POOLS" \
-                                   | egrep -v "$exclude")
+                               ALL_POOLS=$(get_all_pools)
                        done
                done
 
@@ -480,8 +561,7 @@ function default_cleanup_noexit
                typeset fs=""
                for fs in $(zfs list -H -o name \
                    | grep "^$ZONE_POOL/$ZONE_CTR[01234]/"); do
-                       datasetexists $fs && \
-                               log_must zfs destroy -Rf $fs
+                       destroy_dataset "$fs" "-Rf"
                done
 
                # Need cleanup here to avoid garbage dir left.
@@ -525,6 +605,8 @@ function default_cleanup_noexit
        if is_mpath_device $disk1; then
                delete_partitions
        fi
+
+       rm -f $TEST_BASE_DIR/{err,out}
 }
 
 
@@ -542,11 +624,8 @@ function default_container_cleanup
        [[ $? -eq 0 ]] && \
            log_must zfs unmount $TESTPOOL/$TESTCTR/$TESTFS1
 
-       datasetexists $TESTPOOL/$TESTCTR/$TESTFS1 && \
-           log_must zfs destroy -R $TESTPOOL/$TESTCTR/$TESTFS1
-
-       datasetexists $TESTPOOL/$TESTCTR && \
-           log_must zfs destroy -Rf $TESTPOOL/$TESTCTR
+       destroy_dataset "$TESTPOOL/$TESTCTR/$TESTFS1" "-R"
+       destroy_dataset "$TESTPOOL/$TESTCTR" "-Rf"
 
        [[ -e $TESTDIR1 ]] && \
            log_must rm -rf $TESTDIR1 > /dev/null 2>&1
@@ -565,7 +644,7 @@ function destroy_snapshot
        typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}
 
        if ! snapexists $snap; then
-               log_fail "'$snap' does not existed."
+               log_fail "'$snap' does not exist."
        fi
 
        #
@@ -580,7 +659,7 @@ function destroy_snapshot
                        log_fail "get_prop mountpoint $snap failed."
        fi
 
-       log_must zfs destroy $snap
+       destroy_dataset "$snap"
        [[ $mtpt != "" && -d $mtpt ]] && \
                log_must rm -rf $mtpt
 }
@@ -606,7 +685,7 @@ function destroy_clone
                        log_fail "get_prop mountpoint $clone failed."
        fi
 
-       log_must zfs destroy $clone
+       destroy_dataset "$clone"
        [[ $mtpt != "" && -d $mtpt ]] && \
                log_must rm -rf $mtpt
 }
@@ -625,7 +704,7 @@ function destroy_bookmark
                log_fail "'$bkmarkp' does not existed."
        fi
 
-       log_must zfs destroy $bkmark
+       destroy_dataset "$bkmark"
 }
 
 # Return 0 if a snapshot exists; $? otherwise
@@ -744,11 +823,15 @@ function zero_partitions #<whole_disk_name>
        typeset i
 
        if is_linux; then
-               log_must parted $DEV_DSKDIR/$diskname -s -- mklabel gpt
+               DSK=$DEV_DSKDIR/$diskname
+               DSK=$(echo $DSK | sed -e "s|//|/|g")
+               log_must parted $DSK -s -- mklabel gpt
+               blockdev --rereadpt $DSK 2>/dev/null
+               block_device_wait
        else
                for i in 0 1 3 4 5 6 7
                do
-                       set_partition $i "" 0mb $diskname
+                       log_must set_partition $i "" 0mb $diskname
                done
        fi
 
@@ -772,10 +855,11 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
        typeset start=$2
        typeset size=$3
        typeset disk=$4
-       [[ -z $slicenum || -z $size || -z $disk ]] && \
-           log_fail "The slice, size or disk name is unspecified."
 
        if is_linux; then
+               if [[ -z $size || -z $disk ]]; then
+                       log_fail "The size or disk name is unspecified."
+               fi
                typeset size_mb=${size%%[mMgG]}
 
                size_mb=${size_mb%%[mMgG][bB]}
@@ -788,7 +872,11 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
                parted $DEV_DSKDIR/$disk -s -- print 1 >/dev/null
                typeset ret_val=$?
                if [[ $slicenum -eq 0 || $ret_val -ne 0 ]]; then
-                       log_must parted $DEV_DSKDIR/$disk -s -- mklabel gpt
+                       parted $DEV_DSKDIR/$disk -s -- mklabel gpt
+                       if [[ $? -ne 0 ]]; then
+                               log_note "Failed to create GPT partition table on $disk"
+                               return 1
+                       fi
                fi
 
                # When no start is given align on the first cylinder.
@@ -804,12 +892,20 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
                        awk -F '[:k.]' '{print $4}')
                ((end = (size_mb * 1024 / cly_size_kb) + start))
 
-               log_must parted $DEV_DSKDIR/$disk -s -- \
+               parted $DEV_DSKDIR/$disk -s -- \
                    mkpart part$slicenum ${start}cyl ${end}cyl
+               if [[ $? -ne 0 ]]; then
+                       log_note "Failed to create partition $slicenum on $disk"
+                       return 1
+               fi
 
                blockdev --rereadpt $DEV_DSKDIR/$disk 2>/dev/null
                block_device_wait
        else
+               if [[ -z $slicenum || -z $size || -z $disk ]]; then
+                       log_fail "The slice, size or disk name is unspecified."
+               fi
+
                typeset format_file=/var/tmp/format_in.$$
 
                echo "partition" >$format_file
@@ -828,8 +924,10 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
 
        typeset ret_val=$?
        rm -f $format_file
-       [[ $ret_val -ne 0 ]] && \
-           log_fail "Unable to format $disk slice $slicenum to $size"
+       if [[ $ret_val -ne 0 ]]; then
+               log_note "Unable to format $disk slice $slicenum to $size"
+               return 1
+       fi
        return 0
 }
 
@@ -933,7 +1031,7 @@ function get_endslice #<disk> <slice>
 
                ((endcyl = (endcyl + 1) / ratio))
        fi
-       
+
        echo $endcyl
 }
 
@@ -959,7 +1057,7 @@ function partition_disk    #<slice_size> <whole_disk_name> <total_slices>
                                continue
                        fi
                fi
-               set_partition $i "$cyl" $slice_size $disk_name
+               log_must set_partition $i "$cyl" $slice_size $disk_name
                cyl=$(get_endslice $disk_name $i)
                ((i = i+1))
        done
@@ -996,14 +1094,14 @@ function fill_fs # destdir dirnum filenum bytes num_writes data
        typeset -i filenum=${3:-50}
        typeset -i bytes=${4:-8192}
        typeset -i num_writes=${5:-10240}
-       typeset -i data=${6:-0}
+       typeset data=${6:-0}
 
        typeset -i odirnum=1
        typeset -i idirnum=0
        typeset -i fn=0
        typeset -i retval=0
 
-       log_must mkdir -p $destdir/$idirnum
+       mkdir -p $destdir/$idirnum
        while (($odirnum > 0)); do
                if ((dirnum >= 0 && idirnum >= dirnum)); then
                        odirnum=0
@@ -1019,7 +1117,7 @@ function fill_fs # destdir dirnum filenum bytes num_writes data
                if (($fn >= $filenum)); then
                        fn=0
                        ((idirnum = idirnum + 1))
-                       log_must mkdir -p $destdir/$idirnum
+                       mkdir -p $destdir/$idirnum
                else
                        ((fn = fn + 1))
                fi
@@ -1134,30 +1232,11 @@ function datasetnonexists
        return 0
 }
 
-#
-# Given a mountpoint, or a dataset name, determine if it is shared via NFS.
-#
-# Returns 0 if shared, 1 otherwise.
-#
-function is_shared
+function is_shared_impl
 {
        typeset fs=$1
        typeset mtpt
 
-       if [[ $fs != "/"* ]] ; then
-               if datasetnonexists "$fs" ; then
-                       return 1
-               else
-                       mtpt=$(get_prop mountpoint "$fs")
-                       case $mtpt in
-                               none|legacy|-) return 1
-                                       ;;
-                               *)      fs=$mtpt
-                                       ;;
-                       esac
-               fi
-       fi
-
        if is_linux; then
                for mtpt in `share | awk '{print $1}'` ; do
                        if [[ $mtpt == $fs ]] ; then
@@ -1181,6 +1260,33 @@ function is_shared
        return 1
 }
 
+#
+# Given a mountpoint, or a dataset name, determine if it is shared via NFS.
+#
+# Returns 0 if shared, 1 otherwise.
+#
+function is_shared
+{
+       typeset fs=$1
+       typeset mtpt
+
+       if [[ $fs != "/"* ]] ; then
+               if datasetnonexists "$fs" ; then
+                       return 1
+               else
+                       mtpt=$(get_prop mountpoint "$fs")
+                       case $mtpt in
+                               none|legacy|-) return 1
+                                       ;;
+                               *)      fs=$mtpt
+                                       ;;
+                       esac
+               fi
+       fi
+
+       is_shared_impl "$fs"
+}
+
 #
 # Given a dataset name determine if it is shared via SMB.
 #
@@ -1344,7 +1450,13 @@ function setup_nfs_server
        fi
 
        if is_linux; then
-               log_note "NFS server must started prior to running test framework."
+               #
+               # Re-synchronize /var/lib/nfs/etab with /etc/exports and
+               # /etc/exports.d./* to provide a clean test environment.
+               #
+               log_must share -r
+
+               log_note "NFS server must be started prior to running ZTS."
                return
        fi
 
@@ -1513,6 +1625,65 @@ function destroy_pool #pool
        return 0
 }
 
+# Return 0 if created successfully; $? otherwise
+#
+# $1 - dataset name
+# $2-n - dataset options
+
+function create_dataset #dataset dataset_options
+{
+       typeset dataset=$1
+
+       shift
+
+       if [[ -z $dataset ]]; then
+               log_note "Missing dataset name."
+               return 1
+       fi
+
+       if datasetexists $dataset ; then
+               destroy_dataset $dataset
+       fi
+
+       log_must zfs create $@ $dataset
+
+       return 0
+}
+
+# Return 0 if destroy successfully or the dataset exists; $? otherwise
+# Note: In local zones, this function should return 0 silently.
+#
+# $1 - dataset name
+# $2 - custom arguments for zfs destroy
+# Destroy dataset with the given parameters.
+
+function destroy_dataset #dataset #args
+{
+       typeset dataset=$1
+       typeset mtpt
+       typeset args=${2:-""}
+
+       if [[ -z $dataset ]]; then
+               log_note "No dataset name given."
+               return 1
+       fi
+
+       if is_global_zone ; then
+               if datasetexists "$dataset" ; then
+                       mtpt=$(get_prop mountpoint "$dataset")
+                       log_must_busy zfs destroy $args $dataset
+
+                       [[ -d $mtpt ]] && \
+                               log_must rm -rf $mtpt
+               else
+                       log_note "Dataset does not exist. ($dataset)"
+                       return 1
+               fi
+       fi
+
+       return 0
+}
+
 #
 # Firstly, create a pool with 5 datasets. Then, create a single zone and
 # export the 5 datasets to it. In addition, we also add a ZFS filesystem
@@ -1549,7 +1720,7 @@ function zfs_zones_setup #zone_name zone_root zone_ip
        # If current system support slog, add slog device for pool
        #
        if verify_slog_support ; then
-               typeset sdevs="/var/tmp/sdev1 /var/tmp/sdev2"
+               typeset sdevs="$TEST_BASE_DIR/sdev1 $TEST_BASE_DIR/sdev2"
                log_must mkfile $MINVDEVSIZE $sdevs
                log_must zpool add $pool_name log mirror $sdevs
        fi
@@ -1677,124 +1848,6 @@ function check_state # pool disk state{online,offline,degraded}
        return $?
 }
 
-#
-# Cause a scan of all scsi host adapters by default
-#
-# $1 optional host number
-#
-function scan_scsi_hosts
-{
-       typeset hostnum=${1}
-
-       if is_linux; then
-               if [[ -z $hostnum ]]; then
-                       for host in /sys/class/scsi_host/host*; do
-                               log_must eval "echo '- - -' > $host/scan"
-                       done
-               else
-                       log_must eval \
-                           "echo /sys/class/scsi_host/host$hostnum/scan" \
-                           > /dev/null
-                       log_must eval \
-                           "echo '- - -' > /sys/class/scsi_host/host$hostnum/scan"
-               fi
-       fi
-}
-#
-# Wait for newly created block devices to have their minors created.
-#
-function block_device_wait
-{
-       if is_linux; then
-               udevadm trigger
-               udevadm settle
-       fi
-}
-
-#
-# Online or offline a disk on the system
-#
-# First checks state of disk. Test will fail if disk is not properly onlined
-# or offlined. Online is a full rescan of SCSI disks by echoing to every
-# host entry.
-#
-function on_off_disk # disk state{online,offline} host
-{
-       typeset disk=$1
-       typeset state=$2
-       typeset host=$3
-
-       [[ -z $disk ]] || [[ -z $state ]] &&  \
-           log_fail "Arguments invalid or missing"
-
-       if is_linux; then
-               if [[ $state == "offline" ]] && ( is_mpath_device $disk ); then
-                       dm_name="$(readlink $DEV_DSKDIR/$disk \
-                           | nawk -F / '{print $2}')"
-                       slave="$(ls /sys/block/${dm_name}/slaves \
-                           | nawk '{print $1}')"
-                       while [[ -n $slave ]]; do
-                               #check if disk is online
-                               lsscsi | egrep $slave > /dev/null
-                               if (($? == 0)); then
-                                       slave_dir="/sys/block/${dm_name}"
-                                       slave_dir+="/slaves/${slave}/device"
-                                       ss="${slave_dir}/state"
-                                       sd="${slave_dir}/delete"
-                                       log_must eval "echo 'offline' > ${ss}"
-                                       log_must eval "echo '1' > ${sd}"
-                                       lsscsi | egrep $slave > /dev/null
-                                               if (($? == 0)); then
-                                                       log_fail "Offlining" \
-                                                           "$disk failed"
-                                               fi
-                               fi
-                               slave="$(ls /sys/block/$dm_name/slaves \
-                                   2>/dev/null | nawk '{print $1}')"
-                       done
-               elif [[ $state == "offline" ]] && ( is_real_device $disk ); then
-                       #check if disk is online
-                       lsscsi | egrep $disk > /dev/null
-                       if (($? == 0)); then
-                               dev_state="/sys/block/$disk/device/state"
-                               dev_delete="/sys/block/$disk/device/delete"
-                               log_must eval "echo 'offline' > ${dev_state}"
-                               log_must eval "echo '1' > ${dev_delete}"
-                               lsscsi | egrep $disk > /dev/null
-                                       if (($? == 0)); then
-                                               log_fail "Offlining $disk" \
-                                                   "failed"
-                                       fi
-                       else
-                               log_note "$disk is already offline"
-                       fi
-               elif [[ $state == "online" ]]; then
-                       #force a full rescan
-                       scan_scsi_hosts $host
-                       block_device_wait
-                       if is_mpath_device $disk; then
-                               dm_name="$(readlink $DEV_DSKDIR/$disk \
-                                   | nawk -F / '{print $2}')"
-                               slave="$(ls /sys/block/$dm_name/slaves \
-                                   | nawk '{print $1}')"
-                               lsscsi | egrep $slave > /dev/null
-                               if (($? != 0)); then
-                                       log_fail "Onlining $disk failed"
-                               fi
-                       elif is_real_device $disk; then
-                               lsscsi | egrep $disk > /dev/null
-                               if (($? != 0)); then
-                                       log_fail "Onlining $disk failed"
-                               fi
-                       else
-                               log_fail "$disk is not a real dev"
-                       fi
-               else
-                       log_fail "$disk failed to $state"
-               fi
-       fi
-}
-
 #
 # Get the mountpoint of snapshot
 # For the snapshot use <mp_filesystem>/.zfs/snapshot/<snap>
@@ -1880,6 +1933,23 @@ function verify_filesys # pool filesystem dir
        log_must rm -rf $zdbout
 }
 
+#
+# Given a pool issue a scrub and verify that no checksum errors are reported.
+#
+function verify_pool
+{
+       typeset pool=${1:-$TESTPOOL}
+
+       log_must zpool scrub $pool
+       log_must wait_scrubbed $pool
+
+       cksum=$(zpool status $pool | awk 'L{print $NF;L=0} /CKSUM$/{L=1}')
+       if [[ $cksum != 0 ]]; then
+               log_must zpool status -v
+               log_fail "Unexpected CKSUM errors found on $pool ($cksum)"
+       fi
+}
+
 #
 # Given a pool, and this function list all disks in the pool
 #
@@ -1889,7 +1959,7 @@ function get_disklist # pool
 
        disklist=$(zpool iostat -v $1 | nawk '(NR >4) {print $1}' | \
            grep -v "\-\-\-\-\-" | \
-           egrep -v -e "^(mirror|raidz1|raidz2|spare|log|cache)$")
+           egrep -v -e "^(mirror|raidz[1-3]|spare|log|cache|special|dedup)$")
 
        echo $disklist
 }
@@ -1955,6 +2025,31 @@ function check_hotspare_state # pool disk state{inuse,avail}
        return 0
 }
 
+#
+# Wait until a hotspare transitions to a given state or times out.
+#
+# Return 0 when  pool/disk matches expected state, 1 on timeout.
+#
+function wait_hotspare_state # pool disk state timeout
+{
+       typeset pool=$1
+       typeset disk=${2#*$DEV_DSKDIR/}
+       typeset state=$3
+       typeset timeout=${4:-60}
+       typeset -i i=0
+
+       while [[ $i -lt $timeout ]]; do
+               if check_hotspare_state $pool $disk $state; then
+                       return 0
+               fi
+
+               i=$((i+1))
+               sleep 1
+       done
+
+       return 1
+}
+
 #
 # Verify a given slog disk is inuse or avail
 #
@@ -1982,7 +2077,7 @@ function check_slog_state # pool disk state{online,offline,unavail}
 function check_vdev_state # pool disk state{online,offline,unavail}
 {
        typeset pool=$1
-       typeset disk=${2#$/DEV_DSKDIR/}
+       typeset disk=${2#*$DEV_DSKDIR/}
        typeset state=$3
 
        cur_state=$(get_device_state $pool $disk)
@@ -1993,63 +2088,132 @@ function check_vdev_state # pool disk state{online,offline,unavail}
        return 0
 }
 
+#
+# Wait until a vdev transitions to a given state or times out.
+#
+# Return 0 when  pool/disk matches expected state, 1 on timeout.
+#
+function wait_vdev_state # pool disk state timeout
+{
+       typeset pool=$1
+       typeset disk=${2#*$DEV_DSKDIR/}
+       typeset state=$3
+       typeset timeout=${4:-60}
+       typeset -i i=0
+
+       while [[ $i -lt $timeout ]]; do
+               if check_vdev_state $pool $disk $state; then
+                       return 0
+               fi
+
+               i=$((i+1))
+               sleep 1
+       done
+
+       return 1
+}
+
 #
 # Check the output of 'zpool status -v <pool>',
 # and to see if the content of <token> contain the <keyword> specified.
 #
 # Return 0 is contain, 1 otherwise
 #
-function check_pool_status # pool token keyword
+function check_pool_status # pool token keyword <verbose>
 {
        typeset pool=$1
        typeset token=$2
        typeset keyword=$3
+       typeset verbose=${4:-false}
 
-       zpool status -v "$pool" 2>/dev/null | nawk -v token="$token:" '
-               ($1==token) {print $0}' \
-       | grep -i "$keyword" > /dev/null 2>&1
+       scan=$(zpool status -v "$pool" 2>/dev/null | nawk -v token="$token:" '
+               ($1==token) {print $0}')
+       if [[ $verbose == true ]]; then
+               log_note $scan
+       fi
+       echo $scan | grep -i "$keyword" > /dev/null 2>&1
 
        return $?
 }
 
 #
-# These 5 following functions are instance of check_pool_status()
+# These 6 following functions are instance of check_pool_status()
 #      is_pool_resilvering - to check if the pool is resilver in progress
 #      is_pool_resilvered - to check if the pool is resilver completed
 #      is_pool_scrubbing - to check if the pool is scrub in progress
 #      is_pool_scrubbed - to check if the pool is scrub completed
 #      is_pool_scrub_stopped - to check if the pool is scrub stopped
+#      is_pool_scrub_paused - to check if the pool has scrub paused
+#      is_pool_removing - to check if the pool is removing a vdev
+#      is_pool_removed - to check if the pool is remove completed
 #
-function is_pool_resilvering #pool
+function is_pool_resilvering #pool <verbose>
+{
+       check_pool_status "$1" "scan" "resilver in progress since " $2
+       return $?
+}
+
+function is_pool_resilvered #pool <verbose>
 {
-       check_pool_status "$1" "scan" "resilver in progress since "
+       check_pool_status "$1" "scan" "resilvered " $2
        return $?
 }
 
-function is_pool_resilvered #pool
+function is_pool_scrubbing #pool <verbose>
 {
-       check_pool_status "$1" "scan" "resilvered "
+       check_pool_status "$1" "scan" "scrub in progress since " $2
        return $?
 }
 
-function is_pool_scrubbing #pool
+function is_pool_scrubbed #pool <verbose>
 {
-       check_pool_status "$1" "scan" "scrub in progress since "
+       check_pool_status "$1" "scan" "scrub repaired" $2
        return $?
 }
 
-function is_pool_scrubbed #pool
+function is_pool_scrub_stopped #pool <verbose>
 {
-       check_pool_status "$1" "scan" "scrub repaired"
+       check_pool_status "$1" "scan" "scrub canceled" $2
        return $?
 }
 
-function is_pool_scrub_stopped #pool
+function is_pool_scrub_paused #pool <verbose>
 {
-       check_pool_status "$1" "scan" "scrub canceled"
+       check_pool_status "$1" "scan" "scrub paused since " $2
        return $?
 }
 
+function is_pool_removing #pool
+{
+       check_pool_status "$1" "remove" "in progress since "
+       return $?
+}
+
+function is_pool_removed #pool
+{
+       check_pool_status "$1" "remove" "completed on"
+       return $?
+}
+
+function wait_for_degraded
+{
+       typeset pool=$1
+       typeset timeout=${2:-30}
+       typeset t0=$SECONDS
+
+       while :; do
+               [[ $(get_pool_prop health $pool) == "DEGRADED" ]] && break
+               log_note "$pool is not yet degraded."
+               sleep 1
+               if ((SECONDS - t0 > $timeout)); then
+                       log_note "$pool not degraded after $timeout seconds."
+                       return 1
+               fi
+       done
+
+       return 0
+}
+
 #
 # Use create_pool()/destroy_pool() to clean up the information in
 # in the given disk to avoid slice overlapping.
@@ -2211,7 +2375,7 @@ function del_user #<logname> <basedir>
        fi
 
        if id $user > /dev/null 2>&1; then
-               log_must userdel $user
+               log_must_retry "currently used" 5 userdel $user
        fi
 
        [[ -d $basedir/$user ]] && rm -fr $basedir/$user
@@ -2430,7 +2594,6 @@ function verify_opt_p_ops
                                        "when ops is $ops."
                        fi
                        log_must datasetexists $dataset
-                       log_mustnot snapexists $dataset
                        ;;
                *)
                        log_fail "$ops is not supported."
@@ -2438,9 +2601,7 @@ function verify_opt_p_ops
        esac
 
        # make sure the upper level filesystem does not exist
-       if datasetexists ${newdataset%/*} ; then
-               log_must zfs destroy -rRf ${newdataset%/*}
-       fi
+       destroy_dataset "${newdataset%/*}" "-rRf"
 
        # without -p option, operation will fail
        log_mustnot zfs $ops $dataset $newdataset
@@ -2536,7 +2697,7 @@ function random_get
 #
 function verify_slog_support
 {
-       typeset dir=/tmp/disk.$$
+       typeset dir=$TEST_BASE_DIR/disk.$$
        typeset pool=foo.$$
        typeset vdev=$dir/a
        typeset sdev=$dir/b
@@ -2762,217 +2923,6 @@ function get_rootpool
        fi
 }
 
-#
-# Check if the given device is physical device
-#
-function is_physical_device #device
-{
-       typeset device=${1#$DEV_DSKDIR}
-       device=${device#$DEV_RDSKDIR}
-
-       if is_linux; then
-               [[ -b "$DEV_DSKDIR/$device" ]] && \
-               [[ -f /sys/module/loop/parameters/max_part ]]
-               return $?
-       else
-               echo $device | egrep "^c[0-F]+([td][0-F]+)+$" > /dev/null 2>&1
-               return $?
-       fi
-}
-
-#
-# Check if the given device is a real device (ie SCSI device)
-#
-function is_real_device #disk
-{
-       typeset disk=$1
-       [[ -z $disk ]] && log_fail "No argument for disk given."
-
-       if is_linux; then
-               lsblk $DEV_RDSKDIR/$disk -o TYPE 2>/dev/null | \
-                   egrep disk >/dev/null
-               return $?
-       fi
-}
-
-#
-# Check if the given device is a loop device
-#
-function is_loop_device #disk
-{
-       typeset disk=$1
-       [[ -z $disk ]] && log_fail "No argument for disk given."
-
-       if is_linux; then
-               lsblk $DEV_RDSKDIR/$disk -o TYPE 2>/dev/null | \
-                   egrep loop >/dev/null
-               return $?
-       fi
-}
-
-#
-# Check if the given device is a multipath device and if there is a sybolic
-# link to a device mapper and to a disk
-# Currently no support for dm devices alone without multipath
-#
-function is_mpath_device #disk
-{
-       typeset disk=$1
-       [[ -z $disk ]] && log_fail "No argument for disk given."
-
-       if is_linux; then
-               lsblk $DEV_MPATHDIR/$disk -o TYPE 2>/dev/null | \
-                  egrep mpath >/dev/null
-               if (($? == 0)); then
-                       readlink $DEV_MPATHDIR/$disk > /dev/null 2>&1
-                       return $?
-               else
-                       return $?
-               fi
-       fi
-}
-
-# Set the slice prefix for disk partitioning depending
-# on whether the device is a real, multipath, or loop device.
-# Currently all disks have to be of the same type, so only
-# checks first disk to determine slice prefix.
-#
-function set_slice_prefix
-{
-       typeset disk
-       typeset -i i=0
-
-       if is_linux; then
-               while (( i < $DISK_ARRAY_NUM )); do
-                       disk="$(echo $DISKS | nawk '{print $(i + 1)}')"
-                       if ( is_mpath_device $disk ) && [[ -z $(echo $disk | awk 'substr($1,18,1)\
-                            ~ /^[[:digit:]]+$/') ]] || ( is_real_device $disk ); then
-                               export SLICE_PREFIX=""
-                               return 0
-                       elif ( is_mpath_device $disk || is_loop_device \
-                           $disk ); then
-                               export SLICE_PREFIX="p"
-                               return 0
-                       else
-                               log_fail "$disk not supported for partitioning."
-                       fi
-                       (( i = i + 1))
-               done
-       fi
-}
-
-#
-# Set the directory path of the listed devices in $DISK_ARRAY_NUM
-# Currently all disks have to be of the same type, so only
-# checks first disk to determine device directory
-# default = /dev (linux)
-# real disk = /dev (linux)
-# multipath device = /dev/mapper (linux)
-#
-function set_device_dir
-{
-       typeset disk
-       typeset -i i=0
-
-       if is_linux; then
-               while (( i < $DISK_ARRAY_NUM )); do
-                       disk="$(echo $DISKS | nawk '{print $(i + 1)}')"
-                       if is_mpath_device $disk; then
-                               export DEV_DSKDIR=$DEV_MPATHDIR
-                               return 0
-                       else
-                               export DEV_DSKDIR=$DEV_RDSKDIR
-                               return 0
-                       fi
-                       (( i = i + 1))
-               done
-       else
-               export DEV_DSKDIR=$DEV_RDSKDIR
-       fi
-}
-
-#
-# Get the directory path of given device
-#
-function get_device_dir #device
-{
-       typeset device=$1
-
-       if ! $(is_physical_device $device) ; then
-               if [[ $device != "/" ]]; then
-                       device=${device%/*}
-               fi
-               if [[ -b "$DEV_DSKDIR/$device" ]]; then
-                       device="$DEV_DSKDIR"
-               fi
-               echo $device
-       else
-               echo "$DEV_DSKDIR"
-       fi
-}
-
-#
-# Get persistent name for given disk
-#
-function get_persistent_disk_name #device
-{
-       typeset device=$1
-       typeset dev_id
-
-       if is_linux; then
-               if is_real_device $device; then
-                       dev_id="$(udevadm info -q all -n $DEV_DSKDIR/$device \
-                           | egrep disk/by-id | nawk '{print $2; exit}' \
-                           | nawk -F / '{print $3}')"
-                       echo $dev_id
-               elif is_mpath_device $device; then
-                       dev_id="$(udevadm info -q all -n $DEV_DSKDIR/$device \
-                           | egrep disk/by-id/dm-uuid \
-                           | nawk '{print $2; exit}' \
-                           | nawk -F / '{print $3}')"
-                       echo $dev_id
-               else
-                       echo $device
-               fi
-       else
-               echo $device
-       fi
-}
-
-#
-# Load scsi_debug module with specified parameters
-#
-function load_scsi_debug # dev_size_mb add_host num_tgts max_luns
-{
-       typeset devsize=$1
-       typeset hosts=$2
-       typeset tgts=$3
-       typeset luns=$4
-
-       [[ -z $devsize ]] || [[ -z $hosts ]] || [[ -z $tgts ]] || \
-           [[ -z $luns ]] && log_fail "Arguments invalid or missing"
-
-       if is_linux; then
-               modprobe -n scsi_debug
-               if (($? != 0)); then
-                       log_unsupported "Platform does not have scsi_debug"
-                           "module"
-               fi
-               lsmod | egrep scsi_debug > /dev/null
-               if (($? == 0)); then
-                       log_fail "scsi_debug module already installed"
-               else
-                       log_must modprobe scsi_debug dev_size_mb=$devsize \
-                           add_host=$hosts num_tgts=$tgts max_luns=$luns
-                       block_device_wait
-                       lsscsi | egrep scsi_debug > /dev/null
-                       if (($? == 1)); then
-                               log_fail "scsi_debug module install failed"
-                       fi
-               fi
-       fi
-}
-
 #
 # Get the package name
 #
@@ -3068,7 +3018,7 @@ function user_run
        shift
 
        log_note "user:$user $@"
-       eval su - \$user -c \"$@\" > /tmp/out 2>/tmp/err
+       eval su - \$user -c \"$@\" > $TEST_BASE_DIR/out 2>$TEST_BASE_DIR/err
        return $?
 }
 
@@ -3093,8 +3043,11 @@ function vdevs_in_pool
 
        shift
 
+       # We could use 'zpool list' to only get the vdevs of the pool but we
+       # can't reference a mirror/raidz vdev using its ID (i.e mirror-0),
+       # therefore we use the 'zpool status' output.
        typeset tmpfile=$(mktemp)
-       zpool list -Hv "$pool" >$tmpfile
+       zpool status -v "$pool" | grep -A 1000 "config:" >$tmpfile
        for vdev in $@; do
                grep -w ${vdev##*/} $tmpfile >/dev/null 2>&1
                [[ $? -ne 0 ]] && return 1
@@ -3245,9 +3198,43 @@ function wait_replacing #pool
        done
 }
 
+#
+# Wait for a pool to be scrubbed
+#
+# $1 pool name
+# $2 number of seconds to wait (optional)
+#
+# Returns true when pool has been scrubbed, or false if there's a timeout or if
+# no scrub was done.
+#
+function wait_scrubbed
+{
+       typeset pool=${1:-$TESTPOOL}
+       while true ; do
+               is_pool_scrubbed $pool && break
+               log_must sleep 1
+       done
+}
+
+# Backup the zed.rc in our test directory so that we can edit it for our test.
+#
+# Returns: Backup file name.  You will need to pass this to zed_rc_restore().
+function zed_rc_backup
+{
+       zedrc_backup="$(mktemp)"
+       cp $ZEDLET_DIR/zed.rc $zedrc_backup
+       echo $zedrc_backup
+}
+
+function zed_rc_restore
+{
+       mv $1 $ZEDLET_DIR/zed.rc
+}
+
 #
 # Setup custom environment for the ZED.
 #
+# $@ Optional list of zedlets to run under zed.
 function zed_setup
 {
        if ! is_linux; then
@@ -3265,6 +3252,7 @@ function zed_setup
        if [[ -e $VDEVID_CONF_ETC ]]; then
                log_fail "Must not have $VDEVID_CONF_ETC file present on system"
        fi
+       EXTRA_ZEDLETS=$@
 
        # Create a symlink for /etc/zfs/vdev_id.conf file.
        log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC
@@ -3274,32 +3262,46 @@ function zed_setup
        log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
        log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR
 
+       # Scripts must only be user writable.
+       if [[ -n "$EXTRA_ZEDLETS" ]] ; then
+               saved_umask=$(umask)
+               log_must umask 0022
+               for i in $EXTRA_ZEDLETS ; do
+                       log_must cp ${ZEDLET_LIBEXEC_DIR}/$i $ZEDLET_DIR
+               done
+               log_must umask $saved_umask
+       fi
+
        # Customize the zed.rc file to enable the full debug log.
        log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
-       echo "ZED_DEBUG_LOG=$ZEDLET_DIR/zed.debug.log" >>$ZEDLET_DIR/zed.rc
+       echo "ZED_DEBUG_LOG=$ZED_DEBUG_LOG" >>$ZEDLET_DIR/zed.rc
 
-       log_must cp ${ZEDLET_LIBEXEC_DIR}/all-syslog.sh $ZEDLET_DIR
-       log_must cp ${ZEDLET_LIBEXEC_DIR}/all-debug.sh $ZEDLET_DIR
-       log_must touch $ZEDLET_DIR/zed.debug.log
 }
 
 #
 # Cleanup custom ZED environment.
 #
+# $@ Optional list of zedlets to remove from our test zed.d directory.
 function zed_cleanup
 {
        if ! is_linux; then
                return
        fi
+       EXTRA_ZEDLETS=$@
 
        log_must rm -f ${ZEDLET_DIR}/zed.rc
        log_must rm -f ${ZEDLET_DIR}/zed-functions.sh
        log_must rm -f ${ZEDLET_DIR}/all-syslog.sh
        log_must rm -f ${ZEDLET_DIR}/all-debug.sh
-       log_must rm -f ${ZEDLET_DIR}/zed.pid
-       log_must rm -f ${ZEDLET_DIR}/zedlog
-       log_must rm -f ${ZEDLET_DIR}/zed.debug.log
        log_must rm -f ${ZEDLET_DIR}/state
+
+       if [[ -n "$EXTRA_ZEDLETS" ]] ; then
+               for i in $EXTRA_ZEDLETS ; do
+                       log_must rm -f ${ZEDLET_DIR}/$i
+               done
+       fi
+       log_must rm -f $ZED_LOG
+       log_must rm -f $ZED_DEBUG_LOG
        log_must rm -f $VDEVID_CONF_ETC
        log_must rm -f $VDEVID_CONF
        rmdir $ZEDLET_DIR
@@ -3327,9 +3329,10 @@ function zed_start
 
        log_note "Starting ZED"
        # run ZED in the background and redirect foreground logging
-       # output to zedlog
-       log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid" \
-           "-s $ZEDLET_DIR/state 2>${ZEDLET_DIR}/zedlog &"
+       # output to $ZED_LOG.
+       log_must truncate -s 0 $ZED_DEBUG_LOG
+       log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid -P $PATH" \
+           "-s $ZEDLET_DIR/state 2>$ZED_LOG &"
 
        return 0
 }
@@ -3345,13 +3348,44 @@ function zed_stop
 
        log_note "Stopping ZED"
        if [[ -f ${ZEDLET_DIR}/zed.pid ]]; then
-               zedpid=$(cat ${ZEDLET_DIR}/zed.pid)
-               log_must kill $zedpid
+               zedpid=$(<${ZEDLET_DIR}/zed.pid)
+               kill $zedpid
+               while ps -p $zedpid > /dev/null; do
+                       sleep 1
+               done
+               rm -f ${ZEDLET_DIR}/zed.pid
        fi
-
        return 0
 }
 
+#
+# Drain all zevents
+#
+function zed_events_drain
+{
+       while [ $(zpool events -H | wc -l) -ne 0 ]; do
+               sleep 1
+               zpool events -c >/dev/null
+       done
+}
+
+# Set a variable in zed.rc to something, un-commenting it in the process.
+#
+# $1 variable
+# $2 value
+function zed_rc_set
+{
+       var="$1"
+       val="$2"
+       # Remove the line
+       cmd="'/$var/d'"
+       eval sed -i $cmd $ZEDLET_DIR/zed.rc
+
+       # Add it at the end
+       echo "$var=$val" >> $ZEDLET_DIR/zed.rc
+}
+
+
 #
 # Check is provided device is being active used as a swap device.
 #
@@ -3381,7 +3415,7 @@ function swap_setup
        typeset swapdev=$1
 
        if is_linux; then
-               log_must mkswap $swapdev > /dev/null 2>&1
+               log_must eval "mkswap $swapdev > /dev/null 2>&1"
                log_must swapon $swapdev
        else
                log_must swap -a $swapdev
@@ -3407,3 +3441,144 @@ function swap_cleanup
 
        return 0
 }
+
+#
+# Set a global system tunable (64-bit value)
+#
+# $1 tunable name
+# $2 tunable values
+#
+function set_tunable64
+{
+       set_tunable_impl "$1" "$2" Z
+}
+
+#
+# Set a global system tunable (32-bit value)
+#
+# $1 tunable name
+# $2 tunable values
+#
+function set_tunable32
+{
+       set_tunable_impl "$1" "$2" W
+}
+
+function set_tunable_impl
+{
+       typeset tunable="$1"
+       typeset value="$2"
+       typeset mdb_cmd="$3"
+       typeset module="${4:-zfs}"
+
+       [[ -z "$tunable" ]] && return 1
+       [[ -z "$value" ]] && return 1
+       [[ -z "$mdb_cmd" ]] && return 1
+
+       case "$(uname)" in
+       Linux)
+               typeset zfs_tunables="/sys/module/$module/parameters"
+               [[ -w "$zfs_tunables/$tunable" ]] || return 1
+               echo -n "$value" > "$zfs_tunables/$tunable"
+               return "$?"
+               ;;
+       SunOS)
+               [[ "$module" -eq "zfs" ]] || return 1
+               echo "${tunable}/${mdb_cmd}0t${value}" | mdb -kw
+               return "$?"
+               ;;
+       esac
+}
+
+#
+# Get a global system tunable
+#
+# $1 tunable name
+#
+function get_tunable
+{
+       get_tunable_impl "$1"
+}
+
+function get_tunable_impl
+{
+       typeset tunable="$1"
+       typeset module="${2:-zfs}"
+
+       [[ -z "$tunable" ]] && return 1
+
+       case "$(uname)" in
+       Linux)
+               typeset zfs_tunables="/sys/module/$module/parameters"
+               [[ -f "$zfs_tunables/$tunable" ]] || return 1
+               cat $zfs_tunables/$tunable
+               return "$?"
+               ;;
+       SunOS)
+               [[ "$module" -eq "zfs" ]] || return 1
+               ;;
+       esac
+
+       return 1
+}
+
+#
+# Prints the current time in seconds since UNIX Epoch.
+#
+function current_epoch
+{
+       printf '%(%s)T'
+}
+
+#
+# Get decimal value of global uint32_t variable using mdb.
+#
+function mdb_get_uint32
+{
+       typeset variable=$1
+       typeset value
+
+       value=$(mdb -k -e "$variable/X | ::eval .=U")
+       if [[ $? -ne 0 ]]; then
+               log_fail "Failed to get value of '$variable' from mdb."
+               return 1
+       fi
+
+       echo $value
+       return 0
+}
+
+#
+# Set global uint32_t variable to a decimal value using mdb.
+#
+function mdb_set_uint32
+{
+       typeset variable=$1
+       typeset value=$2
+
+       mdb -kw -e "$variable/W 0t$value" > /dev/null
+       if [[ $? -ne 0 ]]; then
+               echo "Failed to set '$variable' to '$value' in mdb."
+               return 1
+       fi
+
+       return 0
+}
+
+#
+# Set global scalar integer variable to a hex value using mdb.
+# Note: Target should have CTF data loaded.
+#
+function mdb_ctf_set_int
+{
+       typeset variable=$1
+       typeset value=$2
+
+       mdb -kw -e "$variable/z $value" > /dev/null
+       if [[ $? -ne 0 ]]; then
+               echo "Failed to set '$variable' to '$value' in mdb."
+               return 1
+       fi
+
+       return 0
+}