etc/init.d: decide which variant to use at build time.

[mirror_zfs.git] / tests / zfs-tests / include / libtest.shlib

#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or https://opensource.org/licenses/CDDL-1.0.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2012, 2020, Delphix. All rights reserved.
# Copyright (c) 2017, Tim Chase. All rights reserved.
# Copyright (c) 2017, Nexenta Systems Inc. All rights reserved.
# Copyright (c) 2017, Lawrence Livermore National Security LLC.
# Copyright (c) 2017, Datto Inc. All rights reserved.
# Copyright (c) 2017, Open-E Inc. All rights reserved.
# Copyright (c) 2021, The FreeBSD Foundation.
# Use is subject to license terms.
#

. ${STF_SUITE}/include/tunables.cfg

. ${STF_TOOLS}/include/logapi.shlib
. ${STF_SUITE}/include/math.shlib
. ${STF_SUITE}/include/blkdev.shlib

# On AlmaLinux 9 we will see $PWD = '.' instead of the full path.  This causes
# some tests to fail.  Fix it up here.
if [ "$PWD" = "." ] ; then
        PWD="$(readlink -f $PWD)"
fi

#
# Apply constrained path when available.  This is required since the
# PATH may have been modified by sudo's secure_path behavior.
#
if [ -n "$STF_PATH" ]; then
        export PATH="$STF_PATH"
fi

#
# Generic dot version comparison function
#
# Returns success when version $1 is greater than or equal to $2.
#
function compare_version_gte
{
        [ "$(printf "$1\n$2" | sort -V | tail -n1)" = "$1" ]
}

# Helper function used by linux_version() and freebsd_version()
function kernel_version
{
        typeset ver="$1"

        [ -z "$ver" ] && ver=$(uname -r | grep -Eo "^[0-9]+\.[0-9]+\.[0-9]+")

        typeset version major minor _
        IFS='.' read -r version major minor _ <<<"$ver"

        [ -z "$version" ] && version=0
        [ -z "$major" ] && major=0
        [ -z "$minor" ] && minor=0

        echo $((version * 100000 + major * 1000 + minor))
}

# Linux kernel version comparison function
#
# $1 Linux version ("4.10", "2.6.32") or blank for installed Linux version
#
# Used for comparison: if [ $(linux_version) -ge $(linux_version "2.6.32") ]
function linux_version {
        kernel_version "$1"
}

# FreeBSD version comparison function
#
# $1 FreeBSD version ("13.2", "14.0") or blank for installed FreeBSD version
#
# Used for comparison: if [ $(freebsd_version) -ge $(freebsd_version "13.2") ]
function freebsd_version {
        kernel_version "$1"
}

# Determine if this is a Linux test system
#
# Return 0 if platform Linux, 1 if otherwise

function is_linux
{
        [ "$UNAME" = "Linux" ]
}

# Determine if this is an illumos test system
#
# Return 0 if platform illumos, 1 if otherwise
function is_illumos
{
        [ "$UNAME" = "illumos" ]
}

# Determine if this is a FreeBSD test system
#
# Return 0 if platform FreeBSD, 1 if otherwise

function is_freebsd
{
        [ "$UNAME" = "FreeBSD" ]
}

# Determine if this is a 32-bit system
#
# Return 0 if platform is 32-bit, 1 if otherwise

function is_32bit
{
        [ $(getconf LONG_BIT) = "32" ]
}

# Determine if kmemleak is enabled
#
# Return 0 if kmemleak is enabled, 1 if otherwise

function is_kmemleak
{
        is_linux && [ -e /sys/kernel/debug/kmemleak ]
}

# Determine whether a dataset is mounted
#
# $1 dataset name
# $2 filesystem type; optional - defaulted to zfs
#
# Return 0 if dataset is mounted; 1 if unmounted; 2 on error

function ismounted
{
        typeset fstype=$2
        [[ -z $fstype ]] && fstype=zfs
        typeset out dir name

        case $fstype in
                zfs)
                        if [[ "$1" == "/"* ]] ; then
                                ! zfs mount | awk -v fs="$1" '$2 == fs {exit 1}'
                        else
                                ! zfs mount | awk -v ds="$1" '$1 == ds {exit 1}'
                        fi
                ;;
                ufs|nfs)
                        if is_freebsd; then
                                mount -pt $fstype | while read dev dir _t _flags; do
                                        [[ "$1" == "$dev" || "$1" == "$dir" ]] && return 0
                                done
                        else
                                out=$(df -F $fstype $1 2>/dev/null) || return

                                dir=${out%%\(*}
                                dir=${dir%% *}
                                name=${out##*\(}
                                name=${name%%\)*}
                                name=${name%% *}

                                [[ "$1" == "$dir" || "$1" == "$name" ]] && return 0
                        fi
                ;;
                ext*)
                        df -t $fstype $1 > /dev/null 2>&1
                ;;
                zvol)
                        if [[ -L "$ZVOL_DEVDIR/$1" ]]; then
                                link=$(readlink -f $ZVOL_DEVDIR/$1)
                                [[ -n "$link" ]] && \
                                        mount | grep -q "^$link" && \
                                                return 0
                        fi
                ;;
                *)
                        false
                ;;
        esac
}

# Return 0 if a dataset is mounted; 1 otherwise
#
# $1 dataset name
# $2 filesystem type; optional - defaulted to zfs

function mounted
{
        ismounted $1 $2
}

# Return 0 if a dataset is unmounted; 1 otherwise
#
# $1 dataset name
# $2 filesystem type; optional - defaulted to zfs

function unmounted
{
        ! ismounted $1 $2
}

function default_setup
{
        default_setup_noexit "$@"

        log_pass
}

function default_setup_no_mountpoint
{
        default_setup_noexit "$1" "$2" "$3" "yes"

        log_pass
}

#
# Given a list of disks, setup storage pools and datasets.
#
function default_setup_noexit
{
        typeset disklist=$1
        typeset container=$2
        typeset volume=$3
        typeset no_mountpoint=$4
        log_note begin default_setup_noexit

        if is_global_zone; then
                if poolexists $TESTPOOL ; then
                        destroy_pool $TESTPOOL
                fi
                [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL
                log_must zpool create -f $TESTPOOL $disklist
        else
                reexport_pool
        fi

        rm -rf $TESTDIR  || log_unresolved Could not remove $TESTDIR
        mkdir -p $TESTDIR || log_unresolved Could not create $TESTDIR

        log_must zfs create $TESTPOOL/$TESTFS
        if [[ -z $no_mountpoint ]]; then
                log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
        fi

        if [[ -n $container ]]; then
                rm -rf $TESTDIR1  || \
                        log_unresolved Could not remove $TESTDIR1
                mkdir -p $TESTDIR1 || \
                        log_unresolved Could not create $TESTDIR1

                log_must zfs create $TESTPOOL/$TESTCTR
                log_must zfs set canmount=off $TESTPOOL/$TESTCTR
                log_must zfs create $TESTPOOL/$TESTCTR/$TESTFS1
                if [[ -z $no_mountpoint ]]; then
                        log_must zfs set mountpoint=$TESTDIR1 \
                            $TESTPOOL/$TESTCTR/$TESTFS1
                fi
        fi

        if [[ -n $volume ]]; then
                if is_global_zone ; then
                        log_must zfs create -V $VOLSIZE $TESTPOOL/$TESTVOL
                        block_device_wait
                else
                        log_must zfs create $TESTPOOL/$TESTVOL
                fi
        fi
}

#
# Given a list of disks, setup a storage pool, file system and
# a container.
#
function default_container_setup
{
        typeset disklist=$1

        default_setup "$disklist" "true"
}

#
# Given a list of disks, setup a storage pool,file system
# and a volume.
#
function default_volume_setup
{
        typeset disklist=$1

        default_setup "$disklist" "" "true"
}

#
# Given a list of disks, setup a storage pool,file system,
# a container and a volume.
#
function default_container_volume_setup
{
        typeset disklist=$1

        default_setup "$disklist" "true" "true"
}

#
# Create a snapshot on a filesystem or volume. Defaultly create a snapshot on
# filesystem
#
# $1 Existing filesystem or volume name. Default, $TESTPOOL/$TESTFS
# $2 snapshot name. Default, $TESTSNAP
#
function create_snapshot
{
        typeset fs_vol=${1:-$TESTPOOL/$TESTFS}
        typeset snap=${2:-$TESTSNAP}

        [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined."
        [[ -z $snap ]] && log_fail "Snapshot's name is undefined."

        if snapexists $fs_vol@$snap; then
                log_fail "$fs_vol@$snap already exists."
        fi
        datasetexists $fs_vol || \
                log_fail "$fs_vol must exist."

        log_must zfs snapshot $fs_vol@$snap
}

#
# Create a clone from a snapshot, default clone name is $TESTCLONE.
#
# $1 Existing snapshot, $TESTPOOL/$TESTFS@$TESTSNAP is default.
# $2 Clone name, $TESTPOOL/$TESTCLONE is default.
#
function create_clone   # snapshot clone
{
        typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}
        typeset clone=${2:-$TESTPOOL/$TESTCLONE}

        [[ -z $snap ]] && \
                log_fail "Snapshot name is undefined."
        [[ -z $clone ]] && \
                log_fail "Clone name is undefined."

        log_must zfs clone $snap $clone
}

#
# Create a bookmark of the given snapshot.  Defaultly create a bookmark on
# filesystem.
#
# $1 Existing filesystem or volume name. Default, $TESTFS
# $2 Existing snapshot name. Default, $TESTSNAP
# $3 bookmark name. Default, $TESTBKMARK
#
function create_bookmark
{
        typeset fs_vol=${1:-$TESTFS}
        typeset snap=${2:-$TESTSNAP}
        typeset bkmark=${3:-$TESTBKMARK}

        [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined."
        [[ -z $snap ]] && log_fail "Snapshot's name is undefined."
        [[ -z $bkmark ]] && log_fail "Bookmark's name is undefined."

        if bkmarkexists $fs_vol#$bkmark; then
                log_fail "$fs_vol#$bkmark already exists."
        fi
        datasetexists $fs_vol || \
                log_fail "$fs_vol must exist."
        snapexists $fs_vol@$snap || \
                log_fail "$fs_vol@$snap must exist."

        log_must zfs bookmark $fs_vol@$snap $fs_vol#$bkmark
}

#
# Create a temporary clone result of an interrupted resumable 'zfs receive'
# $1 Destination filesystem name. Must not exist, will be created as the result
#    of this function along with its %recv temporary clone
# $2 Source filesystem name. Must not exist, will be created and destroyed
#
function create_recv_clone
{
        typeset recvfs="$1"
        typeset sendfs="${2:-$TESTPOOL/create_recv_clone}"
        typeset snap="$sendfs@snap1"
        typeset incr="$sendfs@snap2"
        typeset mountpoint="$TESTDIR/create_recv_clone"
        typeset sendfile="$TESTDIR/create_recv_clone.zsnap"

        [[ -z $recvfs ]] && log_fail "Recv filesystem's name is undefined."

        datasetexists $recvfs && log_fail "Recv filesystem must not exist."
        datasetexists $sendfs && log_fail "Send filesystem must not exist."

        log_must zfs create -o compression=off -o mountpoint="$mountpoint" $sendfs
        log_must zfs snapshot $snap
        log_must eval "zfs send $snap | zfs recv -u $recvfs"
        log_must mkfile 1m "$mountpoint/data"
        log_must zfs snapshot $incr
        log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 \
            iflag=fullblock > $sendfile"
        log_mustnot eval "zfs recv -su $recvfs < $sendfile"
        destroy_dataset "$sendfs" "-r"
        log_must rm -f "$sendfile"

        if [[ $(get_prop 'inconsistent' "$recvfs/%recv") -ne 1 ]]; then
                log_fail "Error creating temporary $recvfs/%recv clone"
        fi
}

function default_mirror_setup
{
        default_mirror_setup_noexit $1 $2 $3

        log_pass
}

#
# Given a pair of disks, set up a storage pool and dataset for the mirror
# @parameters: $1 the primary side of the mirror
#   $2 the secondary side of the mirror
# @uses: ZPOOL ZFS TESTPOOL TESTFS
function default_mirror_setup_noexit
{
        readonly func="default_mirror_setup_noexit"
        typeset primary=$1
        typeset secondary=$2

        [[ -z $primary ]] && \
                log_fail "$func: No parameters passed"
        [[ -z $secondary ]] && \
                log_fail "$func: No secondary partition passed"
        [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL
        log_must zpool create -f $TESTPOOL mirror $@
        log_must zfs create $TESTPOOL/$TESTFS
        log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
}

#
# Destroy the configured testpool mirrors.
# the mirrors are of the form ${TESTPOOL}{number}
# @uses: ZPOOL ZFS TESTPOOL
function destroy_mirrors
{
        default_cleanup_noexit

        log_pass
}

function default_raidz_setup
{
        default_raidz_setup_noexit "$*"

        log_pass
}

#
# Given a minimum of two disks, set up a storage pool and dataset for the raid-z
# $1 the list of disks
#
function default_raidz_setup_noexit
{
        typeset disklist="$*"
        disks=(${disklist[*]})

        if [[ ${#disks[*]} -lt 2 ]]; then
                log_fail "A raid-z requires a minimum of two disks."
        fi

        [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL
        log_must zpool create -f $TESTPOOL raidz $disklist
        log_must zfs create $TESTPOOL/$TESTFS
        log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
}

#
# Common function used to cleanup storage pools and datasets.
#
# Invoked at the start of the test suite to ensure the system
# is in a known state, and also at the end of each set of
# sub-tests to ensure errors from one set of tests doesn't
# impact the execution of the next set.

function default_cleanup
{
        default_cleanup_noexit

        log_pass
}

#
# Utility function used to list all available pool names.
#
# NOTE: $KEEP is a variable containing pool names, separated by a newline
# character, that must be excluded from the returned list.
#
function get_all_pools
{
        zpool list -H -o name | grep -Fvx "$KEEP" | grep -v "$NO_POOLS"
}

function default_cleanup_noexit
{
        typeset pool=""
        #
        # Destroying the pool will also destroy any
        # filesystems it contains.
        #
        if is_global_zone; then
                zfs unmount -a > /dev/null 2>&1
                ALL_POOLS=$(get_all_pools)
                # Here, we loop through the pools we're allowed to
                # destroy, only destroying them if it's safe to do
                # so.
                while [ ! -z ${ALL_POOLS} ]
                do
                        for pool in ${ALL_POOLS}
                        do
                                if safe_to_destroy_pool $pool ;
                                then
                                        destroy_pool $pool
                                fi
                        done
                        ALL_POOLS=$(get_all_pools)
                done

                zfs mount -a
        else
                typeset fs=""
                for fs in $(zfs list -H -o name \
                    | grep "^$ZONE_POOL/$ZONE_CTR[01234]/"); do
                        destroy_dataset "$fs" "-Rf"
                done

                # Need cleanup here to avoid garbage dir left.
                for fs in $(zfs list -H -o name); do
                        [[ $fs == /$ZONE_POOL ]] && continue
                        [[ -d $fs ]] && log_must rm -rf $fs/*
                done

                #
                # Reset the $ZONE_POOL/$ZONE_CTR[01234] file systems property to
                # the default value
                #
                for fs in $(zfs list -H -o name); do
                        if [[ $fs == $ZONE_POOL/$ZONE_CTR[01234] ]]; then
                                log_must zfs set reservation=none $fs
                                log_must zfs set recordsize=128K $fs
                                log_must zfs set mountpoint=/$fs $fs
                                typeset enc=$(get_prop encryption $fs)
                                if [ -z "$enc" ] || [ "$enc" = "off" ]; then
                                        log_must zfs set checksum=on $fs
                                fi
                                log_must zfs set compression=off $fs
                                log_must zfs set atime=on $fs
                                log_must zfs set devices=off $fs
                                log_must zfs set exec=on $fs
                                log_must zfs set setuid=on $fs
                                log_must zfs set readonly=off $fs
                                log_must zfs set snapdir=hidden $fs
                                log_must zfs set aclmode=groupmask $fs
                                log_must zfs set aclinherit=secure $fs
                        fi
                done
        fi

        [[ -d $TESTDIR ]] && \
                log_must rm -rf $TESTDIR

        disk1=${DISKS%% *}
        if is_mpath_device $disk1; then
                delete_partitions
        fi

        rm -f $TEST_BASE_DIR/{err,out}
}


#
# Common function used to cleanup storage pools, file systems
# and containers.
#
function default_container_cleanup
{
        if ! is_global_zone; then
                reexport_pool
        fi

        ismounted $TESTPOOL/$TESTCTR/$TESTFS1 &&
            log_must zfs unmount $TESTPOOL/$TESTCTR/$TESTFS1

        destroy_dataset "$TESTPOOL/$TESTCTR/$TESTFS1" "-R"
        destroy_dataset "$TESTPOOL/$TESTCTR" "-Rf"

        [[ -e $TESTDIR1 ]] && \
            log_must rm -rf $TESTDIR1

        default_cleanup
}

#
# Common function used to cleanup snapshot of file system or volume. Default to
# delete the file system's snapshot
#
# $1 snapshot name
#
function destroy_snapshot
{
        typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}

        if ! snapexists $snap; then
                log_fail "'$snap' does not exist."
        fi

        #
        # For the sake of the value which come from 'get_prop' is not equal
        # to the really mountpoint when the snapshot is unmounted. So, firstly
        # check and make sure this snapshot's been mounted in current system.
        #
        typeset mtpt=""
        if ismounted $snap; then
                mtpt=$(get_prop mountpoint $snap)
        fi

        destroy_dataset "$snap"
        [[ $mtpt != "" && -d $mtpt ]] && \
                log_must rm -rf $mtpt
}

#
# Common function used to cleanup clone.
#
# $1 clone name
#
function destroy_clone
{
        typeset clone=${1:-$TESTPOOL/$TESTCLONE}

        if ! datasetexists $clone; then
                log_fail "'$clone' does not existed."
        fi

        # With the same reason in destroy_snapshot
        typeset mtpt=""
        if ismounted $clone; then
                mtpt=$(get_prop mountpoint $clone)
        fi

        destroy_dataset "$clone"
        [[ $mtpt != "" && -d $mtpt ]] && \
                log_must rm -rf $mtpt
}

#
# Common function used to cleanup bookmark of file system or volume.  Default
# to delete the file system's bookmark.
#
# $1 bookmark name
#
function destroy_bookmark
{
        typeset bkmark=${1:-$TESTPOOL/$TESTFS#$TESTBKMARK}

        if ! bkmarkexists $bkmark; then
                log_fail "'$bkmarkp' does not existed."
        fi

        destroy_dataset "$bkmark"
}

# Return 0 if a snapshot exists; $? otherwise
#
# $1 - snapshot name

function snapexists
{
        zfs list -H -t snapshot "$1" > /dev/null 2>&1
}

#
# Return 0 if a bookmark exists; $? otherwise
#
# $1 - bookmark name
#
function bkmarkexists
{
        zfs list -H -t bookmark "$1" > /dev/null 2>&1
}

#
# Return 0 if a hold exists; $? otherwise
#
# $1 - hold tag
# $2 - snapshot name
#
function holdexists
{
        ! zfs holds "$2" | awk -v t="$1" '$2 ~ t { exit 1 }'
}

#
# Set a property to a certain value on a dataset.
# Sets a property of the dataset to the value as passed in.
# @param:
#       $1 dataset who's property is being set
#       $2 property to set
#       $3 value to set property to
# @return:
#       0 if the property could be set.
#       non-zero otherwise.
# @use: ZFS
#
function dataset_setprop
{
        typeset fn=dataset_setprop

        if (($# < 3)); then
                log_note "$fn: Insufficient parameters (need 3, had $#)"
                return 1
        fi
        typeset output=
        output=$(zfs set $2=$3 $1 2>&1)
        typeset rv=$?
        if ((rv != 0)); then
                log_note "Setting property on $1 failed."
                log_note "property $2=$3"
                log_note "Return Code: $rv"
                log_note "Output: $output"
                return $rv
        fi
        return 0
}

#
# Check a numeric assertion
# @parameter: $@ the assertion to check
# @output: big loud notice if assertion failed
# @use: log_fail
#
function assert
{
        (($@)) || log_fail "$@"
}

#
# Function to format partition size of a disk
# Given a disk cxtxdx reduces all partitions
# to 0 size
#
function zero_partitions #<whole_disk_name>
{
        typeset diskname=$1
        typeset i

        if is_freebsd; then
                gpart destroy -F $diskname
        elif is_linux; then
                DSK=$DEV_DSKDIR/$diskname
                DSK=$(echo $DSK | sed -e "s|//|/|g")
                log_must parted $DSK -s -- mklabel gpt
                blockdev --rereadpt $DSK 2>/dev/null
                block_device_wait
        else
                for i in 0 1 3 4 5 6 7
                do
                        log_must set_partition $i "" 0mb $diskname
                done
        fi

        return 0
}

#
# Given a slice, size and disk, this function
# formats the slice to the specified size.
# Size should be specified with units as per
# the `format` command requirements eg. 100mb 3gb
#
# NOTE: This entire interface is problematic for the Linux parted utility
# which requires the end of the partition to be specified.  It would be
# best to retire this interface and replace it with something more flexible.
# At the moment a best effort is made.
#
# arguments: <slice_num> <slice_start> <size_plus_units>  <whole_disk_name>
function set_partition
{
        typeset -i slicenum=$1
        typeset start=$2
        typeset size=$3
        typeset disk=${4#$DEV_DSKDIR/}
        disk=${disk#$DEV_RDSKDIR/}

        case "$UNAME" in
        Linux)
                if [[ -z $size || -z $disk ]]; then
                        log_fail "The size or disk name is unspecified."
                fi
                disk=$DEV_DSKDIR/$disk
                typeset size_mb=${size%%[mMgG]}

                size_mb=${size_mb%%[mMgG][bB]}
                if [[ ${size:1:1} == 'g' ]]; then
                        ((size_mb = size_mb * 1024))
                fi

                # Create GPT partition table when setting slice 0 or
                # when the device doesn't already contain a GPT label.
                parted $disk -s -- print 1 >/dev/null
                typeset ret_val=$?
                if [[ $slicenum -eq 0 || $ret_val -ne 0 ]]; then
                        if ! parted $disk -s -- mklabel gpt; then
                                log_note "Failed to create GPT partition table on $disk"
                                return 1
                        fi
                fi

                # When no start is given align on the first cylinder.
                if [[ -z "$start" ]]; then
                        start=1
                fi

                # Determine the cylinder size for the device and using
                # that calculate the end offset in cylinders.
                typeset -i cly_size_kb=0
                cly_size_kb=$(parted -m $disk -s -- unit cyl print |
                        awk -F '[:k.]' 'NR == 3 {print $4}')
                ((end = (size_mb * 1024 / cly_size_kb) + start))

                parted $disk -s -- \
                    mkpart part$slicenum ${start}cyl ${end}cyl
                typeset ret_val=$?
                if [[ $ret_val -ne 0 ]]; then
                        log_note "Failed to create partition $slicenum on $disk"
                        return 1
                fi

                blockdev --rereadpt $disk 2>/dev/null
                block_device_wait $disk
                ;;
        FreeBSD)
                if [[ -z $size || -z $disk ]]; then
                        log_fail "The size or disk name is unspecified."
                fi
                disk=$DEV_DSKDIR/$disk

                if [[ $slicenum -eq 0 ]] || ! gpart show $disk >/dev/null 2>&1; then
                        gpart destroy -F $disk >/dev/null 2>&1
                        if ! gpart create -s GPT $disk; then
                                log_note "Failed to create GPT partition table on $disk"
                                return 1
                        fi
                fi

                typeset index=$((slicenum + 1))

                if [[ -n $start ]]; then
                        start="-b $start"
                fi
                gpart add -t freebsd-zfs $start -s $size -i $index $disk
                if [[ $ret_val -ne 0 ]]; then
                        log_note "Failed to create partition $slicenum on $disk"
                        return 1
                fi

                block_device_wait $disk
                ;;
        *)
                if [[ -z $slicenum || -z $size || -z $disk ]]; then
                        log_fail "The slice, size or disk name is unspecified."
                fi

                typeset format_file=/var/tmp/format_in.$$

                echo "partition" >$format_file
                echo "$slicenum" >> $format_file
                echo "" >> $format_file
                echo "" >> $format_file
                echo "$start" >> $format_file
                echo "$size" >> $format_file
                echo "label" >> $format_file
                echo "" >> $format_file
                echo "q" >> $format_file
                echo "q" >> $format_file

                format -e -s -d $disk -f $format_file
                typeset ret_val=$?
                rm -f $format_file
                ;;
        esac

        if [[ $ret_val -ne 0 ]]; then
                log_note "Unable to format $disk slice $slicenum to $size"
                return 1
        fi
        return 0
}

#
# Delete all partitions on all disks - this is specifically for the use of multipath
# devices which currently can only be used in the test suite as raw/un-partitioned
# devices (ie a zpool cannot be created on a whole mpath device that has partitions)
#
function delete_partitions
{
        typeset disk

        if [[ -z $DISKSARRAY ]]; then
                DISKSARRAY=$DISKS
        fi

        if is_linux; then
                typeset -i part
                for disk in $DISKSARRAY; do
                        for (( part = 1; part < MAX_PARTITIONS; part++ )); do
                                typeset partition=${disk}${SLICE_PREFIX}${part}
                                parted $DEV_DSKDIR/$disk -s rm $part > /dev/null 2>&1
                                if lsblk | grep -qF ${partition}; then
                                        log_fail "Partition ${partition} not deleted"
                                else
                                        log_note "Partition ${partition} deleted"
                                fi
                        done
                done
        elif is_freebsd; then
                for disk in $DISKSARRAY; do
                        if gpart destroy -F $disk; then
                                log_note "Partitions for ${disk} deleted"
                        else
                                log_fail "Partitions for ${disk} not deleted"
                        fi
                done
        fi
}

#
# Get the end cyl of the given slice
#
function get_endslice #<disk> <slice>
{
        typeset disk=$1
        typeset slice=$2
        if [[ -z $disk || -z $slice ]] ; then
                log_fail "The disk name or slice number is unspecified."
        fi

        case "$UNAME" in
        Linux)
                endcyl=$(parted -s $DEV_DSKDIR/$disk -- unit cyl print | \
                        awk "/part${slice}/"' {sub(/cyl/, "", $3); print $3}')
                ((endcyl = (endcyl + 1)))
                ;;
        FreeBSD)
                disk=${disk#/dev/zvol/}
                disk=${disk%p*}
                slice=$((slice + 1))
                endcyl=$(gpart show $disk | \
                        awk -v slice=$slice '$3 == slice { print $1 + $2 }')
                ;;
        *)
                disk=${disk#/dev/dsk/}
                disk=${disk#/dev/rdsk/}
                disk=${disk%s*}

                typeset -i ratio=0
                ratio=$(prtvtoc /dev/rdsk/${disk}s2 | \
                    awk '/sectors\/cylinder/ {print $2}')

                if ((ratio == 0)); then
                        return
                fi

                typeset -i endcyl=$(prtvtoc -h /dev/rdsk/${disk}s2 |
                    awk -v token="$slice" '$1 == token {print $6}')

                ((endcyl = (endcyl + 1) / ratio))
                ;;
        esac

        echo $endcyl
}


#
# Given a size,disk and total slice number,  this function formats the
# disk slices from 0 to the total slice number with the same specified
# size.
#
function partition_disk #<slice_size> <whole_disk_name> <total_slices>
{
        typeset -i i=0
        typeset slice_size=$1
        typeset disk_name=$2
        typeset total_slices=$3
        typeset cyl

        zero_partitions $disk_name
        while ((i < $total_slices)); do
                if ! is_linux; then
                        if ((i == 2)); then
                                ((i = i + 1))
                                continue
                        fi
                fi
                log_must set_partition $i "$cyl" $slice_size $disk_name
                cyl=$(get_endslice $disk_name $i)
                ((i = i+1))
        done
}

#
# This function continues to write to a filenum number of files into dirnum
# number of directories until either file_write returns an error or the
# maximum number of files per directory have been written.
#
# Usage:
# fill_fs [destdir] [dirnum] [filenum] [bytes] [num_writes] [data]
#
# Return value: 0 on success
#               non 0 on error
#
# Where :
#       destdir:    is the directory where everything is to be created under
#       dirnum:     the maximum number of subdirectories to use, -1 no limit
#       filenum:    the maximum number of files per subdirectory
#       bytes:      number of bytes to write
#       num_writes: number of types to write out bytes
#       data:       the data that will be written
#
#       E.g.
#       fill_fs /testdir 20 25 1024 256 0
#
# Note: bytes * num_writes equals the size of the testfile
#
function fill_fs # destdir dirnum filenum bytes num_writes data
{
        typeset destdir=${1:-$TESTDIR}
        typeset -i dirnum=${2:-50}
        typeset -i filenum=${3:-50}
        typeset -i bytes=${4:-8192}
        typeset -i num_writes=${5:-10240}
        typeset data=${6:-0}

        mkdir -p $destdir/{1..$dirnum}
        for f in $destdir/{1..$dirnum}/$TESTFILE{1..$filenum}; do
                file_write -o create -f $f -b $bytes -c $num_writes -d $data \
                || return
        done
}

# Get the specified dataset property in parsable format or fail
function get_prop # property dataset
{
        typeset prop=$1
        typeset dataset=$2

        zfs get -Hpo value "$prop" "$dataset" || log_fail "zfs get $prop $dataset"
}

# Get the specified pool property in parsable format or fail
function get_pool_prop # property pool
{
        typeset prop=$1
        typeset pool=$2

        zpool get -Hpo value "$prop" "$pool" || log_fail "zpool get $prop $pool"
}

# Return 0 if a pool exists; $? otherwise
#
# $1 - pool name

function poolexists
{
        typeset pool=$1

        if [[ -z $pool ]]; then
                log_note "No pool name given."
                return 1
        fi

        zpool get name "$pool" > /dev/null 2>&1
}

# Return 0 if all the specified datasets exist; $? otherwise
#
# $1-n  dataset name
function datasetexists
{
        if (($# == 0)); then
                log_note "No dataset name given."
                return 1
        fi

        zfs get name "$@" > /dev/null 2>&1
}

# return 0 if none of the specified datasets exists, otherwise return 1.
#
# $1-n  dataset name
function datasetnonexists
{
        if (($# == 0)); then
                log_note "No dataset name given."
                return 1
        fi

        while (($# > 0)); do
                zfs list -H -t filesystem,snapshot,volume $1 > /dev/null 2>&1 \
                    && return 1
                shift
        done

        return 0
}

# FreeBSD breaks exports(5) at whitespace and doesn't process escapes
# Solaris just breaks
#
# cf. https://github.com/openzfs/zfs/pull/13165#issuecomment-1059845807
#
# Linux can have spaces (which are \OOO-escaped),
# but can't have backslashes because they're parsed recursively
function shares_can_have_whitespace
{
        is_linux
}

function is_shared_freebsd
{
        typeset fs=$1

        pgrep -q mountd && showmount -E | grep -qx "$fs"
}

function is_shared_illumos
{
        typeset fs=$1
        typeset mtpt

        for mtpt in `share | awk '{print $2}'` ; do
                if [[ $mtpt == $fs ]] ; then
                        return 0
                fi
        done

        typeset stat=$(svcs -H -o STA nfs/server:default)
        if [[ $stat != "ON" ]]; then
                log_note "Current nfs/server status: $stat"
        fi

        return 1
}

function is_shared_linux
{
        typeset fs=$1
        ! exportfs -s | awk -v fs="${fs//\\/\\\\}" '/^\// && $1 == fs {exit 1}'
}

#
# Given a mountpoint, or a dataset name, determine if it is shared via NFS.
#
# Returns 0 if shared, 1 otherwise.
#
function is_shared
{
        typeset fs=$1
        typeset mtpt

        if [[ $fs != "/"* ]] ; then
                if datasetnonexists "$fs" ; then
                        return 1
                else
                        mtpt=$(get_prop mountpoint "$fs")
                        case "$mtpt" in
                                none|legacy|-) return 1
                                        ;;
                                *)      fs=$mtpt
                                        ;;
                        esac
                fi
        fi

        case "$UNAME" in
        FreeBSD)        is_shared_freebsd "$fs" ;;
        Linux)          is_shared_linux "$fs"   ;;
        *)              is_shared_illumos "$fs" ;;
        esac
}

function is_exported_illumos
{
        typeset fs=$1
        typeset mtpt _

        while read -r mtpt _; do
                [ "$mtpt" = "$fs" ] && return
        done < /etc/dfs/sharetab

        return 1
}

function is_exported_freebsd
{
        typeset fs=$1
        typeset mtpt _

        while read -r mtpt _; do
                [ "$mtpt" = "$fs" ] && return
        done < /etc/zfs/exports

        return 1
}

function is_exported_linux
{
        typeset fs=$1
        typeset mtpt _

        while read -r mtpt _; do
                [ "$(printf "$mtpt")" = "$fs" ] && return
        done < /etc/exports.d/zfs.exports

        return 1
}

#
# Given a mountpoint, or a dataset name, determine if it is exported via
# the os-specific NFS exports file.
#
# Returns 0 if exported, 1 otherwise.
#
function is_exported
{
        typeset fs=$1
        typeset mtpt

        if [[ $fs != "/"* ]] ; then
                if datasetnonexists "$fs" ; then
                        return 1
                else
                        mtpt=$(get_prop mountpoint "$fs")
                        case $mtpt in
                                none|legacy|-) return 1
                                        ;;
                                *)      fs=$mtpt
                                        ;;
                        esac
                fi
        fi

        case "$UNAME" in
        FreeBSD)        is_exported_freebsd "$fs"       ;;
        Linux)          is_exported_linux "$fs" ;;
        *)              is_exported_illumos "$fs"       ;;
        esac
}

#
# Given a dataset name determine if it is shared via SMB.
#
# Returns 0 if shared, 1 otherwise.
#
function is_shared_smb
{
        typeset fs=$1

        datasetexists "$fs" || return

        if is_linux; then
                net usershare list | grep -xFq "${fs//[-\/]/_}"
        else
                log_note "SMB on $UNAME currently unsupported by the test framework"
                return 1
        fi
}

#
# Given a mountpoint, determine if it is not shared via NFS.
#
# Returns 0 if not shared, 1 otherwise.
#
function not_shared
{
        ! is_shared $1
}

#
# Given a dataset determine if it is not shared via SMB.
#
# Returns 0 if not shared, 1 otherwise.
#
function not_shared_smb
{
        ! is_shared_smb $1
}

#
# Helper function to unshare a mountpoint.
#
function unshare_fs #fs
{
        typeset fs=$1

        if is_shared $fs || is_shared_smb $fs; then
                log_must zfs unshare $fs
        fi
}

#
# Helper function to share a NFS mountpoint.
#
function share_nfs #fs
{
        typeset fs=$1

        is_shared "$fs" && return

        case "$UNAME" in
        Linux)
                log_must exportfs "*:$fs"
                ;;
        FreeBSD)
                typeset mountd
                read -r mountd < /var/run/mountd.pid
                log_must eval "printf '%s\t\n' \"$fs\" >> /etc/zfs/exports"
                log_must kill -s HUP "$mountd"
                ;;
        *)
                log_must share -F nfs "$fs"
                ;;
        esac

        return 0
}

#
# Helper function to unshare a NFS mountpoint.
#
function unshare_nfs #fs
{
        typeset fs=$1

        ! is_shared "$fs" && return

        case "$UNAME" in
        Linux)
                log_must exportfs -u "*:$fs"
                ;;
        FreeBSD)
                typeset mountd
                read -r mountd < /var/run/mountd.pid
                awk -v fs="${fs//\\/\\\\}" '$1 != fs' /etc/zfs/exports > /etc/zfs/exports.$$
                log_must mv /etc/zfs/exports.$$ /etc/zfs/exports
                log_must kill -s HUP "$mountd"
                ;;
        *)
                log_must unshare -F nfs $fs
                ;;
        esac

        return 0
}

#
# Helper function to show NFS shares.
#
function showshares_nfs
{
        case "$UNAME" in
        Linux)
                exportfs -v
                ;;
        FreeBSD)
                showmount
                ;;
        *)
                share -F nfs
                ;;
        esac
}

function check_nfs
{
        case "$UNAME" in
        Linux)
                exportfs -s
                ;;
        FreeBSD)
                showmount -e
                ;;
        *)
                log_unsupported "Unknown platform"
                ;;
        esac || log_unsupported "The NFS utilities are not installed"
}

#
# Check NFS server status and trigger it online.
#
function setup_nfs_server
{
        # Cannot share directory in non-global zone.
        #
        if ! is_global_zone; then
                log_note "Cannot trigger NFS server by sharing in LZ."
                return
        fi

        if is_linux; then
                #
                # Re-synchronize /var/lib/nfs/etab with /etc/exports and
                # /etc/exports.d./* to provide a clean test environment.
                #
                log_must exportfs -r

                log_note "NFS server must be started prior to running ZTS."
                return
        elif is_freebsd; then
                log_must kill -s HUP $(</var/run/mountd.pid)

                log_note "NFS server must be started prior to running ZTS."
                return
        fi

        typeset nfs_fmri="svc:/network/nfs/server:default"
        if [[ $(svcs -Ho STA $nfs_fmri) != "ON" ]]; then
                #
                # Only really sharing operation can enable NFS server
                # to online permanently.
                #
                typeset dummy=/tmp/dummy

                if [[ -d $dummy ]]; then
                        log_must rm -rf $dummy
                fi

                log_must mkdir $dummy
                log_must share $dummy

                #
                # Waiting for fmri's status to be the final status.
                # Otherwise, in transition, an asterisk (*) is appended for
                # instances, unshare will reverse status to 'DIS' again.
                #
                # Waiting for 1's at least.
                #
                log_must sleep 1
                timeout=10
                while [[ timeout -ne 0 && $(svcs -Ho STA $nfs_fmri) == *'*' ]]
                do
                        log_must sleep 1

                        ((timeout -= 1))
                done

                log_must unshare $dummy
                log_must rm -rf $dummy
        fi

        log_note "Current NFS status: '$(svcs -Ho STA,FMRI $nfs_fmri)'"
}

#
# To verify whether calling process is in global zone
#
# Return 0 if in global zone, 1 in non-global zone
#
function is_global_zone
{
        if is_linux || is_freebsd; then
                return 0
        else
                typeset cur_zone=$(zonename 2>/dev/null)
                [ $cur_zone = "global" ]
        fi
}

#
# Verify whether test is permitted to run from
# global zone, local zone, or both
#
# $1 zone limit, could be "global", "local", or "both"(no limit)
#
# Return 0 if permitted, otherwise exit with log_unsupported
#
function verify_runnable # zone limit
{
        typeset limit=$1

        [[ -z $limit ]] && return 0

        if is_global_zone ; then
                case $limit in
                        global|both)
                                ;;
                        local)  log_unsupported "Test is unable to run from "\
                                        "global zone."
                                ;;
                        *)      log_note "Warning: unknown limit $limit - " \
                                        "use both."
                                ;;
                esac
        else
                case $limit in
                        local|both)
                                ;;
                        global) log_unsupported "Test is unable to run from "\
                                        "local zone."
                                ;;
                        *)      log_note "Warning: unknown limit $limit - " \
                                        "use both."
                                ;;
                esac

                reexport_pool
        fi

        return 0
}

# Return 0 if create successfully or the pool exists; $? otherwise
# Note: In local zones, this function should return 0 silently.
#
# $1 - pool name
# $2-n - [keyword] devs_list

function create_pool #pool devs_list
{
        typeset pool=${1%%/*}

        shift

        if [[ -z $pool ]]; then
                log_note "Missing pool name."
                return 1
        fi

        if poolexists $pool ; then
                destroy_pool $pool
        fi

        if is_global_zone ; then
                [[ -d /$pool ]] && rm -rf /$pool
                log_must zpool create -f $pool $@
        fi

        return 0
}

# Return 0 if destroy successfully or the pool exists; $? otherwise
# Note: In local zones, this function should return 0 silently.
#
# $1 - pool name
# Destroy pool with the given parameters.

function destroy_pool #pool
{
        typeset pool=${1%%/*}
        typeset mtpt

        if [[ -z $pool ]]; then
                log_note "No pool name given."
                return 1
        fi

        if is_global_zone ; then
                if poolexists "$pool" ; then
                        mtpt=$(get_prop mountpoint "$pool")

                        # At times, syseventd/udev activity can cause attempts
                        # to destroy a pool to fail with EBUSY. We retry a few
                        # times allowing failures before requiring the destroy
                        # to succeed.
                        log_must_busy zpool destroy -f $pool

                        [[ -d $mtpt ]] && \
                                log_must rm -rf $mtpt
                else
                        log_note "Pool does not exist. ($pool)"
                        return 1
                fi
        fi

        return 0
}

# Return 0 if created successfully; $? otherwise
#
# $1 - dataset name
# $2-n - dataset options

function create_dataset #dataset dataset_options
{
        typeset dataset=$1

        shift

        if [[ -z $dataset ]]; then
                log_note "Missing dataset name."
                return 1
        fi

        if datasetexists $dataset ; then
                destroy_dataset $dataset
        fi

        log_must zfs create $@ $dataset

        return 0
}

# Return 0 if destroy successfully or the dataset exists; $? otherwise
# Note: In local zones, this function should return 0 silently.
#
# $1 - dataset name
# $2 - custom arguments for zfs destroy
# Destroy dataset with the given parameters.

function destroy_dataset # dataset [args]
{
        typeset dataset=$1
        typeset mtpt
        typeset args=${2:-""}

        if [[ -z $dataset ]]; then
                log_note "No dataset name given."
                return 1
        fi

        if is_global_zone ; then
                if datasetexists "$dataset" ; then
                        mtpt=$(get_prop mountpoint "$dataset")
                        log_must_busy zfs destroy $args $dataset

                        [ -d $mtpt ] && log_must rm -rf $mtpt
                else
                        log_note "Dataset does not exist. ($dataset)"
                        return 1
                fi
        fi

        return 0
}

#
# Reexport TESTPOOL & TESTPOOL(1-4)
#
function reexport_pool
{
        typeset -i cntctr=5
        typeset -i i=0

        while ((i < cntctr)); do
                if ((i == 0)); then
                        TESTPOOL=$ZONE_POOL/$ZONE_CTR$i
                        if ! ismounted $TESTPOOL; then
                                log_must zfs mount $TESTPOOL
                        fi
                else
                        eval TESTPOOL$i=$ZONE_POOL/$ZONE_CTR$i
                        if eval ! ismounted \$TESTPOOL$i; then
                                log_must eval zfs mount \$TESTPOOL$i
                        fi
                fi
                ((i += 1))
        done
}

#
# Verify a given disk or pool state
#
# Return 0 is pool/disk matches expected state, 1 otherwise
#
function check_state # pool disk state{online,offline,degraded}
{
        typeset pool=$1
        typeset disk=${2#$DEV_DSKDIR/}
        typeset state=$3

        [[ -z $pool ]] || [[ -z $state ]] \
            && log_fail "Arguments invalid or missing"

        if [[ -z $disk ]]; then
                #check pool state only
                zpool get -H -o value health $pool | grep -qi "$state"
        else
                zpool status -v $pool | grep "$disk" | grep -qi "$state"
        fi
}

#
# Get the mountpoint of snapshot
# For the snapshot use <mp_filesystem>/.zfs/snapshot/<snap>
# as its mountpoint
#
function snapshot_mountpoint
{
        typeset dataset=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}

        if [[ $dataset != *@* ]]; then
                log_fail "Error name of snapshot '$dataset'."
        fi

        typeset fs=${dataset%@*}
        typeset snap=${dataset#*@}

        if [[ -z $fs || -z $snap ]]; then
                log_fail "Error name of snapshot '$dataset'."
        fi

        echo $(get_prop mountpoint $fs)/.zfs/snapshot/$snap
}

#
# Given a device and 'ashift' value verify it's correctly set on every label
#
function verify_ashift # device ashift
{
        typeset device="$1"
        typeset ashift="$2"

        zdb -e -lll $device | awk -v ashift=$ashift '
            /ashift: / {
                if (ashift != $2)
                    exit 1;
                else
                    count++;
            }
            END {
                exit (count != 4);
            }'
}

#
# Given a pool and file system, this function will verify the file system
# using the zdb internal tool. Note that the pool is exported and imported
# to ensure it has consistent state.
#
function verify_filesys # pool filesystem dir
{
        typeset pool="$1"
        typeset filesys="$2"
        typeset zdbout="/tmp/zdbout.$$"

        shift
        shift
        typeset dirs=$@
        typeset search_path=""

        log_note "Calling zdb to verify filesystem '$filesys'"
        zfs unmount -a > /dev/null 2>&1
        log_must zpool export $pool

        if [[ -n $dirs ]] ; then
                for dir in $dirs ; do
                        search_path="$search_path -d $dir"
                done
        fi

        log_must zpool import $search_path $pool

        if ! zdb -cudi $filesys > $zdbout 2>&1; then
                log_note "Output: zdb -cudi $filesys"
                cat $zdbout
                rm -f $zdbout
                log_fail "zdb detected errors with: '$filesys'"
        fi

        log_must zfs mount -a
        log_must rm -rf $zdbout
}

#
# Given a pool issue a scrub and verify that no checksum errors are reported.
#
function verify_pool
{
        typeset pool=${1:-$TESTPOOL}

        log_must zpool scrub $pool
        log_must wait_scrubbed $pool

        typeset -i cksum=$(zpool status $pool | awk '
            !NF { isvdev = 0 }
            isvdev { errors += $NF }
            /CKSUM$/ { isvdev = 1 }
            END { print errors }
        ')
        if [[ $cksum != 0 ]]; then
                log_must zpool status -v
                log_fail "Unexpected CKSUM errors found on $pool ($cksum)"
        fi
}

#
# Given a pool, and this function list all disks in the pool
#
function get_disklist # pool
{
        echo $(zpool iostat -v $1 | awk '(NR > 4) {print $1}' | \
            grep -vEe '^-----' -e "^(mirror|raidz[1-3]|draid[1-3]|spare|log|cache|special|dedup)|\-[0-9]$")
}

#
# Given a pool, and this function list all disks in the pool with their full
# path (like "/dev/sda" instead of "sda").
#
function get_disklist_fullpath # pool
{
        get_disklist "-P $1"
}



# /**
#  This function kills a given list of processes after a time period. We use
#  this in the stress tests instead of STF_TIMEOUT so that we can have processes
#  run for a fixed amount of time, yet still pass. Tests that hit STF_TIMEOUT
#  would be listed as FAIL, which we don't want : we're happy with stress tests
#  running for a certain amount of time, then finishing.
#
# @param $1 the time in seconds after which we should terminate these processes
# @param $2..$n the processes we wish to terminate.
# */
function stress_timeout
{
        typeset -i TIMEOUT=$1
        shift
        typeset cpids="$@"

        log_note "Waiting for child processes($cpids). " \
                "It could last dozens of minutes, please be patient ..."
        log_must sleep $TIMEOUT

        log_note "Killing child processes after ${TIMEOUT} stress timeout."
        typeset pid
        for pid in $cpids; do
                ps -p $pid > /dev/null 2>&1 &&
                        log_must kill -USR1 $pid
        done
}

#
# Verify a given hotspare disk is inuse or avail
#
# Return 0 is pool/disk matches expected state, 1 otherwise
#
function check_hotspare_state # pool disk state{inuse,avail}
{
        typeset pool=$1
        typeset disk=${2#$DEV_DSKDIR/}
        typeset state=$3

        cur_state=$(get_device_state $pool $disk "spares")

        [ $state = $cur_state ]
}

#
# Wait until a hotspare transitions to a given state or times out.
#
# Return 0 when  pool/disk matches expected state, 1 on timeout.
#
function wait_hotspare_state # pool disk state timeout
{
        typeset pool=$1
        typeset disk=${2#*$DEV_DSKDIR/}
        typeset state=$3
        typeset timeout=${4:-60}
        typeset -i i=0

        while [[ $i -lt $timeout ]]; do
                if check_hotspare_state $pool $disk $state; then
                        return 0
                fi

                i=$((i+1))
                sleep 1
        done

        return 1
}

#
# Verify a given vdev disk is inuse or avail
#
# Return 0 is pool/disk matches expected state, 1 otherwise
#
function check_vdev_state # pool disk state{online,offline,unavail,removed}
{
        typeset pool=$1
        typeset disk=${2#*$DEV_DSKDIR/}
        typeset state=$3

        cur_state=$(get_device_state $pool $disk)

        [ $state = $cur_state ]
}

#
# Wait until a vdev transitions to a given state or times out.
#
# Return 0 when  pool/disk matches expected state, 1 on timeout.
#
function wait_vdev_state # pool disk state timeout
{
        typeset pool=$1
        typeset disk=${2#*$DEV_DSKDIR/}
        typeset state=$3
        typeset timeout=${4:-60}
        typeset -i i=0

        while [[ $i -lt $timeout ]]; do
                if check_vdev_state $pool $disk $state; then
                        return 0
                fi

                i=$((i+1))
                sleep 1
        done

        return 1
}

#
# Check the output of 'zpool status -v <pool>',
# and to see if the content of <token> contain the <keyword> specified.
#
# Return 0 is contain, 1 otherwise
#
function check_pool_status # pool token keyword <verbose>
{
        typeset pool=$1
        typeset token=$2
        typeset keyword=$3
        typeset verbose=${4:-false}

        scan=$(zpool status -v "$pool" 2>/dev/null | awk -v token="$token:" '$1==token')
        if [[ $verbose == true ]]; then
                log_note $scan
        fi
        echo $scan | grep -qi "$keyword"
}

#
# The following functions are instance of check_pool_status()
#       is_pool_resilvering - to check if the pool resilver is in progress
#       is_pool_resilvered - to check if the pool resilver is completed
#       is_pool_scrubbing - to check if the pool scrub is in progress
#       is_pool_scrubbed - to check if the pool scrub is completed
#       is_pool_scrub_stopped - to check if the pool scrub is stopped
#       is_pool_scrub_paused - to check if the pool scrub has paused
#       is_pool_removing - to check if the pool removing is a vdev
#       is_pool_removed - to check if the pool remove is completed
#       is_pool_discarding - to check if the pool checkpoint is being discarded
#       is_pool_replacing - to check if the pool is performing a replacement
#
function is_pool_resilvering #pool <verbose>
{
        check_pool_status "$1" "scan" \
            "resilver[ ()0-9A-Za-z:_-]* in progress since" $2
}

function is_pool_resilvered #pool <verbose>
{
        check_pool_status "$1" "scan" "resilvered " $2
}

function is_pool_scrubbing #pool <verbose>
{
        check_pool_status "$1" "scan" "scrub in progress since " $2
}

function is_pool_error_scrubbing #pool <verbose>
{
        check_pool_status "$1" "scrub" "error scrub in progress since " $2
        return $?
}

function is_pool_scrubbed #pool <verbose>
{
        check_pool_status "$1" "scan" "scrub repaired" $2
}

function is_pool_scrub_stopped #pool <verbose>
{
        check_pool_status "$1" "scan" "scrub canceled" $2
}

function is_pool_error_scrub_stopped #pool <verbose>
{
        check_pool_status "$1" "scrub" "error scrub canceled on " $2
        return $?
}

function is_pool_scrub_paused #pool <verbose>
{
        check_pool_status "$1" "scan" "scrub paused since " $2
}

function is_pool_error_scrub_paused #pool <verbose>
{
        check_pool_status "$1" "scrub" "error scrub paused since " $2
        return $?
}

function is_pool_removing #pool
{
        check_pool_status "$1" "remove" "in progress since "
}

function is_pool_removed #pool
{
        check_pool_status "$1" "remove" "completed on"
}

function is_pool_discarding #pool
{
        check_pool_status "$1" "checkpoint" "discarding"
}
function is_pool_replacing #pool
{
        zpool status "$1" | grep -qE 'replacing-[0-9]+'
}

function wait_for_degraded
{
        typeset pool=$1
        typeset timeout=${2:-30}
        typeset t0=$SECONDS

        while :; do
                [[ $(get_pool_prop health $pool) == "DEGRADED" ]] && break
                log_note "$pool is not yet degraded."
                sleep 1
                if ((SECONDS - t0 > $timeout)); then
                        log_note "$pool not degraded after $timeout seconds."
                        return 1
                fi
        done

        return 0
}

#
# Use create_pool()/destroy_pool() to clean up the information in
# in the given disk to avoid slice overlapping.
#
function cleanup_devices #vdevs
{
        typeset pool="foopool$$"

        for vdev in $@; do
                zero_partitions $vdev
        done

        poolexists $pool && destroy_pool $pool
        create_pool $pool $@
        destroy_pool $pool

        return 0
}

#/**
# A function to find and locate free disks on a system or from given
# disks as the parameter. It works by locating disks that are in use
# as swap devices and dump devices, and also disks listed in /etc/vfstab
#
# $@ given disks to find which are free, default is all disks in
# the test system
#
# @return a string containing the list of available disks
#*/
function find_disks
{
        # Trust provided list, no attempt is made to locate unused devices.
        if is_linux || is_freebsd; then
                echo "$@"
                return
        fi


        sfi=/tmp/swaplist.$$
        dmpi=/tmp/dumpdev.$$
        max_finddisksnum=${MAX_FINDDISKSNUM:-6}

        swap -l > $sfi
        dumpadm > $dmpi 2>/dev/null

        disks=${@:-$(echo "" | format -e 2>/dev/null | awk '
BEGIN { FS="."; }

/^Specify disk/{
        searchdisks=0;
}

{
        if (searchdisks && $2 !~ "^$"){
                split($2,arr," ");
                print arr[1];
        }
}

/^AVAILABLE DISK SELECTIONS:/{
        searchdisks=1;
}
')}

        unused=""
        for disk in $disks; do
        # Check for mounted
                grep -q "${disk}[sp]" /etc/mnttab && continue
        # Check for swap
                grep -q "${disk}[sp]" $sfi && continue
        # check for dump device
                grep -q "${disk}[sp]" $dmpi && continue
        # check to see if this disk hasn't been explicitly excluded
        # by a user-set environment variable
                echo "${ZFS_HOST_DEVICES_IGNORE}" | grep -q "${disk}" && continue
                unused_candidates="$unused_candidates $disk"
        done
        rm $sfi $dmpi

# now just check to see if those disks do actually exist
# by looking for a device pointing to the first slice in
# each case. limit the number to max_finddisksnum
        count=0
        for disk in $unused_candidates; do
                if is_disk_device $DEV_DSKDIR/${disk}s0 && \
                    [ $count -lt $max_finddisksnum ]; then
                        unused="$unused $disk"
                        # do not impose limit if $@ is provided
                        [[ -z $@ ]] && ((count = count + 1))
                fi
        done

# finally, return our disk list
        echo $unused
}

function add_user_freebsd #<group_name> <user_name> <basedir>
{
        typeset group=$1
        typeset user=$2
        typeset basedir=$3

        # Check to see if the user exists.
        if id $user > /dev/null 2>&1; then
                return 0
        fi

        # Assign 1000 as the base uid
        typeset -i uid=1000
        while true; do
                pw useradd -u $uid -g $group -d $basedir/$user -m -n $user
                case $? in
                        0) break ;;
                        # The uid is not unique
                        65) ((uid += 1)) ;;
                        *) return 1 ;;
                esac
                if [[ $uid == 65000 ]]; then
                        log_fail "No user id available under 65000 for $user"
                fi
        done

        # Silence MOTD
        touch $basedir/$user/.hushlogin

        return 0
}

#
# Delete the specified user.
#
# $1 login name
#
function del_user_freebsd #<logname>
{
        typeset user=$1

        if id $user > /dev/null 2>&1; then
                log_must pw userdel $user
        fi

        return 0
}

#
# Select valid gid and create specified group.
#
# $1 group name
#
function add_group_freebsd #<group_name>
{
        typeset group=$1

        # See if the group already exists.
        if pw groupshow $group >/dev/null 2>&1; then
                return 0
        fi

        # Assign 1000 as the base gid
        typeset -i gid=1000
        while true; do
                pw groupadd -g $gid -n $group > /dev/null 2>&1
                case $? in
                        0) return 0 ;;
                        # The gid is not  unique
                        65) ((gid += 1)) ;;
                        *) return 1 ;;
                esac
                if [[ $gid == 65000 ]]; then
                        log_fail "No user id available under 65000 for $group"
                fi
        done
}

#
# Delete the specified group.
#
# $1 group name
#
function del_group_freebsd #<group_name>
{
        typeset group=$1

        pw groupdel -n $group > /dev/null 2>&1
        case $? in
                # Group does not exist, or was deleted successfully.
                0|6|65) return 0 ;;
                # Name already exists as a group name
                9) log_must pw groupdel $group ;;
                *) return 1 ;;
        esac

        return 0
}

function add_user_illumos #<group_name> <user_name> <basedir>
{
        typeset group=$1
        typeset user=$2
        typeset basedir=$3

        log_must useradd -g $group -d $basedir/$user -m $user

        return 0
}

function del_user_illumos #<user_name>
{
        typeset user=$1

        if id $user > /dev/null 2>&1; then
                log_must_retry "currently used" 6 userdel $user
        fi

        return 0
}

function add_group_illumos #<group_name>
{
        typeset group=$1

        typeset -i gid=100
        while true; do
                groupadd -g $gid $group > /dev/null 2>&1
                case $? in
                        0) return 0 ;;
                        # The gid is not  unique
                        4) ((gid += 1)) ;;
                        *) return 1 ;;
                esac
        done
}

function del_group_illumos #<group_name>
{
        typeset group=$1

        groupmod -n $grp $grp > /dev/null 2>&1
        case $? in
                # Group does not exist.
                6) return 0 ;;
                # Name already exists as a group name
                9) log_must groupdel $grp ;;
                *) return 1 ;;
        esac
}

function add_user_linux #<group_name> <user_name> <basedir>
{
        typeset group=$1
        typeset user=$2
        typeset basedir=$3

        log_must useradd -g $group -d $basedir/$user -m $user

        # Add new users to the same group and the command line utils.
        # This allows them to be run out of the original users home
        # directory as long as it permissioned to be group readable.
        cmd_group=$(stat --format="%G" $(command -v zfs))
        log_must usermod -a -G $cmd_group $user

        return 0
}

function del_user_linux #<user_name>
{
        typeset user=$1

        if id $user > /dev/null 2>&1; then
                log_must_retry "currently used" 6 userdel $user
        fi
}

function add_group_linux #<group_name>
{
        typeset group=$1

        # Assign 100 as the base gid, a larger value is selected for
        # Linux because for many distributions 1000 and under are reserved.
        while true; do
                groupadd $group > /dev/null 2>&1
                case $? in
                        0) return 0 ;;
                        *) return 1 ;;
                esac
        done
}

function del_group_linux #<group_name>
{
        typeset group=$1

        getent group $group > /dev/null 2>&1
        case $? in
                # Group does not exist.
                2) return 0 ;;
                # Name already exists as a group name
                0) log_must groupdel $group ;;
                *) return 1 ;;
        esac

        return 0
}

#
# Add specified user to specified group
#
# $1 group name
# $2 user name
# $3 base of the homedir (optional)
#
function add_user #<group_name> <user_name> <basedir>
{
        typeset group=$1
        typeset user=$2
        typeset basedir=${3:-"/var/tmp"}

        if ((${#group} == 0 || ${#user} == 0)); then
                log_fail "group name or user name are not defined."
        fi

        case "$UNAME" in
        FreeBSD)
                add_user_freebsd "$group" "$user" "$basedir"
                ;;
        Linux)
                add_user_linux "$group" "$user" "$basedir"
                ;;
        *)
                add_user_illumos "$group" "$user" "$basedir"
                ;;
        esac

        return 0
}

#
# Delete the specified user.
#
# $1 login name
# $2 base of the homedir (optional)
#
function del_user #<logname> <basedir>
{
        typeset user=$1
        typeset basedir=${2:-"/var/tmp"}

        if ((${#user} == 0)); then
                log_fail "login name is necessary."
        fi

        case "$UNAME" in
        FreeBSD)
                del_user_freebsd "$user"
                ;;
        Linux)
                del_user_linux "$user"
                ;;
        *)
                del_user_illumos "$user"
                ;;
        esac

        [[ -d $basedir/$user ]] && rm -fr $basedir/$user

        return 0
}

#
# Select valid gid and create specified group.
#
# $1 group name
#
function add_group #<group_name>
{
        typeset group=$1

        if ((${#group} == 0)); then
                log_fail "group name is necessary."
        fi

        case "$UNAME" in
        FreeBSD)
                add_group_freebsd "$group"
                ;;
        Linux)
                add_group_linux "$group"
                ;;
        *)
                add_group_illumos "$group"
                ;;
        esac

        return 0
}

#
# Delete the specified group.
#
# $1 group name
#
function del_group #<group_name>
{
        typeset group=$1

        if ((${#group} == 0)); then
                log_fail "group name is necessary."
        fi

        case "$UNAME" in
        FreeBSD)
                del_group_freebsd "$group"
                ;;
        Linux)
                del_group_linux "$group"
                ;;
        *)
                del_group_illumos "$group"
                ;;
        esac

        return 0
}

#
# This function will return true if it's safe to destroy the pool passed
# as argument 1. It checks for pools based on zvols and files, and also
# files contained in a pool that may have a different mountpoint.
#
function safe_to_destroy_pool { # $1 the pool name

        typeset pool=""
        typeset DONT_DESTROY=""

        # We check that by deleting the $1 pool, we're not
        # going to pull the rug out from other pools. Do this
        # by looking at all other pools, ensuring that they
        # aren't built from files or zvols contained in this pool.

        for pool in $(zpool list -H -o name)
        do
                ALTMOUNTPOOL=""

                # this is a list of the top-level directories in each of the
                # files that make up the path to the files the pool is based on
                FILEPOOL=$(zpool status -v $pool | awk -v pool="/$1/" '$0 ~ pool {print $1}')

                # this is a list of the zvols that make up the pool
                ZVOLPOOL=$(zpool status -v $pool | awk -v zvols="$ZVOL_DEVDIR/$1$" '$0 ~ zvols {print $1}')

                # also want to determine if it's a file-based pool using an
                # alternate mountpoint...
                POOL_FILE_DIRS=$(zpool status -v $pool | \
                                        awk '/\// {print $1}' | \
                                        awk -F/ '!/dev/ {print $2}')

                for pooldir in $POOL_FILE_DIRS
                do
                        OUTPUT=$(zfs list -H -r -o mountpoint $1 | \
                                        awk -v pd="${pooldir}$" '$0 ~ pd {print $1}')

                        ALTMOUNTPOOL="${ALTMOUNTPOOL}${OUTPUT}"
                done


                if [ ! -z "$ZVOLPOOL" ]
                then
                        DONT_DESTROY="true"
                        log_note "Pool $pool is built from $ZVOLPOOL on $1"
                fi

                if [ ! -z "$FILEPOOL" ]
                then
                        DONT_DESTROY="true"
                        log_note "Pool $pool is built from $FILEPOOL on $1"
                fi

                if [ ! -z "$ALTMOUNTPOOL" ]
                then
                        DONT_DESTROY="true"
                        log_note "Pool $pool is built from $ALTMOUNTPOOL on $1"
                fi
        done

        if [ -z "${DONT_DESTROY}" ]
        then
                return 0
        else
                log_note "Warning: it is not safe to destroy $1!"
                return 1
        fi
}

#
# Verify zfs operation with -p option work as expected
# $1 operation, value could be create, clone or rename
# $2 dataset type, value could be fs or vol
# $3 dataset name
# $4 new dataset name
#
function verify_opt_p_ops
{
        typeset ops=$1
        typeset datatype=$2
        typeset dataset=$3
        typeset newdataset=$4

        if [[ $datatype != "fs" && $datatype != "vol" ]]; then
                log_fail "$datatype is not supported."
        fi

        # check parameters accordingly
        case $ops in
                create)
                        newdataset=$dataset
                        dataset=""
                        if [[ $datatype == "vol" ]]; then
                                ops="create -V $VOLSIZE"
                        fi
                        ;;
                clone)
                        if [[ -z $newdataset ]]; then
                                log_fail "newdataset should not be empty" \
                                        "when ops is $ops."
                        fi
                        log_must datasetexists $dataset
                        log_must snapexists $dataset
                        ;;
                rename)
                        if [[ -z $newdataset ]]; then
                                log_fail "newdataset should not be empty" \
                                        "when ops is $ops."
                        fi
                        log_must datasetexists $dataset
                        ;;
                *)
                        log_fail "$ops is not supported."
                        ;;
        esac

        # make sure the upper level filesystem does not exist
        destroy_dataset "${newdataset%/*}" "-rRf"

        # without -p option, operation will fail
        log_mustnot zfs $ops $dataset $newdataset
        log_mustnot datasetexists $newdataset ${newdataset%/*}

        # with -p option, operation should succeed
        log_must zfs $ops -p $dataset $newdataset
        block_device_wait

        if ! datasetexists $newdataset ; then
                log_fail "-p option does not work for $ops"
        fi

        # when $ops is create or clone, redo the operation still return zero
        if [[ $ops != "rename" ]]; then
                log_must zfs $ops -p $dataset $newdataset
        fi

        return 0
}

#
# Get configuration of pool
# $1 pool name
# $2 config name
#
function get_config
{
        typeset pool=$1
        typeset config=$2

        if ! poolexists "$pool" ; then
                return 1
        fi
        if [ "$(get_pool_prop cachefile "$pool")" = "none" ]; then
                zdb -e $pool
        else
                zdb -C $pool
        fi | awk -F: -v cfg="$config:" '$0 ~ cfg {sub(/^'\''/, $2); sub(/'\''$/, $2); print $2}'
}

#
# Privated function. Random select one of items from arguments.
#
# $1 count
# $2-n string
#
function _random_get
{
        typeset cnt=$1
        shift

        typeset str="$@"
        typeset -i ind
        ((ind = RANDOM % cnt + 1))

        echo "$str" | cut -f $ind -d ' '
}

#
# Random select one of item from arguments which include NONE string
#
function random_get_with_non
{
        typeset -i cnt=$#
        ((cnt =+ 1))

        _random_get "$cnt" "$@"
}

#
# Random select one of item from arguments which doesn't include NONE string
#
function random_get
{
        _random_get "$#" "$@"
}

#
# The function will generate a dataset name with specific length
# $1, the length of the name
# $2, the base string to construct the name
#
function gen_dataset_name
{
        typeset -i len=$1
        typeset basestr="$2"
        typeset -i baselen=${#basestr}
        typeset -i iter=0
        typeset l_name=""

        if ((len % baselen == 0)); then
                ((iter = len / baselen))
        else
                ((iter = len / baselen + 1))
        fi
        while ((iter > 0)); do
                l_name="${l_name}$basestr"

                ((iter -= 1))
        done

        echo $l_name
}

#
# Get cksum tuple of dataset
# $1 dataset name
#
# sample zdb output:
# Dataset data/test [ZPL], ID 355, cr_txg 2413856, 31.0K, 7 objects, rootbp
# DVA[0]=<0:803046400:200> DVA[1]=<0:81199000:200> [L0 DMU objset] fletcher4
# lzjb LE contiguous unique double size=800L/200P birth=2413856L/2413856P
# fill=7 cksum=11ce125712:643a9c18ee2:125e25238fca0:254a3f74b59744
function datasetcksum
{
        typeset cksum
        sync
        sync_all_pools
        zdb -vvv $1 | awk -F= -v ds="^Dataset $1 "'\\[' '$0 ~ ds && /cksum/ {print $7}'
}

#
# Get the given disk/slice state from the specific field of the pool
#
function get_device_state #pool disk field("", "spares","logs")
{
        typeset pool=$1
        typeset disk=${2#$DEV_DSKDIR/}
        typeset field=${3:-$pool}

        zpool status -v "$pool" 2>/dev/null | \
                awk -v device=$disk -v pool=$pool -v field=$field \
                'BEGIN {startconfig=0; startfield=0; }
                /config:/ {startconfig=1}
                (startconfig==1) && ($1==field) {startfield=1; next;}
                (startfield==1) && ($1==device) {print $2; exit;}
                (startfield==1) &&
                ($1==field || $1 ~ "^spares$" || $1 ~ "^logs$") {startfield=0}'
}

#
# get the root filesystem name if it's zfsroot system.
#
# return: root filesystem name
function get_rootfs
{
        typeset rootfs=""

        if is_freebsd; then
                rootfs=$(mount -p | awk '$2 == "/" && $3 == "zfs" {print $1}')
        elif ! is_linux; then
                rootfs=$(awk '$2 == "/" && $3 == "zfs" {print $1}' \
                        /etc/mnttab)
        fi
        if [[ -z "$rootfs" ]]; then
                log_fail "Can not get rootfs"
        fi
        if datasetexists $rootfs; then
                echo $rootfs
        else
                log_fail "This is not a zfsroot system."
        fi
}

#
# get the rootfs's pool name
# return:
#       rootpool name
#
function get_rootpool
{
        typeset rootfs=$(get_rootfs)
        echo ${rootfs%%/*}
}

#
# To verify if the require numbers of disks is given
#
function verify_disk_count
{
        typeset -i min=${2:-1}

        typeset -i count=$(echo "$1" | wc -w)

        if ((count < min)); then
                log_untested "A minimum of $min disks is required to run." \
                        " You specified $count disk(s)"
        fi
}

function ds_is_volume
{
        typeset type=$(get_prop type $1)
        [ $type = "volume" ]
}

function ds_is_filesystem
{
        typeset type=$(get_prop type $1)
        [ $type = "filesystem" ]
}

#
# Check if Trusted Extensions are installed and enabled
#
function is_te_enabled
{
        svcs -H -o state labeld 2>/dev/null | grep -q "enabled"
}

# Return the number of CPUs (cross-platform)
function get_num_cpus
{
        if is_linux ; then
                grep -c '^processor' /proc/cpuinfo
        elif is_freebsd; then
                sysctl -n kern.smp.cpus
        else
                psrinfo | wc -l
        fi
}

# Utility function to determine if a system has multiple cpus.
function is_mp
{
        [[ $(get_num_cpus) -gt 1 ]]
}

function get_cpu_freq
{
        if is_linux; then
                lscpu | awk '/CPU MHz/ { print $3 }'
        elif is_freebsd; then
                sysctl -n hw.clockrate
        else
                psrinfo -v 0 | awk '/processor operates at/ {print $6}'
        fi
}

# Run the given command as the user provided.
function user_run
{
        typeset user=$1
        shift

        log_note "user: $user"
        log_note "cmd: $*"

        typeset out=$TEST_BASE_DIR/out
        typeset err=$TEST_BASE_DIR/err

        sudo -Eu $user env PATH="$PATH" ksh <<<"$*" >$out 2>$err
        typeset res=$?
        log_note "out: $(<$out)"
        log_note "err: $(<$err)"
        return $res
}

#
# Check if the pool contains the specified vdevs
#
# $1 pool
# $2..n <vdev> ...
#
# Return 0 if the vdevs are contained in the pool, 1 if any of the specified
# vdevs is not in the pool, and 2 if pool name is missing.
#
function vdevs_in_pool
{
        typeset pool=$1
        typeset vdev

        if [[ -z $pool ]]; then
                log_note "Missing pool name."
                return 2
        fi

        shift

        # We could use 'zpool list' to only get the vdevs of the pool but we
        # can't reference a mirror/raidz vdev using its ID (i.e mirror-0),
        # therefore we use the 'zpool status' output.
        typeset tmpfile=$(mktemp)
        zpool status -v "$pool" | grep -A 1000 "config:" >$tmpfile
        for vdev in "$@"; do
                grep -wq ${vdev##*/} $tmpfile || return 1
        done

        rm -f $tmpfile
        return 0
}

function get_max
{
        typeset -l i max=$1
        shift

        for i in "$@"; do
                max=$((max > i ? max : i))
        done

        echo $max
}

# Write data that can be compressed into a directory
function write_compressible
{
        typeset dir=$1
        typeset megs=$2
        typeset nfiles=${3:-1}
        typeset bs=${4:-1024k}
        typeset fname=${5:-file}

        [[ -d $dir ]] || log_fail "No directory: $dir"

        # Under Linux fio is not currently used since its behavior can
        # differ significantly across versions.  This includes missing
        # command line options and cases where the --buffer_compress_*
        # options fail to behave as expected.
        if is_linux; then
                typeset file_bytes=$(to_bytes $megs)
                typeset bs_bytes=4096
                typeset blocks=$(($file_bytes / $bs_bytes))

                for (( i = 0; i < $nfiles; i++ )); do
                        truncate -s $file_bytes $dir/$fname.$i

                        # Write every third block to get 66% compression.
                        for (( j = 0; j < $blocks; j += 3 )); do
                                dd if=/dev/urandom of=$dir/$fname.$i \
                                    seek=$j bs=$bs_bytes count=1 \
                                    conv=notrunc >/dev/null 2>&1
                        done
                done
        else
                command -v fio > /dev/null || log_unsupported "fio missing"
                log_must eval fio \
                    --name=job \
                    --fallocate=0 \
                    --minimal \
                    --randrepeat=0 \
                    --buffer_compress_percentage=66 \
                    --buffer_compress_chunk=4096 \
                    --directory="$dir" \
                    --numjobs="$nfiles" \
                    --nrfiles="$nfiles" \
                    --rw=write \
                    --bs="$bs" \
                    --filesize="$megs" \
                    "--filename_format='$fname.\$jobnum' >/dev/null"
        fi
}

function get_objnum
{
        typeset pathname=$1
        typeset objnum

        [[ -e $pathname ]] || log_fail "No such file or directory: $pathname"
        if is_freebsd; then
                objnum=$(stat -f "%i" $pathname)
        else
                objnum=$(stat -c %i $pathname)
        fi
        echo $objnum
}

#
# Sync data to the pool
#
# $1 pool name
# $2 boolean to force uberblock (and config including zpool cache file) update
#
function sync_pool #pool <force>
{
        typeset pool=${1:-$TESTPOOL}
        typeset force=${2:-false}

        if [[ $force == true ]]; then
                log_must zpool sync -f $pool
        else
                log_must zpool sync $pool
        fi

        return 0
}

#
# Sync all pools
#
# $1 boolean to force uberblock (and config including zpool cache file) update
#
function sync_all_pools #<force>
{
        typeset force=${1:-false}

        if [[ $force == true ]]; then
                log_must zpool sync -f
        else
                log_must zpool sync
        fi

        return 0
}

#
# Wait for zpool 'freeing' property drops to zero.
#
# $1 pool name
#
function wait_freeing #pool
{
        typeset pool=${1:-$TESTPOOL}
        while true; do
                [[ "0" == "$(zpool list -Ho freeing $pool)" ]] && break
                log_must sleep 1
        done
}

#
# Wait for every device replace operation to complete
#
# $1 pool name
# $2 timeout
#
function wait_replacing #pool timeout
{
        typeset timeout=${2:-300}
        typeset pool=${1:-$TESTPOOL}
        for (( timer = 0; timer < $timeout; timer++ )); do
                is_pool_replacing $pool || break;
                sleep 1;
        done
}

# Wait for a pool to be scrubbed
#
# $1 pool name
# $2 timeout
#
function wait_scrubbed #pool timeout
{
       typeset timeout=${2:-300}
       typeset pool=${1:-$TESTPOOL}
       for (( timer = 0; timer < $timeout; timer++ )); do
               is_pool_scrubbed $pool && break;
               sleep 1;
       done
}

# Backup the zed.rc in our test directory so that we can edit it for our test.
#
# Returns: Backup file name.  You will need to pass this to zed_rc_restore().
function zed_rc_backup
{
        zedrc_backup="$(mktemp)"
        cp $ZEDLET_DIR/zed.rc $zedrc_backup
        echo $zedrc_backup
}

function zed_rc_restore
{
        mv $1 $ZEDLET_DIR/zed.rc
}

#
# Setup custom environment for the ZED.
#
# $@ Optional list of zedlets to run under zed.
function zed_setup
{
        if ! is_linux; then
                log_unsupported "No zed on $UNAME"
        fi

        if [[ ! -d $ZEDLET_DIR ]]; then
                log_must mkdir $ZEDLET_DIR
        fi

        if [[ ! -e $VDEVID_CONF ]]; then
                log_must touch $VDEVID_CONF
        fi

        if [[ -e $VDEVID_CONF_ETC ]]; then
                log_fail "Must not have $VDEVID_CONF_ETC file present on system"
        fi
        EXTRA_ZEDLETS=$@

        # Create a symlink for /etc/zfs/vdev_id.conf file.
        log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC

        # Setup minimal ZED configuration.  Individual test cases should
        # add additional ZEDLETs as needed for their specific test.
        log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
        log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR

        # Scripts must only be user writable.
        if [[ -n "$EXTRA_ZEDLETS" ]] ; then
                saved_umask=$(umask)
                log_must umask 0022
                for i in $EXTRA_ZEDLETS ; do
                        log_must cp ${ZEDLET_LIBEXEC_DIR}/$i $ZEDLET_DIR
                done
                log_must umask $saved_umask
        fi

        # Customize the zed.rc file to enable the full debug log.
        log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
        echo "ZED_DEBUG_LOG=$ZED_DEBUG_LOG" >>$ZEDLET_DIR/zed.rc

}

#
# Cleanup custom ZED environment.
#
# $@ Optional list of zedlets to remove from our test zed.d directory.
function zed_cleanup
{
        if ! is_linux; then
                return
        fi

        for extra_zedlet; do
                log_must rm -f ${ZEDLET_DIR}/$extra_zedlet
        done
        log_must rm -fd ${ZEDLET_DIR}/zed.rc ${ZEDLET_DIR}/zed-functions.sh ${ZEDLET_DIR}/all-syslog.sh ${ZEDLET_DIR}/all-debug.sh ${ZEDLET_DIR}/state \
                        $ZED_LOG $ZED_DEBUG_LOG $VDEVID_CONF_ETC $VDEVID_CONF \
                        $ZEDLET_DIR
}

#
# Check if ZED is currently running; if so, returns PIDs
#
function zed_check
{
        if ! is_linux; then
                return
        fi
        zedpids="$(pgrep -x zed)"
        zedpids2="$(pgrep -x lt-zed)"
        echo ${zedpids} ${zedpids2}
}

#
# Check if ZED is currently running, if not start ZED.
#
function zed_start
{
        if ! is_linux; then
                return
        fi

        # ZEDLET_DIR=/var/tmp/zed
        if [[ ! -d $ZEDLET_DIR ]]; then
                log_must mkdir $ZEDLET_DIR
        fi

        # Verify the ZED is not already running.
        zedpids=$(zed_check)
        if [ -n "$zedpids" ]; then
                # We never, ever, really want it to just keep going if zed
                # is already running - usually this implies our test cases
                # will break very strangely because whatever we wanted to
                # configure zed for won't be listening to our changes in the
                # tmpdir
                log_fail "ZED already running - ${zedpids}"
        else
                log_note "Starting ZED"
                # run ZED in the background and redirect foreground logging
                # output to $ZED_LOG.
                log_must truncate -s 0 $ZED_DEBUG_LOG
                log_must eval "zed -vF -d $ZEDLET_DIR -P $PATH" \
                    "-s $ZEDLET_DIR/state -j 1 2>$ZED_LOG &"
        fi

        return 0
}

#
# Kill ZED process
#
function zed_stop
{
        if ! is_linux; then
                return ""
        fi

        log_note "Stopping ZED"
        while true; do
                zedpids=$(zed_check)
                [ ! -n "$zedpids" ] && break

                log_must kill $zedpids
                sleep 1
        done
        return 0
}

#
# Drain all zevents
#
function zed_events_drain
{
        while [ $(zpool events -H | wc -l) -ne 0 ]; do
                sleep 1
                zpool events -c >/dev/null
        done
}

# Set a variable in zed.rc to something, un-commenting it in the process.
#
# $1 variable
# $2 value
function zed_rc_set
{
        var="$1"
        val="$2"
        # Remove the line
        cmd="'/$var/d'"
        eval sed -i $cmd $ZEDLET_DIR/zed.rc

        # Add it at the end
        echo "$var=$val" >> $ZEDLET_DIR/zed.rc
}


#
# Check is provided device is being active used as a swap device.
#
function is_swap_inuse
{
        typeset device=$1

        if [[ -z $device ]] ; then
                log_note "No device specified."
                return 1
        fi

        case "$UNAME" in
        Linux)
                swapon -s | grep -wq $(readlink -f $device)
                ;;
        FreeBSD)
                swapctl -l | grep -wq $device
                ;;
        *)
                swap -l | grep -wq $device
                ;;
        esac
}

#
# Setup a swap device using the provided device.
#
function swap_setup
{
        typeset swapdev=$1

        case "$UNAME" in
        Linux)
                log_must eval "mkswap $swapdev > /dev/null 2>&1"
                log_must swapon $swapdev
                ;;
        FreeBSD)
                log_must swapctl -a $swapdev
                ;;
        *)
    log_must swap -a $swapdev
                ;;
        esac

        return 0
}

#
# Cleanup a swap device on the provided device.
#
function swap_cleanup
{
        typeset swapdev=$1

        if is_swap_inuse $swapdev; then
                if is_linux; then
                        log_must swapoff $swapdev
                elif is_freebsd; then
                        log_must swapoff $swapdev
                else
                        log_must swap -d $swapdev
                fi
        fi

        return 0
}

#
# Set a global system tunable (64-bit value)
#
# $1 tunable name (use a NAME defined in tunables.cfg)
# $2 tunable values
#
function set_tunable64
{
        set_tunable_impl "$1" "$2" Z
}

#
# Set a global system tunable (32-bit value)
#
# $1 tunable name (use a NAME defined in tunables.cfg)
# $2 tunable values
#
function set_tunable32
{
        set_tunable_impl "$1" "$2" W
}

function set_tunable_impl
{
        typeset name="$1"
        typeset value="$2"
        typeset mdb_cmd="$3"

        eval "typeset tunable=\$$name"
        case "$tunable" in
        UNSUPPORTED)
                log_unsupported "Tunable '$name' is unsupported on $UNAME"
                ;;
        "")
                log_fail "Tunable '$name' must be added to tunables.cfg"
                ;;
        *)
                ;;
        esac

        [[ -z "$value" ]] && return 1
        [[ -z "$mdb_cmd" ]] && return 1

        case "$UNAME" in
        Linux)
                typeset zfs_tunables="/sys/module/zfs/parameters"
                echo "$value" >"$zfs_tunables/$tunable"
                ;;
        FreeBSD)
                sysctl vfs.zfs.$tunable=$value
                ;;
        SunOS)
                echo "${tunable}/${mdb_cmd}0t${value}" | mdb -kw
                ;;
        esac
}

function save_tunable
{
        [[ ! -d $TEST_BASE_DIR ]] && return 1
        [[ -e $TEST_BASE_DIR/tunable-$1 ]] && return 2
        echo "$(get_tunable """$1""")" > "$TEST_BASE_DIR"/tunable-"$1"
}

function restore_tunable
{
        [[ ! -e $TEST_BASE_DIR/tunable-$1 ]] && return 1
        val="$(cat $TEST_BASE_DIR/tunable-"""$1""")"
        set_tunable64 "$1" "$val"
        rm $TEST_BASE_DIR/tunable-$1
}

#
# Get a global system tunable
#
# $1 tunable name (use a NAME defined in tunables.cfg)
#
function get_tunable
{
        get_tunable_impl "$1"
}

function get_tunable_impl
{
        typeset name="$1"
        typeset module="${2:-zfs}"
        typeset check_only="$3"

        eval "typeset tunable=\$$name"
        case "$tunable" in
        UNSUPPORTED)
                if [ -z "$check_only" ] ; then
                        log_unsupported "Tunable '$name' is unsupported on $UNAME"
                else
                        return 1
                fi
                ;;
        "")
                if [ -z "$check_only" ] ; then
                        log_fail "Tunable '$name' must be added to tunables.cfg"
                else
                        return 1
                fi
                ;;
        *)
                ;;
        esac

        case "$UNAME" in
        Linux)
                typeset zfs_tunables="/sys/module/$module/parameters"
                cat $zfs_tunables/$tunable
                ;;
        FreeBSD)
                sysctl -n vfs.zfs.$tunable
                ;;
        SunOS)
                [[ "$module" -eq "zfs" ]] || return 1
                ;;
        esac
}

# Does a tunable exist?
#
# $1: Tunable name
function tunable_exists
{
        get_tunable_impl $1 "zfs" 1
}

#
# Compute MD5 digest for given file or stdin if no file given.
# Note: file path must not contain spaces
#
function md5digest
{
        typeset file=$1

        case "$UNAME" in
        FreeBSD)
                md5 -q $file
                ;;
        *)
                typeset sum _
                read -r sum _ < <(md5sum -b $file)
                echo $sum
                ;;
        esac
}

#
# Compute SHA256 digest for given file or stdin if no file given.
# Note: file path must not contain spaces
#
function sha256digest
{
        typeset file=$1

        case "$UNAME" in
        FreeBSD)
                sha256 -q $file
                ;;
        *)
                typeset sum _
                read -r sum _ < <(sha256sum -b $file)
                echo $sum
                ;;
        esac
}

function new_fs #<args>
{
        case "$UNAME" in
        FreeBSD)
                newfs "$@"
                ;;
        *)
                echo y | newfs -v "$@"
                ;;
        esac
}

function stat_size #<path>
{
        typeset path=$1

        case "$UNAME" in
        FreeBSD)
                stat -f %z "$path"
                ;;
        *)
                stat -c %s "$path"
                ;;
        esac
}

function stat_mtime #<path>
{
        typeset path=$1

        case "$UNAME" in
        FreeBSD)
                stat -f %m "$path"
                ;;
        *)
                stat -c %Y "$path"
                ;;
        esac
}

function stat_ctime #<path>
{
        typeset path=$1

        case "$UNAME" in
        FreeBSD)
                stat -f %c "$path"
                ;;
        *)
                stat -c %Z "$path"
                ;;
        esac
}

function stat_crtime #<path>
{
        typeset path=$1

        case "$UNAME" in
        FreeBSD)
                stat -f %B "$path"
                ;;
        *)
                stat -c %W "$path"
                ;;
        esac
}

function stat_generation #<path>
{
        typeset path=$1

        case "$UNAME" in
        Linux)
                getversion "${path}"
                ;;
        *)
                stat -f %v "${path}"
                ;;
        esac
}

# Run a command as if it was being run in a TTY.
#
# Usage:
#
#    faketty command
#
function faketty
{
    if is_freebsd; then
        script -q /dev/null env "$@"
    else
        script --return --quiet -c "$*" /dev/null
    fi
}

#
# Produce a random permutation of the integers in a given range (inclusive).
#
function range_shuffle # begin end
{
        typeset -i begin=$1
        typeset -i end=$2

        seq ${begin} ${end} | sort -R
}

#
# Cross-platform xattr helpers
#

function get_xattr # name path
{
        typeset name=$1
        typeset path=$2

        case "$UNAME" in
        FreeBSD)
                getextattr -qq user "${name}" "${path}"
                ;;
        *)
                attr -qg "${name}" "${path}"
                ;;
        esac
}

function set_xattr # name value path
{
        typeset name=$1
        typeset value=$2
        typeset path=$3

        case "$UNAME" in
        FreeBSD)
                setextattr user "${name}" "${value}" "${path}"
                ;;
        *)
                attr -qs "${name}" -V "${value}" "${path}"
                ;;
        esac
}

function set_xattr_stdin # name value
{
        typeset name=$1
        typeset path=$2

        case "$UNAME" in
        FreeBSD)
                setextattr -i user "${name}" "${path}"
                ;;
        *)
                attr -qs "${name}" "${path}"
                ;;
        esac
}

function rm_xattr # name path
{
        typeset name=$1
        typeset path=$2

        case "$UNAME" in
        FreeBSD)
                rmextattr -q user "${name}" "${path}"
                ;;
        *)
                attr -qr "${name}" "${path}"
                ;;
        esac
}

function ls_xattr # path
{
        typeset path=$1

        case "$UNAME" in
        FreeBSD)
                lsextattr -qq user "${path}"
                ;;
        *)
                attr -ql "${path}"
                ;;
        esac
}

function kstat # stat flags?
{
        typeset stat=$1
        typeset flags=${2-"-n"}

        case "$UNAME" in
        FreeBSD)
                sysctl $flags kstat.zfs.misc.$stat
                ;;
        Linux)
                cat "/proc/spl/kstat/zfs/$stat" 2>/dev/null
                ;;
        *)
                false
                ;;
        esac
}

function get_arcstat # stat
{
        typeset stat=$1

        case "$UNAME" in
        FreeBSD)
                kstat arcstats.$stat
                ;;
        Linux)
                kstat arcstats | awk "/$stat/"' { print $3 }'
                ;;
        *)
                false
                ;;
        esac
}

function punch_hole # offset length file
{
        typeset offset=$1
        typeset length=$2
        typeset file=$3

        case "$UNAME" in
        FreeBSD)
                truncate -d -o $offset -l $length "$file"
                ;;
        Linux)
                fallocate --punch-hole --offset $offset --length $length "$file"
                ;;
        *)
                false
                ;;
        esac
}

function zero_range # offset length file
{
        typeset offset=$1
        typeset length=$2
        typeset file=$3

        case "$UNAME" in
        Linux)
                fallocate --zero-range --offset $offset --length $length "$file"
                ;;
        *)
                false
                ;;
        esac
}

#
# Wait for the specified arcstat to reach non-zero quiescence.
# If echo is 1 echo the value after reaching quiescence, otherwise
# if echo is 0 print the arcstat we are waiting on.
#
function arcstat_quiescence # stat echo
{
        typeset stat=$1
        typeset echo=$2
        typeset do_once=true

        if [[ $echo -eq 0 ]]; then
                echo "Waiting for arcstat $1 quiescence."
        fi

        while $do_once || [ $stat1 -ne $stat2 ] || [ $stat2 -eq 0 ]; do
                typeset stat1=$(get_arcstat $stat)
                sleep 0.5
                typeset stat2=$(get_arcstat $stat)
                do_once=false
        done

        if [[ $echo -eq 1 ]]; then
                echo $stat2
        fi
}

function arcstat_quiescence_noecho # stat
{
        typeset stat=$1
        arcstat_quiescence $stat 0
}

function arcstat_quiescence_echo # stat
{
        typeset stat=$1
        arcstat_quiescence $stat 1
}

#
# Given an array of pids, wait until all processes
# have completed and check their return status.
#
function wait_for_children #children
{
        rv=0
        children=("$@")
        for child in "${children[@]}"
        do
                child_exit=0
                wait ${child} || child_exit=$?
                if [ $child_exit -ne 0 ]; then
                        echo "child ${child} failed with ${child_exit}"
                        rv=1
                fi
        done
        return $rv
}

#
# Compare two directory trees recursively in a manner similar to diff(1), but
# using rsync. If there are any discrepancies, a summary of the differences are
# output and a non-zero error is returned.
#
# If you're comparing a directory after a ZIL replay, you should set
# LIBTEST_DIFF_ZIL_REPLAY=1 or use replay_directory_diff which will cause
# directory_diff to ignore mtime changes (the ZIL replay won't fix up mtime
# information).
#
function directory_diff # dir_a dir_b
{
        dir_a="$1"
        dir_b="$2"
        zil_replay="${LIBTEST_DIFF_ZIL_REPLAY:-0}"

        # If one of the directories doesn't exist, return 2. This is to match the
        # semantics of diff.
        if ! [ -d "$dir_a" -a -d "$dir_b" ]; then
                return 2
        fi

        # Run rsync with --dry-run --itemize-changes to get something akin to diff
        # output, but rsync is far more thorough in detecting differences (diff
        # doesn't compare file metadata, and cannot handle special files).
        #
        # Also make sure to filter out non-user.* xattrs when comparing. On
        # SELinux-enabled systems the copied tree will probably have different
        # SELinux labels.
        args=("-nicaAHX" '--filter=-x! user.*' "--delete")

        # NOTE: Quite a few rsync builds do not support --crtimes which would be
        # necessary to verify that creation times are being maintained properly.
        # Unfortunately because of this we cannot use it unconditionally but we can
        # check if this rsync build supports it and use it then. This check is
        # based on the same check in the rsync test suite (testsuite/crtimes.test).
        #
        # We check ctimes even with zil_replay=1 because the ZIL does store
        # creation times and we should make sure they match (if the creation times
        # do not match there is a "c" entry in one of the columns).
        if rsync --version | grep -q "[, ] crtimes"; then
                args+=("--crtimes")
        else
                log_note "This rsync package does not support --crtimes (-N)."
        fi

        # If we are testing a ZIL replay, we need to ignore timestamp changes.
        # Unfortunately --no-times doesn't do what we want -- it will still tell
        # you if the timestamps don't match but rsync will set the timestamps to
        # the current time (leading to an itemised change entry). It's simpler to
        # just filter out those lines.
        if [ "$zil_replay" -eq 0 ]; then
                filter=("cat")
        else
                # Different rsync versions have different numbers of columns. So just
                # require that aside from the first two, all other columns must be
                # blank (literal ".") or a timestamp field ("[tT]").
                filter=("grep" "-v" '^\..[.Tt]\+ ')
        fi

        diff="$(rsync "${args[@]}" "$dir_a/" "$dir_b/" | "${filter[@]}")"
        rv=0
        if [ -n "$diff" ]; then
                echo "$diff"
                rv=1
        fi
        return $rv
}

#
# Compare two directory trees recursively, without checking whether the mtimes
# match (creation times will be checked if the available rsync binary supports
# it). This is necessary for ZIL replay checks (because the ZIL does not
# contain mtimes and thus after a ZIL replay, mtimes won't match).
#
# This is shorthand for LIBTEST_DIFF_ZIL_REPLAY=1 directory_diff <...>.
#
function replay_directory_diff # dir_a dir_b
{
        LIBTEST_DIFF_ZIL_REPLAY=1 directory_diff "$@"
}

#
# Put coredumps into $1/core.{basename}
#
# Output must be saved and passed to pop_coredump_pattern on cleanup
#
function push_coredump_pattern # dir
{
        ulimit -c unlimited
        case "$UNAME" in
        Linux)
                cat /proc/sys/kernel/core_pattern /proc/sys/kernel/core_uses_pid
                echo "$1/core.%e" >/proc/sys/kernel/core_pattern &&
                    echo 0 >/proc/sys/kernel/core_uses_pid
                ;;
        FreeBSD)
                sysctl -n kern.corefile
                sysctl kern.corefile="$1/core.%N" >/dev/null
                ;;
        *)
                # Nothing to output – set only for this shell
                coreadm -p "$1/core.%f"
                ;;
        esac
}

#
# Put coredumps back into the default location
#
function pop_coredump_pattern
{
        [ -s "$1" ] || return 0
        case "$UNAME" in
        Linux)
                typeset pat pid
                { read -r pat; read -r pid; } < "$1"
                echo "$pat" >/proc/sys/kernel/core_pattern &&
                    echo "$pid" >/proc/sys/kernel/core_uses_pid
                ;;
        FreeBSD)
                sysctl kern.corefile="$(<"$1")" >/dev/null
                ;;
        esac
}