tests/zfs-tests/include/libtest.shlib

   1 #!/bin/ksh -p
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22
  23 #
  24 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  25 # Use is subject to license terms.
  26 #
  27
  28 #
  29 # Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  30 #
  31
  32 . ${STF_TOOLS}/include/logapi.shlib
  33
  34 # Determine if this is a Linux test system
  35 #
  36 # Return 0 if platform Linux, 1 if otherwise
  37
  38 function is_linux
  39 {
  40         if [[ $($UNAME -o) == "GNU/Linux" ]]; then
  41                 return 0
  42         else
  43                 return 1
  44         fi
  45 }
  46
  47 # Determine if this is a 32-bit system
  48 #
  49 # Return 0 if platform is 32-bit, 1 if otherwise
  50
  51 function is_32bit
  52 {
  53         if [[ $(getconf LONG_BIT) == "32" ]]; then
  54                 return 0
  55         else
  56                 return 1
  57         fi
  58 }
  59
  60 # Determine if kmemleak is enabled
  61 #
  62 # Return 0 if kmemleak is enabled, 1 if otherwise
  63
  64 function is_kmemleak
  65 {
  66         if is_linux && [[ -e /sys/kernel/debug/kmemleak ]]; then
  67                 return 0
  68         else
  69                 return 1
  70         fi
  71 }
  72
  73 # Determine whether a dataset is mounted
  74 #
  75 # $1 dataset name
  76 # $2 filesystem type; optional - defaulted to zfs
  77 #
  78 # Return 0 if dataset is mounted; 1 if unmounted; 2 on error
  79
  80 function ismounted
  81 {
  82         typeset fstype=$2
  83         [[ -z $fstype ]] && fstype=zfs
  84         typeset out dir name ret
  85
  86         case $fstype in
  87                 zfs)
  88                         if [[ "$1" == "/"* ]] ; then
  89                                 for out in $($ZFS mount | $AWK '{print $2}'); do
  90                                         [[ $1 == $out ]] && return 0
  91                                 done
  92                         else
  93                                 for out in $($ZFS mount | $AWK '{print $1}'); do
  94                                         [[ $1 == $out ]] && return 0
  95                                 done
  96                         fi
  97                 ;;
  98                 ufs|nfs)
  99                         out=$($DF -F $fstype $1 2>/dev/null)
 100                         ret=$?
 101                         (($ret != 0)) && return $ret
 102
 103                         dir=${out%%\(*}
 104                         dir=${dir%% *}
 105                         name=${out##*\(}
 106                         name=${name%%\)*}
 107                         name=${name%% *}
 108
 109                         [[ "$1" == "$dir" || "$1" == "$name" ]] && return 0
 110                 ;;
 111                 ext2)
 112                         out=$($DF -t $fstype $1 2>/dev/null)
 113                         return $?
 114                 ;;
 115                 zvol)
 116                         if [[ -L "$ZVOL_DEVDIR/$1" ]]; then
 117                                 link=$(readlink -f $ZVOL_DEVDIR/$1)
 118                                 [[ -n "$link" ]] && \
 119                                         $MOUNT | $GREP -q "^$link" && \
 120                                                 return 0
 121                         fi
 122                 ;;
 123         esac
 124
 125         return 1
 126 }
 127
 128 # Return 0 if a dataset is mounted; 1 otherwise
 129 #
 130 # $1 dataset name
 131 # $2 filesystem type; optional - defaulted to zfs
 132
 133 function mounted
 134 {
 135         ismounted $1 $2
 136         (($? == 0)) && return 0
 137         return 1
 138 }
 139
 140 # Return 0 if a dataset is unmounted; 1 otherwise
 141 #
 142 # $1 dataset name
 143 # $2 filesystem type; optional - defaulted to zfs
 144
 145 function unmounted
 146 {
 147         ismounted $1 $2
 148         (($? == 1)) && return 0
 149         return 1
 150 }
 151
 152 # split line on ","
 153 #
 154 # $1 - line to split
 155
 156 function splitline
 157 {
 158         $ECHO $1 | $SED "s/,/ /g"
 159 }
 160
 161 function default_setup
 162 {
 163         default_setup_noexit "$@"
 164
 165         log_pass
 166 }
 167
 168 #
 169 # Given a list of disks, setup storage pools and datasets.
 170 #
 171 function default_setup_noexit
 172 {
 173         typeset disklist=$1
 174         typeset container=$2
 175         typeset volume=$3
 176         log_note begin default_setup_noexit
 177
 178         if is_global_zone; then
 179                 if poolexists $TESTPOOL ; then
 180                         destroy_pool $TESTPOOL
 181                 fi
 182                 [[ -d /$TESTPOOL ]] && $RM -rf /$TESTPOOL
 183                 log_note creating pool $TESTPOOL $disklist
 184                 log_must $ZPOOL create -f $TESTPOOL $disklist
 185         else
 186                 reexport_pool
 187         fi
 188
 189         $RM -rf $TESTDIR  || log_unresolved Could not remove $TESTDIR
 190         $MKDIR -p $TESTDIR || log_unresolved Could not create $TESTDIR
 191
 192         log_must $ZFS create $TESTPOOL/$TESTFS
 193         log_must $ZFS set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
 194
 195         if [[ -n $container ]]; then
 196                 $RM -rf $TESTDIR1  || \
 197                         log_unresolved Could not remove $TESTDIR1
 198                 $MKDIR -p $TESTDIR1 || \
 199                         log_unresolved Could not create $TESTDIR1
 200
 201                 log_must $ZFS create $TESTPOOL/$TESTCTR
 202                 log_must $ZFS set canmount=off $TESTPOOL/$TESTCTR
 203                 log_must $ZFS create $TESTPOOL/$TESTCTR/$TESTFS1
 204                 log_must $ZFS set mountpoint=$TESTDIR1 \
 205                     $TESTPOOL/$TESTCTR/$TESTFS1
 206         fi
 207
 208         if [[ -n $volume ]]; then
 209                 if is_global_zone ; then
 210                         log_must $ZFS create -V $VOLSIZE $TESTPOOL/$TESTVOL
 211                         block_device_wait
 212                 else
 213                         log_must $ZFS create $TESTPOOL/$TESTVOL
 214                 fi
 215         fi
 216 }
 217
 218 #
 219 # Given a list of disks, setup a storage pool, file system and
 220 # a container.
 221 #
 222 function default_container_setup
 223 {
 224         typeset disklist=$1
 225
 226         default_setup "$disklist" "true"
 227 }
 228
 229 #
 230 # Given a list of disks, setup a storage pool,file system
 231 # and a volume.
 232 #
 233 function default_volume_setup
 234 {
 235         typeset disklist=$1
 236
 237         default_setup "$disklist" "" "true"
 238 }
 239
 240 #
 241 # Given a list of disks, setup a storage pool,file system,
 242 # a container and a volume.
 243 #
 244 function default_container_volume_setup
 245 {
 246         typeset disklist=$1
 247
 248         default_setup "$disklist" "true" "true"
 249 }
 250
 251 #
 252 # Create a snapshot on a filesystem or volume. Defaultly create a snapshot on
 253 # filesystem
 254 #
 255 # $1 Existing filesystem or volume name. Default, $TESTFS
 256 # $2 snapshot name. Default, $TESTSNAP
 257 #
 258 function create_snapshot
 259 {
 260         typeset fs_vol=${1:-$TESTFS}
 261         typeset snap=${2:-$TESTSNAP}
 262
 263         [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined."
 264         [[ -z $snap ]] && log_fail "Snapshot's name is undefined."
 265
 266         if snapexists $fs_vol@$snap; then
 267                 log_fail "$fs_vol@$snap already exists."
 268         fi
 269         datasetexists $fs_vol || \
 270                 log_fail "$fs_vol must exist."
 271
 272         log_must $ZFS snapshot $fs_vol@$snap
 273 }
 274
 275 #
 276 # Create a clone from a snapshot, default clone name is $TESTCLONE.
 277 #
 278 # $1 Existing snapshot, $TESTPOOL/$TESTFS@$TESTSNAP is default.
 279 # $2 Clone name, $TESTPOOL/$TESTCLONE is default.
 280 #
 281 function create_clone   # snapshot clone
 282 {
 283         typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}
 284         typeset clone=${2:-$TESTPOOL/$TESTCLONE}
 285
 286         [[ -z $snap ]] && \
 287                 log_fail "Snapshot name is undefined."
 288         [[ -z $clone ]] && \
 289                 log_fail "Clone name is undefined."
 290
 291         log_must $ZFS clone $snap $clone
 292 }
 293
 294 #
 295 # Create a bookmark of the given snapshot.  Defaultly create a bookmark on
 296 # filesystem.
 297 #
 298 # $1 Existing filesystem or volume name. Default, $TESTFS
 299 # $2 Existing snapshot name. Default, $TESTSNAP
 300 # $3 bookmark name. Default, $TESTBKMARK
 301 #
 302 function create_bookmark
 303 {
 304         typeset fs_vol=${1:-$TESTFS}
 305         typeset snap=${2:-$TESTSNAP}
 306         typeset bkmark=${3:-$TESTBKMARK}
 307
 308         [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined."
 309         [[ -z $snap ]] && log_fail "Snapshot's name is undefined."
 310         [[ -z $bkmark ]] && log_fail "Bookmark's name is undefined."
 311
 312         if bkmarkexists $fs_vol#$bkmark; then
 313                 log_fail "$fs_vol#$bkmark already exists."
 314         fi
 315         datasetexists $fs_vol || \
 316                 log_fail "$fs_vol must exist."
 317         snapexists $fs_vol@$snap || \
 318                 log_fail "$fs_vol@$snap must exist."
 319
 320         log_must $ZFS bookmark $fs_vol@$snap $fs_vol#$bkmark
 321 }
 322
 323 function default_mirror_setup
 324 {
 325         default_mirror_setup_noexit $1 $2 $3
 326
 327         log_pass
 328 }
 329
 330 #
 331 # Given a pair of disks, set up a storage pool and dataset for the mirror
 332 # @parameters: $1 the primary side of the mirror
 333 #   $2 the secondary side of the mirror
 334 # @uses: ZPOOL ZFS TESTPOOL TESTFS
 335 function default_mirror_setup_noexit
 336 {
 337         readonly func="default_mirror_setup_noexit"
 338         typeset primary=$1
 339         typeset secondary=$2
 340
 341         [[ -z $primary ]] && \
 342                 log_fail "$func: No parameters passed"
 343         [[ -z $secondary ]] && \
 344                 log_fail "$func: No secondary partition passed"
 345         [[ -d /$TESTPOOL ]] && $RM -rf /$TESTPOOL
 346         log_must $ZPOOL create -f $TESTPOOL mirror $@
 347         log_must $ZFS create $TESTPOOL/$TESTFS
 348         log_must $ZFS set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
 349 }
 350
 351 #
 352 # create a number of mirrors.
 353 # We create a number($1) of 2 way mirrors using the pairs of disks named
 354 # on the command line. These mirrors are *not* mounted
 355 # @parameters: $1 the number of mirrors to create
 356 #  $... the devices to use to create the mirrors on
 357 # @uses: ZPOOL ZFS TESTPOOL
 358 function setup_mirrors
 359 {
 360         typeset -i nmirrors=$1
 361
 362         shift
 363         while ((nmirrors > 0)); do
 364                 log_must test -n "$1" -a -n "$2"
 365                 [[ -d /$TESTPOOL$nmirrors ]] && $RM -rf /$TESTPOOL$nmirrors
 366                 log_must $ZPOOL create -f $TESTPOOL$nmirrors mirror $1 $2
 367                 shift 2
 368                 ((nmirrors = nmirrors - 1))
 369         done
 370 }
 371
 372 #
 373 # create a number of raidz pools.
 374 # We create a number($1) of 2 raidz pools  using the pairs of disks named
 375 # on the command line. These pools are *not* mounted
 376 # @parameters: $1 the number of pools to create
 377 #  $... the devices to use to create the pools on
 378 # @uses: ZPOOL ZFS TESTPOOL
 379 function setup_raidzs
 380 {
 381         typeset -i nraidzs=$1
 382
 383         shift
 384         while ((nraidzs > 0)); do
 385                 log_must test -n "$1" -a -n "$2"
 386                 [[ -d /$TESTPOOL$nraidzs ]] && $RM -rf /$TESTPOOL$nraidzs
 387                 log_must $ZPOOL create -f $TESTPOOL$nraidzs raidz $1 $2
 388                 shift 2
 389                 ((nraidzs = nraidzs - 1))
 390         done
 391 }
 392
 393 #
 394 # Destroy the configured testpool mirrors.
 395 # the mirrors are of the form ${TESTPOOL}{number}
 396 # @uses: ZPOOL ZFS TESTPOOL
 397 function destroy_mirrors
 398 {
 399         default_cleanup_noexit
 400
 401         log_pass
 402 }
 403
 404 #
 405 # Given a minimum of two disks, set up a storage pool and dataset for the raid-z
 406 # $1 the list of disks
 407 #
 408 function default_raidz_setup
 409 {
 410         typeset disklist="$*"
 411         disks=(${disklist[*]})
 412
 413         if [[ ${#disks[*]} -lt 2 ]]; then
 414                 log_fail "A raid-z requires a minimum of two disks."
 415         fi
 416
 417         [[ -d /$TESTPOOL ]] && $RM -rf /$TESTPOOL
 418         log_must $ZPOOL create -f $TESTPOOL raidz $1 $2 $3
 419         log_must $ZFS create $TESTPOOL/$TESTFS
 420         log_must $ZFS set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
 421
 422         log_pass
 423 }
 424
 425 #
 426 # Common function used to cleanup storage pools and datasets.
 427 #
 428 # Invoked at the start of the test suite to ensure the system
 429 # is in a known state, and also at the end of each set of
 430 # sub-tests to ensure errors from one set of tests doesn't
 431 # impact the execution of the next set.
 432
 433 function default_cleanup
 434 {
 435         default_cleanup_noexit
 436
 437         log_pass
 438 }
 439
 440 function default_cleanup_noexit
 441 {
 442         typeset exclude=""
 443         typeset pool=""
 444         #
 445         # Destroying the pool will also destroy any
 446         # filesystems it contains.
 447         #
 448         if is_global_zone; then
 449                 $ZFS unmount -a > /dev/null 2>&1
 450                 [[ -z "$KEEP" ]] && KEEP="rpool"
 451                 exclude=`eval $ECHO \"'(${KEEP})'\"`
 452                 ALL_POOLS=$($ZPOOL list -H -o name \
 453                     | $GREP -v "$NO_POOLS" | $EGREP -vw "$exclude")
 454                 # Here, we loop through the pools we're allowed to
 455                 # destroy, only destroying them if it's safe to do
 456                 # so.
 457                 while [ ! -z ${ALL_POOLS} ]
 458                 do
 459                         for pool in ${ALL_POOLS}
 460                         do
 461                                 if safe_to_destroy_pool $pool ;
 462                                 then
 463                                         destroy_pool $pool
 464                                 fi
 465                                 ALL_POOLS=$($ZPOOL list -H -o name \
 466                                     | $GREP -v "$NO_POOLS" \
 467                                     | $EGREP -v "$exclude")
 468                         done
 469                 done
 470
 471                 $ZFS mount -a
 472         else
 473                 typeset fs=""
 474                 for fs in $($ZFS list -H -o name \
 475                     | $GREP "^$ZONE_POOL/$ZONE_CTR[01234]/"); do
 476                         datasetexists $fs && \
 477                                 log_must $ZFS destroy -Rf $fs
 478                 done
 479
 480                 # Need cleanup here to avoid garbage dir left.
 481                 for fs in $($ZFS list -H -o name); do
 482                         [[ $fs == /$ZONE_POOL ]] && continue
 483                         [[ -d $fs ]] && log_must $RM -rf $fs/*
 484                 done
 485
 486                 #
 487                 # Reset the $ZONE_POOL/$ZONE_CTR[01234] file systems property to
 488                 # the default value
 489                 #
 490                 for fs in $($ZFS list -H -o name); do
 491                         if [[ $fs == $ZONE_POOL/$ZONE_CTR[01234] ]]; then
 492                                 log_must $ZFS set reservation=none $fs
 493                                 log_must $ZFS set recordsize=128K $fs
 494                                 log_must $ZFS set mountpoint=/$fs $fs
 495                                 typeset enc=""
 496                                 enc=$(get_prop encryption $fs)
 497                                 if [[ $? -ne 0 ]] || [[ -z "$enc" ]] || \
 498                                         [[ "$enc" == "off" ]]; then
 499                                         log_must $ZFS set checksum=on $fs
 500                                 fi
 501                                 log_must $ZFS set compression=off $fs
 502                                 log_must $ZFS set atime=on $fs
 503                                 log_must $ZFS set devices=off $fs
 504                                 log_must $ZFS set exec=on $fs
 505                                 log_must $ZFS set setuid=on $fs
 506                                 log_must $ZFS set readonly=off $fs
 507                                 log_must $ZFS set snapdir=hidden $fs
 508                                 log_must $ZFS set aclmode=groupmask $fs
 509                                 log_must $ZFS set aclinherit=secure $fs
 510                         fi
 511                 done
 512         fi
 513
 514         [[ -d $TESTDIR ]] && \
 515                 log_must $RM -rf $TESTDIR
 516
 517         disk1=${DISKS%% *}
 518         if is_mpath_device $disk1; then
 519                 delete_partitions
 520         fi
 521 }
 522
 523
 524 #
 525 # Common function used to cleanup storage pools, file systems
 526 # and containers.
 527 #
 528 function default_container_cleanup
 529 {
 530         if ! is_global_zone; then
 531                 reexport_pool
 532         fi
 533
 534         ismounted $TESTPOOL/$TESTCTR/$TESTFS1
 535         [[ $? -eq 0 ]] && \
 536             log_must $ZFS unmount $TESTPOOL/$TESTCTR/$TESTFS1
 537
 538         datasetexists $TESTPOOL/$TESTCTR/$TESTFS1 && \
 539             log_must $ZFS destroy -R $TESTPOOL/$TESTCTR/$TESTFS1
 540
 541         datasetexists $TESTPOOL/$TESTCTR && \
 542             log_must $ZFS destroy -Rf $TESTPOOL/$TESTCTR
 543
 544         [[ -e $TESTDIR1 ]] && \
 545             log_must $RM -rf $TESTDIR1 > /dev/null 2>&1
 546
 547         default_cleanup
 548 }
 549
 550 #
 551 # Common function used to cleanup snapshot of file system or volume. Default to
 552 # delete the file system's snapshot
 553 #
 554 # $1 snapshot name
 555 #
 556 function destroy_snapshot
 557 {
 558         typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}
 559
 560         if ! snapexists $snap; then
 561                 log_fail "'$snap' does not existed."
 562         fi
 563
 564         #
 565         # For the sake of the value which come from 'get_prop' is not equal
 566         # to the really mountpoint when the snapshot is unmounted. So, firstly
 567         # check and make sure this snapshot's been mounted in current system.
 568         #
 569         typeset mtpt=""
 570         if ismounted $snap; then
 571                 mtpt=$(get_prop mountpoint $snap)
 572                 (($? != 0)) && \
 573                         log_fail "get_prop mountpoint $snap failed."
 574         fi
 575
 576         log_must $ZFS destroy $snap
 577         [[ $mtpt != "" && -d $mtpt ]] && \
 578                 log_must $RM -rf $mtpt
 579 }
 580
 581 #
 582 # Common function used to cleanup clone.
 583 #
 584 # $1 clone name
 585 #
 586 function destroy_clone
 587 {
 588         typeset clone=${1:-$TESTPOOL/$TESTCLONE}
 589
 590         if ! datasetexists $clone; then
 591                 log_fail "'$clone' does not existed."
 592         fi
 593
 594         # With the same reason in destroy_snapshot
 595         typeset mtpt=""
 596         if ismounted $clone; then
 597                 mtpt=$(get_prop mountpoint $clone)
 598                 (($? != 0)) && \
 599                         log_fail "get_prop mountpoint $clone failed."
 600         fi
 601
 602         log_must $ZFS destroy $clone
 603         [[ $mtpt != "" && -d $mtpt ]] && \
 604                 log_must $RM -rf $mtpt
 605 }
 606
 607 #
 608 # Common function used to cleanup bookmark of file system or volume.  Default
 609 # to delete the file system's bookmark.
 610 #
 611 # $1 bookmark name
 612 #
 613 function destroy_bookmark
 614 {
 615         typeset bkmark=${1:-$TESTPOOL/$TESTFS#$TESTBKMARK}
 616
 617         if ! bkmarkexists $bkmark; then
 618                 log_fail "'$bkmarkp' does not existed."
 619         fi
 620
 621         log_must $ZFS destroy $bkmark
 622 }
 623
 624 # Return 0 if a snapshot exists; $? otherwise
 625 #
 626 # $1 - snapshot name
 627
 628 function snapexists
 629 {
 630         $ZFS list -H -t snapshot "$1" > /dev/null 2>&1
 631         return $?
 632 }
 633
 634 #
 635 # Return 0 if a bookmark exists; $? otherwise
 636 #
 637 # $1 - bookmark name
 638 #
 639 function bkmarkexists
 640 {
 641         $ZFS list -H -t bookmark "$1" > /dev/null 2>&1
 642         return $?
 643 }
 644
 645 #
 646 # Set a property to a certain value on a dataset.
 647 # Sets a property of the dataset to the value as passed in.
 648 # @param:
 649 #       $1 dataset who's property is being set
 650 #       $2 property to set
 651 #       $3 value to set property to
 652 # @return:
 653 #       0 if the property could be set.
 654 #       non-zero otherwise.
 655 # @use: ZFS
 656 #
 657 function dataset_setprop
 658 {
 659         typeset fn=dataset_setprop
 660
 661         if (($# < 3)); then
 662                 log_note "$fn: Insufficient parameters (need 3, had $#)"
 663                 return 1
 664         fi
 665         typeset output=
 666         output=$($ZFS set $2=$3 $1 2>&1)
 667         typeset rv=$?
 668         if ((rv != 0)); then
 669                 log_note "Setting property on $1 failed."
 670                 log_note "property $2=$3"
 671                 log_note "Return Code: $rv"
 672                 log_note "Output: $output"
 673                 return $rv
 674         fi
 675         return 0
 676 }
 677
 678 #
 679 # Assign suite defined dataset properties.
 680 # This function is used to apply the suite's defined default set of
 681 # properties to a dataset.
 682 # @parameters: $1 dataset to use
 683 # @uses: ZFS COMPRESSION_PROP CHECKSUM_PROP
 684 # @returns:
 685 #   0 if the dataset has been altered.
 686 #   1 if no pool name was passed in.
 687 #   2 if the dataset could not be found.
 688 #   3 if the dataset could not have it's properties set.
 689 #
 690 function dataset_set_defaultproperties
 691 {
 692         typeset dataset="$1"
 693
 694         [[ -z $dataset ]] && return 1
 695
 696         typeset confset=
 697         typeset -i found=0
 698         for confset in $($ZFS list); do
 699                 if [[ $dataset = $confset ]]; then
 700                         found=1
 701                         break
 702                 fi
 703         done
 704         [[ $found -eq 0 ]] && return 2
 705         if [[ -n $COMPRESSION_PROP ]]; then
 706                 dataset_setprop $dataset compression $COMPRESSION_PROP || \
 707                         return 3
 708                 log_note "Compression set to '$COMPRESSION_PROP' on $dataset"
 709         fi
 710         if [[ -n $CHECKSUM_PROP ]]; then
 711                 dataset_setprop $dataset checksum $CHECKSUM_PROP || \
 712                         return 3
 713                 log_note "Checksum set to '$CHECKSUM_PROP' on $dataset"
 714         fi
 715         return 0
 716 }
 717
 718 #
 719 # Check a numeric assertion
 720 # @parameter: $@ the assertion to check
 721 # @output: big loud notice if assertion failed
 722 # @use: log_fail
 723 #
 724 function assert
 725 {
 726         (($@)) || log_fail "$@"
 727 }
 728
 729 #
 730 # Function to format partition size of a disk
 731 # Given a disk cxtxdx reduces all partitions
 732 # to 0 size
 733 #
 734 function zero_partitions #<whole_disk_name>
 735 {
 736         typeset diskname=$1
 737         typeset i
 738
 739         if is_linux; then
 740                 log_must $FORMAT $DEV_DSKDIR/$diskname -s -- mklabel gpt
 741         else
 742                 for i in 0 1 3 4 5 6 7
 743                 do
 744                         set_partition $i "" 0mb $diskname
 745                 done
 746         fi
 747 }
 748
 749 #
 750 # Given a slice, size and disk, this function
 751 # formats the slice to the specified size.
 752 # Size should be specified with units as per
 753 # the `format` command requirements eg. 100mb 3gb
 754 #
 755 # NOTE: This entire interface is problematic for the Linux parted utilty
 756 # which requires the end of the partition to be specified.  It would be
 757 # best to retire this interface and replace it with something more flexible.
 758 # At the moment a best effort is made.
 759 #
 760 function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk_name>
 761 {
 762         typeset -i slicenum=$1
 763         typeset start=$2
 764         typeset size=$3
 765         typeset disk=$4
 766         [[ -z $slicenum || -z $size || -z $disk ]] && \
 767             log_fail "The slice, size or disk name is unspecified."
 768
 769         if is_linux; then
 770                 typeset size_mb=${size%%[mMgG]}
 771
 772                 size_mb=${size_mb%%[mMgG][bB]}
 773                 if [[ ${size:1:1} == 'g' ]]; then
 774                         ((size_mb = size_mb * 1024))
 775                 fi
 776
 777                 # Create GPT partition table when setting slice 0 or
 778                 # when the device doesn't already contain a GPT label.
 779                 $FORMAT $DEV_DSKDIR/$disk -s -- print 1 >/dev/null
 780                 typeset ret_val=$?
 781                 if [[ $slicenum -eq 0 || $ret_val -ne 0 ]]; then
 782                         log_must $FORMAT $DEV_DSKDIR/$disk -s -- mklabel gpt
 783                 fi
 784
 785                 # When no start is given align on the first cylinder.
 786                 if [[ -z "$start" ]]; then
 787                         start=1
 788                 fi
 789
 790                 # Determine the cylinder size for the device and using
 791                 # that calculate the end offset in cylinders.
 792                 typeset -i cly_size_kb=0
 793                 cly_size_kb=$($FORMAT -m $DEV_DSKDIR/$disk -s -- \
 794                         unit cyl print | $HEAD -3 | $TAIL -1 | \
 795                         $AWK -F '[:k.]' '{print $4}')
 796                 ((end = (size_mb * 1024 / cly_size_kb) + start))
 797
 798                 log_must $FORMAT $DEV_DSKDIR/$disk -s -- \
 799                     mkpart part$slicenum ${start}cyl ${end}cyl
 800
 801                 $BLOCKDEV --rereadpt $DEV_DSKDIR/$disk 2>/dev/null
 802                 block_device_wait
 803         else
 804                 typeset format_file=/var/tmp/format_in.$$
 805
 806                 $ECHO "partition" >$format_file
 807                 $ECHO "$slicenum" >> $format_file
 808                 $ECHO "" >> $format_file
 809                 $ECHO "" >> $format_file
 810                 $ECHO "$start" >> $format_file
 811                 $ECHO "$size" >> $format_file
 812                 $ECHO "label" >> $format_file
 813                 $ECHO "" >> $format_file
 814                 $ECHO "q" >> $format_file
 815                 $ECHO "q" >> $format_file
 816
 817                 $FORMAT -e -s -d $disk -f $format_file
 818         fi
 819         typeset ret_val=$?
 820         $RM -f $format_file
 821         [[ $ret_val -ne 0 ]] && \
 822             log_fail "Unable to format $disk slice $slicenum to $size"
 823         return 0
 824 }
 825
 826 #
 827 # Delete all partitions on all disks - this is specifically for the use of multipath
 828 # devices which currently can only be used in the test suite as raw/un-partitioned
 829 # devices (ie a zpool cannot be created on a whole mpath device that has partitions)
 830 #
 831 function delete_partitions
 832 {
 833         typeset -i j=1
 834
 835         if [[ -z $DISK_ARRAY_NUM ]]; then
 836                 DISK_ARRAY_NUM=$($ECHO ${DISKS} | $NAWK '{print NF}')
 837         fi
 838         if [[ -z $DISKSARRAY ]]; then
 839                 DISKSARRAY=$DISKS
 840         fi
 841
 842         if is_linux; then
 843                 if (( $DISK_ARRAY_NUM == 1 )); then
 844                         while ((j < MAX_PARTITIONS)); do
 845                                 $FORMAT $DEV_DSKDIR/$DISK -s rm $j > /dev/null 2>&1
 846                                 if (( $? == 1 )); then
 847                                         $LSBLK | $EGREP ${DISK}${SLICE_PREFIX}${j} > /dev/null
 848                                                 if (( $? == 1 )); then
 849                                                         log_note "Partitions for $DISK should be deleted"
 850                                                 else
 851                                                         log_fail "Partition for ${DISK}${SLICE_PREFIX}${j} not deleted"
 852                                                 fi
 853                                                 return 0
 854                                 else
 855                                         $LSBLK | $EGREP ${DISK}${SLICE_PREFIX}${j} > /dev/null
 856                                                 if (( $? == 0 )); then
 857                                                         log_fail "Partition for ${DISK}${SLICE_PREFIX}${j} not deleted"
 858                                                 fi
 859                                 fi
 860                                 ((j = j+1))
 861                         done
 862                 else
 863                         for disk in `$ECHO $DISKSARRAY`; do
 864                                 while ((j < MAX_PARTITIONS)); do
 865                                         $FORMAT $DEV_DSKDIR/$disk -s rm $j > /dev/null 2>&1
 866                                         if (( $? == 1 )); then
 867                                                 $LSBLK | $EGREP ${disk}${SLICE_PREFIX}${j} > /dev/null
 868                                                 if (( $? == 1 )); then
 869                                                         log_note "Partitions for $disk should be deleted"
 870                                                 else
 871                                                         log_fail "Partition for ${disk}${SLICE_PREFIX}${j} not deleted"
 872                                                 fi
 873                                                 j=7
 874                                         else
 875                                                 $LSBLK | $EGREP ${disk}${SLICE_PREFIX}${j} > /dev/null
 876                                                 if (( $? == 0 )); then
 877                                                         log_fail "Partition for ${disk}${SLICE_PREFIX}${j} not deleted"
 878                                                 fi
 879                                         fi
 880                                         ((j = j+1))
 881                                 done
 882                                 j=1
 883                         done
 884                 fi
 885         fi
 886         return 0
 887 }
 888
 889 #
 890 # Get the end cyl of the given slice
 891 #
 892 function get_endslice #<disk> <slice>
 893 {
 894         typeset disk=$1
 895         typeset slice=$2
 896         if [[ -z $disk || -z $slice ]] ; then
 897                 log_fail "The disk name or slice number is unspecified."
 898         fi
 899
 900         if is_linux; then
 901                 endcyl=$($FORMAT -s $DEV_DSKDIR/$disk -- unit cyl print | \
 902                         $GREP "part${slice}" | \
 903                         $AWK '{print $3}' | \
 904                         $SED 's,cyl,,')
 905                 ((endcyl = (endcyl + 1)))
 906         else
 907                 disk=${disk#/dev/dsk/}
 908                 disk=${disk#/dev/rdsk/}
 909                 disk=${disk%s*}
 910
 911                 typeset -i ratio=0
 912                 ratio=$($PRTVTOC /dev/rdsk/${disk}s2 | \
 913                         $GREP "sectors\/cylinder" | \
 914                         $AWK '{print $2}')
 915
 916                 if ((ratio == 0)); then
 917                         return
 918                 fi
 919
 920                 typeset -i endcyl=$($PRTVTOC -h /dev/rdsk/${disk}s2 |
 921                         $NAWK -v token="$slice" '{if ($1==token) print $6}')
 922
 923                 ((endcyl = (endcyl + 1) / ratio))
 924         fi
 925
 926         echo $endcyl
 927 }
 928
 929
 930 #
 931 # Given a size,disk and total slice number,  this function formats the
 932 # disk slices from 0 to the total slice number with the same specified
 933 # size.
 934 #
 935 function partition_disk #<slice_size> <whole_disk_name> <total_slices>
 936 {
 937         typeset -i i=0
 938         typeset slice_size=$1
 939         typeset disk_name=$2
 940         typeset total_slices=$3
 941         typeset cyl
 942
 943         zero_partitions $disk_name
 944         while ((i < $total_slices)); do
 945                 if ! is_linux; then
 946                         if ((i == 2)); then
 947                                 ((i = i + 1))
 948                                 continue
 949                         fi
 950                 fi
 951                 set_partition $i "$cyl" $slice_size $disk_name
 952                 cyl=$(get_endslice $disk_name $i)
 953                 ((i = i+1))
 954         done
 955 }
 956
 957 #
 958 # This function continues to write to a filenum number of files into dirnum
 959 # number of directories until either $FILE_WRITE returns an error or the
 960 # maximum number of files per directory have been written.
 961 #
 962 # Usage:
 963 # fill_fs [destdir] [dirnum] [filenum] [bytes] [num_writes] [data]
 964 #
 965 # Return value: 0 on success
 966 #               non 0 on error
 967 #
 968 # Where :
 969 #       destdir:    is the directory where everything is to be created under
 970 #       dirnum:     the maximum number of subdirectories to use, -1 no limit
 971 #       filenum:    the maximum number of files per subdirectory
 972 #       bytes:      number of bytes to write
 973 #       num_writes: numer of types to write out bytes
 974 #       data:       the data that will be written
 975 #
 976 #       E.g.
 977 #       file_fs /testdir 20 25 1024 256 0
 978 #
 979 # Note: bytes * num_writes equals the size of the testfile
 980 #
 981 function fill_fs # destdir dirnum filenum bytes num_writes data
 982 {
 983         typeset destdir=${1:-$TESTDIR}
 984         typeset -i dirnum=${2:-50}
 985         typeset -i filenum=${3:-50}
 986         typeset -i bytes=${4:-8192}
 987         typeset -i num_writes=${5:-10240}
 988         typeset -i data=${6:-0}
 989
 990         typeset -i odirnum=1
 991         typeset -i idirnum=0
 992         typeset -i fn=0
 993         typeset -i retval=0
 994
 995         log_must $MKDIR -p $destdir/$idirnum
 996         while (($odirnum > 0)); do
 997                 if ((dirnum >= 0 && idirnum >= dirnum)); then
 998                         odirnum=0
 999                         break
1000                 fi
1001                 $FILE_WRITE -o create -f $destdir/$idirnum/$TESTFILE.$fn \
1002                     -b $bytes -c $num_writes -d $data
1003                 retval=$?
1004                 if (($retval != 0)); then
1005                         odirnum=0
1006                         break
1007                 fi
1008                 if (($fn >= $filenum)); then
1009                         fn=0
1010                         ((idirnum = idirnum + 1))
1011                         log_must $MKDIR -p $destdir/$idirnum
1012                 else
1013                         ((fn = fn + 1))
1014                 fi
1015         done
1016         return $retval
1017 }
1018
1019 #
1020 # Simple function to get the specified property. If unable to
1021 # get the property then exits.
1022 #
1023 # Note property is in 'parsable' format (-p)
1024 #
1025 function get_prop # property dataset
1026 {
1027         typeset prop_val
1028         typeset prop=$1
1029         typeset dataset=$2
1030
1031         prop_val=$($ZFS get -pH -o value $prop $dataset 2>/dev/null)
1032         if [[ $? -ne 0 ]]; then
1033                 log_note "Unable to get $prop property for dataset " \
1034                 "$dataset"
1035                 return 1
1036         fi
1037
1038         $ECHO "$prop_val"
1039         return 0
1040 }
1041
1042 #
1043 # Simple function to get the specified property of pool. If unable to
1044 # get the property then exits.
1045 #
1046 function get_pool_prop # property pool
1047 {
1048         typeset prop_val
1049         typeset prop=$1
1050         typeset pool=$2
1051
1052         if poolexists $pool ; then
1053                 prop_val=$($ZPOOL get $prop $pool 2>/dev/null | $TAIL -1 | \
1054                         $AWK '{print $3}')
1055                 if [[ $? -ne 0 ]]; then
1056                         log_note "Unable to get $prop property for pool " \
1057                         "$pool"
1058                         return 1
1059                 fi
1060         else
1061                 log_note "Pool $pool not exists."
1062                 return 1
1063         fi
1064
1065         $ECHO "$prop_val"
1066         return 0
1067 }
1068
1069 # Return 0 if a pool exists; $? otherwise
1070 #
1071 # $1 - pool name
1072
1073 function poolexists
1074 {
1075         typeset pool=$1
1076
1077         if [[ -z $pool ]]; then
1078                 log_note "No pool name given."
1079                 return 1
1080         fi
1081
1082         $ZPOOL get name "$pool" > /dev/null 2>&1
1083         return $?
1084 }
1085
1086 # Return 0 if all the specified datasets exist; $? otherwise
1087 #
1088 # $1-n  dataset name
1089 function datasetexists
1090 {
1091         if (($# == 0)); then
1092                 log_note "No dataset name given."
1093                 return 1
1094         fi
1095
1096         while (($# > 0)); do
1097                 $ZFS get name $1 > /dev/null 2>&1 || \
1098                         return $?
1099                 shift
1100         done
1101
1102         return 0
1103 }
1104
1105 # return 0 if none of the specified datasets exists, otherwise return 1.
1106 #
1107 # $1-n  dataset name
1108 function datasetnonexists
1109 {
1110         if (($# == 0)); then
1111                 log_note "No dataset name given."
1112                 return 1
1113         fi
1114
1115         while (($# > 0)); do
1116                 $ZFS list -H -t filesystem,snapshot,volume $1 > /dev/null 2>&1 \
1117                     && return 1
1118                 shift
1119         done
1120
1121         return 0
1122 }
1123
1124 #
1125 # Given a mountpoint, or a dataset name, determine if it is shared via NFS.
1126 #
1127 # Returns 0 if shared, 1 otherwise.
1128 #
1129 function is_shared
1130 {
1131         typeset fs=$1
1132         typeset mtpt
1133
1134         if [[ $fs != "/"* ]] ; then
1135                 if datasetnonexists "$fs" ; then
1136                         return 1
1137                 else
1138                         mtpt=$(get_prop mountpoint "$fs")
1139                         case $mtpt in
1140                                 none|legacy|-) return 1
1141                                         ;;
1142                                 *)      fs=$mtpt
1143                                         ;;
1144                         esac
1145                 fi
1146         fi
1147
1148         if is_linux; then
1149                 for mtpt in `$SHARE | $AWK '{print $1}'` ; do
1150                         if [[ $mtpt == $fs ]] ; then
1151                                 return 0
1152                         fi
1153                 done
1154                 return 1
1155         fi
1156
1157         for mtpt in `$SHARE | $AWK '{print $2}'` ; do
1158                 if [[ $mtpt == $fs ]] ; then
1159                         return 0
1160                 fi
1161         done
1162
1163         typeset stat=$($SVCS -H -o STA nfs/server:default)
1164         if [[ $stat != "ON" ]]; then
1165                 log_note "Current nfs/server status: $stat"
1166         fi
1167
1168         return 1
1169 }
1170
1171 #
1172 # Given a dataset name determine if it is shared via SMB.
1173 #
1174 # Returns 0 if shared, 1 otherwise.
1175 #
1176 function is_shared_smb
1177 {
1178         typeset fs=$1
1179         typeset mtpt
1180
1181         if datasetnonexists "$fs" ; then
1182                 return 1
1183         else
1184                 fs=$(echo $fs | sed 's@/@_@g')
1185         fi
1186
1187         if is_linux; then
1188                 for mtpt in `$NET usershare list | $AWK '{print $1}'` ; do
1189                         if [[ $mtpt == $fs ]] ; then
1190                                 return 0
1191                         fi
1192                 done
1193                 return 1
1194         else
1195                 log_unsupported "Currently unsupported by the test framework"
1196                 return 1
1197         fi
1198 }
1199
1200 #
1201 # Given a mountpoint, determine if it is not shared via NFS.
1202 #
1203 # Returns 0 if not shared, 1 otherwise.
1204 #
1205 function not_shared
1206 {
1207         typeset fs=$1
1208
1209         is_shared $fs
1210         if (($? == 0)); then
1211                 return 1
1212         fi
1213
1214         return 0
1215 }
1216
1217 #
1218 # Given a dataset determine if it is not shared via SMB.
1219 #
1220 # Returns 0 if not shared, 1 otherwise.
1221 #
1222 function not_shared_smb
1223 {
1224         typeset fs=$1
1225
1226         is_shared_smb $fs
1227         if (($? == 0)); then
1228                 return 1
1229         fi
1230
1231         return 0
1232 }
1233
1234 #
1235 # Helper function to unshare a mountpoint.
1236 #
1237 function unshare_fs #fs
1238 {
1239         typeset fs=$1
1240
1241         is_shared $fs || is_shared_smb $fs
1242         if (($? == 0)); then
1243                 log_must $ZFS unshare $fs
1244         fi
1245
1246         return 0
1247 }
1248
1249 #
1250 # Helper function to share a NFS mountpoint.
1251 #
1252 function share_nfs #fs
1253 {
1254         typeset fs=$1
1255
1256         if is_linux; then
1257                 is_shared $fs
1258                 if (($? != 0)); then
1259                         log_must $SHARE "*:$fs"
1260                 fi
1261         else
1262                 is_shared $fs
1263                 if (($? != 0)); then
1264                         log_must $SHARE -F nfs $fs
1265                 fi
1266         fi
1267
1268         return 0
1269 }
1270
1271 #
1272 # Helper function to unshare a NFS mountpoint.
1273 #
1274 function unshare_nfs #fs
1275 {
1276         typeset fs=$1
1277
1278         if is_linux; then
1279                 is_shared $fs
1280                 if (($? == 0)); then
1281                         log_must $UNSHARE -u "*:$fs"
1282                 fi
1283         else
1284                 is_shared $fs
1285                 if (($? == 0)); then
1286                         log_must $UNSHARE -F nfs $fs
1287                 fi
1288         fi
1289
1290         return 0
1291 }
1292
1293 #
1294 # Helper function to show NFS shares.
1295 #
1296 function showshares_nfs
1297 {
1298         if is_linux; then
1299                 $SHARE -v
1300         else
1301                 $SHARE -F nfs
1302         fi
1303
1304         return 0
1305 }
1306
1307 #
1308 # Helper function to show SMB shares.
1309 #
1310 function showshares_smb
1311 {
1312         if is_linux; then
1313                 $NET usershare list
1314         else
1315                 $SHARE -F smb
1316         fi
1317
1318         return 0
1319 }
1320
1321 #
1322 # Check NFS server status and trigger it online.
1323 #
1324 function setup_nfs_server
1325 {
1326         # Cannot share directory in non-global zone.
1327         #
1328         if ! is_global_zone; then
1329                 log_note "Cannot trigger NFS server by sharing in LZ."
1330                 return
1331         fi
1332
1333         if is_linux; then
1334                 log_note "NFS server must started prior to running test framework."
1335                 return
1336         fi
1337
1338         typeset nfs_fmri="svc:/network/nfs/server:default"
1339         if [[ $($SVCS -Ho STA $nfs_fmri) != "ON" ]]; then
1340                 #
1341                 # Only really sharing operation can enable NFS server
1342                 # to online permanently.
1343                 #
1344                 typeset dummy=/tmp/dummy
1345
1346                 if [[ -d $dummy ]]; then
1347                         log_must $RM -rf $dummy
1348                 fi
1349
1350                 log_must $MKDIR $dummy
1351                 log_must $SHARE $dummy
1352
1353                 #
1354                 # Waiting for fmri's status to be the final status.
1355                 # Otherwise, in transition, an asterisk (*) is appended for
1356                 # instances, unshare will reverse status to 'DIS' again.
1357                 #
1358                 # Waiting for 1's at least.
1359                 #
1360                 log_must $SLEEP 1
1361                 timeout=10
1362                 while [[ timeout -ne 0 && $($SVCS -Ho STA $nfs_fmri) == *'*' ]]
1363                 do
1364                         log_must $SLEEP 1
1365
1366                         ((timeout -= 1))
1367                 done
1368
1369                 log_must $UNSHARE $dummy
1370                 log_must $RM -rf $dummy
1371         fi
1372
1373         log_note "Current NFS status: '$($SVCS -Ho STA,FMRI $nfs_fmri)'"
1374 }
1375
1376 #
1377 # To verify whether calling process is in global zone
1378 #
1379 # Return 0 if in global zone, 1 in non-global zone
1380 #
1381 function is_global_zone
1382 {
1383         typeset cur_zone=$($ZONENAME 2>/dev/null)
1384         if [[ $cur_zone != "global" ]]; then
1385                 return 1
1386         fi
1387         return 0
1388 }
1389
1390 #
1391 # Verify whether test is permitted to run from
1392 # global zone, local zone, or both
1393 #
1394 # $1 zone limit, could be "global", "local", or "both"(no limit)
1395 #
1396 # Return 0 if permitted, otherwise exit with log_unsupported
1397 #
1398 function verify_runnable # zone limit
1399 {
1400         typeset limit=$1
1401
1402         [[ -z $limit ]] && return 0
1403
1404         if is_global_zone ; then
1405                 case $limit in
1406                         global|both)
1407                                 ;;
1408                         local)  log_unsupported "Test is unable to run from "\
1409                                         "global zone."
1410                                 ;;
1411                         *)      log_note "Warning: unknown limit $limit - " \
1412                                         "use both."
1413                                 ;;
1414                 esac
1415         else
1416                 case $limit in
1417                         local|both)
1418                                 ;;
1419                         global) log_unsupported "Test is unable to run from "\
1420                                         "local zone."
1421                                 ;;
1422                         *)      log_note "Warning: unknown limit $limit - " \
1423                                         "use both."
1424                                 ;;
1425                 esac
1426
1427                 reexport_pool
1428         fi
1429
1430         return 0
1431 }
1432
1433 # Return 0 if create successfully or the pool exists; $? otherwise
1434 # Note: In local zones, this function should return 0 silently.
1435 #
1436 # $1 - pool name
1437 # $2-n - [keyword] devs_list
1438
1439 function create_pool #pool devs_list
1440 {
1441         typeset pool=${1%%/*}
1442
1443         shift
1444
1445         if [[ -z $pool ]]; then
1446                 log_note "Missing pool name."
1447                 return 1
1448         fi
1449
1450         if poolexists $pool ; then
1451                 destroy_pool $pool
1452         fi
1453
1454         if is_global_zone ; then
1455                 [[ -d /$pool ]] && $RM -rf /$pool
1456                 log_must $ZPOOL create -f $pool $@
1457         fi
1458
1459         return 0
1460 }
1461
1462 # Return 0 if destroy successfully or the pool exists; $? otherwise
1463 # Note: In local zones, this function should return 0 silently.
1464 #
1465 # $1 - pool name
1466 # Destroy pool with the given parameters.
1467
1468 function destroy_pool #pool
1469 {
1470         typeset pool=${1%%/*}
1471         typeset mtpt
1472
1473         if [[ -z $pool ]]; then
1474                 log_note "No pool name given."
1475                 return 1
1476         fi
1477
1478         if is_global_zone ; then
1479                 if poolexists "$pool" ; then
1480                         mtpt=$(get_prop mountpoint "$pool")
1481
1482                         # At times, syseventd activity can cause attempts to
1483                         # destroy a pool to fail with EBUSY. We retry a few
1484                         # times allowing failures before requiring the destroy
1485                         # to succeed.
1486                         typeset -i wait_time=10 ret=1 count=0
1487                         must=""
1488                         while [[ $ret -ne 0 ]]; do
1489                                 $must $ZPOOL destroy -f $pool
1490                                 ret=$?
1491                                 [[ $ret -eq 0 ]] && break
1492                                 log_note "zpool destroy failed with $ret"
1493                                 [[ count++ -ge 7 ]] && must=log_must
1494                                 $SLEEP $wait_time
1495                         done
1496
1497                         [[ -d $mtpt ]] && \
1498                                 log_must $RM -rf $mtpt
1499                 else
1500                         log_note "Pool does not exist. ($pool)"
1501                         return 1
1502                 fi
1503         fi
1504
1505         return 0
1506 }
1507
1508 #
1509 # Firstly, create a pool with 5 datasets. Then, create a single zone and
1510 # export the 5 datasets to it. In addition, we also add a ZFS filesystem
1511 # and a zvol device to the zone.
1512 #
1513 # $1 zone name
1514 # $2 zone root directory prefix
1515 # $3 zone ip
1516 #
1517 function zfs_zones_setup #zone_name zone_root zone_ip
1518 {
1519         typeset zone_name=${1:-$(hostname)-z}
1520         typeset zone_root=${2:-"/zone_root"}
1521         typeset zone_ip=${3:-"10.1.1.10"}
1522         typeset prefix_ctr=$ZONE_CTR
1523         typeset pool_name=$ZONE_POOL
1524         typeset -i cntctr=5
1525         typeset -i i=0
1526
1527         # Create pool and 5 container within it
1528         #
1529         [[ -d /$pool_name ]] && $RM -rf /$pool_name
1530         log_must $ZPOOL create -f $pool_name $DISKS
1531         while ((i < cntctr)); do
1532                 log_must $ZFS create $pool_name/$prefix_ctr$i
1533                 ((i += 1))
1534         done
1535
1536         # create a zvol
1537         log_must $ZFS create -V 1g $pool_name/zone_zvol
1538         block_device_wait
1539
1540         #
1541         # If current system support slog, add slog device for pool
1542         #
1543         if verify_slog_support ; then
1544                 typeset sdevs="/var/tmp/sdev1 /var/tmp/sdev2"
1545                 log_must $MKFILE 100M $sdevs
1546                 log_must $ZPOOL add $pool_name log mirror $sdevs
1547         fi
1548
1549         # this isn't supported just yet.
1550         # Create a filesystem. In order to add this to
1551         # the zone, it must have it's mountpoint set to 'legacy'
1552         # log_must $ZFS create $pool_name/zfs_filesystem
1553         # log_must $ZFS set mountpoint=legacy $pool_name/zfs_filesystem
1554
1555         [[ -d $zone_root ]] && \
1556                 log_must $RM -rf $zone_root/$zone_name
1557         [[ ! -d $zone_root ]] && \
1558                 log_must $MKDIR -p -m 0700 $zone_root/$zone_name
1559
1560         # Create zone configure file and configure the zone
1561         #
1562         typeset zone_conf=/tmp/zone_conf.$$
1563         $ECHO "create" > $zone_conf
1564         $ECHO "set zonepath=$zone_root/$zone_name" >> $zone_conf
1565         $ECHO "set autoboot=true" >> $zone_conf
1566         i=0
1567         while ((i < cntctr)); do
1568                 $ECHO "add dataset" >> $zone_conf
1569                 $ECHO "set name=$pool_name/$prefix_ctr$i" >> \
1570                         $zone_conf
1571                 $ECHO "end" >> $zone_conf
1572                 ((i += 1))
1573         done
1574
1575         # add our zvol to the zone
1576         $ECHO "add device" >> $zone_conf
1577         $ECHO "set match=$ZVOL_DEVDIR/$pool_name/zone_zvol" >> $zone_conf
1578         $ECHO "end" >> $zone_conf
1579
1580         # add a corresponding zvol rdsk to the zone
1581         $ECHO "add device" >> $zone_conf
1582         $ECHO "set match=$ZVOL_RDEVDIR/$pool_name/zone_zvol" >> $zone_conf
1583         $ECHO "end" >> $zone_conf
1584
1585         # once it's supported, we'll add our filesystem to the zone
1586         # $ECHO "add fs" >> $zone_conf
1587         # $ECHO "set type=zfs" >> $zone_conf
1588         # $ECHO "set special=$pool_name/zfs_filesystem" >> $zone_conf
1589         # $ECHO "set dir=/export/zfs_filesystem" >> $zone_conf
1590         # $ECHO "end" >> $zone_conf
1591
1592         $ECHO "verify" >> $zone_conf
1593         $ECHO "commit" >> $zone_conf
1594         log_must $ZONECFG -z $zone_name -f $zone_conf
1595         log_must $RM -f $zone_conf
1596
1597         # Install the zone
1598         $ZONEADM -z $zone_name install
1599         if (($? == 0)); then
1600                 log_note "SUCCESS: $ZONEADM -z $zone_name install"
1601         else
1602                 log_fail "FAIL: $ZONEADM -z $zone_name install"
1603         fi
1604
1605         # Install sysidcfg file
1606         #
1607         typeset sysidcfg=$zone_root/$zone_name/root/etc/sysidcfg
1608         $ECHO "system_locale=C" > $sysidcfg
1609         $ECHO  "terminal=dtterm" >> $sysidcfg
1610         $ECHO  "network_interface=primary {" >> $sysidcfg
1611         $ECHO  "hostname=$zone_name" >> $sysidcfg
1612         $ECHO  "}" >> $sysidcfg
1613         $ECHO  "name_service=NONE" >> $sysidcfg
1614         $ECHO  "root_password=mo791xfZ/SFiw" >> $sysidcfg
1615         $ECHO  "security_policy=NONE" >> $sysidcfg
1616         $ECHO  "timezone=US/Eastern" >> $sysidcfg
1617
1618         # Boot this zone
1619         log_must $ZONEADM -z $zone_name boot
1620 }
1621
1622 #
1623 # Reexport TESTPOOL & TESTPOOL(1-4)
1624 #
1625 function reexport_pool
1626 {
1627         typeset -i cntctr=5
1628         typeset -i i=0
1629
1630         while ((i < cntctr)); do
1631                 if ((i == 0)); then
1632                         TESTPOOL=$ZONE_POOL/$ZONE_CTR$i
1633                         if ! ismounted $TESTPOOL; then
1634                                 log_must $ZFS mount $TESTPOOL
1635                         fi
1636                 else
1637                         eval TESTPOOL$i=$ZONE_POOL/$ZONE_CTR$i
1638                         if eval ! ismounted \$TESTPOOL$i; then
1639                                 log_must eval $ZFS mount \$TESTPOOL$i
1640                         fi
1641                 fi
1642                 ((i += 1))
1643         done
1644 }
1645
1646 #
1647 # Verify a given disk is online or offline
1648 #
1649 # Return 0 is pool/disk matches expected state, 1 otherwise
1650 #
1651 function check_state # pool disk state{online,offline}
1652 {
1653         typeset pool=$1
1654         typeset disk=${2#$DEV_DSKDIR/}
1655         typeset state=$3
1656
1657         $ZPOOL status -v $pool | grep "$disk"  \
1658             | grep -i "$state" > /dev/null 2>&1
1659
1660         return $?
1661 }
1662
1663 #
1664 # Get the mountpoint of snapshot
1665 # For the snapshot use <mp_filesystem>/.zfs/snapshot/<snap>
1666 # as its mountpoint
1667 #
1668 function snapshot_mountpoint
1669 {
1670         typeset dataset=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}
1671
1672         if [[ $dataset != *@* ]]; then
1673                 log_fail "Error name of snapshot '$dataset'."
1674         fi
1675
1676         typeset fs=${dataset%@*}
1677         typeset snap=${dataset#*@}
1678
1679         if [[ -z $fs || -z $snap ]]; then
1680                 log_fail "Error name of snapshot '$dataset'."
1681         fi
1682
1683         $ECHO $(get_prop mountpoint $fs)/.zfs/snapshot/$snap
1684 }
1685
1686 #
1687 # Given a pool and file system, this function will verify the file system
1688 # using the zdb internal tool. Note that the pool is exported and imported
1689 # to ensure it has consistent state.
1690 #
1691 function verify_filesys # pool filesystem dir
1692 {
1693         typeset pool="$1"
1694         typeset filesys="$2"
1695         typeset zdbout="/tmp/zdbout.$$"
1696
1697         shift
1698         shift
1699         typeset dirs=$@
1700         typeset search_path=""
1701
1702         log_note "Calling $ZDB to verify filesystem '$filesys'"
1703         $ZFS unmount -a > /dev/null 2>&1
1704         log_must $ZPOOL export $pool
1705
1706         if [[ -n $dirs ]] ; then
1707                 for dir in $dirs ; do
1708                         search_path="$search_path -d $dir"
1709                 done
1710         fi
1711
1712         log_must $ZPOOL import $search_path $pool
1713
1714         $ZDB -cudi $filesys > $zdbout 2>&1
1715         if [[ $? != 0 ]]; then
1716                 log_note "Output: $ZDB -cudi $filesys"
1717                 $CAT $zdbout
1718                 log_fail "$ZDB detected errors with: '$filesys'"
1719         fi
1720
1721         log_must $ZFS mount -a
1722         log_must $RM -rf $zdbout
1723 }
1724
1725 #
1726 # Given a pool, and this function list all disks in the pool
1727 #
1728 function get_disklist # pool
1729 {
1730         typeset disklist=""
1731
1732         disklist=$($ZPOOL iostat -v $1 | $NAWK '(NR >4) {print $1}' | \
1733             $GREP -v "\-\-\-\-\-" | \
1734             $EGREP -v -e "^(mirror|raidz1|raidz2|spare|log|cache)$")
1735
1736         $ECHO $disklist
1737 }
1738
1739 #
1740 # Given a pool, and this function list all disks in the pool with their full
1741 # path (like "/dev/sda" instead of "sda").
1742 #
1743 function get_disklist_fullpath # pool
1744 {
1745         args="-P $1"
1746         get_disklist $args
1747 }
1748
1749
1750
1751 # /**
1752 #  This function kills a given list of processes after a time period. We use
1753 #  this in the stress tests instead of STF_TIMEOUT so that we can have processes
1754 #  run for a fixed amount of time, yet still pass. Tests that hit STF_TIMEOUT
1755 #  would be listed as FAIL, which we don't want : we're happy with stress tests
1756 #  running for a certain amount of time, then finishing.
1757 #
1758 # @param $1 the time in seconds after which we should terminate these processes
1759 # @param $2..$n the processes we wish to terminate.
1760 # */
1761 function stress_timeout
1762 {
1763         typeset -i TIMEOUT=$1
1764         shift
1765         typeset cpids="$@"
1766
1767         log_note "Waiting for child processes($cpids). " \
1768                 "It could last dozens of minutes, please be patient ..."
1769         log_must $SLEEP $TIMEOUT
1770
1771         log_note "Killing child processes after ${TIMEOUT} stress timeout."
1772         typeset pid
1773         for pid in $cpids; do
1774                 $PS -p $pid > /dev/null 2>&1
1775                 if (($? == 0)); then
1776                         log_must $KILL -USR1 $pid
1777                 fi
1778         done
1779 }
1780
1781 #
1782 # Verify a given hotspare disk is inuse or avail
1783 #
1784 # Return 0 is pool/disk matches expected state, 1 otherwise
1785 #
1786 function check_hotspare_state # pool disk state{inuse,avail}
1787 {
1788         typeset pool=$1
1789         typeset disk=${2#$DEV_DSKDIR/}
1790         typeset state=$3
1791
1792         cur_state=$(get_device_state $pool $disk "spares")
1793
1794         if [[ $state != ${cur_state} ]]; then
1795                 return 1
1796         fi
1797         return 0
1798 }
1799
1800 #
1801 # Verify a given slog disk is inuse or avail
1802 #
1803 # Return 0 is pool/disk matches expected state, 1 otherwise
1804 #
1805 function check_slog_state # pool disk state{online,offline,unavail}
1806 {
1807         typeset pool=$1
1808         typeset disk=${2#$DEV_DSKDIR/}
1809         typeset state=$3
1810
1811         cur_state=$(get_device_state $pool $disk "logs")
1812
1813         if [[ $state != ${cur_state} ]]; then
1814                 return 1
1815         fi
1816         return 0
1817 }
1818
1819 #
1820 # Verify a given vdev disk is inuse or avail
1821 #
1822 # Return 0 is pool/disk matches expected state, 1 otherwise
1823 #
1824 function check_vdev_state # pool disk state{online,offline,unavail}
1825 {
1826         typeset pool=$1
1827         typeset disk=${2#$/DEV_DSKDIR/}
1828         typeset state=$3
1829
1830         cur_state=$(get_device_state $pool $disk)
1831
1832         if [[ $state != ${cur_state} ]]; then
1833                 return 1
1834         fi
1835         return 0
1836 }
1837
1838 #
1839 # Check the output of 'zpool status -v <pool>',
1840 # and to see if the content of <token> contain the <keyword> specified.
1841 #
1842 # Return 0 is contain, 1 otherwise
1843 #
1844 function check_pool_status # pool token keyword
1845 {
1846         typeset pool=$1
1847         typeset token=$2
1848         typeset keyword=$3
1849
1850         $ZPOOL status -v "$pool" 2>/dev/null | $NAWK -v token="$token:" '
1851                 ($1==token) {print $0}' \
1852         | $GREP -i "$keyword" > /dev/null 2>&1
1853
1854         return $?
1855 }
1856
1857 #
1858 # These 5 following functions are instance of check_pool_status()
1859 #       is_pool_resilvering - to check if the pool is resilver in progress
1860 #       is_pool_resilvered - to check if the pool is resilver completed
1861 #       is_pool_scrubbing - to check if the pool is scrub in progress
1862 #       is_pool_scrubbed - to check if the pool is scrub completed
1863 #       is_pool_scrub_stopped - to check if the pool is scrub stopped
1864 #
1865 function is_pool_resilvering #pool
1866 {
1867         check_pool_status "$1" "scan" "resilver in progress since "
1868         return $?
1869 }
1870
1871 function is_pool_resilvered #pool
1872 {
1873         check_pool_status "$1" "scan" "resilvered "
1874         return $?
1875 }
1876
1877 function is_pool_scrubbing #pool
1878 {
1879         check_pool_status "$1" "scan" "scrub in progress since "
1880         return $?
1881 }
1882
1883 function is_pool_scrubbed #pool
1884 {
1885         check_pool_status "$1" "scan" "scrub repaired"
1886         return $?
1887 }
1888
1889 function is_pool_scrub_stopped #pool
1890 {
1891         check_pool_status "$1" "scan" "scrub canceled"
1892         return $?
1893 }
1894
1895 #
1896 # Use create_pool()/destroy_pool() to clean up the information in
1897 # in the given disk to avoid slice overlapping.
1898 #
1899 function cleanup_devices #vdevs
1900 {
1901         typeset pool="foopool$$"
1902
1903         if poolexists $pool ; then
1904                 destroy_pool $pool
1905         fi
1906
1907         create_pool $pool $@
1908         destroy_pool $pool
1909
1910         return 0
1911 }
1912
1913 #
1914 # Verify the rsh connectivity to each remote host in RHOSTS.
1915 #
1916 # Return 0 if remote host is accessible; otherwise 1.
1917 # $1 remote host name
1918 # $2 username
1919 #
1920 function verify_rsh_connect #rhost, username
1921 {
1922         typeset rhost=$1
1923         typeset username=$2
1924         typeset rsh_cmd="$RSH -n"
1925         typeset cur_user=
1926
1927         $GETENT hosts $rhost >/dev/null 2>&1
1928         if (($? != 0)); then
1929                 log_note "$rhost cannot be found from" \
1930                         "administrative database."
1931                 return 1
1932         fi
1933
1934         $PING $rhost 3 >/dev/null 2>&1
1935         if (($? != 0)); then
1936                 log_note "$rhost is not reachable."
1937                 return 1
1938         fi
1939
1940         if ((${#username} != 0)); then
1941                 rsh_cmd="$rsh_cmd -l $username"
1942                 cur_user="given user \"$username\""
1943         else
1944                 cur_user="current user \"`$LOGNAME`\""
1945         fi
1946
1947          if ! $rsh_cmd $rhost $TRUE; then
1948                 log_note "$RSH to $rhost is not accessible" \
1949                         "with $cur_user."
1950                 return 1
1951         fi
1952
1953         return 0
1954 }
1955
1956 #
1957 # Verify the remote host connection via rsh after rebooting
1958 # $1 remote host
1959 #
1960 function verify_remote
1961 {
1962         rhost=$1
1963
1964         #
1965         # The following loop waits for the remote system rebooting.
1966         # Each iteration will wait for 150 seconds. there are
1967         # total 5 iterations, so the total timeout value will
1968         # be 12.5  minutes for the system rebooting. This number
1969         # is an approxiate number.
1970         #
1971         typeset -i count=0
1972         while ! verify_rsh_connect $rhost; do
1973                 sleep 150
1974                 ((count = count + 1))
1975                 if ((count > 5)); then
1976                         return 1
1977                 fi
1978         done
1979         return 0
1980 }
1981
1982 #
1983 # Replacement function for /usr/bin/rsh. This function will include
1984 # the /usr/bin/rsh and meanwhile return the execution status of the
1985 # last command.
1986 #
1987 # $1 usrname passing down to -l option of /usr/bin/rsh
1988 # $2 remote machine hostname
1989 # $3... command string
1990 #
1991
1992 function rsh_status
1993 {
1994         typeset ruser=$1
1995         typeset rhost=$2
1996         typeset -i ret=0
1997         typeset cmd_str=""
1998         typeset rsh_str=""
1999
2000         shift; shift
2001         cmd_str="$@"
2002
2003         err_file=/tmp/${rhost}.$$.err
2004         if ((${#ruser} == 0)); then
2005                 rsh_str="$RSH -n"
2006         else
2007                 rsh_str="$RSH -n -l $ruser"
2008         fi
2009
2010         $rsh_str $rhost /bin/ksh -c "'$cmd_str; \
2011                 print -u 2 \"status=\$?\"'" \
2012                 >/dev/null 2>$err_file
2013         ret=$?
2014         if (($ret != 0)); then
2015                 $CAT $err_file
2016                 $RM -f $std_file $err_file
2017                 log_fail  "$RSH itself failed with exit code $ret..."
2018         fi
2019
2020          ret=$($GREP -v 'print -u 2' $err_file | $GREP 'status=' | \
2021                 $CUT -d= -f2)
2022         (($ret != 0)) && $CAT $err_file >&2
2023
2024         $RM -f $err_file >/dev/null 2>&1
2025         return $ret
2026 }
2027
2028 #
2029 # Get the SUNWstc-fs-zfs package installation path in a remote host
2030 # $1 remote host name
2031 #
2032 function get_remote_pkgpath
2033 {
2034         typeset rhost=$1
2035         typeset pkgpath=""
2036
2037         pkgpath=$($RSH -n $rhost "$PKGINFO -l SUNWstc-fs-zfs | $GREP BASEDIR: |\
2038                         $CUT -d: -f2")
2039
2040         $ECHO $pkgpath
2041 }
2042
2043 #/**
2044 # A function to find and locate free disks on a system or from given
2045 # disks as the parameter. It works by locating disks that are in use
2046 # as swap devices and dump devices, and also disks listed in /etc/vfstab
2047 #
2048 # $@ given disks to find which are free, default is all disks in
2049 # the test system
2050 #
2051 # @return a string containing the list of available disks
2052 #*/
2053 function find_disks
2054 {
2055         # Trust provided list, no attempt is made to locate unused devices.
2056         if is_linux; then
2057                 $ECHO "$@"
2058                 return
2059         fi
2060
2061
2062         sfi=/tmp/swaplist.$$
2063         dmpi=/tmp/dumpdev.$$
2064         max_finddisksnum=${MAX_FINDDISKSNUM:-6}
2065
2066         $SWAP -l > $sfi
2067         $DUMPADM > $dmpi 2>/dev/null
2068
2069 # write an awk script that can process the output of format
2070 # to produce a list of disks we know about. Note that we have
2071 # to escape "$2" so that the shell doesn't interpret it while
2072 # we're creating the awk script.
2073 # -------------------
2074         $CAT > /tmp/find_disks.awk <<EOF
2075 #!/bin/nawk -f
2076         BEGIN { FS="."; }
2077
2078         /^Specify disk/{
2079                 searchdisks=0;
2080         }
2081
2082         {
2083                 if (searchdisks && \$2 !~ "^$"){
2084                         split(\$2,arr," ");
2085                         print arr[1];
2086                 }
2087         }
2088
2089         /^AVAILABLE DISK SELECTIONS:/{
2090                 searchdisks=1;
2091         }
2092 EOF
2093 #---------------------
2094
2095         $CHMOD 755 /tmp/find_disks.awk
2096         disks=${@:-$($ECHO "" | $FORMAT -e 2>/dev/null | /tmp/find_disks.awk)}
2097         $RM /tmp/find_disks.awk
2098
2099         unused=""
2100         for disk in $disks; do
2101         # Check for mounted
2102                 $GREP "${disk}[sp]" /etc/mnttab >/dev/null
2103                 (($? == 0)) && continue
2104         # Check for swap
2105                 $GREP "${disk}[sp]" $sfi >/dev/null
2106                 (($? == 0)) && continue
2107         # check for dump device
2108                 $GREP "${disk}[sp]" $dmpi >/dev/null
2109                 (($? == 0)) && continue
2110         # check to see if this disk hasn't been explicitly excluded
2111         # by a user-set environment variable
2112                 $ECHO "${ZFS_HOST_DEVICES_IGNORE}" | $GREP "${disk}" > /dev/null
2113                 (($? == 0)) && continue
2114                 unused_candidates="$unused_candidates $disk"
2115         done
2116         $RM $sfi
2117         $RM $dmpi
2118
2119 # now just check to see if those disks do actually exist
2120 # by looking for a device pointing to the first slice in
2121 # each case. limit the number to max_finddisksnum
2122         count=0
2123         for disk in $unused_candidates; do
2124                 if [ -b $DEV_DSKDIR/${disk}s0 ]; then
2125                 if [ $count -lt $max_finddisksnum ]; then
2126                         unused="$unused $disk"
2127                         # do not impose limit if $@ is provided
2128                         [[ -z $@ ]] && ((count = count + 1))
2129                 fi
2130                 fi
2131         done
2132
2133 # finally, return our disk list
2134         $ECHO $unused
2135 }
2136
2137 #
2138 # Add specified user to specified group
2139 #
2140 # $1 group name
2141 # $2 user name
2142 # $3 base of the homedir (optional)
2143 #
2144 function add_user #<group_name> <user_name> <basedir>
2145 {
2146         typeset gname=$1
2147         typeset uname=$2
2148         typeset basedir=${3:-"/var/tmp"}
2149
2150         if ((${#gname} == 0 || ${#uname} == 0)); then
2151                 log_fail "group name or user name are not defined."
2152         fi
2153
2154         log_must $USERADD -g $gname -d $basedir/$uname -m $uname
2155
2156         # Add new users to the same group and the command line utils.
2157         # This allows them to be run out of the original users home
2158         # directory as long as it permissioned to be group readable.
2159         if is_linux; then
2160                 cmd_group=$(stat --format="%G" $ZFS)
2161                 log_must $USERMOD -a -G $cmd_group $uname
2162         fi
2163
2164         return 0
2165 }
2166
2167 #
2168 # Delete the specified user.
2169 #
2170 # $1 login name
2171 # $2 base of the homedir (optional)
2172 #
2173 function del_user #<logname> <basedir>
2174 {
2175         typeset user=$1
2176         typeset basedir=${2:-"/var/tmp"}
2177
2178         if ((${#user} == 0)); then
2179                 log_fail "login name is necessary."
2180         fi
2181
2182         if $ID $user > /dev/null 2>&1; then
2183                 log_must $USERDEL $user
2184         fi
2185
2186         [[ -d $basedir/$user ]] && $RM -fr $basedir/$user
2187
2188         return 0
2189 }
2190
2191 #
2192 # Select valid gid and create specified group.
2193 #
2194 # $1 group name
2195 #
2196 function add_group #<group_name>
2197 {
2198         typeset group=$1
2199
2200         if ((${#group} == 0)); then
2201                 log_fail "group name is necessary."
2202         fi
2203
2204         # Assign 100 as the base gid, a larger value is selected for
2205         # Linux because for many distributions 1000 and under are reserved.
2206         if is_linux; then
2207                 while true; do
2208                         $GROUPADD $group > /dev/null 2>&1
2209                         typeset -i ret=$?
2210                         case $ret in
2211                                 0) return 0 ;;
2212                                 *) return 1 ;;
2213                         esac
2214                 done
2215         else
2216                 typeset -i gid=100
2217
2218                 while true; do
2219                         $GROUPADD -g $gid $group > /dev/null 2>&1
2220                         typeset -i ret=$?
2221                         case $ret in
2222                                 0) return 0 ;;
2223                                 # The gid is not  unique
2224                                 4) ((gid += 1)) ;;
2225                                 *) return 1 ;;
2226                         esac
2227                 done
2228         fi
2229 }
2230
2231 #
2232 # Delete the specified group.
2233 #
2234 # $1 group name
2235 #
2236 function del_group #<group_name>
2237 {
2238         typeset grp=$1
2239         if ((${#grp} == 0)); then
2240                 log_fail "group name is necessary."
2241         fi
2242
2243         if is_linux; then
2244                 $GETENT group $grp > /dev/null 2>&1
2245                 typeset -i ret=$?
2246                 case $ret in
2247                         # Group does not exist.
2248                         2) return 0 ;;
2249                         # Name already exists as a group name
2250                         0) log_must $GROUPDEL $grp ;;
2251                         *) return 1 ;;
2252                 esac
2253         else
2254                 $GROUPMOD -n $grp $grp > /dev/null 2>&1
2255                 typeset -i ret=$?
2256                 case $ret in
2257                         # Group does not exist.
2258                         6) return 0 ;;
2259                         # Name already exists as a group name
2260                         9) log_must $GROUPDEL $grp ;;
2261                         *) return 1 ;;
2262                 esac
2263         fi
2264
2265         return 0
2266 }
2267
2268 #
2269 # This function will return true if it's safe to destroy the pool passed
2270 # as argument 1. It checks for pools based on zvols and files, and also
2271 # files contained in a pool that may have a different mountpoint.
2272 #
2273 function safe_to_destroy_pool { # $1 the pool name
2274
2275         typeset pool=""
2276         typeset DONT_DESTROY=""
2277
2278         # We check that by deleting the $1 pool, we're not
2279         # going to pull the rug out from other pools. Do this
2280         # by looking at all other pools, ensuring that they
2281         # aren't built from files or zvols contained in this pool.
2282
2283         for pool in $($ZPOOL list -H -o name)
2284         do
2285                 ALTMOUNTPOOL=""
2286
2287                 # this is a list of the top-level directories in each of the
2288                 # files that make up the path to the files the pool is based on
2289                 FILEPOOL=$($ZPOOL status -v $pool | $GREP /$1/ | \
2290                         $AWK '{print $1}')
2291
2292                 # this is a list of the zvols that make up the pool
2293                 ZVOLPOOL=$($ZPOOL status -v $pool | $GREP "$ZVOL_DEVDIR/$1$" \
2294                     | $AWK '{print $1}')
2295
2296                 # also want to determine if it's a file-based pool using an
2297                 # alternate mountpoint...
2298                 POOL_FILE_DIRS=$($ZPOOL status -v $pool | \
2299                                         $GREP / | $AWK '{print $1}' | \
2300                                         $AWK -F/ '{print $2}' | $GREP -v "dev")
2301
2302                 for pooldir in $POOL_FILE_DIRS
2303                 do
2304                         OUTPUT=$($ZFS list -H -r -o mountpoint $1 | \
2305                                         $GREP "${pooldir}$" | $AWK '{print $1}')
2306
2307                         ALTMOUNTPOOL="${ALTMOUNTPOOL}${OUTPUT}"
2308                 done
2309
2310
2311                 if [ ! -z "$ZVOLPOOL" ]
2312                 then
2313                         DONT_DESTROY="true"
2314                         log_note "Pool $pool is built from $ZVOLPOOL on $1"
2315                 fi
2316
2317                 if [ ! -z "$FILEPOOL" ]
2318                 then
2319                         DONT_DESTROY="true"
2320                         log_note "Pool $pool is built from $FILEPOOL on $1"
2321                 fi
2322
2323                 if [ ! -z "$ALTMOUNTPOOL" ]
2324                 then
2325                         DONT_DESTROY="true"
2326                         log_note "Pool $pool is built from $ALTMOUNTPOOL on $1"
2327                 fi
2328         done
2329
2330         if [ -z "${DONT_DESTROY}" ]
2331         then
2332                 return 0
2333         else
2334                 log_note "Warning: it is not safe to destroy $1!"
2335                 return 1
2336         fi
2337 }
2338
2339 #
2340 # Get the available ZFS compression options
2341 # $1 option type zfs_set|zfs_compress
2342 #
2343 function get_compress_opts
2344 {
2345         typeset COMPRESS_OPTS
2346         typeset GZIP_OPTS="gzip gzip-1 gzip-2 gzip-3 gzip-4 gzip-5 \
2347                         gzip-6 gzip-7 gzip-8 gzip-9"
2348
2349         if [[ $1 == "zfs_compress" ]] ; then
2350                 COMPRESS_OPTS="on lzjb"
2351         elif [[ $1 == "zfs_set" ]] ; then
2352                 COMPRESS_OPTS="on off lzjb"
2353         fi
2354         typeset valid_opts="$COMPRESS_OPTS"
2355         $ZFS get 2>&1 | $GREP gzip >/dev/null 2>&1
2356         if [[ $? -eq 0 ]]; then
2357                 valid_opts="$valid_opts $GZIP_OPTS"
2358         fi
2359         $ECHO "$valid_opts"
2360 }
2361
2362 #
2363 # Verify zfs operation with -p option work as expected
2364 # $1 operation, value could be create, clone or rename
2365 # $2 dataset type, value could be fs or vol
2366 # $3 dataset name
2367 # $4 new dataset name
2368 #
2369 function verify_opt_p_ops
2370 {
2371         typeset ops=$1
2372         typeset datatype=$2
2373         typeset dataset=$3
2374         typeset newdataset=$4
2375
2376         if [[ $datatype != "fs" && $datatype != "vol" ]]; then
2377                 log_fail "$datatype is not supported."
2378         fi
2379
2380         # check parameters accordingly
2381         case $ops in
2382                 create)
2383                         newdataset=$dataset
2384                         dataset=""
2385                         if [[ $datatype == "vol" ]]; then
2386                                 ops="create -V $VOLSIZE"
2387                         fi
2388                         ;;
2389                 clone)
2390                         if [[ -z $newdataset ]]; then
2391                                 log_fail "newdataset should not be empty" \
2392                                         "when ops is $ops."
2393                         fi
2394                         log_must datasetexists $dataset
2395                         log_must snapexists $dataset
2396                         ;;
2397                 rename)
2398                         if [[ -z $newdataset ]]; then
2399                                 log_fail "newdataset should not be empty" \
2400                                         "when ops is $ops."
2401                         fi
2402                         log_must datasetexists $dataset
2403                         log_mustnot snapexists $dataset
2404                         ;;
2405                 *)
2406                         log_fail "$ops is not supported."
2407                         ;;
2408         esac
2409
2410         # make sure the upper level filesystem does not exist
2411         if datasetexists ${newdataset%/*} ; then
2412                 log_must $ZFS destroy -rRf ${newdataset%/*}
2413         fi
2414
2415         # without -p option, operation will fail
2416         log_mustnot $ZFS $ops $dataset $newdataset
2417         log_mustnot datasetexists $newdataset ${newdataset%/*}
2418
2419         # with -p option, operation should succeed
2420         log_must $ZFS $ops -p $dataset $newdataset
2421         block_device_wait
2422
2423         if ! datasetexists $newdataset ; then
2424                 log_fail "-p option does not work for $ops"
2425         fi
2426
2427         # when $ops is create or clone, redo the operation still return zero
2428         if [[ $ops != "rename" ]]; then
2429                 log_must $ZFS $ops -p $dataset $newdataset
2430         fi
2431
2432         return 0
2433 }
2434
2435 #
2436 # Get configuration of pool
2437 # $1 pool name
2438 # $2 config name
2439 #
2440 function get_config
2441 {
2442         typeset pool=$1
2443         typeset config=$2
2444         typeset alt_root
2445
2446         if ! poolexists "$pool" ; then
2447                 return 1
2448         fi
2449         alt_root=$($ZPOOL list -H $pool | $AWK '{print $NF}')
2450         if [[ $alt_root == "-" ]]; then
2451                 value=$($ZDB -C $pool | $GREP "$config:" | $AWK -F: \
2452                     '{print $2}')
2453         else
2454                 value=$($ZDB -e $pool | $GREP "$config:" | $AWK -F: \
2455                     '{print $2}')
2456         fi
2457         if [[ -n $value ]] ; then
2458                 value=${value#'}
2459                 value=${value%'}
2460         fi
2461         echo $value
2462
2463         return 0
2464 }
2465
2466 #
2467 # Privated function. Random select one of items from arguments.
2468 #
2469 # $1 count
2470 # $2-n string
2471 #
2472 function _random_get
2473 {
2474         typeset cnt=$1
2475         shift
2476
2477         typeset str="$@"
2478         typeset -i ind
2479         ((ind = RANDOM % cnt + 1))
2480
2481         typeset ret=$($ECHO "$str" | $CUT -f $ind -d ' ')
2482         $ECHO $ret
2483 }
2484
2485 #
2486 # Random select one of item from arguments which include NONE string
2487 #
2488 function random_get_with_non
2489 {
2490         typeset -i cnt=$#
2491         ((cnt =+ 1))
2492
2493         _random_get "$cnt" "$@"
2494 }
2495
2496 #
2497 # Random select one of item from arguments which doesn't include NONE string
2498 #
2499 function random_get
2500 {
2501         _random_get "$#" "$@"
2502 }
2503
2504 #
2505 # Detect if the current system support slog
2506 #
2507 function verify_slog_support
2508 {
2509         typeset dir=/tmp/disk.$$
2510         typeset pool=foo.$$
2511         typeset vdev=$dir/a
2512         typeset sdev=$dir/b
2513
2514         $MKDIR -p $dir
2515         $MKFILE 64M $vdev $sdev
2516
2517         typeset -i ret=0
2518         if ! $ZPOOL create -n $pool $vdev log $sdev > /dev/null 2>&1; then
2519                 ret=1
2520         fi
2521         $RM -r $dir
2522
2523         return $ret
2524 }
2525
2526 #
2527 # The function will generate a dataset name with specific length
2528 # $1, the length of the name
2529 # $2, the base string to construct the name
2530 #
2531 function gen_dataset_name
2532 {
2533         typeset -i len=$1
2534         typeset basestr="$2"
2535         typeset -i baselen=${#basestr}
2536         typeset -i iter=0
2537         typeset l_name=""
2538
2539         if ((len % baselen == 0)); then
2540                 ((iter = len / baselen))
2541         else
2542                 ((iter = len / baselen + 1))
2543         fi
2544         while ((iter > 0)); do
2545                 l_name="${l_name}$basestr"
2546
2547                 ((iter -= 1))
2548         done
2549
2550         $ECHO $l_name
2551 }
2552
2553 #
2554 # Get cksum tuple of dataset
2555 # $1 dataset name
2556 #
2557 # sample zdb output:
2558 # Dataset data/test [ZPL], ID 355, cr_txg 2413856, 31.0K, 7 objects, rootbp
2559 # DVA[0]=<0:803046400:200> DVA[1]=<0:81199000:200> [L0 DMU objset] fletcher4
2560 # lzjb LE contiguous unique double size=800L/200P birth=2413856L/2413856P
2561 # fill=7 cksum=11ce125712:643a9c18ee2:125e25238fca0:254a3f74b59744
2562 function datasetcksum
2563 {
2564         typeset cksum
2565         $SYNC
2566         cksum=$($ZDB -vvv $1 | $GREP "^Dataset $1 \[" | $GREP "cksum" \
2567                 | $AWK -F= '{print $7}')
2568         $ECHO $cksum
2569 }
2570
2571 #
2572 # Get cksum of file
2573 # #1 file path
2574 #
2575 function checksum
2576 {
2577         typeset cksum
2578         cksum=$($CKSUM $1 | $AWK '{print $1}')
2579         $ECHO $cksum
2580 }
2581
2582 #
2583 # Get the given disk/slice state from the specific field of the pool
2584 #
2585 function get_device_state #pool disk field("", "spares","logs")
2586 {
2587         typeset pool=$1
2588         typeset disk=${2#$DEV_DSKDIR/}
2589         typeset field=${3:-$pool}
2590
2591         state=$($ZPOOL status -v "$pool" 2>/dev/null | \
2592                 $NAWK -v device=$disk -v pool=$pool -v field=$field \
2593                 'BEGIN {startconfig=0; startfield=0; }
2594                 /config:/ {startconfig=1}
2595                 (startconfig==1) && ($1==field) {startfield=1; next;}
2596                 (startfield==1) && ($1==device) {print $2; exit;}
2597                 (startfield==1) &&
2598                 ($1==field || $1 ~ "^spares$" || $1 ~ "^logs$") {startfield=0}')
2599         echo $state
2600 }
2601
2602
2603 #
2604 # print the given directory filesystem type
2605 #
2606 # $1 directory name
2607 #
2608 function get_fstype
2609 {
2610         typeset dir=$1
2611
2612         if [[ -z $dir ]]; then
2613                 log_fail "Usage: get_fstype <directory>"
2614         fi
2615
2616         #
2617         #  $ df -n /
2618         #  /              : ufs
2619         #
2620         $DF -n $dir | $AWK '{print $3}'
2621 }
2622
2623 #
2624 # Given a disk, label it to VTOC regardless what label was on the disk
2625 # $1 disk
2626 #
2627 function labelvtoc
2628 {
2629         typeset disk=$1
2630         if [[ -z $disk ]]; then
2631                 log_fail "The disk name is unspecified."
2632         fi
2633         typeset label_file=/var/tmp/labelvtoc.$$
2634         typeset arch=$($UNAME -p)
2635
2636         if is_linux; then
2637                 log_note "Currently unsupported by the test framework"
2638                 return 1
2639         fi
2640
2641         if [[ $arch == "i386" ]]; then
2642                 $ECHO "label" > $label_file
2643                 $ECHO "0" >> $label_file
2644                 $ECHO "" >> $label_file
2645                 $ECHO "q" >> $label_file
2646                 $ECHO "q" >> $label_file
2647
2648                 $FDISK -B $disk >/dev/null 2>&1
2649                 # wait a while for fdisk finishes
2650                 $SLEEP 60
2651         elif [[ $arch == "sparc" ]]; then
2652                 $ECHO "label" > $label_file
2653                 $ECHO "0" >> $label_file
2654                 $ECHO "" >> $label_file
2655                 $ECHO "" >> $label_file
2656                 $ECHO "" >> $label_file
2657                 $ECHO "q" >> $label_file
2658         else
2659                 log_fail "unknown arch type"
2660         fi
2661
2662         $FORMAT -e -s -d $disk -f $label_file
2663         typeset -i ret_val=$?
2664         $RM -f $label_file
2665         #
2666         # wait the format to finish
2667         #
2668         $SLEEP 60
2669         if ((ret_val != 0)); then
2670                 log_fail "unable to label $disk as VTOC."
2671         fi
2672
2673         return 0
2674 }
2675
2676 #
2677 # check if the system was installed as zfsroot or not
2678 # return: 0 ture, otherwise false
2679 #
2680 function is_zfsroot
2681 {
2682         $DF -n / | $GREP zfs > /dev/null 2>&1
2683         return $?
2684 }
2685
2686 #
2687 # get the root filesystem name if it's zfsroot system.
2688 #
2689 # return: root filesystem name
2690 function get_rootfs
2691 {
2692         typeset rootfs=""
2693         rootfs=$($AWK '{if ($2 == "/" && $3 == "zfs") print $1}' \
2694                 /etc/mnttab)
2695         if [[ -z "$rootfs" ]]; then
2696                 log_fail "Can not get rootfs"
2697         fi
2698         $ZFS list $rootfs > /dev/null 2>&1
2699         if (($? == 0)); then
2700                 $ECHO $rootfs
2701         else
2702                 log_fail "This is not a zfsroot system."
2703         fi
2704 }
2705
2706 #
2707 # get the rootfs's pool name
2708 # return:
2709 #       rootpool name
2710 #
2711 function get_rootpool
2712 {
2713         typeset rootfs=""
2714         typeset rootpool=""
2715         rootfs=$($AWK '{if ($2 == "/" && $3 =="zfs") print $1}' \
2716                  /etc/mnttab)
2717         if [[ -z "$rootfs" ]]; then
2718                 log_fail "Can not get rootpool"
2719         fi
2720         $ZFS list $rootfs > /dev/null 2>&1
2721         if (($? == 0)); then
2722                 rootpool=`$ECHO $rootfs | awk -F\/ '{print $1}'`
2723                 $ECHO $rootpool
2724         else
2725                 log_fail "This is not a zfsroot system."
2726         fi
2727 }
2728
2729 #
2730 # Get the sub string from specified source string
2731 #
2732 # $1 source string
2733 # $2 start position. Count from 1
2734 # $3 offset
2735 #
2736 function get_substr #src_str pos offset
2737 {
2738         typeset pos offset
2739
2740         $ECHO $1 | \
2741                 $NAWK -v pos=$2 -v offset=$3 '{print substr($0, pos, offset)}'
2742 }
2743
2744 #
2745 # Check if the given device is physical device
2746 #
2747 function is_physical_device #device
2748 {
2749         typeset device=${1#$DEV_DSKDIR}
2750         device=${device#$DEV_RDSKDIR}
2751
2752         if is_linux; then
2753                 [[ -b "$DEV_DSKDIR/$device" ]] && \
2754                 [[ -f /sys/module/loop/parameters/max_part ]]
2755                 return $?
2756         else
2757                 $ECHO $device | $EGREP "^c[0-F]+([td][0-F]+)+$" > /dev/null 2>&1
2758                 return $?
2759         fi
2760 }
2761
2762 #
2763 # Check if the given device is a real device (ie SCSI device)
2764 #
2765 function is_real_device #disk
2766 {
2767         typeset disk=$1
2768         [[ -z $disk ]] && log_fail "No argument for disk given."
2769
2770        if is_linux; then
2771                 $LSBLK $DEV_RDSKDIR/$disk -o TYPE | $EGREP disk > /dev/null 2>&1
2772                 return $?
2773        fi
2774 }
2775
2776 #
2777 # Check if the given device is a loop device
2778 #
2779 function is_loop_device #disk
2780 {
2781         typeset disk=$1
2782         [[ -z $disk ]] && log_fail "No argument for disk given."
2783
2784        if is_linux; then
2785                 $LSBLK $DEV_RDSKDIR/$disk -o TYPE | $EGREP loop > /dev/null 2>&1
2786                 return $?
2787        fi
2788 }
2789
2790 #
2791 # Check if the given device is a multipath device and if there is a sybolic
2792 # link to a device mapper and to a disk
2793 # Currently no support for dm devices alone without multipath
2794 #
2795 function is_mpath_device #disk
2796 {
2797         typeset disk=$1
2798         [[ -z $disk ]] && log_fail "No argument for disk given."
2799
2800         if is_linux; then
2801                 $LSBLK $DEV_MPATHDIR/$disk -o TYPE | $EGREP mpath > /dev/null 2>&1
2802                 if (($? == 0)); then
2803                         $READLINK $DEV_MPATHDIR/$disk > /dev/null 2>&1
2804                         return $?
2805                 else
2806                         return $?
2807                 fi
2808         fi
2809 }
2810
2811 # Set the slice prefix for disk partitioning depending
2812 # on whether the device is a real, multipath, or loop device.
2813 # Currently all disks have to be of the same type, so only
2814 # checks first disk to determine slice prefix.
2815 #
2816 function set_slice_prefix
2817 {
2818         typeset disk
2819         typeset -i i=0
2820
2821         if is_linux; then
2822                 while (( i < $DISK_ARRAY_NUM )); do
2823                         disk="$($ECHO $DISKS | $NAWK '{print $(i + 1)}')"
2824                         if ( is_mpath_device $disk ) && [[ -z $($ECHO $disk | awk 'substr($1,18,1)\
2825                              ~ /^[[:digit:]]+$/') ]] || ( is_real_device $disk ); then
2826                                 export SLICE_PREFIX=""
2827                                 return 0
2828                         elif ( is_mpath_device $disk || is_loop_device $disk ); then
2829                                 export SLICE_PREFIX="p"
2830                                 return 0
2831                         else
2832                                 log_fail "$disk not supported for partitioning."
2833                         fi
2834                         (( i = i + 1))
2835                 done
2836         fi
2837 }
2838
2839 #
2840 # Set the directory path of the listed devices in $DISK_ARRAY_NUM
2841 # Currently all disks have to be of the same type, so only
2842 # checks first disk to determine device directory
2843 # default = /dev (linux)
2844 # real disk = /dev (linux)
2845 # multipath device = /dev/mapper (linux)
2846 #
2847 function set_device_dir
2848 {
2849         typeset disk
2850         typeset -i i=0
2851
2852         if is_linux; then
2853                 while (( i < $DISK_ARRAY_NUM )); do
2854                         disk="$($ECHO $DISKS | $NAWK '{print $(i + 1)}')"
2855                         if is_mpath_device $disk; then
2856                                 export DEV_DSKDIR=$DEV_MPATHDIR
2857                                 return 0
2858                         else
2859                                 export DEV_DSKDIR=$DEV_RDSKDIR
2860                                 return 0
2861                         fi
2862                         (( i = i + 1))
2863                 done
2864         else
2865                 export DEV_DSKDIR=$DEV_RDSKDIR
2866         fi
2867 }
2868
2869 #
2870 # Get the directory path of given device
2871 #
2872 function get_device_dir #device
2873 {
2874         typeset device=$1
2875
2876         if ! $(is_physical_device $device) ; then
2877                 if [[ $device != "/" ]]; then
2878                         device=${device%/*}
2879                 fi
2880                 if [[ -b "$DEV_DSKDIR/$device" ]]; then
2881                         device="$DEV_DSKDIR"
2882                 fi
2883                 $ECHO $device
2884         else
2885                 $ECHO "$DEV_DSKDIR"
2886         fi
2887 }
2888
2889 #
2890 # Get the package name
2891 #
2892 function get_package_name
2893 {
2894         typeset dirpath=${1:-$STC_NAME}
2895
2896         echo "SUNWstc-${dirpath}" | /usr/bin/sed -e "s/\//-/g"
2897 }
2898
2899 #
2900 # Get the word numbers from a string separated by white space
2901 #
2902 function get_word_count
2903 {
2904         $ECHO $1 | $WC -w
2905 }
2906
2907 #
2908 # To verify if the require numbers of disks is given
2909 #
2910 function verify_disk_count
2911 {
2912         typeset -i min=${2:-1}
2913
2914         typeset -i count=$(get_word_count "$1")
2915
2916         if ((count < min)); then
2917                 log_untested "A minimum of $min disks is required to run." \
2918                         " You specified $count disk(s)"
2919         fi
2920 }
2921
2922 function ds_is_volume
2923 {
2924         typeset type=$(get_prop type $1)
2925         [[ $type = "volume" ]] && return 0
2926         return 1
2927 }
2928
2929 function ds_is_filesystem
2930 {
2931         typeset type=$(get_prop type $1)
2932         [[ $type = "filesystem" ]] && return 0
2933         return 1
2934 }
2935
2936 function ds_is_snapshot
2937 {
2938         typeset type=$(get_prop type $1)
2939         [[ $type = "snapshot" ]] && return 0
2940         return 1
2941 }
2942
2943 #
2944 # Check if Trusted Extensions are installed and enabled
2945 #
2946 function is_te_enabled
2947 {
2948         $SVCS -H -o state labeld 2>/dev/null | $GREP "enabled"
2949         if (($? != 0)); then
2950                 return 1
2951         else
2952                 return 0
2953         fi
2954 }
2955
2956 # Utility function to determine if a system has multiple cpus.
2957 function is_mp
2958 {
2959         if is_linux; then
2960                 (($($NPROC) > 1))
2961         else
2962                 (($($PSRINFO | $WC -l) > 1))
2963         fi
2964
2965         return $?
2966 }
2967
2968 function get_cpu_freq
2969 {
2970         if is_linux; then
2971                 lscpu | $AWK '/CPU MHz/ { print $3 }'
2972         else
2973                 $PSRINFO -v 0 | $AWK '/processor operates at/ {print $6}'
2974         fi
2975 }
2976
2977 # Run the given command as the user provided.
2978 function user_run
2979 {
2980         typeset user=$1
2981         shift
2982
2983         log_note "user:$user $@"
2984         eval \$SU \$user -c \"$@\" > /tmp/out 2>/tmp/err
2985         return $?
2986 }
2987
2988 #
2989 # Check if the pool contains the specified vdevs
2990 #
2991 # $1 pool
2992 # $2..n <vdev> ...
2993 #
2994 # Return 0 if the vdevs are contained in the pool, 1 if any of the specified
2995 # vdevs is not in the pool, and 2 if pool name is missing.
2996 #
2997 function vdevs_in_pool
2998 {
2999         typeset pool=$1
3000         typeset vdev
3001
3002         if [[ -z $pool ]]; then
3003                 log_note "Missing pool name."
3004                 return 2
3005         fi
3006
3007         shift
3008
3009         typeset tmpfile=$($MKTEMP)
3010         $ZPOOL list -Hv "$pool" >$tmpfile
3011         for vdev in $@; do
3012                 $GREP -w ${vdev##*/} $tmpfile >/dev/null 2>&1
3013                 [[ $? -ne 0 ]] && return 1
3014         done
3015
3016         $RM -f $tmpfile
3017
3018         return 0;
3019 }
3020
3021 function get_max
3022 {
3023         typeset -l i max=$1
3024         shift
3025
3026         for i in "$@"; do
3027                 max=$(echo $((max > i ? max : i)))
3028         done
3029
3030         echo $max
3031 }
3032
3033 function get_min
3034 {
3035         typeset -l i min=$1
3036         shift
3037
3038         for i in "$@"; do
3039                 min=$(echo $((min < i ? min : i)))
3040         done
3041
3042         echo $min
3043 }
3044
3045 #
3046 # Wait for newly created block devices to have their minors created.
3047 #
3048 function block_device_wait
3049 {
3050         if is_linux; then
3051                 $UDEVADM trigger
3052                 $UDEVADM settle
3053         fi
3054 }
3055
3056 #
3057 # Synchronize all the data in pool
3058 #
3059 # $1 pool name
3060 #
3061 function sync_pool #pool
3062 {
3063         typeset pool=${1:-$TESTPOOL}
3064
3065         log_must $SYNC
3066         log_must $SLEEP 2
3067         # Flush all the pool data.
3068         typeset -i ret
3069         $ZPOOL scrub $pool >/dev/null 2>&1
3070         ret=$?
3071         (( $ret != 0 )) && \
3072         log_fail "$ZPOOL scrub $pool failed."
3073
3074         while ! is_pool_scrubbed $pool; do
3075                 if is_pool_resilvered $pool ; then
3076                         log_fail "$pool should not be resilver completed."
3077                 fi
3078                 log_must $SLEEP 2
3079         done
3080 }