tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib

   1 #
   2 # CDDL HEADER START
   3 #
   4 # The contents of this file are subject to the terms of the
   5 # Common Development and Distribution License (the "License").
   6 # You may not use this file except in compliance with the License.
   7 #
   8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9 # or http://www.opensolaris.org/os/licensing.
  10 # See the License for the specific language governing permissions
  11 # and limitations under the License.
  12 #
  13 # When distributing Covered Code, include this CDDL HEADER in each
  14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15 # If applicable, add the following below this CDDL HEADER, with the
  16 # fields enclosed by brackets "[]" replaced with your own identifying
  17 # information: Portions Copyright [yyyy] [name of copyright owner]
  18 #
  19 # CDDL HEADER END
  20 #
  21
  22 #
  23 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24 # Use is subject to license terms.
  25 #
  26
  27 #
  28 # Copyright (c) 2013 by Delphix. All rights reserved.
  29 #
  30
  31 . $STF_SUITE/include/libtest.shlib
  32 . $STF_SUITE/tests/functional/redundancy/redundancy.cfg
  33
  34 function cleanup
  35 {
  36         if poolexists $TESTPOOL; then
  37                 destroy_pool $TESTPOOL
  38         fi
  39         typeset dir
  40         for dir in $TESTDIR $BASEDIR; do
  41                 if [[ -d $dir ]]; then
  42                         log_must $RM -rf $dir
  43                 fi
  44         done
  45 }
  46
  47 #
  48 # Get random number between min and max number.
  49 #
  50 # $1 Minimal value
  51 # $2 Maximal value
  52 #
  53 function random
  54 {
  55         typeset -i min=$1
  56         typeset -i max=$2
  57         typeset -i value
  58
  59         while true; do
  60                 ((value = RANDOM % (max + 1)))
  61                 if ((value >= min)); then
  62                         break
  63                 fi
  64         done
  65
  66         $ECHO $value
  67 }
  68
  69 #
  70 # Record the directories construction and checksum all the files which reside
  71 # within the specified pool
  72 #
  73 # $1 The specified pool
  74 # $2 The file which save the record.
  75 #
  76 function record_data
  77 {
  78         typeset pool=$1
  79         typeset recordfile=$2
  80
  81         [[ -z $pool ]] && log_fail "No specified pool."
  82         [[ -f $recordfile ]] && log_must $RM -f $recordfile
  83
  84         typeset mntpnt
  85         mntpnt=$(get_prop mountpoint $pool)
  86         log_must eval "$DU -a $mntpnt > $recordfile 2>&1"
  87         #
  88         # When the data was damaged, checksum is failing and return 1
  89         # So, will not use log_must
  90         #
  91         $FIND $mntpnt -type f -exec $CKSUM {} + >> $recordfile 2>&1
  92 }
  93
  94 #
  95 # Create test pool and fill with files and directories.
  96 #
  97 # $1 pool name
  98 # $2 pool type
  99 # $3 virtual devices number
 100 #
 101 function setup_test_env
 102 {
 103         typeset pool=$1
 104         typeset keyword=$2
 105         typeset -i vdev_cnt=$3
 106         typeset vdevs
 107
 108         typeset -i i=0
 109         while (( i < vdev_cnt )); do
 110                 vdevs="$vdevs $BASEDIR/vdev$i"
 111                 ((i += 1))
 112         done
 113
 114         if [[ ! -d $BASEDIR ]]; then
 115                 log_must $MKDIR $BASEDIR
 116         fi
 117
 118         if poolexists $pool ; then
 119                 destroy_pool $pool
 120         fi
 121
 122         log_must $MKFILE $DEV_SIZE $vdevs
 123
 124         log_must $ZPOOL create -m $TESTDIR $pool $keyword $vdevs
 125
 126         log_note "Filling up the filesystem ..."
 127         typeset -i ret=0
 128         typeset -i i=0
 129         typeset file=$TESTDIR/file
 130         while $TRUE ; do
 131                 $FILE_WRITE -o create -f $file.$i \
 132                         -b $BLOCKSZ -c $NUM_WRITES
 133                 ret=$?
 134                 (( $ret != 0 )) && break
 135                 (( i = i + 1 ))
 136         done
 137         (($ret != 28 )) && log_note "$FILE_WRITE return value($ret) is unexpected."
 138
 139         record_data $TESTPOOL $PRE_RECORD_FILE
 140 }
 141
 142 #
 143 # Check pool status is healthy
 144 #
 145 # $1 pool
 146 #
 147 function is_healthy
 148 {
 149         typeset pool=$1
 150
 151         typeset healthy_output="pool '$pool' is healthy"
 152         typeset real_output=$($ZPOOL status -x $pool)
 153
 154         if [[ "$real_output" == "$healthy_output" ]]; then
 155                 return 0
 156         else
 157                 typeset -i ret
 158                 $ZPOOL status -x $pool | $GREP "state:" | \
 159                         $GREP "FAULTED" >/dev/null 2>&1
 160                 ret=$?
 161                 (( $ret == 0 )) && return 1
 162                 typeset l_scan
 163                 typeset errnum
 164                 l_scan=$($ZPOOL status -x $pool | $GREP "scan:")
 165                 l_scan=${l_scan##*"with"}
 166                 errnum=$($ECHO $l_scan | $AWK '{print $1}')
 167
 168                 return $errnum
 169         fi
 170 }
 171
 172 #
 173 # Check pool data is valid
 174 #
 175 # $1 pool
 176 #
 177 function is_data_valid
 178 {
 179         typeset pool=$1
 180
 181         record_data $pool $PST_RECORD_FILE
 182         if ! $DIFF $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then
 183                 return 1
 184         fi
 185
 186         return 0
 187 }
 188
 189 #
 190 # Get the specified count devices name
 191 #
 192 # $1 pool name
 193 # $2 devices count
 194 #
 195 function get_vdevs #pool cnt
 196 {
 197         typeset pool=$1
 198         typeset -i cnt=$2
 199
 200         typeset all_devs=$($ZPOOL iostat -v $pool | $AWK '{print $1}'| \
 201                 $EGREP -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|---" | \
 202                 $EGREP -v "/old$|^$pool$")
 203         typeset -i i=0
 204         typeset vdevs
 205         while ((i < cnt)); do
 206                 typeset dev=$($ECHO $all_devs | $AWK '{print $1}')
 207                 eval all_devs=\${all_devs##*$dev}
 208
 209                 vdevs="$dev $vdevs"
 210                 ((i += 1))
 211         done
 212
 213         $ECHO "$vdevs"
 214 }
 215
 216 #
 217 # Synchronize all the data in pool
 218 #
 219 # $1 pool name
 220 #
 221 function sync_pool #pool
 222 {
 223         typeset pool=$1
 224
 225         log_must $SYNC
 226         log_must $SLEEP 2
 227         # Flush all the pool data.
 228         typeset -i ret
 229         $ZPOOL scrub $pool >/dev/null 2>&1
 230         ret=$?
 231         (( $ret != 0 )) && \
 232                 log_fail "$ZPOOL scrub $pool failed."
 233
 234         while ! is_pool_scrubbed $pool; do
 235                 if is_pool_resilvered $pool ; then
 236                         log_fail "$pool should not be resilver completed."
 237                 fi
 238                 log_must $SLEEP 2
 239         done
 240 }
 241
 242 #
 243 # Create and replace the same name virtual device files
 244 #
 245 # $1 pool name
 246 # $2-n virtual device files
 247 #
 248 function replace_missing_devs
 249 {
 250         typeset pool=$1
 251         shift
 252
 253         typeset vdev
 254         for vdev in $@; do
 255                 log_must $MKFILE $DEV_SIZE $vdev
 256                 log_must $ZPOOL replace -f $pool $vdev $vdev
 257                 while true; do
 258                         if ! is_pool_resilvered $pool ; then
 259                                 log_must $SLEEP 2
 260                         else
 261                                 break
 262                         fi
 263                 done
 264         done
 265 }
 266
 267 #
 268 # Damage the pool's virtual device files.
 269 #
 270 # $1 pool name
 271 # $2 Failing devices count
 272 # $3 damage vdevs method, if not null, we keep
 273 #    the label for the vdevs
 274 #
 275 function damage_devs
 276 {
 277         typeset pool=$1
 278         typeset -i cnt=$2
 279         typeset label="$3"
 280         typeset vdevs
 281         typeset -i bs_count
 282
 283         vdevs=$(get_vdevs $pool $cnt)
 284         if [[ -n $label ]]; then
 285                 typeset dev
 286                 for dev in $vdevs; do
 287                         bs_count=$($LS -l $dev | $AWK '{print $5}')
 288                         (( bs_count = bs_count/1024 - 512 ))
 289                         $DD if=/dev/zero of=$dev seek=512 bs=1024 \
 290                                 count=$bs_count conv=notrunc >/dev/null 2>&1
 291                 done
 292         else
 293                 log_must $MKFILE $DEV_SIZE $vdevs
 294         fi
 295
 296         sync_pool $pool
 297 }
 298
 299 #
 300 # Clear errors in the pool caused by data corruptions
 301 #
 302 # $1 pool name
 303 #
 304 function clear_errors
 305 {
 306         typeset pool=$1
 307
 308         log_must $ZPOOL clear $pool
 309
 310         if ! is_healthy $pool ; then
 311                 log_note "$pool should be healthy."
 312                 return 1
 313         fi
 314         if ! is_data_valid $pool ; then
 315                 log_note "Data should be valid in $pool."
 316                 return 1
 317         fi
 318
 319         return 0
 320 }
 321
 322 #
 323 # Remove the specified pool's virtual device files
 324 #
 325 # $1 Pool name
 326 # $2 Missing devices count
 327 #
 328 function remove_devs
 329 {
 330         typeset pool=$1
 331         typeset -i cnt=$2
 332         typeset vdevs
 333
 334         vdevs=$(get_vdevs $pool $cnt)
 335         log_must $RM -f $vdevs
 336
 337         sync_pool $pool
 338 }
 339
 340 #
 341 # Recover the bad or missing device files in the pool
 342 #
 343 # $1 Pool name
 344 # $2 Missing devices count
 345 #
 346 function recover_bad_missing_devs
 347 {
 348         typeset pool=$1
 349         typeset -i cnt=$2
 350         typeset vdevs
 351
 352         vdevs=$(get_vdevs $pool $cnt)
 353         replace_missing_devs $pool $vdevs
 354
 355         if ! is_healthy $pool ; then
 356                 log_note "$pool should be healthy."
 357                 return 1
 358         fi
 359         if ! is_data_valid $pool ; then
 360                 log_note "Data should be valid in $pool."
 361                 return 1
 362         fi
 363
 364         return 0
 365 }