tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib

   1 #
   2 # CDDL HEADER START
   3 #
   4 # The contents of this file are subject to the terms of the
   5 # Common Development and Distribution License (the "License").
   6 # You may not use this file except in compliance with the License.
   7 #
   8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9 # or http://www.opensolaris.org/os/licensing.
  10 # See the License for the specific language governing permissions
  11 # and limitations under the License.
  12 #
  13 # When distributing Covered Code, include this CDDL HEADER in each
  14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15 # If applicable, add the following below this CDDL HEADER, with the
  16 # fields enclosed by brackets "[]" replaced with your own identifying
  17 # information: Portions Copyright [yyyy] [name of copyright owner]
  18 #
  19 # CDDL HEADER END
  20 #
  21
  22 #
  23 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24 # Use is subject to license terms.
  25 #
  26
  27 #
  28 # Copyright (c) 2013 by Delphix. All rights reserved.
  29 #
  30
  31 . $STF_SUITE/include/libtest.shlib
  32 . $STF_SUITE/tests/functional/redundancy/redundancy.cfg
  33
  34 function cleanup
  35 {
  36         if poolexists $TESTPOOL; then
  37                 destroy_pool $TESTPOOL
  38         fi
  39         typeset dir
  40         for dir in $TESTDIR $BASEDIR; do
  41                 if [[ -d $dir ]]; then
  42                         log_must $RM -rf $dir
  43                 fi
  44         done
  45 }
  46
  47 #
  48 # Get random number between min and max number.
  49 #
  50 # $1 Minimal value
  51 # $2 Maximal value
  52 #
  53 function random
  54 {
  55         typeset -i min=$1
  56         typeset -i max=$2
  57         typeset -i value
  58
  59         while true; do
  60                 ((value = RANDOM % (max + 1)))
  61                 if ((value >= min)); then
  62                         break
  63                 fi
  64         done
  65
  66         $ECHO $value
  67 }
  68
  69 #
  70 # Record the directories construction and checksum all the files which reside
  71 # within the specified pool
  72 #
  73 # $1 The specified pool
  74 # $2 The file which save the record.
  75 #
  76 function record_data
  77 {
  78         typeset pool=$1
  79         typeset recordfile=$2
  80
  81         [[ -z $pool ]] && log_fail "No specified pool."
  82         [[ -f $recordfile ]] && log_must $RM -f $recordfile
  83
  84         typeset mntpnt
  85         mntpnt=$(get_prop mountpoint $pool)
  86         log_must eval "$DU -a $mntpnt > $recordfile 2>&1"
  87         #
  88         # When the data was damaged, checksum is failing and return 1
  89         # So, will not use log_must
  90         #
  91         $FIND $mntpnt -type f -exec $CKSUM {} + >> $recordfile 2>&1
  92 }
  93
  94 #
  95 # Create test pool and fill with files and directories.
  96 #
  97 # $1 pool name
  98 # $2 pool type
  99 # $3 virtual devices number
 100 #
 101 function setup_test_env
 102 {
 103         typeset pool=$1
 104         typeset keyword=$2
 105         typeset -i vdev_cnt=$3
 106         typeset vdevs
 107
 108         typeset -i i=0
 109         while (( i < vdev_cnt )); do
 110                 vdevs="$vdevs $BASEDIR/vdev$i"
 111                 ((i += 1))
 112         done
 113
 114         if [[ ! -d $BASEDIR ]]; then
 115                 log_must $MKDIR $BASEDIR
 116         fi
 117
 118         if poolexists $pool ; then
 119                 destroy_pool $pool
 120         fi
 121
 122         log_must $MKFILE $DEV_SIZE $vdevs
 123
 124         log_must $ZPOOL create -m $TESTDIR $pool $keyword $vdevs
 125
 126         log_note "Filling up the filesystem ..."
 127         typeset -i ret=0
 128         typeset -i i=0
 129         typeset file=$TESTDIR/file
 130         while $TRUE ; do
 131                 $FILE_WRITE -o create -f $file.$i \
 132                         -b $BLOCKSZ -c $NUM_WRITES
 133                 ret=$?
 134                 (( $ret != 0 )) && break
 135                 (( i = i + 1 ))
 136         done
 137         (($ret != 28 )) && log_note "$FILE_WRITE return value($ret) is unexpected."
 138
 139         record_data $TESTPOOL $PRE_RECORD_FILE
 140 }
 141
 142 #
 143 # Check pool status is healthy
 144 #
 145 # $1 pool
 146 #
 147 function is_healthy
 148 {
 149         typeset pool=$1
 150
 151         typeset healthy_output="pool '$pool' is healthy"
 152         typeset real_output=$($ZPOOL status -x $pool)
 153
 154         if [[ "$real_output" == "$healthy_output" ]]; then
 155                 return 0
 156         else
 157                 typeset -i ret
 158                 $ZPOOL status -x $pool | $GREP "state:" | \
 159                         $GREP "FAULTED" >/dev/null 2>&1
 160                 ret=$?
 161                 (( $ret == 0 )) && return 1
 162                 typeset l_scan
 163                 typeset errnum
 164                 l_scan=$($ZPOOL status -x $pool | $GREP "scan:")
 165                 l_scan=${l_scan##*"with"}
 166                 errnum=$($ECHO $l_scan | $AWK '{print $1}')
 167
 168                 return $errnum
 169         fi
 170 }
 171
 172 #
 173 # Check pool data is valid
 174 #
 175 # $1 pool
 176 #
 177 function is_data_valid
 178 {
 179         typeset pool=$1
 180
 181         record_data $pool $PST_RECORD_FILE
 182         if ! $DIFF $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then
 183                 return 1
 184         fi
 185
 186         return 0
 187 }
 188
 189 #
 190 # Get the specified count devices name
 191 #
 192 # $1 pool name
 193 # $2 devices count
 194 #
 195 function get_vdevs #pool cnt
 196 {
 197         typeset pool=$1
 198         typeset -i cnt=$2
 199
 200         typeset all_devs=$($ZPOOL iostat -v $pool | $AWK '{print $1}'| \
 201                 $EGREP -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|---" | \
 202                 $EGREP -v "/old$|^$pool$")
 203         typeset -i i=0
 204         typeset vdevs
 205         while ((i < cnt)); do
 206                 typeset dev=$($ECHO $all_devs | $AWK '{print $1}')
 207                 eval all_devs=\${all_devs##*$dev}
 208
 209                 vdevs="$dev $vdevs"
 210                 ((i += 1))
 211         done
 212
 213         $ECHO "$vdevs"
 214 }
 215
 216 #
 217 # Create and replace the same name virtual device files
 218 #
 219 # $1 pool name
 220 # $2-n virtual device files
 221 #
 222 function replace_missing_devs
 223 {
 224         typeset pool=$1
 225         shift
 226
 227         typeset vdev
 228         for vdev in $@; do
 229                 log_must $MKFILE $DEV_SIZE $vdev
 230                 log_must $ZPOOL replace -f $pool $vdev $vdev
 231                 while true; do
 232                         if ! is_pool_resilvered $pool ; then
 233                                 log_must $SLEEP 2
 234                         else
 235                                 break
 236                         fi
 237                 done
 238         done
 239 }
 240
 241 #
 242 # Damage the pool's virtual device files.
 243 #
 244 # $1 pool name
 245 # $2 Failing devices count
 246 # $3 damage vdevs method, if not null, we keep
 247 #    the label for the vdevs
 248 #
 249 function damage_devs
 250 {
 251         typeset pool=$1
 252         typeset -i cnt=$2
 253         typeset label="$3"
 254         typeset vdevs
 255         typeset -i bs_count
 256
 257         vdevs=$(get_vdevs $pool $cnt)
 258         if [[ -n $label ]]; then
 259                 typeset dev
 260                 for dev in $vdevs; do
 261                         bs_count=$($LS -l $dev | $AWK '{print $5}')
 262                         (( bs_count = bs_count/1024 - 512 ))
 263                         $DD if=/dev/zero of=$dev seek=512 bs=1024 \
 264                                 count=$bs_count conv=notrunc >/dev/null 2>&1
 265                 done
 266         else
 267                 log_must $MKFILE $DEV_SIZE $vdevs
 268         fi
 269
 270         sync_pool $pool
 271 }
 272
 273 #
 274 # Clear errors in the pool caused by data corruptions
 275 #
 276 # $1 pool name
 277 #
 278 function clear_errors
 279 {
 280         typeset pool=$1
 281
 282         log_must $ZPOOL clear $pool
 283
 284         if ! is_healthy $pool ; then
 285                 log_note "$pool should be healthy."
 286                 return 1
 287         fi
 288         if ! is_data_valid $pool ; then
 289                 log_note "Data should be valid in $pool."
 290                 return 1
 291         fi
 292
 293         return 0
 294 }
 295
 296 #
 297 # Remove the specified pool's virtual device files
 298 #
 299 # $1 Pool name
 300 # $2 Missing devices count
 301 #
 302 function remove_devs
 303 {
 304         typeset pool=$1
 305         typeset -i cnt=$2
 306         typeset vdevs
 307
 308         vdevs=$(get_vdevs $pool $cnt)
 309         log_must $RM -f $vdevs
 310
 311         sync_pool $pool
 312 }
 313
 314 #
 315 # Recover the bad or missing device files in the pool
 316 #
 317 # $1 Pool name
 318 # $2 Missing devices count
 319 #
 320 function recover_bad_missing_devs
 321 {
 322         typeset pool=$1
 323         typeset -i cnt=$2
 324         typeset vdevs
 325
 326         vdevs=$(get_vdevs $pool $cnt)
 327         replace_missing_devs $pool $vdevs
 328
 329         if ! is_healthy $pool ; then
 330                 log_note "$pool should be healthy."
 331                 return 1
 332         fi
 333         if ! is_data_valid $pool ; then
 334                 log_note "Data should be valid in $pool."
 335                 return 1
 336         fi
 337
 338         return 0
 339 }