tests/zfs-tests/tests/functional/mmp/mmp.kshlib

   1 #
   2 # CDDL HEADER START
   3 #
   4 # The contents of this file are subject to the terms of the
   5 # Common Development and Distribution License (the "License").
   6 # You may not use this file except in compliance with the License.
   7 #
   8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9 # or https://opensource.org/licenses/CDDL-1.0.
  10 # See the License for the specific language governing permissions
  11 # and limitations under the License.
  12 #
  13 # When distributing Covered Code, include this CDDL HEADER in each
  14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15 # If applicable, add the following below this CDDL HEADER, with the
  16 # fields enclosed by brackets "[]" replaced with your own identifying
  17 # information: Portions Copyright [yyyy] [name of copyright owner]
  18 #
  19 # CDDL HEADER END
  20 #
  21
  22 #
  23 # Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
  24 # Use is subject to license terms.
  25 #
  26
  27 . $STF_SUITE/include/libtest.shlib
  28 . $STF_SUITE/tests/functional/mmp/mmp.cfg
  29
  30
  31 function check_pool_import # pool opts token keyword
  32 {
  33         typeset pool=${1:-$MMP_POOL}
  34         typeset opts=$2
  35         typeset token=$3
  36         typeset keyword=$4
  37
  38         zpool import $opts 2>&1 | \
  39             awk -v token="$token:" '($1==token) {print}' | \
  40             grep -iq "$keyword"
  41 }
  42
  43 function is_pool_imported # pool opts
  44 {
  45         typeset pool=${1:-$MMP_POOL}
  46         typeset opts=$2
  47
  48         check_pool_import "$pool" "$opts" "status" \
  49             "The pool is currently imported"
  50 }
  51
  52 function wait_pool_imported # pool opts
  53 {
  54         typeset pool=${1:-$MMP_POOL}
  55         typeset opts=$2
  56
  57         while is_pool_imported "$pool" "$opts"; do
  58                 log_must sleep 5
  59         done
  60 }
  61
  62 function try_pool_import # pool opts message
  63 {
  64         typeset pool=${1:-$MMP_POOL}
  65         typeset opts=$2
  66         typeset msg=$3
  67
  68         zpool import $opts $pool 2>&1 | grep -i "$msg"
  69 }
  70
  71 function mmp_set_hostid
  72 {
  73         typeset hostid=$1
  74
  75         zgenhostid $1
  76
  77         [ $(hostid) = "$hostid" ]
  78 }
  79
  80 function mmp_clear_hostid
  81 {
  82         rm -f $HOSTID_FILE
  83 }
  84
  85 function mmp_pool_create_simple # pool dir
  86 {
  87         typeset pool=${1:-$MMP_POOL}
  88         typeset dir=${2:-$MMP_DIR}
  89
  90         log_must mkdir -p $dir
  91         log_must rm -f $dir/*
  92         log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2
  93
  94         log_must mmp_clear_hostid
  95         log_must mmp_set_hostid $HOSTID1
  96         log_must zpool create -f -o cachefile=$MMP_CACHE $pool \
  97             mirror $dir/vdev1 $dir/vdev2
  98         log_must zpool set multihost=on $pool
  99 }
 100
 101 function mmp_pool_create # pool dir
 102 {
 103         typeset pool=${1:-$MMP_POOL}
 104         typeset dir=${2:-$MMP_DIR}
 105         typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool"
 106
 107         mmp_pool_create_simple $pool $dir
 108
 109         log_must mv $MMP_CACHE ${MMP_CACHE}.stale
 110         log_must zpool export $pool
 111         log_must mmp_clear_hostid
 112         log_must mmp_set_hostid $HOSTID2
 113
 114         log_note "Starting ztest in the background as hostid $HOSTID1"
 115         log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &"
 116
 117         while ! is_pool_imported "$pool" "-d $dir"; do
 118                 log_must pgrep ztest
 119                 log_must sleep 5
 120         done
 121 }
 122
 123 function mmp_pool_destroy # pool dir
 124 {
 125         typeset pool=${1:-$MMP_POOL}
 126         typeset dir=${2:-$MMP_DIR}
 127
 128         ZTESTPID=$(pgrep ztest)
 129         if [ -n "$ZTESTPID" ]; then
 130                 log_must kill $ZTESTPID
 131                 wait $ZTESTPID
 132         fi
 133
 134         if poolexists $pool; then
 135                 destroy_pool $pool
 136         fi
 137
 138         log_must rm -f $dir/*
 139         mmp_clear_hostid
 140 }
 141
 142 function mmp_pool_set_hostid # pool hostid
 143 {
 144         typeset pool=$1
 145         typeset hostid=$2
 146
 147         log_must mmp_clear_hostid
 148         log_must mmp_set_hostid $hostid
 149         log_must zpool export $pool
 150         log_must zpool import $pool
 151
 152         return 0
 153 }
 154
 155 function import_no_activity_check # pool opts
 156 {
 157         typeset pool=$1
 158         typeset opts=$2
 159
 160         typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1))
 161
 162         SECONDS=0
 163         zpool import $opts $pool
 164         typeset rc=$?
 165
 166         if [[ $SECONDS -gt $max_duration ]]; then
 167                 log_fail "ERROR: import_no_activity_check unexpected activity \
 168 check (${SECONDS}s gt $max_duration)"
 169         fi
 170
 171         return $rc
 172 }
 173
 174 function import_activity_check # pool opts act_test_duration
 175 {
 176         typeset pool=$1
 177         typeset opts=$2
 178         typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT}
 179
 180         SECONDS=0
 181         zpool import $opts $pool
 182         typeset rc=$?
 183
 184         if [[ $SECONDS -le $min_duration ]]; then
 185                 log_fail "ERROR: import_activity_check expected activity check \
 186 (${SECONDS}s le min_duration $min_duration)"
 187         fi
 188
 189         return $rc
 190 }
 191
 192 function clear_mmp_history
 193 {
 194         log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY_OFF
 195         log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY
 196 }
 197
 198 function count_skipped_mmp_writes # pool duration
 199 {
 200         typeset pool=$1
 201         typeset -i duration=$2
 202         typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost"
 203
 204         sleep $duration
 205         awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' "$hist_path"
 206 }
 207
 208 function count_mmp_writes # pool duration
 209 {
 210         typeset pool=$1
 211         typeset -i duration=$2
 212         typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost"
 213
 214         sleep $duration
 215         awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' "$hist_path"
 216 }
 217
 218 function summarize_uberblock_mmp # device
 219 {
 220         typeset device=$1
 221
 222         zdb -luuuu $device | awk '
 223         BEGIN                           {write_fail_present=0; write_fail_missing=0; uber_invalid=0;}
 224         /Uberblock\[[0-9][0-9]*\]/      {delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};};
 225         /mmp_fail/                      {fail=$3};
 226         /mmp_seq/                       {seq=$3};
 227         /mmp_write/                     {write=$3};
 228         /mmp_delay/                     {delay=$3; if (delay==0) {delay_zero++};};
 229         /mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++};
 230         /mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++};
 231         /mmp_valid/ && delay>0 && write<=0 {write_missing++};
 232         /mmp_valid/ && delay>0 && fail<=0 {fail_missing++};
 233         /mmp_valid/ && delay>0 && seq>0 {seq_nonzero++};
 234         END {
 235                 print "total_uberblocks " total;
 236                 print "delay_zero " delay_zero;
 237                 print "write_fail_present " write_fail_present;
 238                 print "write_fail_missing " write_fail_missing;
 239                 print "write_missing " write_missing;
 240                 print "fail_missing " fail_missing;
 241                 print "seq_nonzero " seq_nonzero;
 242                 print "uberblock_invalid " uber_invalid;
 243         }'
 244 }
 245
 246 function count_mmp_write_fail_present # device
 247 {
 248         typeset device=$1
 249
 250         summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}'
 251 }
 252
 253 function count_mmp_write_fail_missing # device
 254 {
 255         typeset device=$1
 256
 257         summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}'
 258 }
 259
 260 function verify_mmp_write_fail_present # device
 261 {
 262         typeset device=$1
 263
 264         count=$(count_mmp_write_fail_present $device)
 265         log_note "present count: $count"
 266         if [ $count -eq 0 ]; then
 267                 summarize_uberblock_mmp $device
 268                 log_note "----- snip -----"
 269                 zdb -luuuu $device
 270                 log_note "----- snip -----"
 271                 log_fail "No Uberblocks contain valid mmp_write and fail values"
 272         fi
 273
 274         count=$(count_mmp_write_fail_missing $device)
 275         log_note "missing count: $count"
 276         if [ $count -gt 0 ]; then
 277                 summarize_uberblock_mmp $device
 278                 log_note "----- snip -----"
 279                 zdb -luuuu $device
 280                 log_note "----- snip -----"
 281                 log_fail "Uberblocks missing mmp_write or mmp_fail"
 282         fi
 283 }