tests/zfs-tests/tests/functional/cli_root/zpool_events/zpool_events_errors.ksh

   1 #!/bin/ksh -p
   2 # CDDL HEADER START
   3 #
   4 # The contents of this file are subject to the terms of the
   5 # Common Development and Distribution License (the "License").
   6 # You may not use this file except in compliance with the License.
   7 #
   8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9 # or http://www.opensolaris.org/os/licensing.
  10 # See the License for the specific language governing permissions
  11 # and limitations under the License.
  12 #
  13 # When distributing Covered Code, include this CDDL HEADER in each
  14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15 # If applicable, add the following below this CDDL HEADER, with the
  16 # fields enclosed by brackets "[]" replaced with your own identifying
  17 # information: Portions Copyright [yyyy] [name of copyright owner]
  18 #
  19 # CDDL HEADER END
  20 #
  21
  22 #
  23 # Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
  24 #
  25
  26 # DESCRIPTION:
  27 #       Verify the number of IO and checksum events match the error counters
  28 #       in zpool status.
  29 #
  30 # STRATEGY:
  31 #       1. Create a mirror, raidz, or draid pool
  32 #       2. Inject read/write IO errors or checksum errors
  33 #       3. Verify the number of errors in zpool status match the corresponding
  34 #          number of error events.
  35 #       4. Repeat for all combinations of mirror/raidz/draid and io/checksum
  36 #          errors.
  37 #
  38
  39 . $STF_SUITE/include/libtest.shlib
  40
  41 verify_runnable "both"
  42
  43 MOUNTDIR=$TEST_BASE_DIR/mount
  44 VDEV1=$TEST_BASE_DIR/file1
  45 VDEV2=$TEST_BASE_DIR/file2
  46 VDEV3=$TEST_BASE_DIR/file3
  47 POOL=error_pool
  48 FILESIZE=$((20 * 1024 * 1024))
  49 OLD_CHECKSUMS=$(get_tunable CHECKSUM_EVENTS_PER_SECOND)
  50 OLD_LEN_MAX=$(get_tunable ZEVENT_LEN_MAX)
  51
  52 function cleanup
  53 {
  54         log_must set_tunable64 CHECKSUM_EVENTS_PER_SECOND $OLD_CHECKSUMS
  55         log_must set_tunable64 ZEVENT_LEN_MAX $OLD_LEN_MAX
  56
  57         log_must zinject -c all
  58         log_must zpool events -c
  59         if poolexists $POOL ; then
  60                 log_must destroy_pool $POOL
  61         fi
  62         log_must rm -f $VDEV1 $VDEV2 $VDEV3
  63 }
  64
  65 log_assert "Check that the number of zpool errors match the number of events"
  66
  67 log_onexit cleanup
  68
  69 # Set our thresholds high so we never ratelimit or drop events.
  70 set_tunable64 CHECKSUM_EVENTS_PER_SECOND 20000
  71 set_tunable64 ZEVENT_LEN_MAX 20000
  72
  73 log_must truncate -s $MINVDEVSIZE $VDEV1 $VDEV2 $VDEV3
  74 log_must mkdir -p $MOUNTDIR
  75
  76 # Run error test on a specific type of pool
  77 #
  78 # $1: pool - mirror, raidz, draid
  79 # $2: test type - corrupt (checksum error), io
  80 # $3: read, write
  81 function do_test
  82 {
  83         POOLTYPE=$1
  84         ERR=$2
  85         RW=$3
  86
  87         log_note "Testing $ERR $RW on $POOLTYPE"
  88         log_must zpool create -f -m $MOUNTDIR -o failmode=continue $POOL $POOLTYPE $VDEV1 $VDEV2 $VDEV3
  89         log_must zpool events -c
  90         log_must zfs set compression=off $POOL
  91
  92         if [ "$RW" == "read" ] ; then
  93                 log_must mkfile $FILESIZE $MOUNTDIR/file
  94         fi
  95
  96         log_must zinject -d $VDEV1 -e $ERR -T $RW -f 100 $POOL
  97
  98         if [ "$RW" == "write" ] ; then
  99                 log_must mkfile $FILESIZE $MOUNTDIR/file
 100                 sync_pool $POOL
 101         else
 102                 log_must zpool scrub $POOL
 103                 wait_scrubbed $POOL
 104         fi
 105
 106         log_must zinject -c all
 107
 108         # Wait for the pool to settle down and finish resilvering (if
 109         # necessary).  We want the errors to stop incrementing before we
 110         # check the error and event counts.
 111         while is_pool_resilvering $POOL ; do
 112                 sleep 1
 113         done
 114
 115         out="$(zpool status -p | grep $VDEV1)"
 116
 117         if [ "$ERR" == "corrupt" ] ; then
 118                 events=$(zpool events | grep checksum | wc -l)
 119                 val=$(echo "$out" | awk '{print $5}')
 120                 str="checksum"
 121         elif [ "$ERR" == "io" ] ; then
 122                 allevents=$(zpool events | grep io)
 123                 events=$(echo "$allevents" | wc -l)
 124                 if [ "$RW" == "read" ] ; then
 125                         str="read IO"
 126                         val=$(echo "$out" | awk '{print $3}')
 127                 else
 128                         str="write IO"
 129                         val=$(echo "$out" | awk '{print $4}')
 130                 fi
 131         fi
 132
 133         if [ -z "$val" -o $val -eq 0 -o -z "$events" -o $events -eq 0 ] ; then
 134                 log_fail "Didn't see any errors or events ($val/$events)"
 135         fi
 136
 137         if [ $val -ne $events ] ; then
 138                 log_fail "$val $POOLTYPE $str errors != $events events"
 139         else
 140                 log_note "$val $POOLTYPE $str errors == $events events"
 141         fi
 142
 143         log_must zpool destroy $POOL
 144 }
 145
 146 # Test all types of errors on mirror, raidz, and draid pools
 147 for pooltype in mirror raidz draid; do
 148         do_test $pooltype corrupt read
 149         do_test $pooltype io read
 150         do_test $pooltype io write
 151 done
 152
 153 log_pass "The number of errors matched the number of events"