scripts/zloop.sh

   1 #!/usr/bin/env bash
   2
   3 #
   4 # CDDL HEADER START
   5 #
   6 # This file and its contents are supplied under the terms of the
   7 # Common Development and Distribution License ("CDDL"), version 1.0.
   8 # You may only use this file in accordance with the terms of version
   9 # 1.0 of the CDDL.
  10 #
  11 # A full copy of the text of the CDDL should have accompanied this
  12 # source.  A copy of the CDDL is also available via the Internet at
  13 # http://www.illumos.org/license/CDDL.
  14 #
  15 # CDDL HEADER END
  16 #
  17
  18 #
  19 # Copyright (c) 2015 by Delphix. All rights reserved.
  20 # Copyright (C) 2016 Lawrence Livermore National Security, LLC.
  21 # Copyright (c) 2017, Intel Corporation.
  22 #
  23
  24 BASE_DIR=$(dirname "$0")
  25 SCRIPT_COMMON=common.sh
  26 if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
  27         . "${BASE_DIR}/${SCRIPT_COMMON}"
  28 else
  29         echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  30 fi
  31
  32 # shellcheck disable=SC2034
  33 PROG=zloop.sh
  34 GDB=${GDB:-gdb}
  35
  36 DEFAULTWORKDIR=/var/tmp
  37 DEFAULTCOREDIR=/var/tmp/zloop
  38
  39 function usage
  40 {
  41         echo -e "\n$0 [-t <timeout>] [ -s <vdev size> ] [-c <dump directory>]" \
  42             "[ -- [extra ztest parameters]]\n" \
  43             "\n" \
  44             "  This script runs ztest repeatedly with randomized arguments.\n" \
  45             "  If a crash is encountered, the ztest logs, any associated\n" \
  46             "  vdev files, and core file (if one exists) are moved to the\n" \
  47             "  output directory ($DEFAULTCOREDIR by default). Any options\n" \
  48             "  after the -- end-of-options marker will be passed to ztest.\n" \
  49             "\n" \
  50             "  Options:\n" \
  51             "    -t  Total time to loop for, in seconds. If not provided,\n" \
  52             "        zloop runs forever.\n" \
  53             "    -s  Size of vdev devices.\n" \
  54             "    -f  Specify working directory for ztest vdev files.\n" \
  55             "    -c  Specify a core dump directory to use.\n" \
  56             "    -m  Max number of core dumps to allow before exiting.\n" \
  57             "    -l  Create 'ztest.core.N' symlink to core directory.\n" \
  58             "    -h  Print this help message.\n" \
  59             "" >&2
  60 }
  61
  62 function or_die
  63 {
  64         # shellcheck disable=SC2068
  65         if ! $@; then
  66                 echo "Command failed: $*"
  67                 exit 1
  68         fi
  69 }
  70
  71 case $(uname) in
  72 FreeBSD)
  73         coreglob="z*.core"
  74         ;;
  75 Linux)
  76         # core file helpers
  77         origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
  78         coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
  79
  80         if [[ $coreglob = "*" ]]; then
  81                 echo "Setting core file pattern..."
  82                 echo "core" > /proc/sys/kernel/core_pattern
  83                 coreglob="$(grep -E -o '^([^|%[:space:]]*)' \
  84                     /proc/sys/kernel/core_pattern)*"
  85         fi
  86         ;;
  87 *)
  88         exit 1
  89         ;;
  90 esac
  91
  92 function core_file
  93 {
  94         # shellcheck disable=SC2012 disable=2086
  95         printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)"
  96 }
  97
  98 function core_prog
  99 {
 100         prog=$ZTEST
 101         core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \
 102             tr  \' ' ')
 103         # shellcheck disable=SC2076
 104         if [[ "$core_id" =~ "zdb "  ]]; then
 105                 prog=$ZDB
 106         fi
 107         printf "%s" "$prog"
 108 }
 109
 110 function store_core
 111 {
 112         core="$(core_file)"
 113         if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
 114                 df -h "$workdir" >>ztest.out
 115                 coreid=$(date "+zloop-%y%m%d-%H%M%S")
 116                 foundcrashes=$((foundcrashes + 1))
 117
 118                 # zdb debugging
 119                 zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest"
 120                 zdbdebug=$($zdbcmd 2>&1)
 121                 echo -e "$zdbcmd\n" >>ztest.zdb
 122                 echo "$zdbdebug" >>ztest.zdb
 123
 124                 dest=$coredir/$coreid
 125                 or_die mkdir -p "$dest"
 126                 or_die mkdir -p "$dest/vdev"
 127
 128                 if [[ $symlink -ne 0 ]]; then
 129                         or_die ln -sf "$dest" ztest.core.$foundcrashes
 130                 fi
 131
 132                 echo "*** ztest crash found - moving logs to $dest"
 133
 134                 or_die mv ztest.history "$dest/"
 135                 or_die mv ztest.zdb "$dest/"
 136                 or_die mv ztest.out "$dest/"
 137                 or_die mv "$workdir/ztest*" "$dest/vdev/"
 138
 139                 if [[ -e "$workdir/zpool.cache" ]]; then
 140                         or_die mv "$workdir/zpool.cache" "$dest/vdev/"
 141                 fi
 142
 143                 # check for core
 144                 if [[ -f "$core" ]]; then
 145                         coreprog=$(core_prog "$core")
 146                         coredebug=$($GDB --batch --quiet \
 147                             -ex "set print thread-events off" \
 148                             -ex "printf \"*\n* Backtrace \n*\n\"" \
 149                             -ex "bt" \
 150                             -ex "printf \"*\n* Libraries \n*\n\"" \
 151                             -ex "info sharedlib" \
 152                             -ex "printf \"*\n* Threads (full) \n*\n\"" \
 153                             -ex "info threads" \
 154                             -ex "printf \"*\n* Backtraces \n*\n\"" \
 155                             -ex "thread apply all bt" \
 156                             -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
 157                             -ex "thread apply all bt full" \
 158                             -ex "quit" "$coreprog" "$core" 2>&1 | \
 159                             grep -v "New LWP")
 160
 161                         # Dump core + logs to stored directory
 162                         echo "$coredebug" >>"$dest/ztest.gdb"
 163                         or_die mv "$core" "$dest/"
 164
 165                         # Record info in cores logfile
 166                         echo "*** core @ $coredir/$coreid/$core:" | \
 167                             tee -a ztest.cores
 168                 fi
 169
 170                 if [[ $coremax -gt 0 ]] &&
 171                    [[ $foundcrashes -ge $coremax ]]; then
 172                         echo "exiting... max $coremax allowed cores"
 173                         exit 1
 174                 else
 175                         echo "continuing..."
 176                 fi
 177         fi
 178 }
 179
 180 # parse arguments
 181 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
 182 coredir=$DEFAULTCOREDIR
 183 basedir=$DEFAULTWORKDIR
 184 rundir="zloop-run"
 185 timeout=0
 186 size="512m"
 187 coremax=0
 188 symlink=0
 189 while getopts ":ht:m:s:c:f:l" opt; do
 190         case $opt in
 191                 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
 192                 m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;;
 193                 s ) [[ $OPTARG ]] && size=$OPTARG ;;
 194                 c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
 195                 f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
 196                 l ) symlink=1 ;;
 197                 h ) usage
 198                     exit 2
 199                     ;;
 200                 * ) echo "Invalid argument: -$OPTARG";
 201                     usage
 202                     exit 1
 203         esac
 204 done
 205 # pass remaining arguments on to ztest
 206 shift $((OPTIND - 1))
 207
 208 # enable core dumps
 209 ulimit -c unlimited
 210 export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0
 211
 212 if [[ -f "$(core_file)" ]]; then
 213         echo -n "There's a core dump here you might want to look at first... "
 214         core_file
 215         echo
 216         exit 1
 217 fi
 218
 219 if [[ ! -d $coredir ]]; then
 220         echo "core dump directory ($coredir) does not exist, creating it."
 221         or_die mkdir -p "$coredir"
 222 fi
 223
 224 if [[ ! -w $coredir ]]; then
 225         echo "core dump directory ($coredir) is not writable."
 226         exit 1
 227 fi
 228
 229 or_die rm -f ztest.history
 230 or_die rm -f ztest.zdb
 231 or_die rm -f ztest.cores
 232
 233 ztrc=0          # ztest return value
 234 foundcrashes=0  # number of crashes found so far
 235 starttime=$(date +%s)
 236 curtime=$starttime
 237
 238 # if no timeout was specified, loop forever.
 239 while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do
 240         zopt="-G -VVVVV"
 241
 242         # start each run with an empty directory
 243         workdir="$basedir/$rundir"
 244         or_die rm -rf "$workdir"
 245         or_die mkdir "$workdir"
 246
 247         # switch between three types of configs
 248         # 1/3 basic, 1/3 raidz mix, and 1/3 draid mix
 249         choice=$((RANDOM % 3))
 250
 251         # ashift range 9 - 15
 252         align=$(((RANDOM % 2) * 3 + 9))
 253
 254         # randomly use special classes
 255         class="special=random"
 256
 257         if [[ $choice -eq 0 ]]; then
 258                 # basic mirror only
 259                 parity=1
 260                 mirrors=2
 261                 draid_data=0
 262                 draid_spares=0
 263                 raid_children=0
 264                 vdevs=2
 265                 raid_type="raidz"
 266         elif [[ $choice -eq 1 ]]; then
 267                 # fully randomized mirror/raidz (sans dRAID)
 268                 parity=$(((RANDOM % 3) + 1))
 269                 mirrors=$(((RANDOM % 3) * 1))
 270                 draid_data=0
 271                 draid_spares=0
 272                 raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
 273                 vdevs=$(((RANDOM % 3) + 3))
 274                 raid_type="raidz"
 275         else
 276                 # fully randomized dRAID (sans mirror/raidz)
 277                 parity=$(((RANDOM % 3) + 1))
 278                 mirrors=0
 279                 draid_data=$(((RANDOM % 8) + 3))
 280                 draid_spares=$(((RANDOM % 2) + parity))
 281                 stripe=$((draid_data + parity))
 282                 extra=$((draid_spares + (RANDOM % 4)))
 283                 raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra))
 284                 vdevs=$((RANDOM % 3))
 285                 raid_type="draid"
 286         fi
 287
 288         # run from 30 to 120 seconds
 289         runtime=$(((RANDOM % 90) + 30))
 290         passtime=$((RANDOM % (runtime / 3 + 1) + 10))
 291
 292         zopt="$zopt -K $raid_type"
 293         zopt="$zopt -m $mirrors"
 294         zopt="$zopt -r $raid_children"
 295         zopt="$zopt -D $draid_data"
 296         zopt="$zopt -S $draid_spares"
 297         zopt="$zopt -R $parity"
 298         zopt="$zopt -v $vdevs"
 299         zopt="$zopt -a $align"
 300         zopt="$zopt -C $class"
 301         zopt="$zopt -T $runtime"
 302         zopt="$zopt -P $passtime"
 303         zopt="$zopt -s $size"
 304         zopt="$zopt -f $workdir"
 305
 306         # shellcheck disable=SC2124
 307         cmd="$ZTEST $zopt $@"
 308         desc="$(date '+%m/%d %T') $cmd"
 309         echo "$desc" | tee -a ztest.history
 310         echo "$desc" >>ztest.out
 311         $cmd >>ztest.out 2>&1
 312         ztrc=$?
 313         grep -E '===|WARNING' ztest.out >>ztest.history
 314
 315         store_core
 316
 317         curtime=$(date +%s)
 318 done
 319
 320 echo "zloop finished, $foundcrashes crashes found"
 321
 322 # restore core pattern.
 323 case $(uname) in
 324 Linux)
 325         echo "$origcorepattern" > /proc/sys/kernel/core_pattern
 326         ;;
 327 *)
 328         ;;
 329 esac
 330
 331 uptime >>ztest.out
 332
 333 if [[ $foundcrashes -gt 0 ]]; then
 334         exit 1
 335 fi