scripts/zloop.sh

   1 #!/bin/bash
   2
   3 #
   4 # CDDL HEADER START
   5 #
   6 # This file and its contents are supplied under the terms of the
   7 # Common Development and Distribution License ("CDDL"), version 1.0.
   8 # You may only use this file in accordance with the terms of version
   9 # 1.0 of the CDDL.
  10 #
  11 # A full copy of the text of the CDDL should have accompanied this
  12 # source.  A copy of the CDDL is also available via the Internet at
  13 # http://www.illumos.org/license/CDDL.
  14 #
  15 # CDDL HEADER END
  16 #
  17
  18 #
  19 # Copyright (c) 2015 by Delphix. All rights reserved.
  20 # Copyright (C) 2016 Lawrence Livermore National Security, LLC.
  21 #
  22
  23 BASE_DIR=$(dirname "$0")
  24 SCRIPT_COMMON=common.sh
  25 if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
  26         . "${BASE_DIR}/${SCRIPT_COMMON}"
  27 else
  28         echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  29 fi
  30
  31 # shellcheck disable=SC2034
  32 PROG=zloop.sh
  33
  34 DEFAULTWORKDIR=/var/tmp
  35 DEFAULTCOREDIR=/var/tmp/zloop
  36
  37 function usage
  38 {
  39         echo -e "\n$0 [-t <timeout>] [ -s <vdev size> ] [-c <dump directory>]" \
  40             "[ -- [extra ztest parameters]]\n" \
  41             "\n" \
  42             "  This script runs ztest repeatedly with randomized arguments.\n" \
  43             "  If a crash is encountered, the ztest logs, any associated\n" \
  44             "  vdev files, and core file (if one exists) are moved to the\n" \
  45             "  output directory ($DEFAULTCOREDIR by default). Any options\n" \
  46             "  after the -- end-of-options marker will be passed to ztest.\n" \
  47             "\n" \
  48             "  Options:\n" \
  49             "    -t  Total time to loop for, in seconds. If not provided,\n" \
  50             "        zloop runs forever.\n" \
  51             "    -s  Size of vdev devices.\n" \
  52             "    -f  Specify working directory for ztest vdev files.\n" \
  53             "    -c  Specify a core dump directory to use.\n" \
  54             "    -h  Print this help message.\n" \
  55             "" >&2
  56 }
  57
  58 function or_die
  59 {
  60         # shellcheck disable=SC2068
  61         $@
  62         # shellcheck disable=SC2181
  63         if [[ $? -ne 0 ]]; then
  64                 # shellcheck disable=SC2145
  65                 echo "Command failed: $@"
  66                 exit 1
  67         fi
  68 }
  69
  70 # core file helpers
  71 origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
  72 coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
  73
  74 if [[ $coreglob = "*" ]]; then
  75         echo "Setting core file pattern..."
  76         echo "core" > /proc/sys/kernel/core_pattern
  77         coreglob="$(egrep -o '^([^|%[:space:]]*)' \
  78             /proc/sys/kernel/core_pattern)*"
  79 fi
  80
  81 function core_file
  82 {
  83         # shellcheck disable=SC2012 disable=2086
  84         printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)"
  85 }
  86
  87 function core_prog
  88 {
  89         prog=$ZTEST
  90         core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \
  91             tr  \' ' ')
  92         # shellcheck disable=SC2076
  93         if [[ "$core_id" =~ "zdb "  ]]; then
  94                 prog=$ZDB
  95         fi
  96         printf "%s" "$prog"
  97 }
  98
  99 function store_core
 100 {
 101         core="$(core_file)"
 102         if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
 103                 df -h "$workdir" >>ztest.out
 104                 coreid=$(date "+zloop-%y%m%d-%H%M%S")
 105                 foundcrashes=$((foundcrashes + 1))
 106
 107                 dest=$coredir/$coreid
 108                 or_die mkdir -p "$dest"
 109                 or_die mkdir -p "$dest/vdev"
 110
 111                 echo "*** ztest crash found - moving logs to $dest"
 112
 113                 or_die mv ztest.history "$dest/"
 114                 or_die mv ztest.ddt "$dest/"
 115                 or_die mv ztest.out "$dest/"
 116                 or_die mv "$workdir/ztest*" "$dest/vdev/"
 117                 or_die mv "$workdir/zpool.cache" "$dest/vdev/"
 118
 119                 # check for core
 120                 if [[ -f "$core" ]]; then
 121                         coreprog=$(core_prog "$core")
 122                         corestatus=$($GDB --batch --quiet \
 123                             -ex "set print thread-events off" \
 124                             -ex "printf \"*\n* Backtrace \n*\n\"" \
 125                             -ex "bt" \
 126                             -ex "printf \"*\n* Libraries \n*\n\"" \
 127                             -ex "info sharedlib" \
 128                             -ex "printf \"*\n* Threads (full) \n*\n\"" \
 129                             -ex "info threads" \
 130                             -ex "printf \"*\n* Backtraces \n*\n\"" \
 131                             -ex "thread apply all bt" \
 132                             -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
 133                             -ex "thread apply all bt full" \
 134                             -ex "quit" "$coreprog" "$core" | grep -v "New LWP")
 135
 136                         # Dump core + logs to stored directory
 137                         echo "$corestatus" >>"$dest/status"
 138                         or_die mv "$core" "$dest/"
 139
 140                         # Record info in cores logfile
 141                         echo "*** core @ $coredir/$coreid/$core:" | \
 142                             tee -a ztest.cores
 143                         echo "$corestatus" | tee -a ztest.cores
 144                         echo "" | tee -a ztest.cores
 145                 fi
 146                 echo "continuing..."
 147         fi
 148 }
 149
 150 rngdpid=""
 151 function on_exit
 152 {
 153         if [ -n "$rngdpid" ]; then
 154                 kill -9 "$rngdpid"
 155         fi
 156 }
 157 trap on_exit EXIT
 158
 159 # parse arguments
 160 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
 161 coredir=$DEFAULTCOREDIR
 162 basedir=$DEFAULTWORKDIR
 163 rundir="zloop-run"
 164 timeout=0
 165 size="512m"
 166 while getopts ":ht:s:c:f:" opt; do
 167         case $opt in
 168                 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
 169                 s ) [[ $OPTARG ]] && size=$OPTARG ;;
 170                 c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
 171                 f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
 172                 h ) usage
 173                     exit 2
 174                     ;;
 175                 * ) echo "Invalid argument: -$OPTARG";
 176                     usage
 177                     exit 1
 178         esac
 179 done
 180 # pass remaining arguments on to ztest
 181 shift $((OPTIND - 1))
 182
 183 # enable core dumps
 184 ulimit -c unlimited
 185
 186 if [[ -f "$(core_file)" ]]; then
 187         echo -n "There's a core dump here you might want to look at first... "
 188         core_file
 189         exit 1
 190 fi
 191
 192 if [[ ! -d $coredir ]]; then
 193         echo "core dump directory ($coredir) does not exist, creating it."
 194         or_die mkdir -p "$coredir"
 195 fi
 196
 197 if [[ ! -w $coredir ]]; then
 198         echo "core dump directory ($coredir) is not writable."
 199         exit 1
 200 fi
 201
 202 or_die rm -f ztest.history
 203 or_die rm -f ztest.ddt
 204 or_die rm -f ztest.cores
 205
 206 # start rngd in the background so we don't run out of entropy
 207 or_die read -r rngdpid < <(rngd -f -r /dev/urandom & echo $!)
 208
 209 ztrc=0          # ztest return value
 210 foundcrashes=0  # number of crashes found so far
 211 starttime=$(date +%s)
 212 curtime=$starttime
 213
 214 # if no timeout was specified, loop forever.
 215 while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do
 216         zopt="-VVVVV"
 217
 218         # start each run with an empty directory
 219         workdir="$basedir/$rundir"
 220         or_die rm -rf "$workdir"
 221         or_die mkdir "$workdir"
 222
 223         # switch between common arrangements & fully randomized
 224         if [[ $((RANDOM % 2)) -eq 0 ]]; then
 225                 mirrors=2
 226                 raidz=0
 227                 parity=1
 228                 vdevs=2
 229         else
 230                 mirrors=$(((RANDOM % 3) * 1))
 231                 parity=$(((RANDOM % 3) + 1))
 232                 raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
 233                 vdevs=$(((RANDOM % 3) + 3))
 234         fi
 235         align=$(((RANDOM % 2) * 3 + 9))
 236         runtime=$((RANDOM % 100))
 237         passtime=$((RANDOM % (runtime / 3 + 1) + 10))
 238
 239         zopt="$zopt -m $mirrors"
 240         zopt="$zopt -r $raidz"
 241         zopt="$zopt -R $parity"
 242         zopt="$zopt -v $vdevs"
 243         zopt="$zopt -a $align"
 244         zopt="$zopt -T $runtime"
 245         zopt="$zopt -P $passtime"
 246         zopt="$zopt -s $size"
 247         zopt="$zopt -f $workdir"
 248
 249         # shellcheck disable=SC2124
 250         cmd="$ZTEST $zopt $@"
 251         desc="$(date '+%m/%d %T') $cmd"
 252         echo "$desc" | tee -a ztest.history
 253         echo "$desc" >>ztest.out
 254         $cmd >>ztest.out 2>&1
 255         ztrc=$?
 256         egrep '===|WARNING' ztest.out >>ztest.history
 257         $ZDB -U "$workdir/zpool.cache" -DD ztest >>ztest.ddt 2>&1
 258
 259         store_core
 260
 261         curtime=$(date +%s)
 262 done
 263
 264 echo "zloop finished, $foundcrashes crashes found"
 265
 266 #restore core pattern
 267 echo "$origcorepattern" > /proc/sys/kernel/core_pattern
 268
 269 uptime >>ztest.out
 270
 271 if [[ $foundcrashes -gt 0 ]]; then
 272         exit 1
 273 fi