scripts/zloop.sh

   1 #!/bin/bash
   2
   3 #
   4 # CDDL HEADER START
   5 #
   6 # This file and its contents are supplied under the terms of the
   7 # Common Development and Distribution License ("CDDL"), version 1.0.
   8 # You may only use this file in accordance with the terms of version
   9 # 1.0 of the CDDL.
  10 #
  11 # A full copy of the text of the CDDL should have accompanied this
  12 # source.  A copy of the CDDL is also available via the Internet at
  13 # http://www.illumos.org/license/CDDL.
  14 #
  15 # CDDL HEADER END
  16 #
  17
  18 #
  19 # Copyright (c) 2015 by Delphix. All rights reserved.
  20 # Copyright (C) 2016 Lawrence Livermore National Security, LLC.
  21 #
  22
  23 basedir="$(dirname $0)"
  24
  25 SCRIPT_COMMON=common.sh
  26 if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then
  27         . "${basedir}/${SCRIPT_COMMON}"
  28 else
  29         echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  30 fi
  31
  32 PROG=zloop.sh
  33
  34 DEFAULTWORKDIR=/var/tmp
  35 DEFAULTCOREDIR=/var/tmp/zloop
  36
  37 function usage
  38 {
  39         echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \
  40             "[ -- [extra ztest parameters]]\n" \
  41             "\n" \
  42             "  This script runs ztest repeatedly with randomized arguments.\n" \
  43             "  If a crash is encountered, the ztest logs, any associated\n" \
  44             "  vdev files, and core file (if one exists) are moved to the\n" \
  45             "  output directory ($DEFAULTCOREDIR by default). Any options\n" \
  46             "  after the -- end-of-options marker will be passed to ztest.\n" \
  47             "\n" \
  48             "  Options:\n" \
  49             "    -t  Total time to loop for, in seconds. If not provided,\n" \
  50             "        zloop runs forever.\n" \
  51             "    -f  Specify working directory for ztest vdev files.\n" \
  52             "    -c  Specify a core dump directory to use.\n" \
  53             "    -h  Print this help message.\n" \
  54             "" >&2
  55 }
  56
  57 function or_die
  58 {
  59         $@
  60         if [[ $? -ne 0 ]]; then
  61                 echo "Command failed: $@"
  62                 exit 1
  63         fi
  64 }
  65
  66 # core file helpers
  67 origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
  68 coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
  69
  70 if [[ $coreglob = "*" ]]; then
  71         echo "Setting core file pattern..."
  72         echo "core" > /proc/sys/kernel/core_pattern
  73         coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
  74 fi
  75
  76 function core_file
  77 {
  78         printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)"
  79 }
  80
  81 function store_core
  82 {
  83         core="$(core_file)"
  84         if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
  85                 coreid=$(date "+zloop-%y%m%d-%H%M%S")
  86                 foundcrashes=$(($foundcrashes + 1))
  87
  88                 dest=$coredir/$coreid
  89                 or_die mkdir -p $dest
  90                 or_die mkdir -p $dest/vdev
  91
  92                 echo "*** ztest crash found - moving logs to $dest"
  93
  94                 or_die mv ztest.history $dest/
  95                 or_die mv ztest.ddt $dest/
  96                 or_die mv ztest.out $dest/
  97                 or_die mv $workdir/ztest* $dest/vdev/
  98                 or_die mv $workdir/zpool.cache $dest/vdev/
  99
 100                 # check for core
 101                 if [[ -f "$core" ]]; then
 102                         corestatus=$($GDB --batch --quiet \
 103                             -ex "set print thread-events off" \
 104                             -ex "printf \"*\n* Backtrace \n*\n\"" \
 105                             -ex "bt" \
 106                             -ex "printf \"*\n* Libraries \n*\n\"" \
 107                             -ex "info sharedlib" \
 108                             -ex "printf \"*\n* Threads (full) \n*\n\"" \
 109                             -ex "info threads" \
 110                             -ex "printf \"*\n* Backtraces \n*\n\"" \
 111                             -ex "thread apply all bt" \
 112                             -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
 113                             -ex "thread apply all bt full" \
 114                             -ex "quit" $ZTEST "$core" | grep -v "New LWP")
 115
 116                         # Dump core + logs to stored directory
 117                         echo "$corestatus" >>$dest/status
 118                         or_die mv "$core" $dest/
 119
 120                         # Record info in cores logfile
 121                         echo "*** core @ $coredir/$coreid/$core:" | \
 122                             tee -a ztest.cores
 123                         echo "$corestatus" | tee -a ztest.cores
 124                         echo "" | tee -a ztest.cores
 125                 fi
 126                 echo "continuing..."
 127         fi
 128 }
 129
 130 # parse arguments
 131 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
 132 coredir=$DEFAULTCOREDIR
 133 workdir=$DEFAULTWORKDIR
 134 timeout=0
 135 while getopts ":ht:c:f:" opt; do
 136         case $opt in
 137                 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
 138                 c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
 139                 f ) [[ $OPTARG ]] && workdir=$(readlink -f $OPTARG) ;;
 140                 h ) usage
 141                     exit 2
 142                     ;;
 143                 * ) echo "Invalid argument: -$OPTARG";
 144                     usage
 145                     exit 1
 146         esac
 147 done
 148 # pass remaining arguments on to ztest
 149 shift $((OPTIND - 1))
 150
 151 # enable core dumps
 152 ulimit -c unlimited
 153
 154 if [[ -f "$(core_file)" ]]; then
 155         echo -n "There's a core dump here you might want to look at first... "
 156         echo "$(core_file)"
 157         exit 1
 158 fi
 159
 160 if [[ ! -d $coredir ]]; then
 161         echo "core dump directory ($coredir) does not exist, creating it."
 162         or_die mkdir -p $coredir
 163 fi
 164
 165 if [[ ! -w $coredir ]]; then
 166         echo "core dump directory ($coredir) is not writable."
 167         exit 1
 168 fi
 169
 170 or_die rm -f ztest.history
 171 or_die rm -f ztest.ddt
 172 or_die rm -f ztest.cores
 173
 174 ztrc=0          # ztest return value
 175 foundcrashes=0  # number of crashes found so far
 176 starttime=$(date +%s)
 177 curtime=$starttime
 178
 179 # if no timeout was specified, loop forever.
 180 while [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do
 181         zopt="-VVVVV"
 182
 183         # switch between common arrangements & fully randomized
 184         if [[ $((RANDOM % 2)) -eq 0 ]]; then
 185                 mirrors=2
 186                 raidz=0
 187                 parity=1
 188                 vdevs=2
 189         else
 190                 mirrors=$(((RANDOM % 3) * 1))
 191                 parity=$(((RANDOM % 3) + 1))
 192                 raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
 193                 vdevs=$(((RANDOM % 3) + 3))
 194         fi
 195         align=$(((RANDOM % 2) * 3 + 9))
 196         runtime=$((RANDOM % 100))
 197         passtime=$((RANDOM % (runtime / 3 + 1) + 10))
 198         size=128m
 199
 200         zopt="$zopt -m $mirrors"
 201         zopt="$zopt -r $raidz"
 202         zopt="$zopt -R $parity"
 203         zopt="$zopt -v $vdevs"
 204         zopt="$zopt -a $align"
 205         zopt="$zopt -T $runtime"
 206         zopt="$zopt -P $passtime"
 207         zopt="$zopt -s $size"
 208         zopt="$zopt -f $workdir"
 209
 210         cmd="$ZTEST $zopt $@"
 211         desc="$(date '+%m/%d %T') $cmd"
 212         echo "$desc" | tee -a ztest.history
 213         echo "$desc" >>ztest.out
 214         $cmd >>ztest.out 2>&1
 215         ztrc=$?
 216         egrep '===|WARNING' ztest.out >>ztest.history
 217         $ZDB -U $workdir/zpool.cache -DD ztest >>ztest.ddt
 218
 219         store_core
 220
 221         curtime=$(date +%s)
 222 done
 223
 224 echo "zloop finished, $foundcrashes crashes found"
 225
 226 #restore core pattern
 227 echo "$origcorepattern" > /proc/sys/kernel/core_pattern
 228
 229 uptime >>ztest.out
 230
 231 if [[ $foundcrashes -gt 0 ]]; then
 232         exit 1
 233 fi