zfs-patches/0034-Extend-zloop.sh-for-automated-testing.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Brian Behlendorf <behlendorf1@llnl.gov>
   3 Date: Mon, 22 Jan 2018 12:48:39 -0800
   4 Subject: [PATCH] Extend zloop.sh for automated testing
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 In order to debug issues encountered by ztest during automated
  10 testing it's important that as much debugging information as
  11 possible by dumped at the time of the failure.  The following
  12 changes extend the zloop.sh script in order to make it easier
  13 to integrate with buildbot.
  14
  15 * Add the `-m <maximum cores>` option to zloop.sh to place a
  16   limit of the number of core dumps generated.  By default, the
  17   existing behavior is maintained and no limit is set.
  18
  19 * Add the `-l` option to create a 'ztest.core.N' symlink in the
  20   current directory to the core directory. This functionality
  21   is provided primarily for buildbot which expects log files to
  22   have well known names.
  23
  24 * Rename 'ztest.ddt' to 'ztest.zdb' and extend it to dump
  25   additional basic information on failure for latter analysis.
  26
  27 Reviewed-by: Tim Chase <tim@chase2k.com>
  28 Reviewed by: Thomas Caputi <tcaputi@datto.com>
  29 Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov>
  30 Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
  31 Closes #6999
  32
  33 Conflicts:
  34         scripts/zloop.sh
  35 (cherry picked from commit 137b3e6cff5552c5b0e137008fd274ad9a6f7a0d)
  36 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  37 ---
  38  scripts/zloop.sh | 42 ++++++++++++++++++++++++++++++++----------
  39  1 file changed, 32 insertions(+), 10 deletions(-)
  40
  41 diff --git a/scripts/zloop.sh b/scripts/zloop.sh
  42 index f0af87553..f39e91ef9 100755
  43 --- a/scripts/zloop.sh
  44 +++ b/scripts/zloop.sh
  45 @@ -52,6 +52,8 @@ function usage
  46             "    -s  Size of vdev devices.\n" \
  47             "    -f  Specify working directory for ztest vdev files.\n" \
  48             "    -c  Specify a core dump directory to use.\n" \
  49 +           "    -m  Max number of core dumps to allow before exiting.\n" \
  50 +           "    -l  Create 'ztest.core.N' symlink to core directory.\n" \
  51             "    -h  Print this help message.\n" \
  52             "" >&2
  53  }
  54 @@ -105,14 +107,24 @@ function store_core
  55                 coreid=$(date "+zloop-%y%m%d-%H%M%S")
  56                 foundcrashes=$((foundcrashes + 1))
  57
  58 +               # zdb debugging
  59 +               zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest"
  60 +               zdbdebug=$($zdbcmd 2>&1)
  61 +               echo -e "$zdbcmd\n" >>ztest.zdb
  62 +               echo "$zdbdebug" >>ztest.zdb
  63 +
  64                 dest=$coredir/$coreid
  65                 or_die mkdir -p "$dest"
  66                 or_die mkdir -p "$dest/vdev"
  67
  68 +               if [[ $symlink -ne 0 ]]; then
  69 +                       or_die ln -sf "$dest" ztest.core.$foundcrashes
  70 +               fi
  71 +
  72                 echo "*** ztest crash found - moving logs to $dest"
  73
  74                 or_die mv ztest.history "$dest/"
  75 -               or_die mv ztest.ddt "$dest/"
  76 +               or_die mv ztest.zdb "$dest/"
  77                 or_die mv ztest.out "$dest/"
  78                 or_die mv "$workdir/ztest*" "$dest/vdev/"
  79                 or_die mv "$workdir/zpool.cache" "$dest/vdev/"
  80 @@ -120,7 +132,7 @@ function store_core
  81                 # check for core
  82                 if [[ -f "$core" ]]; then
  83                         coreprog=$(core_prog "$core")
  84 -                       corestatus=$($GDB --batch --quiet \
  85 +                       coredebug=$($GDB --batch --quiet \
  86                             -ex "set print thread-events off" \
  87                             -ex "printf \"*\n* Backtrace \n*\n\"" \
  88                             -ex "bt" \
  89 @@ -132,19 +144,25 @@ function store_core
  90                             -ex "thread apply all bt" \
  91                             -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
  92                             -ex "thread apply all bt full" \
  93 -                           -ex "quit" "$coreprog" "$core" | grep -v "New LWP")
  94 +                           -ex "quit" "$coreprog" "$core" 2>&1 | \
  95 +                           grep -v "New LWP")
  96
  97                         # Dump core + logs to stored directory
  98 -                       echo "$corestatus" >>"$dest/status"
  99 +                       echo "$coredebug" >>"$dest/ztest.gdb"
 100                         or_die mv "$core" "$dest/"
 101
 102                         # Record info in cores logfile
 103                         echo "*** core @ $coredir/$coreid/$core:" | \
 104                             tee -a ztest.cores
 105 -                       echo "$corestatus" | tee -a ztest.cores
 106 -                       echo "" | tee -a ztest.cores
 107                 fi
 108 -               echo "continuing..."
 109 +
 110 +               if [[ $coremax -gt 0 ]] &&
 111 +                  [[ $foundcrashes -ge $coremax ]]; then
 112 +                       echo "exiting... max $coremax allowed cores"
 113 +                       exit 1
 114 +               else
 115 +                       echo "continuing..."
 116 +               fi
 117         fi
 118  }
 119
 120 @@ -155,12 +173,16 @@ basedir=$DEFAULTWORKDIR
 121  rundir="zloop-run"
 122  timeout=0
 123  size="512m"
 124 -while getopts ":ht:s:c:f:" opt; do
 125 +coremax=0
 126 +symlink=0
 127 +while getopts ":ht:m:s:c:f:l" opt; do
 128         case $opt in
 129                 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
 130 +               m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;;
 131                 s ) [[ $OPTARG ]] && size=$OPTARG ;;
 132                 c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
 133                 f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
 134 +               l ) symlink=1 ;;
 135                 h ) usage
 136                     exit 2
 137                     ;;
 138 @@ -178,6 +200,7 @@ ulimit -c unlimited
 139  if [[ -f "$(core_file)" ]]; then
 140         echo -n "There's a core dump here you might want to look at first... "
 141         core_file
 142 +       echo
 143         exit 1
 144  fi
 145
 146 @@ -192,7 +215,7 @@ if [[ ! -w $coredir ]]; then
 147  fi
 148
 149  or_die rm -f ztest.history
 150 -or_die rm -f ztest.ddt
 151 +or_die rm -f ztest.zdb
 152  or_die rm -f ztest.cores
 153
 154  ztrc=0         # ztest return value
 155 @@ -243,7 +266,6 @@ while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do
 156         $cmd >>ztest.out 2>&1
 157         ztrc=$?
 158         grep -E '===|WARNING' ztest.out >>ztest.history
 159 -       $ZDB -U "$workdir/zpool.cache" -DD ztest >>ztest.ddt
 160
 161         store_core
 162
 163 --
 164 2.14.2
 165