X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=scripts%2Fzloop.sh;h=1f36f865b5946da610012332bc3cae0966973408;hb=83c796c5e9d9ecb28e9553338f079a5d6b015b10;hp=f285d6a252365e12679868b45733242da7c5a84f;hpb=5df5d06a8d86af475e90fe7dd86ea044f937be66;p=mirror_zfs.git diff --git a/scripts/zloop.sh b/scripts/zloop.sh index f285d6a25..1f36f865b 100755 --- a/scripts/zloop.sh +++ b/scripts/zloop.sh @@ -30,13 +30,14 @@ fi # shellcheck disable=SC2034 PROG=zloop.sh +GDB=${GDB:-gdb} DEFAULTWORKDIR=/var/tmp DEFAULTCOREDIR=/var/tmp/zloop function usage { - echo -e "\n$0 [-t ] [-c ]" \ + echo -e "\n$0 [-t ] [ -s ] [-c ]" \ "[ -- [extra ztest parameters]]\n" \ "\n" \ " This script runs ztest repeatedly with randomized arguments.\n" \ @@ -48,8 +49,11 @@ function usage " Options:\n" \ " -t Total time to loop for, in seconds. If not provided,\n" \ " zloop runs forever.\n" \ + " -s Size of vdev devices.\n" \ " -f Specify working directory for ztest vdev files.\n" \ " -c Specify a core dump directory to use.\n" \ + " -m Max number of core dumps to allow before exiting.\n" \ + " -l Create 'ztest.core.N' symlink to core directory.\n" \ " -h Print this help message.\n" \ "" >&2 } @@ -68,12 +72,12 @@ function or_die # core file helpers origcorepattern="$(cat /proc/sys/kernel/core_pattern)" -coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" +coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" if [[ $coreglob = "*" ]]; then echo "Setting core file pattern..." echo "core" > /proc/sys/kernel/core_pattern - coreglob="$(egrep -o '^([^|%[:space:]]*)' \ + coreglob="$(grep -E -o '^([^|%[:space:]]*)' \ /proc/sys/kernel/core_pattern)*" fi @@ -103,22 +107,35 @@ function store_core coreid=$(date "+zloop-%y%m%d-%H%M%S") foundcrashes=$((foundcrashes + 1)) + # zdb debugging + zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest" + zdbdebug=$($zdbcmd 2>&1) + echo -e "$zdbcmd\n" >>ztest.zdb + echo "$zdbdebug" >>ztest.zdb + dest=$coredir/$coreid or_die mkdir -p "$dest" or_die mkdir -p "$dest/vdev" + if [[ $symlink -ne 0 ]]; then + or_die ln -sf "$dest" ztest.core.$foundcrashes + fi + echo "*** ztest crash found - moving logs to $dest" or_die mv ztest.history "$dest/" - or_die mv ztest.ddt "$dest/" + or_die mv ztest.zdb "$dest/" or_die mv ztest.out "$dest/" or_die mv "$workdir/ztest*" "$dest/vdev/" - or_die mv "$workdir/zpool.cache" "$dest/vdev/" + + if [[ -e "$workdir/zpool.cache" ]]; then + or_die mv "$workdir/zpool.cache" "$dest/vdev/" + fi # check for core if [[ -f "$core" ]]; then coreprog=$(core_prog "$core") - corestatus=$($GDB --batch --quiet \ + coredebug=$($GDB --batch --quiet \ -ex "set print thread-events off" \ -ex "printf \"*\n* Backtrace \n*\n\"" \ -ex "bt" \ @@ -130,19 +147,25 @@ function store_core -ex "thread apply all bt" \ -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ -ex "thread apply all bt full" \ - -ex "quit" "$coreprog" "$core" | grep -v "New LWP") + -ex "quit" "$coreprog" "$core" 2>&1 | \ + grep -v "New LWP") # Dump core + logs to stored directory - echo "$corestatus" >>"$dest/status" + echo "$coredebug" >>"$dest/ztest.gdb" or_die mv "$core" "$dest/" # Record info in cores logfile echo "*** core @ $coredir/$coreid/$core:" | \ tee -a ztest.cores - echo "$corestatus" | tee -a ztest.cores - echo "" | tee -a ztest.cores fi - echo "continuing..." + + if [[ $coremax -gt 0 ]] && + [[ $foundcrashes -ge $coremax ]]; then + echo "exiting... max $coremax allowed cores" + exit 1 + else + echo "continuing..." + fi fi } @@ -152,11 +175,17 @@ coredir=$DEFAULTCOREDIR basedir=$DEFAULTWORKDIR rundir="zloop-run" timeout=0 -while getopts ":ht:c:f:" opt; do +size="512m" +coremax=0 +symlink=0 +while getopts ":ht:m:s:c:f:l" opt; do case $opt in t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; + m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;; + s ) [[ $OPTARG ]] && size=$OPTARG ;; c ) [[ $OPTARG ]] && coredir=$OPTARG ;; f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; + l ) symlink=1 ;; h ) usage exit 2 ;; @@ -170,10 +199,12 @@ shift $((OPTIND - 1)) # enable core dumps ulimit -c unlimited +export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0 if [[ -f "$(core_file)" ]]; then echo -n "There's a core dump here you might want to look at first... " core_file + echo exit 1 fi @@ -188,7 +219,7 @@ if [[ ! -w $coredir ]]; then fi or_die rm -f ztest.history -or_die rm -f ztest.ddt +or_die rm -f ztest.zdb or_die rm -f ztest.cores ztrc=0 # ztest return value @@ -198,7 +229,7 @@ curtime=$starttime # if no timeout was specified, loop forever. while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do - zopt="-VVVVV" + zopt="-G -VVVVV" # start each run with an empty directory workdir="$basedir/$rundir" @@ -220,7 +251,6 @@ while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do align=$(((RANDOM % 2) * 3 + 9)) runtime=$((RANDOM % 100)) passtime=$((RANDOM % (runtime / 3 + 1) + 10)) - size=128m zopt="$zopt -m $mirrors" zopt="$zopt -r $raidz" @@ -239,8 +269,7 @@ while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do echo "$desc" >>ztest.out $cmd >>ztest.out 2>&1 ztrc=$? - egrep '===|WARNING' ztest.out >>ztest.history - $ZDB -U "$workdir/zpool.cache" -DD ztest >>ztest.ddt + grep -E '===|WARNING' ztest.out >>ztest.history store_core