[mirror_zfs.git] / scripts / zloop.sh

#!/bin/bash

#
# CDDL HEADER START
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# CDDL HEADER END
#

#
# Copyright (c) 2015 by Delphix. All rights reserved.
# Copyright (C) 2016 Lawrence Livermore National Security, LLC.
#

BASE_DIR=$(dirname "$0")
SCRIPT_COMMON=common.sh
if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
	. "${BASE_DIR}/${SCRIPT_COMMON}"
else
	echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
fi

# shellcheck disable=SC2034
PROG=zloop.sh

DEFAULTWORKDIR=/var/tmp
DEFAULTCOREDIR=/var/tmp/zloop

function usage
{
	echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \
	    "[ -- [extra ztest parameters]]\n" \
	    "\n" \
	    "  This script runs ztest repeatedly with randomized arguments.\n" \
	    "  If a crash is encountered, the ztest logs, any associated\n" \
	    "  vdev files, and core file (if one exists) are moved to the\n" \
	    "  output directory ($DEFAULTCOREDIR by default). Any options\n" \
	    "  after the -- end-of-options marker will be passed to ztest.\n" \
	    "\n" \
	    "  Options:\n" \
	    "    -t  Total time to loop for, in seconds. If not provided,\n" \
	    "        zloop runs forever.\n" \
	    "    -f  Specify working directory for ztest vdev files.\n" \
	    "    -c  Specify a core dump directory to use.\n" \
	    "    -h  Print this help message.\n" \
	    "" >&2
}

function or_die
{
	# shellcheck disable=SC2068
	$@
	# shellcheck disable=SC2181
	if [[ $? -ne 0 ]]; then
		# shellcheck disable=SC2145
		echo "Command failed: $@"
		exit 1
	fi
}

# core file helpers
origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"

if [[ $coreglob = "*" ]]; then
        echo "Setting core file pattern..."
        echo "core" > /proc/sys/kernel/core_pattern
        coreglob="$(egrep -o '^([^|%[:space:]]*)' \
            /proc/sys/kernel/core_pattern)*"
fi

function core_file
{
	# shellcheck disable=SC2012 disable=2086
        printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)"
}

function core_prog
{
	prog=$ZTEST
	core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \
	    tr  \' ' ')
	# shellcheck disable=SC2076
	if [[ "$core_id" =~ "zdb "  ]]; then
		prog=$ZDB
	fi
	printf "%s" "$prog"
}

function store_core
{
	core="$(core_file)"
	if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
		df -h "$workdir" >>ztest.out
		coreid=$(date "+zloop-%y%m%d-%H%M%S")
		foundcrashes=$((foundcrashes + 1))

		dest=$coredir/$coreid
		or_die mkdir -p "$dest"
		or_die mkdir -p "$dest/vdev"

		echo "*** ztest crash found - moving logs to $dest"

		or_die mv ztest.history "$dest/"
		or_die mv ztest.ddt "$dest/"
		or_die mv ztest.out "$dest/"
		or_die mv "$workdir/ztest*" "$dest/vdev/"
		or_die mv "$workdir/zpool.cache" "$dest/vdev/"

		# check for core
		if [[ -f "$core" ]]; then
			coreprog=$(core_prog "$core")
			corestatus=$($GDB --batch --quiet \
			    -ex "set print thread-events off" \
			    -ex "printf \"*\n* Backtrace \n*\n\"" \
			    -ex "bt" \
			    -ex "printf \"*\n* Libraries \n*\n\"" \
			    -ex "info sharedlib" \
			    -ex "printf \"*\n* Threads (full) \n*\n\"" \
			    -ex "info threads" \
			    -ex "printf \"*\n* Backtraces \n*\n\"" \
			    -ex "thread apply all bt" \
			    -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
			    -ex "thread apply all bt full" \
			    -ex "quit" "$coreprog" "$core" | grep -v "New LWP")

			# Dump core + logs to stored directory
			echo "$corestatus" >>"$dest/status"
			or_die mv "$core" "$dest/"

			# Record info in cores logfile
			echo "*** core @ $coredir/$coreid/$core:" | \
			    tee -a ztest.cores
			echo "$corestatus" | tee -a ztest.cores
			echo "" | tee -a ztest.cores
		fi
		echo "continuing..."
	fi
}

rngdpid=""
function on_exit
{
	if [ -n "$rngdpid" ]; then
		kill -9 "$rngdpid"
	fi
}
trap on_exit EXIT

# parse arguments
# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
coredir=$DEFAULTCOREDIR
basedir=$DEFAULTWORKDIR
rundir="zloop-run"
timeout=0
while getopts ":ht:c:f:" opt; do
	case $opt in
		t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
		c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
		f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
		h ) usage
		    exit 2
		    ;;
		* ) echo "Invalid argument: -$OPTARG";
		    usage
		    exit 1
	esac
done
# pass remaining arguments on to ztest
shift $((OPTIND - 1))

# enable core dumps
ulimit -c unlimited

if [[ -f "$(core_file)" ]]; then
	echo -n "There's a core dump here you might want to look at first... "
	core_file
	exit 1
fi

if [[ ! -d $coredir ]]; then
	echo "core dump directory ($coredir) does not exist, creating it."
	or_die mkdir -p "$coredir"
fi

if [[ ! -w $coredir ]]; then
	echo "core dump directory ($coredir) is not writable."
	exit 1
fi

or_die rm -f ztest.history
or_die rm -f ztest.ddt
or_die rm -f ztest.cores

# start rngd in the background so we don't run out of entropy
or_die read -r rngdpid < <(rngd -f -r /dev/urandom & echo $!)

ztrc=0		# ztest return value
foundcrashes=0	# number of crashes found so far
starttime=$(date +%s)
curtime=$starttime

# if no timeout was specified, loop forever.
while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do
	zopt="-VVVVV"

	# start each run with an empty directory
	workdir="$basedir/$rundir"
	or_die rm -rf "$workdir"
	or_die mkdir "$workdir"

	# switch between common arrangements & fully randomized
	if [[ $((RANDOM % 2)) -eq 0 ]]; then
		mirrors=2
		raidz=0
		parity=1
		vdevs=2
	else
		mirrors=$(((RANDOM % 3) * 1))
		parity=$(((RANDOM % 3) + 1))
		raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
		vdevs=$(((RANDOM % 3) + 3))
	fi
	align=$(((RANDOM % 2) * 3 + 9))
	runtime=$((RANDOM % 100))
	passtime=$((RANDOM % (runtime / 3 + 1) + 10))
	size=128m

	zopt="$zopt -m $mirrors"
	zopt="$zopt -r $raidz"
	zopt="$zopt -R $parity"
	zopt="$zopt -v $vdevs"
	zopt="$zopt -a $align"
	zopt="$zopt -T $runtime"
	zopt="$zopt -P $passtime"
	zopt="$zopt -s $size"
	zopt="$zopt -f $workdir"

	# shellcheck disable=SC2124
	cmd="$ZTEST $zopt $@"
	desc="$(date '+%m/%d %T') $cmd"
	echo "$desc" | tee -a ztest.history
	echo "$desc" >>ztest.out
	$cmd >>ztest.out 2>&1
	ztrc=$?
	egrep '===|WARNING' ztest.out >>ztest.history
	$ZDB -U "$workdir/zpool.cache" -DD ztest >>ztest.ddt

	store_core

	curtime=$(date +%s)
done

echo "zloop finished, $foundcrashes crashes found"

#restore core pattern
echo "$origcorepattern" > /proc/sys/kernel/core_pattern

uptime >>ztest.out

if [[ $foundcrashes -gt 0 ]]; then
	exit 1
fi
Commit	Line	Data
9c13b489	1	#!/bin/bash
541a0901 BB	2
	3	#
	4	# CDDL HEADER START
	5	#
	6	# This file and its contents are supplied under the terms of the
	7	# Common Development and Distribution License ("CDDL"), version 1.0.
	8	# You may only use this file in accordance with the terms of version
	9	# 1.0 of the CDDL.
	10	#
	11	# A full copy of the text of the CDDL should have accompanied this
	12	# source. A copy of the CDDL is also available via the Internet at
	13	# http://www.illumos.org/license/CDDL.
	14	#
	15	# CDDL HEADER END
	16	#
	17
	18	#
	19	# Copyright (c) 2015 by Delphix. All rights reserved.
	20	# Copyright (C) 2016 Lawrence Livermore National Security, LLC.
	21	#
	22
c8f9061f	23	BASE_DIR=$(dirname "$0")
541a0901	24	SCRIPT_COMMON=common.sh
c8f9061f BB	25	if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
c8f9061f BB	26	. "${BASE_DIR}/${SCRIPT_COMMON}"
541a0901 BB	27	else
	28	echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
	29	fi
	30
c552fbc5	31	# shellcheck disable=SC2034
541a0901 BB	32	PROG=zloop.sh
	33
	34	DEFAULTWORKDIR=/var/tmp
	35	DEFAULTCOREDIR=/var/tmp/zloop
	36
	37	function usage
	38	{
	39	echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \
	40	"[ -- [extra ztest parameters]]\n" \
	41	"\n" \
	42	" This script runs ztest repeatedly with randomized arguments.\n" \
	43	" If a crash is encountered, the ztest logs, any associated\n" \
	44	" vdev files, and core file (if one exists) are moved to the\n" \
	45	" output directory ($DEFAULTCOREDIR by default). Any options\n" \
	46	" after the -- end-of-options marker will be passed to ztest.\n" \
	47	"\n" \
	48	" Options:\n" \
	49	" -t Total time to loop for, in seconds. If not provided,\n" \
	50	" zloop runs forever.\n" \
	51	" -f Specify working directory for ztest vdev files.\n" \
	52	" -c Specify a core dump directory to use.\n" \
	53	" -h Print this help message.\n" \
	54	"" >&2
	55	}
	56
	57	function or_die
	58	{
c552fbc5	59	# shellcheck disable=SC2068
541a0901	60	$@
c552fbc5	61	# shellcheck disable=SC2181
541a0901	62	if [[ $? -ne 0 ]]; then
c552fbc5	63	# shellcheck disable=SC2145
541a0901 BB	64	echo "Command failed: $@"
	65	exit 1
	66	fi
	67	}
	68
20da0566 GN	69	# core file helpers
	70	origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
	71	coreglob="$(egrep -o '^([^\|%[:space:]])' /proc/sys/kernel/core_pattern)"
	72
	73	if [[ $coreglob = "*" ]]; then
	74	echo "Setting core file pattern..."
	75	echo "core" > /proc/sys/kernel/core_pattern
0c313d2f GN	76	coreglob="$(egrep -o '^([^\|%[:space:]]*)' \
0c313d2f GN	77	/proc/sys/kernel/core_pattern)*"
20da0566 GN	78	fi
	79
	80	function core_file
	81	{
c552fbc5	82	# shellcheck disable=SC2012 disable=2086
20da0566 GN	83	printf "%s" "$(ls -tr1 $coreglob 2> /dev/null \| head -1)"
	84	}
	85
0c313d2f GN	86	function core_prog
	87	{
	88	prog=$ZTEST
c552fbc5	89	core_id=$($GDB --batch -c "$1" \| grep "Core was generated by" \| \
0c313d2f	90	tr \' ' ')
c552fbc5	91	# shellcheck disable=SC2076
0c313d2f GN	92	if [[ "$core_id" =~ "zdb " ]]; then
	93	prog=$ZDB
	94	fi
	95	printf "%s" "$prog"
	96	}
	97
541a0901 BB	98	function store_core
541a0901 BB	99	{
20da0566 GN	100	core="$(core_file)"
20da0566 GN	101	if [[ $ztrc -ne 0 ]] \|\| [[ -f "$core" ]]; then
5df5d06a	102	df -h "$workdir" >>ztest.out
541a0901	103	coreid=$(date "+zloop-%y%m%d-%H%M%S")
c552fbc5	104	foundcrashes=$((foundcrashes + 1))
541a0901 BB	105
541a0901 BB	106	dest=$coredir/$coreid
c552fbc5 GDN	107	or_die mkdir -p "$dest"
c552fbc5 GDN	108	or_die mkdir -p "$dest/vdev"
541a0901 BB	109
	110	echo "*** ztest crash found - moving logs to $dest"
	111
c552fbc5 GDN	112	or_die mv ztest.history "$dest/"
	113	or_die mv ztest.ddt "$dest/"
	114	or_die mv ztest.out "$dest/"
	115	or_die mv "$workdir/ztest*" "$dest/vdev/"
	116	or_die mv "$workdir/zpool.cache" "$dest/vdev/"
541a0901 BB	117
541a0901 BB	118	# check for core
20da0566	119	if [[ -f "$core" ]]; then
c552fbc5	120	coreprog=$(core_prog "$core")
541a0901 BB	121	corestatus=$($GDB --batch --quiet \
	122	-ex "set print thread-events off" \
	123	-ex "printf \"\n Backtrace \n*\n\"" \
	124	-ex "bt" \
	125	-ex "printf \"\n Libraries \n*\n\"" \
	126	-ex "info sharedlib" \
	127	-ex "printf \"\n Threads (full) \n*\n\"" \
	128	-ex "info threads" \
	129	-ex "printf \"\n Backtraces \n*\n\"" \
	130	-ex "thread apply all bt" \
	131	-ex "printf \"\n Backtraces (full) \n*\n\"" \
	132	-ex "thread apply all bt full" \
c552fbc5	133	-ex "quit" "$coreprog" "$core" \| grep -v "New LWP")
541a0901 BB	134
541a0901 BB	135	# Dump core + logs to stored directory
c552fbc5 GDN	136	echo "$corestatus" >>"$dest/status"
c552fbc5 GDN	137	or_die mv "$core" "$dest/"
541a0901 BB	138
541a0901 BB	139	# Record info in cores logfile
20da0566	140	echo "*** core @ $coredir/$coreid/$core:" \| \
541a0901 BB	141	tee -a ztest.cores
	142	echo "$corestatus" \| tee -a ztest.cores
	143	echo "" \| tee -a ztest.cores
	144	fi
	145	echo "continuing..."
	146	fi
	147	}
	148
4807c0ba TC	149	rngdpid=""
	150	function on_exit
	151	{
	152	if [ -n "$rngdpid" ]; then
	153	kill -9 "$rngdpid"
	154	fi
	155	}
	156	trap on_exit EXIT
	157
541a0901 BB	158	# parse arguments
	159	# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
	160	coredir=$DEFAULTCOREDIR
5df5d06a DB	161	basedir=$DEFAULTWORKDIR
5df5d06a DB	162	rundir="zloop-run"
541a0901 BB	163	timeout=0
	164	while getopts ":ht:c:f:" opt; do
	165	case $opt in
	166	t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
	167	c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
5df5d06a	168	f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
541a0901 BB	169	h ) usage
	170	exit 2
	171	;;
	172	* ) echo "Invalid argument: -$OPTARG";
	173	usage
	174	exit 1
	175	esac
	176	done
	177	# pass remaining arguments on to ztest
	178	shift $((OPTIND - 1))
	179
	180	# enable core dumps
	181	ulimit -c unlimited
	182
20da0566 GN	183	if [[ -f "$(core_file)" ]]; then
20da0566 GN	184	echo -n "There's a core dump here you might want to look at first... "
c552fbc5	185	core_file
541a0901 BB	186	exit 1
	187	fi
	188
	189	if [[ ! -d $coredir ]]; then
	190	echo "core dump directory ($coredir) does not exist, creating it."
c552fbc5	191	or_die mkdir -p "$coredir"
541a0901 BB	192	fi
	193
	194	if [[ ! -w $coredir ]]; then
	195	echo "core dump directory ($coredir) is not writable."
	196	exit 1
	197	fi
	198
	199	or_die rm -f ztest.history
	200	or_die rm -f ztest.ddt
	201	or_die rm -f ztest.cores
	202
4807c0ba TC	203	# start rngd in the background so we don't run out of entropy
	204	or_die read -r rngdpid < <(rngd -f -r /dev/urandom & echo $!)
	205
541a0901 BB	206	ztrc=0 # ztest return value
	207	foundcrashes=0 # number of crashes found so far
	208	starttime=$(date +%s)
	209	curtime=$starttime
	210
	211	# if no timeout was specified, loop forever.
c552fbc5	212	while [[ $timeout -eq 0 ]] \|\| [[ $curtime -le $((starttime + timeout)) ]]; do
541a0901 BB	213	zopt="-VVVVV"
541a0901 BB	214
5df5d06a DB	215	# start each run with an empty directory
	216	workdir="$basedir/$rundir"
	217	or_die rm -rf "$workdir"
	218	or_die mkdir "$workdir"
	219
541a0901 BB	220	# switch between common arrangements & fully randomized
	221	if [[ $((RANDOM % 2)) -eq 0 ]]; then
	222	mirrors=2
	223	raidz=0
	224	parity=1
	225	vdevs=2
	226	else
	227	mirrors=$(((RANDOM % 3) * 1))
	228	parity=$(((RANDOM % 3) + 1))
	229	raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
	230	vdevs=$(((RANDOM % 3) + 3))
	231	fi
	232	align=$(((RANDOM % 2) * 3 + 9))
	233	runtime=$((RANDOM % 100))
	234	passtime=$((RANDOM % (runtime / 3 + 1) + 10))
	235	size=128m
	236
	237	zopt="$zopt -m $mirrors"
	238	zopt="$zopt -r $raidz"
	239	zopt="$zopt -R $parity"
	240	zopt="$zopt -v $vdevs"
	241	zopt="$zopt -a $align"
	242	zopt="$zopt -T $runtime"
	243	zopt="$zopt -P $passtime"
	244	zopt="$zopt -s $size"
	245	zopt="$zopt -f $workdir"
	246
c552fbc5	247	# shellcheck disable=SC2124
541a0901 BB	248	cmd="$ZTEST $zopt $@"
	249	desc="$(date '+%m/%d %T') $cmd"
	250	echo "$desc" \| tee -a ztest.history
	251	echo "$desc" >>ztest.out
	252	$cmd >>ztest.out 2>&1
	253	ztrc=$?
	254	egrep '===\|WARNING' ztest.out >>ztest.history
c552fbc5	255	$ZDB -U "$workdir/zpool.cache" -DD ztest >>ztest.ddt
541a0901 BB	256
	257	store_core
	258
	259	curtime=$(date +%s)
	260	done
	261
	262	echo "zloop finished, $foundcrashes crashes found"
	263
20da0566 GN	264	#restore core pattern
	265	echo "$origcorepattern" > /proc/sys/kernel/core_pattern
	266
541a0901 BB	267	uptime >>ztest.out
	268
	269	if [[ $foundcrashes -gt 0 ]]; then
	270	exit 1
	271	fi