]>
Commit | Line | Data |
---|---|---|
541a0901 BB |
1 | #!/bin/sh |
2 | ||
3 | # | |
4 | # CDDL HEADER START | |
5 | # | |
6 | # This file and its contents are supplied under the terms of the | |
7 | # Common Development and Distribution License ("CDDL"), version 1.0. | |
8 | # You may only use this file in accordance with the terms of version | |
9 | # 1.0 of the CDDL. | |
10 | # | |
11 | # A full copy of the text of the CDDL should have accompanied this | |
12 | # source. A copy of the CDDL is also available via the Internet at | |
13 | # http://www.illumos.org/license/CDDL. | |
14 | # | |
15 | # CDDL HEADER END | |
16 | # | |
17 | ||
18 | # | |
19 | # Copyright (c) 2015 by Delphix. All rights reserved. | |
20 | # Copyright (C) 2016 Lawrence Livermore National Security, LLC. | |
21 | # | |
22 | ||
23 | basedir="$(dirname $0)" | |
24 | ||
25 | SCRIPT_COMMON=common.sh | |
26 | if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then | |
27 | . "${basedir}/${SCRIPT_COMMON}" | |
28 | else | |
29 | echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 | |
30 | fi | |
31 | ||
32 | PROG=zloop.sh | |
33 | ||
34 | DEFAULTWORKDIR=/var/tmp | |
35 | DEFAULTCOREDIR=/var/tmp/zloop | |
36 | ||
37 | function usage | |
38 | { | |
39 | echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \ | |
40 | "[ -- [extra ztest parameters]]\n" \ | |
41 | "\n" \ | |
42 | " This script runs ztest repeatedly with randomized arguments.\n" \ | |
43 | " If a crash is encountered, the ztest logs, any associated\n" \ | |
44 | " vdev files, and core file (if one exists) are moved to the\n" \ | |
45 | " output directory ($DEFAULTCOREDIR by default). Any options\n" \ | |
46 | " after the -- end-of-options marker will be passed to ztest.\n" \ | |
47 | "\n" \ | |
48 | " Options:\n" \ | |
49 | " -t Total time to loop for, in seconds. If not provided,\n" \ | |
50 | " zloop runs forever.\n" \ | |
51 | " -f Specify working directory for ztest vdev files.\n" \ | |
52 | " -c Specify a core dump directory to use.\n" \ | |
53 | " -h Print this help message.\n" \ | |
54 | "" >&2 | |
55 | } | |
56 | ||
57 | function or_die | |
58 | { | |
59 | $@ | |
60 | if [[ $? -ne 0 ]]; then | |
61 | echo "Command failed: $@" | |
62 | exit 1 | |
63 | fi | |
64 | } | |
65 | ||
66 | function store_core | |
67 | { | |
68 | if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then | |
69 | coreid=$(date "+zloop-%y%m%d-%H%M%S") | |
70 | foundcrashes=$(($foundcrashes + 1)) | |
71 | ||
72 | dest=$coredir/$coreid | |
73 | or_die mkdir -p $dest | |
74 | or_die mkdir -p $dest/vdev | |
75 | ||
76 | echo "*** ztest crash found - moving logs to $dest" | |
77 | ||
78 | or_die mv ztest.history $dest/ | |
79 | or_die mv ztest.ddt $dest/ | |
80 | or_die mv ztest.out $dest/ | |
81 | or_die mv $workdir/ztest* $dest/vdev/ | |
82 | or_die mv $workdir/zpool.cache $dest/vdev/ | |
83 | ||
84 | # check for core | |
85 | if [[ -f core ]]; then | |
86 | corestatus=$($GDB --batch --quiet \ | |
87 | -ex "set print thread-events off" \ | |
88 | -ex "printf \"*\n* Backtrace \n*\n\"" \ | |
89 | -ex "bt" \ | |
90 | -ex "printf \"*\n* Libraries \n*\n\"" \ | |
91 | -ex "info sharedlib" \ | |
92 | -ex "printf \"*\n* Threads (full) \n*\n\"" \ | |
93 | -ex "info threads" \ | |
94 | -ex "printf \"*\n* Backtraces \n*\n\"" \ | |
95 | -ex "thread apply all bt" \ | |
96 | -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ | |
97 | -ex "thread apply all bt full" \ | |
98 | -ex "quit" $ZTEST core | grep -v "New LWP") | |
99 | ||
100 | # Dump core + logs to stored directory | |
101 | echo "$corestatus" >>$dest/status | |
102 | or_die mv core $dest/ | |
103 | ||
104 | # Record info in cores logfile | |
105 | echo "*** core @ $coredir/$coreid/core:" | \ | |
106 | tee -a ztest.cores | |
107 | echo "$corestatus" | tee -a ztest.cores | |
108 | echo "" | tee -a ztest.cores | |
109 | fi | |
110 | echo "continuing..." | |
111 | fi | |
112 | } | |
113 | ||
114 | # parse arguments | |
115 | # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] | |
116 | coredir=$DEFAULTCOREDIR | |
117 | workdir=$DEFAULTWORKDIR | |
118 | timeout=0 | |
119 | while getopts ":ht:c:f:" opt; do | |
120 | case $opt in | |
121 | t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; | |
122 | c ) [[ $OPTARG ]] && coredir=$OPTARG ;; | |
123 | f ) [[ $OPTARG ]] && workdir=$(readlink -f $OPTARG) ;; | |
124 | h ) usage | |
125 | exit 2 | |
126 | ;; | |
127 | * ) echo "Invalid argument: -$OPTARG"; | |
128 | usage | |
129 | exit 1 | |
130 | esac | |
131 | done | |
132 | # pass remaining arguments on to ztest | |
133 | shift $((OPTIND - 1)) | |
134 | ||
135 | # enable core dumps | |
136 | ulimit -c unlimited | |
137 | ||
138 | if [[ -f core ]]; then | |
139 | echo "There's a core dump here you might want to look at first." | |
140 | exit 1 | |
141 | fi | |
142 | ||
143 | if [[ ! -d $coredir ]]; then | |
144 | echo "core dump directory ($coredir) does not exist, creating it." | |
145 | or_die mkdir -p $coredir | |
146 | fi | |
147 | ||
148 | if [[ ! -w $coredir ]]; then | |
149 | echo "core dump directory ($coredir) is not writable." | |
150 | exit 1 | |
151 | fi | |
152 | ||
153 | or_die rm -f ztest.history | |
154 | or_die rm -f ztest.ddt | |
155 | or_die rm -f ztest.cores | |
156 | ||
157 | ztrc=0 # ztest return value | |
158 | foundcrashes=0 # number of crashes found so far | |
159 | starttime=$(date +%s) | |
160 | curtime=$starttime | |
161 | ||
162 | # if no timeout was specified, loop forever. | |
163 | while [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do | |
164 | zopt="-VVVVV" | |
165 | ||
166 | # switch between common arrangements & fully randomized | |
167 | if [[ $((RANDOM % 2)) -eq 0 ]]; then | |
168 | mirrors=2 | |
169 | raidz=0 | |
170 | parity=1 | |
171 | vdevs=2 | |
172 | else | |
173 | mirrors=$(((RANDOM % 3) * 1)) | |
174 | parity=$(((RANDOM % 3) + 1)) | |
175 | raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) | |
176 | vdevs=$(((RANDOM % 3) + 3)) | |
177 | fi | |
178 | align=$(((RANDOM % 2) * 3 + 9)) | |
179 | runtime=$((RANDOM % 100)) | |
180 | passtime=$((RANDOM % (runtime / 3 + 1) + 10)) | |
181 | size=128m | |
182 | ||
183 | zopt="$zopt -m $mirrors" | |
184 | zopt="$zopt -r $raidz" | |
185 | zopt="$zopt -R $parity" | |
186 | zopt="$zopt -v $vdevs" | |
187 | zopt="$zopt -a $align" | |
188 | zopt="$zopt -T $runtime" | |
189 | zopt="$zopt -P $passtime" | |
190 | zopt="$zopt -s $size" | |
191 | zopt="$zopt -f $workdir" | |
192 | ||
193 | cmd="$ZTEST $zopt $@" | |
194 | desc="$(date '+%m/%d %T') $cmd" | |
195 | echo "$desc" | tee -a ztest.history | |
196 | echo "$desc" >>ztest.out | |
197 | $cmd >>ztest.out 2>&1 | |
198 | ztrc=$? | |
199 | egrep '===|WARNING' ztest.out >>ztest.history | |
200 | $ZDB -U $workdir/zpool.cache -DD ztest >>ztest.ddt | |
201 | ||
202 | store_core | |
203 | ||
204 | curtime=$(date +%s) | |
205 | done | |
206 | ||
207 | echo "zloop finished, $foundcrashes crashes found" | |
208 | ||
209 | uptime >>ztest.out | |
210 | ||
211 | if [[ $foundcrashes -gt 0 ]]; then | |
212 | exit 1 | |
213 | fi |