]>
Commit | Line | Data |
---|---|---|
54007c79 | 1 | #!/usr/bin/env bash |
541a0901 BB |
2 | |
3 | # | |
4 | # CDDL HEADER START | |
5 | # | |
6 | # This file and its contents are supplied under the terms of the | |
7 | # Common Development and Distribution License ("CDDL"), version 1.0. | |
8 | # You may only use this file in accordance with the terms of version | |
9 | # 1.0 of the CDDL. | |
10 | # | |
11 | # A full copy of the text of the CDDL should have accompanied this | |
12 | # source. A copy of the CDDL is also available via the Internet at | |
13 | # http://www.illumos.org/license/CDDL. | |
14 | # | |
15 | # CDDL HEADER END | |
16 | # | |
17 | ||
18 | # | |
19 | # Copyright (c) 2015 by Delphix. All rights reserved. | |
20 | # Copyright (C) 2016 Lawrence Livermore National Security, LLC. | |
b2255edc | 21 | # Copyright (c) 2017, Intel Corporation. |
541a0901 BB |
22 | # |
23 | ||
c8f9061f | 24 | BASE_DIR=$(dirname "$0") |
541a0901 | 25 | SCRIPT_COMMON=common.sh |
c8f9061f BB |
26 | if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then |
27 | . "${BASE_DIR}/${SCRIPT_COMMON}" | |
541a0901 BB |
28 | else |
29 | echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 | |
30 | fi | |
31 | ||
c552fbc5 | 32 | # shellcheck disable=SC2034 |
541a0901 | 33 | PROG=zloop.sh |
fed90353 | 34 | GDB=${GDB:-gdb} |
541a0901 BB |
35 | |
36 | DEFAULTWORKDIR=/var/tmp | |
37 | DEFAULTCOREDIR=/var/tmp/zloop | |
38 | ||
39 | function usage | |
40 | { | |
14b43fbd RM |
41 | cat >&2 <<EOF |
42 | ||
43 | $0 [-hl] [-c <dump directory>] [-f <vdev directory>] | |
44 | [-m <max core dumps>] [-s <vdev size>] [-t <timeout>] | |
45 | [-I <max iterations>] [-- [extra ztest parameters]] | |
46 | ||
47 | This script runs ztest repeatedly with randomized arguments. | |
48 | If a crash is encountered, the ztest logs, any associated | |
49 | vdev files, and core file (if one exists) are moved to the | |
50 | output directory ($DEFAULTCOREDIR by default). Any options | |
51 | after the -- end-of-options marker will be passed to ztest. | |
52 | ||
53 | Options: | |
54 | -c Specify a core dump directory to use. | |
55 | -f Specify working directory for ztest vdev files. | |
56 | -h Print this help message. | |
57 | -l Create 'ztest.core.N' symlink to core directory. | |
58 | -m Max number of core dumps to allow before exiting. | |
59 | -s Size of vdev devices. | |
60 | -t Total time to loop for, in seconds. If not provided, | |
61 | zloop runs forever. | |
62 | -I Max number of iterations to loop before exiting. | |
63 | ||
64 | EOF | |
541a0901 BB |
65 | } |
66 | ||
67 | function or_die | |
68 | { | |
c552fbc5 | 69 | # shellcheck disable=SC2068 |
2ca77988 AZ |
70 | if ! $@; then |
71 | echo "Command failed: $*" | |
541a0901 BB |
72 | exit 1 |
73 | fi | |
74 | } | |
75 | ||
54007c79 RM |
76 | case $(uname) in |
77 | FreeBSD) | |
78 | coreglob="z*.core" | |
79 | ;; | |
80 | Linux) | |
81 | # core file helpers | |
82 | origcorepattern="$(cat /proc/sys/kernel/core_pattern)" | |
83 | coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" | |
84 | ||
85 | if [[ $coreglob = "*" ]]; then | |
86 | echo "Setting core file pattern..." | |
87 | echo "core" > /proc/sys/kernel/core_pattern | |
88 | coreglob="$(grep -E -o '^([^|%[:space:]]*)' \ | |
89 | /proc/sys/kernel/core_pattern)*" | |
90 | fi | |
91 | ;; | |
92 | *) | |
93 | exit 1 | |
94 | ;; | |
95 | esac | |
20da0566 GN |
96 | |
97 | function core_file | |
98 | { | |
6fc30992 AZ |
99 | # shellcheck disable=SC2012,SC2086 |
100 | ls -tr1 $coreglob 2>/dev/null | head -1 | |
20da0566 GN |
101 | } |
102 | ||
0c313d2f GN |
103 | function core_prog |
104 | { | |
83613696 | 105 | # shellcheck disable=SC2154 |
0c313d2f | 106 | prog=$ZTEST |
c552fbc5 | 107 | core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \ |
0c313d2f | 108 | tr \' ' ') |
6fc30992 | 109 | if [[ "$core_id" == *"zdb "* ]]; then |
83613696 | 110 | # shellcheck disable=SC2154 |
0c313d2f GN |
111 | prog=$ZDB |
112 | fi | |
113 | printf "%s" "$prog" | |
114 | } | |
115 | ||
541a0901 BB |
116 | function store_core |
117 | { | |
20da0566 GN |
118 | core="$(core_file)" |
119 | if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then | |
5df5d06a | 120 | df -h "$workdir" >>ztest.out |
541a0901 | 121 | coreid=$(date "+zloop-%y%m%d-%H%M%S") |
c552fbc5 | 122 | foundcrashes=$((foundcrashes + 1)) |
541a0901 | 123 | |
cd0a89de | 124 | # zdb debugging |
61871518 | 125 | zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest" |
cd0a89de BB |
126 | zdbdebug=$($zdbcmd 2>&1) |
127 | echo -e "$zdbcmd\n" >>ztest.zdb | |
128 | echo "$zdbdebug" >>ztest.zdb | |
129 | ||
541a0901 | 130 | dest=$coredir/$coreid |
c552fbc5 GDN |
131 | or_die mkdir -p "$dest" |
132 | or_die mkdir -p "$dest/vdev" | |
541a0901 | 133 | |
cd0a89de | 134 | if [[ $symlink -ne 0 ]]; then |
83613696 | 135 | or_die ln -sf "$dest" "ztest.core.${foundcrashes}" |
cd0a89de BB |
136 | fi |
137 | ||
541a0901 BB |
138 | echo "*** ztest crash found - moving logs to $dest" |
139 | ||
c552fbc5 | 140 | or_die mv ztest.history "$dest/" |
cd0a89de | 141 | or_die mv ztest.zdb "$dest/" |
c552fbc5 GDN |
142 | or_die mv ztest.out "$dest/" |
143 | or_die mv "$workdir/ztest*" "$dest/vdev/" | |
8fb1ede1 BB |
144 | |
145 | if [[ -e "$workdir/zpool.cache" ]]; then | |
146 | or_die mv "$workdir/zpool.cache" "$dest/vdev/" | |
147 | fi | |
541a0901 BB |
148 | |
149 | # check for core | |
20da0566 | 150 | if [[ -f "$core" ]]; then |
c552fbc5 | 151 | coreprog=$(core_prog "$core") |
cd0a89de | 152 | coredebug=$($GDB --batch --quiet \ |
541a0901 BB |
153 | -ex "set print thread-events off" \ |
154 | -ex "printf \"*\n* Backtrace \n*\n\"" \ | |
155 | -ex "bt" \ | |
156 | -ex "printf \"*\n* Libraries \n*\n\"" \ | |
157 | -ex "info sharedlib" \ | |
158 | -ex "printf \"*\n* Threads (full) \n*\n\"" \ | |
159 | -ex "info threads" \ | |
160 | -ex "printf \"*\n* Backtraces \n*\n\"" \ | |
161 | -ex "thread apply all bt" \ | |
162 | -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ | |
163 | -ex "thread apply all bt full" \ | |
cd0a89de BB |
164 | -ex "quit" "$coreprog" "$core" 2>&1 | \ |
165 | grep -v "New LWP") | |
541a0901 BB |
166 | |
167 | # Dump core + logs to stored directory | |
cd0a89de | 168 | echo "$coredebug" >>"$dest/ztest.gdb" |
c552fbc5 | 169 | or_die mv "$core" "$dest/" |
541a0901 BB |
170 | |
171 | # Record info in cores logfile | |
20da0566 | 172 | echo "*** core @ $coredir/$coreid/$core:" | \ |
541a0901 | 173 | tee -a ztest.cores |
541a0901 | 174 | fi |
cd0a89de BB |
175 | |
176 | if [[ $coremax -gt 0 ]] && | |
177 | [[ $foundcrashes -ge $coremax ]]; then | |
178 | echo "exiting... max $coremax allowed cores" | |
179 | exit 1 | |
180 | else | |
181 | echo "continuing..." | |
182 | fi | |
541a0901 BB |
183 | fi |
184 | } | |
185 | ||
186 | # parse arguments | |
187 | # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] | |
188 | coredir=$DEFAULTCOREDIR | |
5df5d06a DB |
189 | basedir=$DEFAULTWORKDIR |
190 | rundir="zloop-run" | |
541a0901 | 191 | timeout=0 |
aea899a6 | 192 | size="512m" |
cd0a89de BB |
193 | coremax=0 |
194 | symlink=0 | |
14b43fbd RM |
195 | iterations=0 |
196 | while getopts ":ht:m:I:s:c:f:l" opt; do | |
541a0901 BB |
197 | case $opt in |
198 | t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; | |
cd0a89de | 199 | m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;; |
83613696 DS |
200 | I ) [[ -n $OPTARG ]] && iterations=$OPTARG ;; |
201 | s ) [[ -n $OPTARG ]] && size=$OPTARG ;; | |
202 | c ) [[ -n $OPTARG ]] && coredir=$OPTARG ;; | |
203 | f ) [[ -n $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; | |
cd0a89de | 204 | l ) symlink=1 ;; |
541a0901 BB |
205 | h ) usage |
206 | exit 2 | |
207 | ;; | |
208 | * ) echo "Invalid argument: -$OPTARG"; | |
209 | usage | |
210 | exit 1 | |
211 | esac | |
212 | done | |
213 | # pass remaining arguments on to ztest | |
214 | shift $((OPTIND - 1)) | |
215 | ||
216 | # enable core dumps | |
217 | ulimit -c unlimited | |
63652e15 DS |
218 | export ASAN_OPTIONS=abort_on_error=true:halt_on_error=true:allocator_may_return_null=true:disable_coredump=false:detect_stack_use_after_return=true |
219 | export UBSAN_OPTIONS=abort_on_error=true:halt_on_error=true:print_stacktrace=true | |
541a0901 | 220 | |
20da0566 GN |
221 | if [[ -f "$(core_file)" ]]; then |
222 | echo -n "There's a core dump here you might want to look at first... " | |
c552fbc5 | 223 | core_file |
cd0a89de | 224 | echo |
541a0901 BB |
225 | exit 1 |
226 | fi | |
227 | ||
228 | if [[ ! -d $coredir ]]; then | |
229 | echo "core dump directory ($coredir) does not exist, creating it." | |
c552fbc5 | 230 | or_die mkdir -p "$coredir" |
541a0901 BB |
231 | fi |
232 | ||
233 | if [[ ! -w $coredir ]]; then | |
234 | echo "core dump directory ($coredir) is not writable." | |
235 | exit 1 | |
236 | fi | |
237 | ||
238 | or_die rm -f ztest.history | |
cd0a89de | 239 | or_die rm -f ztest.zdb |
541a0901 BB |
240 | or_die rm -f ztest.cores |
241 | ||
242 | ztrc=0 # ztest return value | |
243 | foundcrashes=0 # number of crashes found so far | |
244 | starttime=$(date +%s) | |
245 | curtime=$starttime | |
14b43fbd | 246 | iteration=0 |
541a0901 BB |
247 | |
248 | # if no timeout was specified, loop forever. | |
14b43fbd RM |
249 | while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do |
250 | if (( iterations > 0 )) && (( iteration++ == iterations )); then | |
251 | break | |
252 | fi | |
253 | ||
8fb1ede1 | 254 | zopt="-G -VVVVV" |
541a0901 | 255 | |
5df5d06a DB |
256 | # start each run with an empty directory |
257 | workdir="$basedir/$rundir" | |
258 | or_die rm -rf "$workdir" | |
259 | or_die mkdir "$workdir" | |
260 | ||
b2255edc BB |
261 | # switch between three types of configs |
262 | # 1/3 basic, 1/3 raidz mix, and 1/3 draid mix | |
263 | choice=$((RANDOM % 3)) | |
264 | ||
265 | # ashift range 9 - 15 | |
266 | align=$(((RANDOM % 2) * 3 + 9)) | |
267 | ||
268 | # randomly use special classes | |
269 | class="special=random" | |
270 | ||
271 | if [[ $choice -eq 0 ]]; then | |
272 | # basic mirror only | |
541a0901 | 273 | parity=1 |
b2255edc BB |
274 | mirrors=2 |
275 | draid_data=0 | |
276 | draid_spares=0 | |
277 | raid_children=0 | |
541a0901 | 278 | vdevs=2 |
b2255edc BB |
279 | raid_type="raidz" |
280 | elif [[ $choice -eq 1 ]]; then | |
281 | # fully randomized mirror/raidz (sans dRAID) | |
541a0901 | 282 | parity=$(((RANDOM % 3) + 1)) |
b2255edc BB |
283 | mirrors=$(((RANDOM % 3) * 1)) |
284 | draid_data=0 | |
285 | draid_spares=0 | |
286 | raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) | |
541a0901 | 287 | vdevs=$(((RANDOM % 3) + 3)) |
b2255edc BB |
288 | raid_type="raidz" |
289 | else | |
290 | # fully randomized dRAID (sans mirror/raidz) | |
291 | parity=$(((RANDOM % 3) + 1)) | |
292 | mirrors=0 | |
293 | draid_data=$(((RANDOM % 8) + 3)) | |
294 | draid_spares=$(((RANDOM % 2) + parity)) | |
295 | stripe=$((draid_data + parity)) | |
296 | extra=$((draid_spares + (RANDOM % 4))) | |
297 | raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra)) | |
298 | vdevs=$((RANDOM % 3)) | |
299 | raid_type="draid" | |
541a0901 | 300 | fi |
b2255edc | 301 | |
b2255edc | 302 | zopt="$zopt -K $raid_type" |
541a0901 | 303 | zopt="$zopt -m $mirrors" |
b2255edc BB |
304 | zopt="$zopt -r $raid_children" |
305 | zopt="$zopt -D $draid_data" | |
306 | zopt="$zopt -S $draid_spares" | |
541a0901 BB |
307 | zopt="$zopt -R $parity" |
308 | zopt="$zopt -v $vdevs" | |
309 | zopt="$zopt -a $align" | |
b2255edc | 310 | zopt="$zopt -C $class" |
541a0901 BB |
311 | zopt="$zopt -s $size" |
312 | zopt="$zopt -f $workdir" | |
313 | ||
6fc30992 | 314 | cmd="$ZTEST $zopt $*" |
541a0901 BB |
315 | desc="$(date '+%m/%d %T') $cmd" |
316 | echo "$desc" | tee -a ztest.history | |
317 | echo "$desc" >>ztest.out | |
318 | $cmd >>ztest.out 2>&1 | |
319 | ztrc=$? | |
3da3488e | 320 | grep -E '===|WARNING' ztest.out >>ztest.history |
541a0901 BB |
321 | |
322 | store_core | |
323 | ||
324 | curtime=$(date +%s) | |
325 | done | |
326 | ||
327 | echo "zloop finished, $foundcrashes crashes found" | |
328 | ||
54007c79 RM |
329 | # restore core pattern. |
330 | case $(uname) in | |
331 | Linux) | |
332 | echo "$origcorepattern" > /proc/sys/kernel/core_pattern | |
333 | ;; | |
334 | *) | |
335 | ;; | |
336 | esac | |
20da0566 | 337 | |
541a0901 BB |
338 | uptime >>ztest.out |
339 | ||
340 | if [[ $foundcrashes -gt 0 ]]; then | |
341 | exit 1 | |
342 | fi |