]>
Commit | Line | Data |
---|---|---|
cae5b340 AX |
1 | #!/bin/bash |
2 | ||
3 | # | |
4 | # CDDL HEADER START | |
5 | # | |
6 | # This file and its contents are supplied under the terms of the | |
7 | # Common Development and Distribution License ("CDDL"), version 1.0. | |
8 | # You may only use this file in accordance with the terms of version | |
9 | # 1.0 of the CDDL. | |
10 | # | |
11 | # A full copy of the text of the CDDL should have accompanied this | |
12 | # source. A copy of the CDDL is also available via the Internet at | |
13 | # http://www.illumos.org/license/CDDL. | |
14 | # | |
15 | # CDDL HEADER END | |
16 | # | |
17 | ||
18 | # | |
19 | # Copyright (c) 2015 by Delphix. All rights reserved. | |
20 | # Copyright (C) 2016 Lawrence Livermore National Security, LLC. | |
21 | # | |
22 | ||
23 | basedir=$(dirname "$0") | |
24 | ||
25 | SCRIPT_COMMON=common.sh | |
26 | if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then | |
27 | . "${basedir}/${SCRIPT_COMMON}" | |
28 | else | |
29 | echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 | |
30 | fi | |
31 | ||
32 | # shellcheck disable=SC2034 | |
33 | PROG=zloop.sh | |
34 | ||
35 | DEFAULTWORKDIR=/var/tmp | |
36 | DEFAULTCOREDIR=/var/tmp/zloop | |
37 | ||
38 | function usage | |
39 | { | |
8ec27e97 | 40 | echo -e "\n$0 [-t <timeout>] [ -s <vdev size> ] [-c <dump directory>]" \ |
cae5b340 AX |
41 | "[ -- [extra ztest parameters]]\n" \ |
42 | "\n" \ | |
43 | " This script runs ztest repeatedly with randomized arguments.\n" \ | |
44 | " If a crash is encountered, the ztest logs, any associated\n" \ | |
45 | " vdev files, and core file (if one exists) are moved to the\n" \ | |
46 | " output directory ($DEFAULTCOREDIR by default). Any options\n" \ | |
47 | " after the -- end-of-options marker will be passed to ztest.\n" \ | |
48 | "\n" \ | |
49 | " Options:\n" \ | |
50 | " -t Total time to loop for, in seconds. If not provided,\n" \ | |
51 | " zloop runs forever.\n" \ | |
8ec27e97 | 52 | " -s Size of vdev devices.\n" \ |
cae5b340 AX |
53 | " -f Specify working directory for ztest vdev files.\n" \ |
54 | " -c Specify a core dump directory to use.\n" \ | |
047218e2 AX |
55 | " -m Max number of core dumps to allow before exiting.\n" \ |
56 | " -l Create 'ztest.core.N' symlink to core directory.\n" \ | |
cae5b340 AX |
57 | " -h Print this help message.\n" \ |
58 | "" >&2 | |
59 | } | |
60 | ||
61 | function or_die | |
62 | { | |
63 | # shellcheck disable=SC2068 | |
64 | $@ | |
65 | # shellcheck disable=SC2181 | |
66 | if [[ $? -ne 0 ]]; then | |
67 | # shellcheck disable=SC2145 | |
68 | echo "Command failed: $@" | |
69 | exit 1 | |
70 | fi | |
71 | } | |
72 | ||
73 | # core file helpers | |
74 | origcorepattern="$(cat /proc/sys/kernel/core_pattern)" | |
047218e2 | 75 | coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" |
cae5b340 AX |
76 | |
77 | if [[ $coreglob = "*" ]]; then | |
78 | echo "Setting core file pattern..." | |
79 | echo "core" > /proc/sys/kernel/core_pattern | |
047218e2 | 80 | coreglob="$(grep -E -o '^([^|%[:space:]]*)' \ |
cae5b340 AX |
81 | /proc/sys/kernel/core_pattern)*" |
82 | fi | |
83 | ||
84 | function core_file | |
85 | { | |
86 | # shellcheck disable=SC2012 disable=2086 | |
87 | printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)" | |
88 | } | |
89 | ||
90 | function core_prog | |
91 | { | |
92 | prog=$ZTEST | |
93 | core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \ | |
94 | tr \' ' ') | |
95 | # shellcheck disable=SC2076 | |
96 | if [[ "$core_id" =~ "zdb " ]]; then | |
97 | prog=$ZDB | |
98 | fi | |
99 | printf "%s" "$prog" | |
100 | } | |
101 | ||
102 | function store_core | |
103 | { | |
104 | core="$(core_file)" | |
105 | if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then | |
047218e2 | 106 | df -h "$workdir" >>ztest.out |
cae5b340 AX |
107 | coreid=$(date "+zloop-%y%m%d-%H%M%S") |
108 | foundcrashes=$((foundcrashes + 1)) | |
109 | ||
047218e2 AX |
110 | # zdb debugging |
111 | zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest" | |
112 | zdbdebug=$($zdbcmd 2>&1) | |
113 | echo -e "$zdbcmd\n" >>ztest.zdb | |
114 | echo "$zdbdebug" >>ztest.zdb | |
115 | ||
cae5b340 AX |
116 | dest=$coredir/$coreid |
117 | or_die mkdir -p "$dest" | |
118 | or_die mkdir -p "$dest/vdev" | |
119 | ||
047218e2 AX |
120 | if [[ $symlink -ne 0 ]]; then |
121 | or_die ln -sf "$dest" ztest.core.$foundcrashes | |
122 | fi | |
123 | ||
cae5b340 AX |
124 | echo "*** ztest crash found - moving logs to $dest" |
125 | ||
126 | or_die mv ztest.history "$dest/" | |
047218e2 | 127 | or_die mv ztest.zdb "$dest/" |
cae5b340 AX |
128 | or_die mv ztest.out "$dest/" |
129 | or_die mv "$workdir/ztest*" "$dest/vdev/" | |
130 | or_die mv "$workdir/zpool.cache" "$dest/vdev/" | |
131 | ||
132 | # check for core | |
133 | if [[ -f "$core" ]]; then | |
134 | coreprog=$(core_prog "$core") | |
047218e2 | 135 | coredebug=$($GDB --batch --quiet \ |
cae5b340 AX |
136 | -ex "set print thread-events off" \ |
137 | -ex "printf \"*\n* Backtrace \n*\n\"" \ | |
138 | -ex "bt" \ | |
139 | -ex "printf \"*\n* Libraries \n*\n\"" \ | |
140 | -ex "info sharedlib" \ | |
141 | -ex "printf \"*\n* Threads (full) \n*\n\"" \ | |
142 | -ex "info threads" \ | |
143 | -ex "printf \"*\n* Backtraces \n*\n\"" \ | |
144 | -ex "thread apply all bt" \ | |
145 | -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ | |
146 | -ex "thread apply all bt full" \ | |
047218e2 AX |
147 | -ex "quit" "$coreprog" "$core" 2>&1 | \ |
148 | grep -v "New LWP") | |
cae5b340 AX |
149 | |
150 | # Dump core + logs to stored directory | |
047218e2 | 151 | echo "$coredebug" >>"$dest/ztest.gdb" |
cae5b340 AX |
152 | or_die mv "$core" "$dest/" |
153 | ||
154 | # Record info in cores logfile | |
155 | echo "*** core @ $coredir/$coreid/$core:" | \ | |
156 | tee -a ztest.cores | |
cae5b340 | 157 | fi |
047218e2 AX |
158 | |
159 | if [[ $coremax -gt 0 ]] && | |
160 | [[ $foundcrashes -ge $coremax ]]; then | |
161 | echo "exiting... max $coremax allowed cores" | |
162 | exit 1 | |
163 | else | |
164 | echo "continuing..." | |
165 | fi | |
cae5b340 AX |
166 | fi |
167 | } | |
168 | ||
169 | # parse arguments | |
170 | # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] | |
171 | coredir=$DEFAULTCOREDIR | |
047218e2 AX |
172 | basedir=$DEFAULTWORKDIR |
173 | rundir="zloop-run" | |
cae5b340 | 174 | timeout=0 |
8ec27e97 | 175 | size="512m" |
047218e2 AX |
176 | coremax=0 |
177 | symlink=0 | |
178 | while getopts ":ht:m:s:c:f:l" opt; do | |
cae5b340 AX |
179 | case $opt in |
180 | t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; | |
047218e2 | 181 | m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;; |
8ec27e97 | 182 | s ) [[ $OPTARG ]] && size=$OPTARG ;; |
cae5b340 | 183 | c ) [[ $OPTARG ]] && coredir=$OPTARG ;; |
047218e2 AX |
184 | f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; |
185 | l ) symlink=1 ;; | |
cae5b340 AX |
186 | h ) usage |
187 | exit 2 | |
188 | ;; | |
189 | * ) echo "Invalid argument: -$OPTARG"; | |
190 | usage | |
191 | exit 1 | |
192 | esac | |
193 | done | |
194 | # pass remaining arguments on to ztest | |
195 | shift $((OPTIND - 1)) | |
196 | ||
197 | # enable core dumps | |
198 | ulimit -c unlimited | |
199 | ||
200 | if [[ -f "$(core_file)" ]]; then | |
201 | echo -n "There's a core dump here you might want to look at first... " | |
202 | core_file | |
047218e2 | 203 | echo |
cae5b340 AX |
204 | exit 1 |
205 | fi | |
206 | ||
207 | if [[ ! -d $coredir ]]; then | |
208 | echo "core dump directory ($coredir) does not exist, creating it." | |
209 | or_die mkdir -p "$coredir" | |
210 | fi | |
211 | ||
212 | if [[ ! -w $coredir ]]; then | |
213 | echo "core dump directory ($coredir) is not writable." | |
214 | exit 1 | |
215 | fi | |
216 | ||
217 | or_die rm -f ztest.history | |
047218e2 | 218 | or_die rm -f ztest.zdb |
cae5b340 AX |
219 | or_die rm -f ztest.cores |
220 | ||
221 | ztrc=0 # ztest return value | |
222 | foundcrashes=0 # number of crashes found so far | |
223 | starttime=$(date +%s) | |
224 | curtime=$starttime | |
225 | ||
226 | # if no timeout was specified, loop forever. | |
227 | while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do | |
228 | zopt="-VVVVV" | |
229 | ||
047218e2 AX |
230 | # start each run with an empty directory |
231 | workdir="$basedir/$rundir" | |
232 | or_die rm -rf "$workdir" | |
233 | or_die mkdir "$workdir" | |
234 | ||
cae5b340 AX |
235 | # switch between common arrangements & fully randomized |
236 | if [[ $((RANDOM % 2)) -eq 0 ]]; then | |
237 | mirrors=2 | |
238 | raidz=0 | |
239 | parity=1 | |
240 | vdevs=2 | |
241 | else | |
242 | mirrors=$(((RANDOM % 3) * 1)) | |
243 | parity=$(((RANDOM % 3) + 1)) | |
244 | raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) | |
245 | vdevs=$(((RANDOM % 3) + 3)) | |
246 | fi | |
247 | align=$(((RANDOM % 2) * 3 + 9)) | |
248 | runtime=$((RANDOM % 100)) | |
249 | passtime=$((RANDOM % (runtime / 3 + 1) + 10)) | |
cae5b340 AX |
250 | |
251 | zopt="$zopt -m $mirrors" | |
252 | zopt="$zopt -r $raidz" | |
253 | zopt="$zopt -R $parity" | |
254 | zopt="$zopt -v $vdevs" | |
255 | zopt="$zopt -a $align" | |
256 | zopt="$zopt -T $runtime" | |
257 | zopt="$zopt -P $passtime" | |
258 | zopt="$zopt -s $size" | |
259 | zopt="$zopt -f $workdir" | |
260 | ||
261 | # shellcheck disable=SC2124 | |
262 | cmd="$ZTEST $zopt $@" | |
263 | desc="$(date '+%m/%d %T') $cmd" | |
264 | echo "$desc" | tee -a ztest.history | |
265 | echo "$desc" >>ztest.out | |
266 | $cmd >>ztest.out 2>&1 | |
267 | ztrc=$? | |
047218e2 | 268 | grep -E '===|WARNING' ztest.out >>ztest.history |
cae5b340 AX |
269 | |
270 | store_core | |
271 | ||
272 | curtime=$(date +%s) | |
273 | done | |
274 | ||
275 | echo "zloop finished, $foundcrashes crashes found" | |
276 | ||
277 | #restore core pattern | |
278 | echo "$origcorepattern" > /proc/sys/kernel/core_pattern | |
279 | ||
280 | uptime >>ztest.out | |
281 | ||
282 | if [[ $foundcrashes -gt 0 ]]; then | |
283 | exit 1 | |
284 | fi |