]>
Commit | Line | Data |
---|---|---|
9c13b489 | 1 | #!/bin/bash |
541a0901 BB |
2 | |
3 | # | |
4 | # CDDL HEADER START | |
5 | # | |
6 | # This file and its contents are supplied under the terms of the | |
7 | # Common Development and Distribution License ("CDDL"), version 1.0. | |
8 | # You may only use this file in accordance with the terms of version | |
9 | # 1.0 of the CDDL. | |
10 | # | |
11 | # A full copy of the text of the CDDL should have accompanied this | |
12 | # source. A copy of the CDDL is also available via the Internet at | |
13 | # http://www.illumos.org/license/CDDL. | |
14 | # | |
15 | # CDDL HEADER END | |
16 | # | |
17 | ||
18 | # | |
19 | # Copyright (c) 2015 by Delphix. All rights reserved. | |
20 | # Copyright (C) 2016 Lawrence Livermore National Security, LLC. | |
21 | # | |
22 | ||
c8f9061f | 23 | BASE_DIR=$(dirname "$0") |
541a0901 | 24 | SCRIPT_COMMON=common.sh |
c8f9061f BB |
25 | if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then |
26 | . "${BASE_DIR}/${SCRIPT_COMMON}" | |
541a0901 BB |
27 | else |
28 | echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 | |
29 | fi | |
30 | ||
c552fbc5 | 31 | # shellcheck disable=SC2034 |
541a0901 BB |
32 | PROG=zloop.sh |
33 | ||
34 | DEFAULTWORKDIR=/var/tmp | |
35 | DEFAULTCOREDIR=/var/tmp/zloop | |
36 | ||
37 | function usage | |
38 | { | |
aea899a6 | 39 | echo -e "\n$0 [-t <timeout>] [ -s <vdev size> ] [-c <dump directory>]" \ |
541a0901 BB |
40 | "[ -- [extra ztest parameters]]\n" \ |
41 | "\n" \ | |
42 | " This script runs ztest repeatedly with randomized arguments.\n" \ | |
43 | " If a crash is encountered, the ztest logs, any associated\n" \ | |
44 | " vdev files, and core file (if one exists) are moved to the\n" \ | |
45 | " output directory ($DEFAULTCOREDIR by default). Any options\n" \ | |
46 | " after the -- end-of-options marker will be passed to ztest.\n" \ | |
47 | "\n" \ | |
48 | " Options:\n" \ | |
49 | " -t Total time to loop for, in seconds. If not provided,\n" \ | |
50 | " zloop runs forever.\n" \ | |
aea899a6 | 51 | " -s Size of vdev devices.\n" \ |
541a0901 BB |
52 | " -f Specify working directory for ztest vdev files.\n" \ |
53 | " -c Specify a core dump directory to use.\n" \ | |
54 | " -h Print this help message.\n" \ | |
55 | "" >&2 | |
56 | } | |
57 | ||
58 | function or_die | |
59 | { | |
c552fbc5 | 60 | # shellcheck disable=SC2068 |
541a0901 | 61 | $@ |
c552fbc5 | 62 | # shellcheck disable=SC2181 |
541a0901 | 63 | if [[ $? -ne 0 ]]; then |
c552fbc5 | 64 | # shellcheck disable=SC2145 |
541a0901 BB |
65 | echo "Command failed: $@" |
66 | exit 1 | |
67 | fi | |
68 | } | |
69 | ||
20da0566 GN |
70 | # core file helpers |
71 | origcorepattern="$(cat /proc/sys/kernel/core_pattern)" | |
72 | coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" | |
73 | ||
74 | if [[ $coreglob = "*" ]]; then | |
75 | echo "Setting core file pattern..." | |
76 | echo "core" > /proc/sys/kernel/core_pattern | |
0c313d2f GN |
77 | coreglob="$(egrep -o '^([^|%[:space:]]*)' \ |
78 | /proc/sys/kernel/core_pattern)*" | |
20da0566 GN |
79 | fi |
80 | ||
81 | function core_file | |
82 | { | |
c552fbc5 | 83 | # shellcheck disable=SC2012 disable=2086 |
20da0566 GN |
84 | printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)" |
85 | } | |
86 | ||
0c313d2f GN |
87 | function core_prog |
88 | { | |
89 | prog=$ZTEST | |
c552fbc5 | 90 | core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \ |
0c313d2f | 91 | tr \' ' ') |
c552fbc5 | 92 | # shellcheck disable=SC2076 |
0c313d2f GN |
93 | if [[ "$core_id" =~ "zdb " ]]; then |
94 | prog=$ZDB | |
95 | fi | |
96 | printf "%s" "$prog" | |
97 | } | |
98 | ||
541a0901 BB |
99 | function store_core |
100 | { | |
20da0566 GN |
101 | core="$(core_file)" |
102 | if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then | |
5df5d06a | 103 | df -h "$workdir" >>ztest.out |
541a0901 | 104 | coreid=$(date "+zloop-%y%m%d-%H%M%S") |
c552fbc5 | 105 | foundcrashes=$((foundcrashes + 1)) |
541a0901 BB |
106 | |
107 | dest=$coredir/$coreid | |
c552fbc5 GDN |
108 | or_die mkdir -p "$dest" |
109 | or_die mkdir -p "$dest/vdev" | |
541a0901 BB |
110 | |
111 | echo "*** ztest crash found - moving logs to $dest" | |
112 | ||
c552fbc5 GDN |
113 | or_die mv ztest.history "$dest/" |
114 | or_die mv ztest.ddt "$dest/" | |
115 | or_die mv ztest.out "$dest/" | |
116 | or_die mv "$workdir/ztest*" "$dest/vdev/" | |
117 | or_die mv "$workdir/zpool.cache" "$dest/vdev/" | |
541a0901 BB |
118 | |
119 | # check for core | |
20da0566 | 120 | if [[ -f "$core" ]]; then |
c552fbc5 | 121 | coreprog=$(core_prog "$core") |
541a0901 BB |
122 | corestatus=$($GDB --batch --quiet \ |
123 | -ex "set print thread-events off" \ | |
124 | -ex "printf \"*\n* Backtrace \n*\n\"" \ | |
125 | -ex "bt" \ | |
126 | -ex "printf \"*\n* Libraries \n*\n\"" \ | |
127 | -ex "info sharedlib" \ | |
128 | -ex "printf \"*\n* Threads (full) \n*\n\"" \ | |
129 | -ex "info threads" \ | |
130 | -ex "printf \"*\n* Backtraces \n*\n\"" \ | |
131 | -ex "thread apply all bt" \ | |
132 | -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ | |
133 | -ex "thread apply all bt full" \ | |
c552fbc5 | 134 | -ex "quit" "$coreprog" "$core" | grep -v "New LWP") |
541a0901 BB |
135 | |
136 | # Dump core + logs to stored directory | |
c552fbc5 GDN |
137 | echo "$corestatus" >>"$dest/status" |
138 | or_die mv "$core" "$dest/" | |
541a0901 BB |
139 | |
140 | # Record info in cores logfile | |
20da0566 | 141 | echo "*** core @ $coredir/$coreid/$core:" | \ |
541a0901 BB |
142 | tee -a ztest.cores |
143 | echo "$corestatus" | tee -a ztest.cores | |
144 | echo "" | tee -a ztest.cores | |
145 | fi | |
146 | echo "continuing..." | |
147 | fi | |
148 | } | |
149 | ||
4807c0ba TC |
150 | rngdpid="" |
151 | function on_exit | |
152 | { | |
153 | if [ -n "$rngdpid" ]; then | |
154 | kill -9 "$rngdpid" | |
155 | fi | |
156 | } | |
157 | trap on_exit EXIT | |
158 | ||
541a0901 BB |
159 | # parse arguments |
160 | # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] | |
161 | coredir=$DEFAULTCOREDIR | |
5df5d06a DB |
162 | basedir=$DEFAULTWORKDIR |
163 | rundir="zloop-run" | |
541a0901 | 164 | timeout=0 |
aea899a6 BB |
165 | size="512m" |
166 | while getopts ":ht:s:c:f:" opt; do | |
541a0901 BB |
167 | case $opt in |
168 | t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; | |
aea899a6 | 169 | s ) [[ $OPTARG ]] && size=$OPTARG ;; |
541a0901 | 170 | c ) [[ $OPTARG ]] && coredir=$OPTARG ;; |
5df5d06a | 171 | f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; |
541a0901 BB |
172 | h ) usage |
173 | exit 2 | |
174 | ;; | |
175 | * ) echo "Invalid argument: -$OPTARG"; | |
176 | usage | |
177 | exit 1 | |
178 | esac | |
179 | done | |
180 | # pass remaining arguments on to ztest | |
181 | shift $((OPTIND - 1)) | |
182 | ||
183 | # enable core dumps | |
184 | ulimit -c unlimited | |
185 | ||
20da0566 GN |
186 | if [[ -f "$(core_file)" ]]; then |
187 | echo -n "There's a core dump here you might want to look at first... " | |
c552fbc5 | 188 | core_file |
541a0901 BB |
189 | exit 1 |
190 | fi | |
191 | ||
192 | if [[ ! -d $coredir ]]; then | |
193 | echo "core dump directory ($coredir) does not exist, creating it." | |
c552fbc5 | 194 | or_die mkdir -p "$coredir" |
541a0901 BB |
195 | fi |
196 | ||
197 | if [[ ! -w $coredir ]]; then | |
198 | echo "core dump directory ($coredir) is not writable." | |
199 | exit 1 | |
200 | fi | |
201 | ||
202 | or_die rm -f ztest.history | |
203 | or_die rm -f ztest.ddt | |
204 | or_die rm -f ztest.cores | |
205 | ||
4807c0ba TC |
206 | # start rngd in the background so we don't run out of entropy |
207 | or_die read -r rngdpid < <(rngd -f -r /dev/urandom & echo $!) | |
208 | ||
541a0901 BB |
209 | ztrc=0 # ztest return value |
210 | foundcrashes=0 # number of crashes found so far | |
211 | starttime=$(date +%s) | |
212 | curtime=$starttime | |
213 | ||
214 | # if no timeout was specified, loop forever. | |
c552fbc5 | 215 | while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do |
541a0901 BB |
216 | zopt="-VVVVV" |
217 | ||
5df5d06a DB |
218 | # start each run with an empty directory |
219 | workdir="$basedir/$rundir" | |
220 | or_die rm -rf "$workdir" | |
221 | or_die mkdir "$workdir" | |
222 | ||
541a0901 BB |
223 | # switch between common arrangements & fully randomized |
224 | if [[ $((RANDOM % 2)) -eq 0 ]]; then | |
225 | mirrors=2 | |
226 | raidz=0 | |
227 | parity=1 | |
228 | vdevs=2 | |
229 | else | |
230 | mirrors=$(((RANDOM % 3) * 1)) | |
231 | parity=$(((RANDOM % 3) + 1)) | |
232 | raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) | |
233 | vdevs=$(((RANDOM % 3) + 3)) | |
234 | fi | |
235 | align=$(((RANDOM % 2) * 3 + 9)) | |
236 | runtime=$((RANDOM % 100)) | |
237 | passtime=$((RANDOM % (runtime / 3 + 1) + 10)) | |
541a0901 BB |
238 | |
239 | zopt="$zopt -m $mirrors" | |
240 | zopt="$zopt -r $raidz" | |
241 | zopt="$zopt -R $parity" | |
242 | zopt="$zopt -v $vdevs" | |
243 | zopt="$zopt -a $align" | |
244 | zopt="$zopt -T $runtime" | |
245 | zopt="$zopt -P $passtime" | |
246 | zopt="$zopt -s $size" | |
247 | zopt="$zopt -f $workdir" | |
248 | ||
c552fbc5 | 249 | # shellcheck disable=SC2124 |
541a0901 BB |
250 | cmd="$ZTEST $zopt $@" |
251 | desc="$(date '+%m/%d %T') $cmd" | |
252 | echo "$desc" | tee -a ztest.history | |
253 | echo "$desc" >>ztest.out | |
254 | $cmd >>ztest.out 2>&1 | |
255 | ztrc=$? | |
256 | egrep '===|WARNING' ztest.out >>ztest.history | |
c552fbc5 | 257 | $ZDB -U "$workdir/zpool.cache" -DD ztest >>ztest.ddt |
541a0901 BB |
258 | |
259 | store_core | |
260 | ||
261 | curtime=$(date +%s) | |
262 | done | |
263 | ||
264 | echo "zloop finished, $foundcrashes crashes found" | |
265 | ||
20da0566 GN |
266 | #restore core pattern |
267 | echo "$origcorepattern" > /proc/sys/kernel/core_pattern | |
268 | ||
541a0901 BB |
269 | uptime >>ztest.out |
270 | ||
271 | if [[ $foundcrashes -gt 0 ]]; then | |
272 | exit 1 | |
273 | fi |