]>
Commit | Line | Data |
---|---|---|
0ee8118b BB |
1 | #!/bin/bash |
2 | # | |
3 | # ZPOOL fault verification test script. | |
4 | # | |
5 | # The current suite of fault tests should not be thought of an exhaustive | |
6 | # list of failure modes. Rather it is simply an starting point which trys | |
7 | # to cover the bulk the of the 'easy' and hopefully common, failure modes. | |
8 | # | |
9 | # Additional tests should be added but the current suite as new interesting | |
10 | # failures modes are observed. Additional failure modes I'd like to see | |
11 | # tests for include, but are not limited too: | |
12 | # | |
13 | # * Slow but successful IO. | |
14 | # * SCSI sense codes generated as zevents. | |
15 | # * 4k sectors | |
16 | # * noise | |
17 | # * medium error | |
18 | # * recovered error | |
19 | # | |
20 | # The current infrastructure using the 'mdadm' faulty device and the | |
21 | # 'scsi_debug' simulated scsi devices. The idea is to inject the error | |
22 | # below the zfs stack to validate all the error paths. More targeted | |
23 | # failure testing should be added using the 'zinject' command line util. | |
24 | # | |
25 | # Requires the following packages: | |
26 | # * mdadm | |
27 | # * lsscsi | |
28 | # * sg3-utils | |
29 | # | |
30 | ||
31 | basedir="$(dirname $0)" | |
32 | ||
33 | SCRIPT_COMMON=common.sh | |
34 | if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then | |
35 | . "${basedir}/${SCRIPT_COMMON}" | |
36 | else | |
37 | echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 | |
38 | fi | |
39 | ||
40 | PROG=zfault.sh | |
41 | ||
42 | usage() { | |
43 | cat << EOF | |
44 | USAGE: | |
e0f3df67 | 45 | $0 [hvcts] |
0ee8118b BB |
46 | |
47 | DESCRIPTION: | |
48 | ZPOOL fault verification tests | |
49 | ||
50 | OPTIONS: | |
51 | -h Show this message | |
52 | -v Verbose | |
53 | -c Cleanup md+lo+file devices at start | |
54 | -t <#> Run listed tests | |
55 | -s <#> Skip listed tests | |
56 | ||
57 | EOF | |
58 | } | |
59 | ||
60 | while getopts 'hvct:s:?' OPTION; do | |
61 | case $OPTION in | |
62 | h) | |
63 | usage | |
64 | exit 1 | |
65 | ;; | |
66 | v) | |
67 | VERBOSE=1 | |
68 | ;; | |
69 | c) | |
70 | CLEANUP=1 | |
71 | ;; | |
72 | t) | |
73 | TESTS_RUN=($OPTARG) | |
74 | ;; | |
75 | s) | |
76 | TESTS_SKIP=($OPTARG) | |
77 | ;; | |
78 | ?) | |
79 | usage | |
80 | exit | |
81 | ;; | |
82 | esac | |
83 | done | |
84 | ||
85 | if [ $(id -u) != 0 ]; then | |
86 | die "Must run as root" | |
87 | fi | |
88 | ||
5cbf6db9 BB |
89 | # Initialize the test suite |
90 | init | |
91 | ||
0ee8118b BB |
92 | # Perform pre-cleanup is requested |
93 | if [ ${CLEANUP} ]; then | |
7dc3830c | 94 | ${ZFS_SH} -u |
0ee8118b BB |
95 | cleanup_md_devices |
96 | cleanup_loop_devices | |
97 | rm -f /tmp/zpool.cache.* | |
98 | fi | |
99 | ||
100 | # Check if we need to skip all md based tests. | |
101 | MD_PARTITIONABLE=0 | |
102 | check_md_partitionable && MD_PARTITIONABLE=1 | |
103 | if [ ${MD_PARTITIONABLE} -eq 0 ]; then | |
104 | echo "Skipping tests 1-7 which require partitionable md devices" | |
105 | fi | |
106 | ||
107 | # Check if we need to skip all the scsi_debug tests. | |
108 | SCSI_DEBUG=0 | |
109 | ${INFOMOD} scsi_debug &>/dev/null && SCSI_DEBUG=1 | |
110 | if [ ${SCSI_DEBUG} -eq 0 ]; then | |
111 | echo "Skipping tests 8-9 which require the scsi_debug module" | |
112 | fi | |
113 | ||
114 | if [ ${MD_PARTITIONABLE} -eq 0 ] || [ ${SCSI_DEBUG} -eq 0 ]; then | |
115 | echo | |
116 | fi | |
117 | ||
118 | printf "%40s%s\t%s\t%s\t%s\t%s\n" "" "raid0" "raid10" "raidz" "raidz2" "raidz3" | |
119 | ||
120 | pass_nonewline() { | |
121 | echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t" | |
122 | } | |
123 | ||
124 | skip_nonewline() { | |
125 | echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t" | |
126 | } | |
127 | ||
128 | nth_zpool_vdev() { | |
129 | local POOL_NAME=$1 | |
130 | local DEVICE_TYPE=$2 | |
131 | local DEVICE_NTH=$3 | |
132 | ||
133 | ${ZPOOL} status ${POOL_NAME} | grep ${DEVICE_TYPE} ${TMP_STATUS} | \ | |
134 | head -n${DEVICE_NTH} | tail -n1 | ${AWK} "{ print \$1 }" | |
135 | } | |
136 | ||
137 | vdev_status() { | |
138 | local POOL_NAME=$1 | |
139 | local VDEV_NAME=$2 | |
140 | ||
141 | ${ZPOOL} status ${POOL_NAME} | ${AWK} "/${VDEV_NAME}/ { print \$2 }" | |
142 | } | |
143 | ||
144 | # Required format is x.yz[KMGTP] | |
145 | expand_numeric_suffix() { | |
146 | local VALUE=$1 | |
147 | ||
148 | VALUE=`echo "${VALUE/%K/*1000}"` | |
149 | VALUE=`echo "${VALUE/%M/*1000000}"` | |
150 | VALUE=`echo "${VALUE/%G/*1000000000}"` | |
151 | VALUE=`echo "${VALUE/%T/*1000000000000}"` | |
152 | VALUE=`echo "${VALUE/%P/*1000000000000000}"` | |
153 | VALUE=`echo "${VALUE}" | bc | cut -d'.' -f1` | |
154 | ||
155 | echo "${VALUE}" | |
156 | } | |
157 | ||
158 | vdev_read_errors() { | |
159 | local POOL_NAME=$1 | |
160 | local VDEV_NAME=$2 | |
161 | local VDEV_ERRORS=`${ZPOOL} status ${POOL_NAME} | | |
162 | ${AWK} "/${VDEV_NAME}/ { print \\$3 }"` | |
163 | ||
164 | expand_numeric_suffix ${VDEV_ERRORS} | |
165 | } | |
166 | ||
167 | vdev_write_errors() { | |
168 | local POOL_NAME=$1 | |
169 | local VDEV_NAME=$2 | |
170 | local VDEV_ERRORS=`${ZPOOL} status ${POOL_NAME} | | |
171 | ${AWK} "/${VDEV_NAME}/ { print \\$4 }"` | |
172 | ||
173 | expand_numeric_suffix ${VDEV_ERRORS} | |
174 | } | |
175 | ||
176 | vdev_cksum_errors() { | |
177 | local POOL_NAME=$1 | |
178 | local VDEV_NAME=$2 | |
179 | local VDEV_ERRORS=`${ZPOOL} status ${POOL_NAME} | | |
180 | ${AWK} "/${VDEV_NAME}/ { print \\$5 }"` | |
181 | ||
182 | expand_numeric_suffix ${VDEV_ERRORS} | |
183 | } | |
184 | ||
185 | zpool_state() { | |
186 | local POOL_NAME=$1 | |
187 | ||
188 | ${ZPOOL} status ${POOL_NAME} | ${AWK} "/state/ { print \$2; exit }" | |
189 | } | |
190 | ||
191 | zpool_event() { | |
192 | local EVENT_NAME=$1 | |
193 | local EVENT_KEY=$2 | |
194 | ||
195 | SCRIPT1="BEGIN {RS=\"\"; FS=\"\n\"} /${EVENT_NAME}/ { print \$0; exit }" | |
196 | SCRIPT2="BEGIN {FS=\"=\"} /${EVENT_KEY}/ { print \$2; exit }" | |
197 | ||
198 | ${ZPOOL} events -vH | ${AWK} "${SCRIPT1}" | ${AWK} "${SCRIPT2}" | |
199 | } | |
200 | ||
201 | zpool_scan_errors() { | |
202 | local POOL_NAME=$1 | |
203 | ||
204 | ${ZPOOL} status ${POOL_NAME} | ${AWK} "/scan: scrub/ { print \$8 }" | |
205 | ${ZPOOL} status ${POOL_NAME} | ${AWK} "/scan: resilver/ { print \$7 }" | |
206 | } | |
207 | ||
208 | pattern_create() { | |
209 | local PATTERN_BLOCK_SIZE=$1 | |
210 | local PATTERN_BLOCK_COUNT=$2 | |
211 | local PATTERN_NAME=`mktemp -p /tmp zpool.pattern.XXXXXXXX` | |
212 | ||
213 | echo ${PATTERN_NAME} | |
214 | dd if=/dev/urandom of=${PATTERN_NAME} bs=${PATTERN_BLOCK_SIZE} \ | |
215 | count=${PATTERN_BLOCK_COUNT} &>/dev/null | |
216 | return $? | |
217 | } | |
218 | ||
219 | pattern_write() { | |
220 | local PATTERN_NAME=$1 | |
221 | local PATTERN_BLOCK_SIZE=$2 | |
222 | local PATTERN_BLOCK_COUNT=$3 | |
223 | local DEVICE_NAME=$4 | |
224 | ||
225 | dd if=${PATTERN_NAME} of=${DEVICE_NAME} bs=${PATTERN_BLOCK_SIZE} \ | |
226 | count=${PATTERN_BLOCK_COUNT} oflag=direct &>/dev/null | |
227 | return $? | |
228 | } | |
229 | ||
230 | pattern_write_bg() { | |
231 | local PATTERN_NAME=$1 | |
232 | local PATTERN_BLOCK_SIZE=$2 | |
233 | local PATTERN_BLOCK_COUNT=$3 | |
234 | local DEVICE_NAME=$4 | |
235 | ||
236 | dd if=${PATTERN_NAME} of=${DEVICE_NAME} bs=${PATTERN_BLOCK_SIZE} \ | |
237 | count=${PATTERN_BLOCK_COUNT} oflag=direct &>/dev/null & | |
238 | return $? | |
239 | } | |
240 | ||
241 | pattern_verify() { | |
242 | local PATTERN_NAME=$1 | |
243 | local PATTERN_BLOCK_SIZE=$2 | |
244 | local PATTERN_BLOCK_COUNT=$3 | |
245 | local DEVICE_NAME=$4 | |
246 | local DEVICE_FILE=`mktemp -p /tmp zpool.pattern.XXXXXXXX` | |
247 | ||
248 | dd if=${DEVICE_NAME} of=${DEVICE_FILE} bs=${PATTERN_BLOCK_SIZE} \ | |
249 | count=${PATTERN_BLOCK_COUNT} iflag=direct &>/dev/null | |
250 | cmp -s ${PATTERN_NAME} ${DEVICE_FILE} | |
251 | RC=$? | |
252 | rm -f ${DEVICE_FILE} | |
253 | ||
254 | return ${RC} | |
255 | } | |
256 | ||
257 | pattern_remove() { | |
258 | local PATTERN_NAME=$1 | |
259 | ||
260 | rm -f ${PATTERN_NAME} | |
261 | return $? | |
262 | } | |
263 | ||
264 | fault_set_md() { | |
265 | local VDEV_FAULTY=$1 | |
266 | local FAULT_TYPE=$2 | |
267 | ||
268 | ${MDADM} /dev/${VDEV_FAULTY} --grow --level=faulty \ | |
269 | --layout=${FAULT_TYPE} >/dev/null | |
270 | return $? | |
271 | } | |
272 | ||
273 | fault_clear_md() { | |
274 | local VDEV_FAULTY=$1 | |
275 | ||
276 | # Clear all failure injection. | |
277 | ${MDADM} /dev/${VDEV_FAULTY} --grow --level=faulty \ | |
278 | --layout=clear >/dev/null || return $? | |
279 | ${MDADM} /dev/${VDEV_FAULTY} --grow --level=faulty \ | |
280 | --layout=flush >/dev/null || return $? | |
281 | return $? | |
282 | } | |
283 | ||
284 | fault_set_sd() { | |
285 | local OPTS=$1 | |
286 | local NTH=$2 | |
287 | ||
288 | echo ${OPTS} >/sys/bus/pseudo/drivers/scsi_debug/opts | |
289 | echo ${NTH} >/sys/bus/pseudo/drivers/scsi_debug/every_nth | |
290 | } | |
291 | ||
292 | fault_clear_sd() { | |
293 | echo 0 >/sys/bus/pseudo/drivers/scsi_debug/every_nth | |
294 | echo 0 >/sys/bus/pseudo/drivers/scsi_debug/opts | |
295 | } | |
296 | ||
297 | test_setup() { | |
298 | local POOL_NAME=$1 | |
299 | local POOL_CONFIG=$2 | |
300 | local ZVOL_NAME=$3 | |
301 | local TMP_CACHE=$4 | |
302 | ||
303 | ${ZFS_SH} zfs="spa_config_path=${TMP_CACHE}" || fail 1 | |
304 | ${ZPOOL_CREATE_SH} -p ${POOL_NAME} -c ${POOL_CONFIG} || fail 2 | |
305 | ${ZFS} create -V 64M ${POOL_NAME}/${ZVOL_NAME} || fail 3 | |
306 | ||
307 | # Trigger udev and re-read the partition table to ensure all of | |
308 | # this IO is out of the way before we begin injecting failures. | |
309 | udev_trigger || fail 4 | |
310 | ${BLOCKDEV} --rereadpt /dev/${POOL_NAME}/${ZVOL_NAME} || fail 5 | |
311 | } | |
312 | ||
313 | test_cleanup() { | |
314 | local POOL_NAME=$1 | |
315 | local POOL_CONFIG=$2 | |
316 | local ZVOL_NAME=$3 | |
317 | local TMP_CACHE=$4 | |
318 | ||
319 | ${ZFS} destroy ${POOL_NAME}/${ZVOL_NAME} || fail 101 | |
320 | ${ZPOOL_CREATE_SH} -p ${POOL_NAME} -c ${POOL_CONFIG} -d || fail 102 | |
321 | ${ZFS_SH} -u || fail 103 | |
322 | rm -f ${TMP_CACHE} || fail 104 | |
323 | } | |
324 | ||
325 | test_write_soft() { | |
326 | local POOL_NAME=$1 | |
327 | local POOL_CONFIG=$2 | |
328 | local POOL_REDUNDANT=$3 | |
329 | local ZVOL_NAME="zvol" | |
330 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
331 | ||
332 | if [ ${MD_PARTITIONABLE} -eq 0 ]; then | |
333 | skip_nonewline | |
334 | return | |
335 | fi | |
336 | ||
337 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
338 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
339 | ||
340 | # Set soft write failure for first vdev device. | |
341 | local VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} md 1` | |
342 | fault_set_md ${VDEV_FAULTY} write-transient | |
343 | ||
344 | # The application must not observe an error. | |
345 | local TMP_PATTERN=`pattern_create 1M 8` || fail 11 | |
346 | pattern_write ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 12 | |
347 | fault_clear_md ${VDEV_FAULTY} | |
348 | ||
349 | # Soft errors will not be logged to 'zpool status' | |
350 | local WRITE_ERRORS=`vdev_write_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
351 | test ${WRITE_ERRORS} -eq 0 || fail 13 | |
352 | ||
353 | # Soft errors will still generate an EIO (5) event. | |
354 | test `zpool_event "zfs.io" "zio_err"` = "0x5" || fail 14 | |
355 | ||
356 | # Verify the known pattern. | |
357 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 15 | |
358 | pattern_remove ${TMP_PATTERN} || fail 16 | |
359 | ||
360 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
361 | pass_nonewline | |
362 | } | |
363 | ||
364 | # Soft write error. | |
365 | test_1() { | |
366 | test_write_soft tank lo-faulty-raid0 0 | |
367 | test_write_soft tank lo-faulty-raid10 1 | |
368 | test_write_soft tank lo-faulty-raidz 1 | |
369 | test_write_soft tank lo-faulty-raidz2 1 | |
370 | test_write_soft tank lo-faulty-raidz3 1 | |
371 | echo | |
372 | } | |
373 | run_test 1 "soft write error" | |
374 | ||
375 | test_write_hard() { | |
376 | local POOL_NAME=$1 | |
377 | local POOL_CONFIG=$2 | |
378 | local POOL_REDUNDANT=$3 | |
379 | local ZVOL_NAME="zvol" | |
380 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
381 | ||
382 | if [ ${MD_PARTITIONABLE} -eq 0 ]; then | |
383 | skip_nonewline | |
384 | return | |
385 | fi | |
386 | ||
387 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
388 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
389 | ||
390 | # Set hard write failure for first vdev device. | |
391 | local VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} md 1` | |
392 | fault_set_md ${VDEV_FAULTY} write-persistent | |
393 | ||
394 | # The application must not observe an error. | |
395 | local TMP_PATTERN=`pattern_create 1M 8` || fail 11 | |
396 | pattern_write ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 12 | |
397 | fault_clear_md ${VDEV_FAULTY} | |
398 | ||
399 | local WRITE_ERRORS=`vdev_write_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
400 | if [ ${POOL_REDUNDANT} -eq 1 ]; then | |
401 | # For redundant configurations hard errors will not be | |
402 | # logged to 'zpool status' but will generate EIO events. | |
403 | test ${WRITE_ERRORS} -eq 0 || fail 21 | |
404 | test `zpool_event "zfs.io" "zio_err"` = "0x5" || fail 22 | |
405 | else | |
406 | # For non-redundant configurations hard errors will be | |
407 | # logged to 'zpool status' and generate EIO events. They | |
408 | # will also trigger a scrub of the impacted sectors. | |
409 | sleep 10 | |
410 | test ${WRITE_ERRORS} -gt 0 || fail 31 | |
411 | test `zpool_event "zfs.io" "zio_err"` = "0x5" || fail 32 | |
412 | test `zpool_event "zfs.resilver.start" "ena"` != "" || fail 33 | |
413 | test `zpool_event "zfs.resilver.finish" "ena"` != "" || fail 34 | |
414 | test `zpool_scan_errors ${POOL_NAME}` -eq 0 || fail 35 | |
415 | fi | |
416 | ||
417 | # Verify the known pattern. | |
418 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 41 | |
419 | pattern_remove ${TMP_PATTERN} || fail 42 | |
420 | ||
421 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
422 | pass_nonewline | |
423 | } | |
424 | ||
425 | # Hard write error. | |
426 | test_2() { | |
427 | test_write_hard tank lo-faulty-raid0 0 | |
428 | test_write_hard tank lo-faulty-raid10 1 | |
429 | test_write_hard tank lo-faulty-raidz 1 | |
430 | test_write_hard tank lo-faulty-raidz2 1 | |
431 | test_write_hard tank lo-faulty-raidz3 1 | |
432 | echo | |
433 | } | |
434 | run_test 2 "hard write error" | |
435 | ||
436 | test_write_all() { | |
437 | local POOL_NAME=$1 | |
438 | local POOL_CONFIG=$2 | |
439 | local POOL_REDUNDANT=$3 | |
440 | local ZVOL_NAME="zvol" | |
441 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
442 | ||
443 | if [ ${MD_PARTITIONABLE} -eq 0 ]; then | |
444 | skip_nonewline | |
445 | return | |
446 | fi | |
447 | ||
448 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
449 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
450 | ||
451 | # Set all write failures for first vdev device. | |
452 | local VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} md 1` | |
453 | fault_set_md ${VDEV_FAULTY} write-all | |
454 | ||
455 | local TMP_PATTERN=`pattern_create 1M 8` || fail 11 | |
456 | if [ ${POOL_REDUNDANT} -eq 1 ]; then | |
457 | # The application must not observe an error. | |
458 | pattern_write ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 12 | |
459 | else | |
460 | # The application is expected to hang in the background until | |
461 | # the faulty device is repaired and 'zpool clear' is run. | |
462 | pattern_write_bg ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 13 | |
463 | sleep 10 | |
464 | fi | |
465 | fault_clear_md ${VDEV_FAULTY} | |
466 | ||
467 | local WRITE_ERRORS=`vdev_write_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
468 | local VDEV_STATUS=`vdev_status ${POOL_NAME} ${VDEV_FAULTY}` | |
469 | local POOL_STATE=`zpool_state ${POOL_NAME}` | |
470 | # For all configurations write errors are logged to 'zpool status', | |
471 | # and EIO events are generated. However, only a redundant config | |
472 | # will cause the vdev to be FAULTED and pool DEGRADED. In a non- | |
473 | # redundant config the IO will hang until 'zpool clear' is run. | |
474 | test ${WRITE_ERRORS} -gt 0 || fail 14 | |
475 | test `zpool_event "zfs.io" "zio_err"` = "0x5" || fail 15 | |
476 | ||
477 | if [ ${POOL_REDUNDANT} -eq 1 ]; then | |
478 | test "${VDEV_STATUS}" = "FAULTED" || fail 21 | |
479 | test "${POOL_STATE}" = "DEGRADED" || fail 22 | |
480 | else | |
481 | BLOCKED=`ps a | grep "${ZVOL_DEVICE}" | grep -c -v "grep"` | |
482 | ${ZPOOL} clear ${POOL_NAME} || fail 31 | |
483 | test ${BLOCKED} -eq 1 || fail 32 | |
484 | wait | |
485 | fi | |
486 | ||
487 | # Verify the known pattern. | |
488 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 41 | |
489 | pattern_remove ${TMP_PATTERN} || fail 42 | |
490 | ||
491 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
492 | pass_nonewline | |
493 | } | |
494 | ||
495 | # All write errors. | |
496 | test_3() { | |
497 | test_write_all tank lo-faulty-raid0 0 | |
498 | test_write_all tank lo-faulty-raid10 1 | |
499 | test_write_all tank lo-faulty-raidz 1 | |
500 | test_write_all tank lo-faulty-raidz2 1 | |
501 | test_write_all tank lo-faulty-raidz3 1 | |
502 | echo | |
503 | } | |
504 | run_test 3 "all write errors" | |
505 | ||
506 | test_read_soft() { | |
507 | local POOL_NAME=$1 | |
508 | local POOL_CONFIG=$2 | |
509 | local POOL_REDUNDANT=$3 | |
510 | local ZVOL_NAME="zvol" | |
511 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
512 | local READ_ERRORS=0 | |
513 | ||
514 | if [ ${MD_PARTITIONABLE} -eq 0 ]; then | |
515 | skip_nonewline | |
516 | return | |
517 | fi | |
518 | ||
519 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
520 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
521 | ||
522 | # Create a pattern to be verified during a read error. | |
523 | local TMP_PATTERN=`pattern_create 1M 8` || fail 11 | |
524 | pattern_write ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 12 | |
525 | ||
526 | # Set soft read failure for all the vdevs to ensure we hit it. | |
527 | for (( i=1; i<=4; i++ )); do | |
528 | fault_set_md `nth_zpool_vdev ${POOL_NAME} md $i` read-transient | |
529 | done | |
530 | ||
531 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 13 | |
532 | pattern_remove ${TMP_PATTERN} || fail 14 | |
533 | ||
534 | # Clear all failure injection and sum read errors. | |
535 | for (( i=1; i<=4; i++ )); do | |
536 | local VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} md $i` | |
537 | local VDEV_ERRORS=`vdev_read_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
538 | let READ_ERRORS=${READ_ERRORS}+${VDEV_ERRORS} | |
539 | fault_clear_md ${VDEV_FAULTY} | |
540 | done | |
541 | ||
542 | # Soft errors will not be logged to 'zpool status'. | |
543 | test ${READ_ERRORS} -eq 0 || fail 15 | |
544 | ||
545 | # Soft errors will still generate an EIO (5) event. | |
546 | test `zpool_event "zfs.io" "zio_err"` = "0x5" || fail 16 | |
547 | ||
548 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
549 | pass_nonewline | |
550 | } | |
551 | ||
552 | # Soft read error. | |
553 | test_4() { | |
554 | test_read_soft tank lo-faulty-raid0 0 | |
555 | test_read_soft tank lo-faulty-raid10 1 | |
556 | test_read_soft tank lo-faulty-raidz 1 | |
557 | test_read_soft tank lo-faulty-raidz2 1 | |
558 | test_read_soft tank lo-faulty-raidz3 1 | |
559 | echo | |
560 | } | |
561 | run_test 4 "soft read error" | |
562 | ||
563 | test_read_hard() { | |
564 | local POOL_NAME=$1 | |
565 | local POOL_CONFIG=$2 | |
566 | local POOL_REDUNDANT=$3 | |
567 | local ZVOL_NAME="zvol" | |
568 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
569 | local READ_ERRORS=0 | |
570 | ||
571 | if [ ${MD_PARTITIONABLE} -eq 0 ]; then | |
572 | skip_nonewline | |
573 | return | |
574 | fi | |
575 | ||
576 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
577 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
578 | ||
579 | # Create a pattern to be verified during a read error. | |
580 | local TMP_PATTERN=`pattern_create 1M 8` || fail 11 | |
581 | pattern_write ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 12 | |
582 | ||
583 | # Set hard read failure for the fourth vdev. | |
584 | local VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} md 4` | |
585 | fault_set_md ${VDEV_FAULTY} read-persistent | |
586 | ||
587 | # For a redundant pool there must be no IO error, for a non-redundant | |
588 | # pool we expect permanent damage and an IO error during verify, unless | |
589 | # we get exceptionally lucky and have just damaged redundant metadata. | |
590 | if [ ${POOL_REDUNDANT} -eq 1 ]; then | |
591 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 21 | |
592 | local READ_ERRORS=`vdev_read_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
593 | test ${READ_ERRORS} -eq 0 || fail 22 | |
594 | else | |
595 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} | |
596 | ${ZPOOL} scrub ${POOL_NAME} || fail 32 | |
597 | local READ_ERRORS=`vdev_read_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
598 | test ${READ_ERRORS} -gt 0 || fail 33 | |
599 | ${ZPOOL} status -v ${POOL_NAME} | \ | |
600 | grep -A8 "Permanent errors" | \ | |
601 | grep -q "${POOL_NAME}" || fail 34 | |
602 | fi | |
603 | pattern_remove ${TMP_PATTERN} || fail 41 | |
604 | ||
605 | # Clear all failure injection and sum read errors. | |
606 | fault_clear_md ${VDEV_FAULTY} | |
607 | ||
608 | # Hard errors will generate an EIO (5) event. | |
609 | test `zpool_event "zfs.io" "zio_err"` = "0x5" || fail 42 | |
610 | ||
611 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
612 | pass_nonewline | |
613 | } | |
614 | ||
615 | # Hard read error. | |
616 | test_5() { | |
617 | test_read_hard tank lo-faulty-raid0 0 | |
618 | test_read_hard tank lo-faulty-raid10 1 | |
619 | test_read_hard tank lo-faulty-raidz 1 | |
620 | test_read_hard tank lo-faulty-raidz2 1 | |
621 | test_read_hard tank lo-faulty-raidz3 1 | |
622 | echo | |
623 | } | |
624 | run_test 5 "hard read error" | |
625 | ||
626 | # Fixable read error. | |
627 | test_read_fixable() { | |
628 | local POOL_NAME=$1 | |
629 | local POOL_CONFIG=$2 | |
630 | local POOL_REDUNDANT=$3 | |
631 | local ZVOL_NAME="zvol" | |
632 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
633 | local READ_ERRORS=0 | |
634 | ||
635 | if [ ${MD_PARTITIONABLE} -eq 0 ]; then | |
636 | skip_nonewline | |
637 | return | |
638 | fi | |
639 | ||
640 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
641 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
642 | ||
643 | # Create a pattern to be verified during a read error. | |
644 | local TMP_PATTERN=`pattern_create 1M 8` || fail 11 | |
645 | pattern_write ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 12 | |
646 | ||
647 | # Set hard read failure for the fourth vdev. | |
648 | local VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} md 4` | |
649 | fault_set_md ${VDEV_FAULTY} read-fixable | |
650 | ||
651 | # For a redundant pool there must be no IO error, for a non-redundant | |
652 | # pool we expect permanent damage and an IO error during verify, unless | |
653 | # we get exceptionally lucky and have just damaged redundant metadata. | |
654 | if [ ${POOL_REDUNDANT} -eq 1 ]; then | |
655 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 21 | |
656 | local READ_ERRORS=`vdev_read_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
657 | test ${READ_ERRORS} -eq 0 || fail 22 | |
658 | else | |
659 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} | |
660 | ${ZPOOL} scrub ${POOL_NAME} || fail 32 | |
661 | local READ_ERRORS=`vdev_read_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
662 | test ${READ_ERRORS} -gt 0 || fail 33 | |
663 | ${ZPOOL} status -v ${POOL_NAME} | \ | |
664 | grep -A8 "Permanent errors" | \ | |
665 | grep -q "${POOL_NAME}" || fail 34 | |
666 | fi | |
667 | pattern_remove ${TMP_PATTERN} || fail 41 | |
668 | ||
669 | # Clear all failure injection and sum read errors. | |
670 | fault_clear_md ${VDEV_FAULTY} | |
671 | ||
672 | # Hard errors will generate an EIO (5) event. | |
673 | test `zpool_event "zfs.io" "zio_err"` = "0x5" || fail 42 | |
674 | ||
675 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
676 | pass_nonewline | |
677 | } | |
678 | ||
679 | # Read errors fixable with a write. | |
680 | test_6() { | |
681 | test_read_fixable tank lo-faulty-raid0 0 | |
682 | test_read_fixable tank lo-faulty-raid10 1 | |
683 | test_read_fixable tank lo-faulty-raidz 1 | |
684 | test_read_fixable tank lo-faulty-raidz2 1 | |
685 | test_read_fixable tank lo-faulty-raidz3 1 | |
686 | echo | |
687 | } | |
688 | run_test 6 "fixable read error" | |
689 | ||
690 | test_cksum() { | |
691 | local POOL_NAME=$1 | |
692 | local POOL_CONFIG=$2 | |
693 | local POOL_REDUNDANT=$3 | |
694 | local VDEV_DAMAGE="$4" | |
695 | local ZVOL_NAME="zvol" | |
696 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
697 | ||
698 | if [ ${MD_PARTITIONABLE} -eq 0 ]; then | |
699 | skip_nonewline | |
700 | return | |
701 | fi | |
702 | ||
703 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
704 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
705 | ||
706 | # Create a pattern to be verified. | |
707 | local TMP_PATTERN=`pattern_create 1M 8` || fail 11 | |
708 | pattern_write ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 12 | |
709 | ||
710 | # Verify the pattern and that no vdev has cksum errors. | |
711 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 13 | |
712 | for (( i=1; i<4; i++ )); do | |
713 | VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} md ${i}` | |
714 | CKSUM_ERRORS=`vdev_cksum_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
715 | test ${CKSUM_ERRORS} -eq 0 || fail 14 | |
716 | done | |
717 | ||
718 | # Corrupt the bulk of a vdev with random garbage, we damage as many | |
719 | # vdevs as we have levels of redundancy. For example for a raidz3 | |
720 | # configuration we can trash 3 vdevs and still expect correct data. | |
721 | # This improves the odds that we read one of the damaged vdevs. | |
722 | for VDEV in ${VDEV_DAMAGE}; do | |
723 | VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} md $VDEV` | |
724 | pattern_write /dev/urandom 1M 64 /dev/${VDEV_FAULTY}p1 | |
725 | done | |
726 | ||
727 | # Verify the pattern is still correct. For non-redundant pools | |
728 | # expect failure and for redundant pools success due to resilvering. | |
729 | if [ ${POOL_REDUNDANT} -eq 1 ]; then | |
730 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 16 | |
731 | else | |
732 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} && fail 17 | |
733 | fi | |
734 | ||
735 | CKSUM_ERRORS=`vdev_cksum_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
736 | test ${CKSUM_ERRORS} -gt 0 || fail 18 | |
737 | STATUS=`vdev_status ${POOL_NAME} ${VDEV_FAULTY}` | |
738 | test "${STATUS}" = "ONLINE" || fail 19 | |
739 | ||
740 | # The checksum errors must be logged as an event. | |
741 | local CKSUM_ERRORS=`zpool_event "zfs.checksum" "zio_err"` | |
742 | test ${CKSUM_ERRORS} = "0x34" || test ${CKSUM_ERRORS} = "0x0" || fail 20 | |
743 | ||
744 | # Verify permant errors for non-redundant pools, and for redundant | |
745 | # pools trigger a scrub and check that all checksums have been fixed. | |
746 | if [ ${POOL_REDUNDANT} -eq 1 ]; then | |
747 | # Scrub the checksum errors and clear the faults. | |
748 | ${ZPOOL} scrub ${POOL_NAME} || fail 21 | |
749 | sleep 3 | |
750 | ${ZPOOL} clear ${POOL_NAME} || fail 22 | |
751 | ||
752 | # Re-verify the pattern for fixed checksums. | |
753 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 23 | |
754 | CKSUM_ERRORS=`vdev_cksum_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
755 | test ${CKSUM_ERRORS} -eq 0 || fail 24 | |
756 | ||
757 | # Re-verify the entire pool for fixed checksums. | |
758 | ${ZPOOL} scrub ${POOL_NAME} || fail 25 | |
759 | CKSUM_ERRORS=`vdev_cksum_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
760 | test ${CKSUM_ERRORS} -eq 0 || fail 26 | |
761 | else | |
762 | ${ZPOOL} status -v ${POOL_NAME} | \ | |
763 | grep -A8 "Permanent errors" | \ | |
764 | grep -q "${POOL_NAME}/${ZVOL_NAME}" || fail 31 | |
765 | ${ZPOOL} clear ${POOL_NAME} || fail 32 | |
766 | fi | |
767 | pattern_remove ${TMP_PATTERN} || fail 41 | |
768 | ||
769 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
770 | pass_nonewline | |
771 | } | |
772 | ||
773 | # Silent data corruption | |
774 | test_7() { | |
775 | test_cksum tank lo-faulty-raid0 0 "1" | |
776 | test_cksum tank lo-faulty-raid10 1 "1 3" | |
777 | test_cksum tank lo-faulty-raidz 1 "4" | |
778 | test_cksum tank lo-faulty-raidz2 1 "3 4" | |
779 | test_cksum tank lo-faulty-raidz3 1 "2 3 4" | |
780 | echo | |
781 | } | |
782 | run_test 7 "silent data corruption" | |
783 | ||
784 | # Soft write timeout at the scsi device layer. | |
785 | test_write_timeout_soft() { | |
786 | local POOL_NAME=$1 | |
787 | local POOL_CONFIG=$2 | |
788 | local POOL_REDUNDANT=$3 | |
789 | local POOL_NTH=$4 | |
790 | local ZVOL_NAME="zvol" | |
791 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
792 | ||
793 | if [ ${SCSI_DEBUG} -eq 0 ]; then | |
794 | skip_nonewline | |
795 | return | |
796 | fi | |
797 | ||
798 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
799 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
800 | ||
801 | # Set timeout(0x4) for every nth command. | |
802 | fault_set_sd 4 ${POOL_NTH} | |
803 | ||
804 | # The application must not observe an error. | |
805 | local TMP_PATTERN=`pattern_create 1M 8` || fail 11 | |
806 | pattern_write ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 12 | |
807 | fault_clear_sd | |
808 | ||
809 | # Intermittent write timeouts even with FAILFAST set may not cause | |
810 | # an EIO (5) event. This is because how FAILFAST is handled depends | |
811 | # a log on the low level driver and the exact nature of the failure. | |
812 | # We will however see a 'zfs.delay' event logged due to the timeout. | |
813 | VDEV_DELAY=`zpool_event "zfs.delay" "zio_delay"` | |
814 | test `printf "%d" ${VDEV_DELAY}` -ge 30000 || fail 13 | |
815 | ||
816 | # Verify the known pattern. | |
817 | pattern_verify ${TMP_PATTERN} 1M 8 ${ZVOL_DEVICE} || fail 14 | |
818 | pattern_remove ${TMP_PATTERN} || fail 15 | |
819 | ||
820 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
821 | pass_nonewline | |
822 | } | |
823 | ||
824 | test_8() { | |
825 | test_write_timeout_soft tank scsi_debug-raid0 0 50 | |
826 | test_write_timeout_soft tank scsi_debug-raid10 1 100 | |
827 | test_write_timeout_soft tank scsi_debug-raidz 1 75 | |
828 | test_write_timeout_soft tank scsi_debug-raidz2 1 150 | |
829 | test_write_timeout_soft tank scsi_debug-raidz3 1 300 | |
830 | echo | |
831 | } | |
832 | run_test 8 "soft write timeout" | |
833 | ||
834 | # Persistent write timeout at the scsi device layer. | |
835 | test_write_timeout_hard() { | |
836 | local POOL_NAME=$1 | |
837 | local POOL_CONFIG=$2 | |
838 | local POOL_REDUNDANT=$3 | |
839 | local POOL_NTH=$4 | |
840 | local ZVOL_NAME="zvol" | |
841 | local ZVOL_DEVICE="/dev/${POOL_NAME}/${ZVOL_NAME}" | |
842 | local RESCAN=1 | |
843 | ||
844 | if [ ${SCSI_DEBUG} -eq 0 ]; then | |
845 | skip_nonewline | |
846 | return | |
847 | fi | |
848 | ||
849 | local TMP_CACHE=`mktemp -p /tmp zpool.cache.XXXXXXXX` | |
850 | test_setup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
851 | ||
852 | local TMP_PATTERN1=`pattern_create 1M 8` | |
853 | local TMP_PATTERN2=`pattern_create 1M 8` | |
854 | local TMP_PATTERN3=`pattern_create 1M 8` | |
855 | ||
856 | # Create three partitions each one gets a unique pattern. The first | |
857 | # pattern is written before the failure, the second pattern during | |
858 | # the failure, and the third pattern while the vdev is degraded. | |
859 | # All three patterns are verified while the vdev is degraded and | |
860 | # then again once it is brought back online. | |
861 | ${PARTED} -s ${ZVOL_DEVICE} mklabel gpt || fail 11 | |
862 | ${PARTED} -s ${ZVOL_DEVICE} mkpart primary 1M 16M || fail 12 | |
863 | ${PARTED} -s ${ZVOL_DEVICE} mkpart primary 16M 32M || fail 13 | |
864 | ${PARTED} -s ${ZVOL_DEVICE} mkpart primary 32M 48M || fail 14 | |
865 | ||
866 | wait_udev ${ZVOL_DEVICE}1 30 | |
867 | wait_udev ${ZVOL_DEVICE}2 30 | |
868 | wait_udev ${ZVOL_DEVICE}3 30 | |
869 | ||
870 | # Before the failure. | |
871 | pattern_write ${TMP_PATTERN1} 1M 8 ${ZVOL_DEVICE}1 || fail 15 | |
872 | ||
873 | # Get the faulty vdev name. | |
874 | local VDEV_FAULTY=`nth_zpool_vdev ${POOL_NAME} sd 1` | |
875 | ||
876 | # Set timeout(0x4) for every nth command. | |
877 | fault_set_sd 4 ${POOL_NTH} | |
878 | ||
879 | # During the failure. | |
880 | pattern_write ${TMP_PATTERN2} 1M 8 ${ZVOL_DEVICE}2 || fail 21 | |
881 | ||
882 | # Expect write errors to be logged to 'zpool status' | |
883 | local WRITE_ERRORS=`vdev_write_errors ${POOL_NAME} ${VDEV_FAULTY}` | |
884 | test ${WRITE_ERRORS} -gt 0 || fail 22 | |
885 | ||
886 | local VDEV_STATUS=`vdev_status ${POOL_NAME} ${VDEV_FAULTY}` | |
887 | test "${VDEV_STATUS}" = "UNAVAIL" || fail 23 | |
888 | ||
889 | # Clear the error and remove it from /dev/. | |
890 | fault_clear_sd | |
891 | rm -f /dev/${VDEV_FAULTY}[0-9] | |
892 | ||
893 | # Verify the first two patterns and write out the third. | |
894 | pattern_write ${TMP_PATTERN3} 1M 8 ${ZVOL_DEVICE}3 || fail 31 | |
895 | pattern_verify ${TMP_PATTERN1} 1M 8 ${ZVOL_DEVICE}1 || fail 32 | |
896 | pattern_verify ${TMP_PATTERN2} 1M 8 ${ZVOL_DEVICE}2 || fail 33 | |
897 | pattern_verify ${TMP_PATTERN3} 1M 8 ${ZVOL_DEVICE}3 || fail 34 | |
898 | ||
899 | # Bring the device back online by rescanning for it. It must appear | |
900 | # in lsscsi and be available to dd before allowing ZFS to bring it | |
901 | # online. This is not required but provides additional sanity. | |
902 | while [ ${RESCAN} -eq 1 ]; do | |
903 | scsi_rescan | |
904 | wait_udev /dev/${VDEV_FAULTY} 30 | |
905 | ||
906 | if [ `${LSSCSI} | grep -c "/dev/${VDEV_FAULTY}"` -eq 0 ]; then | |
907 | continue | |
908 | fi | |
909 | ||
910 | dd if=/dev/${VDEV_FAULTY} of=/dev/null bs=8M count=1 &>/dev/null | |
911 | if [ $? -ne 0 ]; then | |
912 | continue | |
913 | fi | |
914 | ||
915 | RESCAN=0 | |
916 | done | |
917 | ||
918 | # Bring the device back online. We expect it to be automatically | |
919 | # resilvered without error and we should see minimally the zfs.io, | |
920 | # zfs.statechange (VDEV_STATE_HEALTHY (0x7)), and zfs.resilver.* | |
921 | # events posted. | |
31165fd9 | 922 | ${ZPOOL} online ${POOL_NAME} ${VDEV_FAULTY} || fail 51 |
0ee8118b BB |
923 | sleep 3 |
924 | test `zpool_event "zfs.io" "zio_err"` = "0x5" || fail 52 | |
925 | test `zpool_event "zfs.statechange" "vdev_state"` = "0x7" || fail 53 | |
926 | test `zpool_event "zfs.resilver.start" "ena"` != "" || fail 54 | |
927 | test `zpool_event "zfs.resilver.finish" "ena"` != "" || fail 55 | |
928 | test `zpool_scan_errors ${POOL_NAME}` -eq 0 || fail 56 | |
929 | ||
930 | local VDEV_STATUS=`vdev_status ${POOL_NAME} ${VDEV_FAULTY}` | |
931 | test "${VDEV_STATUS}" = "ONLINE" || fail 57 | |
932 | ||
933 | # Verify the known pattern. | |
934 | pattern_verify ${TMP_PATTERN1} 1M 8 ${ZVOL_DEVICE}1 || fail 61 | |
935 | pattern_verify ${TMP_PATTERN2} 1M 8 ${ZVOL_DEVICE}2 || fail 62 | |
936 | pattern_verify ${TMP_PATTERN3} 1M 8 ${ZVOL_DEVICE}3 || fail 63 | |
937 | pattern_remove ${TMP_PATTERN1} || fail 64 | |
938 | pattern_remove ${TMP_PATTERN2} || fail 65 | |
939 | pattern_remove ${TMP_PATTERN3} || fail 66 | |
940 | ||
941 | test_cleanup ${POOL_NAME} ${POOL_CONFIG} ${ZVOL_NAME} ${TMP_CACHE} | |
942 | pass_nonewline | |
943 | } | |
944 | ||
945 | test_9() { | |
946 | skip_nonewline # Skip non-redundant config | |
947 | test_write_timeout_hard tank scsi_debug-raid10 1 -50 | |
948 | test_write_timeout_hard tank scsi_debug-raidz 1 -50 | |
949 | test_write_timeout_hard tank scsi_debug-raidz2 1 -50 | |
950 | test_write_timeout_hard tank scsi_debug-raidz3 1 -50 | |
951 | echo | |
952 | } | |
953 | run_test 9 "hard write timeout" | |
954 | ||
955 | exit 0 |