g_zfs_hdl = NULL;
}
-
-/*
- * In ZED context, all the FMA agents run in the same thread
- * and do not require a unique libzfs instance. Modules should
- * use these stubs.
- */
-libzfs_handle_t *
-__libzfs_init(void)
-{
- return (g_zfs_hdl);
-}
-
-void
-__libzfs_fini(libzfs_handle_t *hdl)
-{
-}
extern void zfs_slm_fini(void);
extern void zfs_slm_event(const char *, const char *, nvlist_t *);
-/*
- * In ZED context, all the FMA agents run in the same thread
- * and do not require a unique libzfs instance.
- */
-extern libzfs_handle_t *__libzfs_init(void);
-extern void __libzfs_fini(libzfs_handle_t *);
-
#ifdef __cplusplus
}
#endif
{
libzfs_handle_t *zhdl;
- if ((zhdl = __libzfs_init()) == NULL)
+ if ((zhdl = libzfs_init()) == NULL)
return;
if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
NULL, UU_LIST_POOL_DEBUG)) == NULL) {
- __libzfs_fini(zhdl);
+ libzfs_fini(zhdl);
return;
}
if ((zfs_cases = uu_list_create(zfs_case_pool, NULL,
UU_LIST_DEBUG)) == NULL) {
uu_list_pool_destroy(zfs_case_pool);
- __libzfs_fini(zhdl);
+ libzfs_fini(zhdl);
return;
}
if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
uu_list_destroy(zfs_cases);
uu_list_pool_destroy(zfs_case_pool);
- __libzfs_fini(zhdl);
+ libzfs_fini(zhdl);
return;
}
uu_list_pool_destroy(zfs_case_pool);
zhdl = fmd_hdl_getspecific(hdl);
- __libzfs_fini(zhdl);
+ libzfs_fini(zhdl);
}
* trigger the FMA fault that we skipped earlier.
*
* ZFS on Linux porting notes:
- * In lieu of a thread pool, just spawn a thread on demmand.
* Linux udev provides a disk insert for both the disk and the partition
*
*/
#include <sys/sunddi.h>
#include <sys/sysevent/eventdefs.h>
#include <sys/sysevent/dev.h>
+#include <thread_pool.h>
#include <pthread.h>
#include <unistd.h>
#include "zfs_agents.h"
libzfs_handle_t *g_zfshdl;
list_t g_pool_list; /* list of unavailable pools at initialization */
list_t g_device_list; /* list of disks with asynchronous label request */
+tpool_t *g_tpool;
boolean_t g_enumeration_done;
-pthread_t g_zfs_tid;
+pthread_t g_zfs_tid; /* zfs_enum_pools() thread */
typedef struct unavailpool {
zpool_handle_t *uap_zhp;
- pthread_t uap_enable_tid; /* dataset enable thread if activated */
list_node_t uap_node;
} unavailpool_t;
unavailpool_t *uap;
uap = malloc(sizeof (unavailpool_t));
uap->uap_zhp = zhp;
- uap->uap_enable_tid = 0;
list_insert_tail((list_t *)data, uap);
} else {
zpool_close(zhp);
(dp->dd_func)(zhp, nvl, dp->dd_islabeled);
}
-static void *
+void
zfs_enable_ds(void *arg)
{
unavailpool_t *pool = (unavailpool_t *)arg;
- assert(pool->uap_enable_tid = pthread_self());
-
(void) zpool_enable_datasets(pool->uap_zhp, NULL, 0);
zpool_close(pool->uap_zhp);
- pool->uap_zhp = NULL;
-
- /* Note: zfs_slm_fini() will cleanup this pool entry on exit */
- return (NULL);
+ free(pool);
}
static int
for (pool = list_head(&g_pool_list); pool != NULL;
pool = list_next(&g_pool_list, pool)) {
- if (pool->uap_enable_tid != 0)
- continue; /* entry already processed */
if (strcmp(zpool_get_name(zhp),
zpool_get_name(pool->uap_zhp)))
continue;
if (zfs_toplevel_state(zhp) >= VDEV_STATE_DEGRADED) {
- /* send to a background thread; keep on list */
- (void) pthread_create(&pool->uap_enable_tid,
- NULL, zfs_enable_ds, pool);
+ list_remove(&g_pool_list, pool);
+ (void) tpool_dispatch(g_tpool, zfs_enable_ds,
+ pool);
break;
}
}
int
zfs_slm_init()
{
- if ((g_zfshdl = __libzfs_init()) == NULL)
+ if ((g_zfshdl = libzfs_init()) == NULL)
return (-1);
/*
if (pthread_create(&g_zfs_tid, NULL, zfs_enum_pools, NULL) != 0) {
list_destroy(&g_pool_list);
- __libzfs_fini(g_zfshdl);
+ libzfs_fini(g_zfshdl);
return (-1);
}
/* wait for zfs_enum_pools thread to complete */
(void) pthread_join(g_zfs_tid, NULL);
+ /* destroy the thread pool */
+ if (g_tpool != NULL) {
+ tpool_wait(g_tpool);
+ tpool_destroy(g_tpool);
+ }
while ((pool = (list_head(&g_pool_list))) != NULL) {
- /*
- * each pool entry has two possibilities
- * 1. was made available (so wait for zfs_enable_ds thread)
- * 2. still unavailable (just close the pool)
- */
- if (pool->uap_enable_tid)
- (void) pthread_join(pool->uap_enable_tid, NULL);
- else if (pool->uap_zhp != NULL)
- zpool_close(pool->uap_zhp);
-
list_remove(&g_pool_list, pool);
+ zpool_close(pool->uap_zhp);
free(pool);
}
list_destroy(&g_pool_list);
}
list_destroy(&g_device_list);
- __libzfs_fini(g_zfshdl);
+ libzfs_fini(g_zfshdl);
}
void
nvlist_t **spares;
uint_t s, nspares;
char *dev_name;
+ zprop_source_t source;
+ int ashift;
config = zpool_get_config(zhp, NULL);
if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
&spares, &nspares) != 0)
return;
+ /*
+ * lookup "ashift" pool property, we may need it for the replacement
+ */
+ ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &source);
+
replacement = fmd_nvl_alloc(hdl, FMD_SLEEP);
(void) nvlist_add_string(replacement, ZPOOL_CONFIG_TYPE,
&spare_name) != 0)
continue;
+ /* if set, add the "ashift" pool property to the spare nvlist */
+ if (source != ZPROP_SRC_DEFAULT)
+ (void) nvlist_add_uint64(spares[s],
+ ZPOOL_CONFIG_ASHIFT, ashift);
+
(void) nvlist_add_nvlist_array(replacement,
ZPOOL_CONFIG_CHILDREN, &spares[s], 1);
zfs_retire_data_t *zdp;
libzfs_handle_t *zhdl;
- if ((zhdl = __libzfs_init()) == NULL)
+ if ((zhdl = libzfs_init()) == NULL)
return;
if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
if (zdp != NULL) {
zfs_retire_clear_data(hdl, zdp);
- __libzfs_fini(zdp->zrd_hdl);
+ libzfs_fini(zdp->zrd_hdl);
fmd_hdl_free(hdl, zdp, sizeof (zfs_retire_data_t));
}
}
[tests/functional/fault]
tests = ['auto_online_001_pos', 'auto_replace_001_pos', 'auto_spare_001_pos',
- 'auto_spare_002_pos.ksh']
+ 'auto_spare_002_pos', 'auto_spare_ashift', 'auto_spare_multiple']
tags = ['functional', 'fault']
[tests/functional/features/async_destroy]
#
# Load scsi_debug module with specified parameters
+# $blksz can be either one of: < 512b | 512e | 4Kn >
#
-function load_scsi_debug # dev_size_mb add_host num_tgts max_luns
+function load_scsi_debug # dev_size_mb add_host num_tgts max_luns blksz
{
typeset devsize=$1
typeset hosts=$2
typeset tgts=$3
typeset luns=$4
+ typeset blksz=$5
[[ -z $devsize ]] || [[ -z $hosts ]] || [[ -z $tgts ]] || \
- [[ -z $luns ]] && log_fail "Arguments invalid or missing"
+ [[ -z $luns ]] || [[ -z $blksz ]] && \
+ log_fail "Arguments invalid or missing"
+
+ case "$5" in
+ '512b')
+ typeset sector=512
+ typeset blkexp=0
+ ;;
+ '512e')
+ typeset sector=512
+ typeset blkexp=3
+ ;;
+ '4Kn')
+ typeset sector=4096
+ typeset blkexp=0
+ ;;
+ *) log_fail "Unsupported blksz value: $5" ;;
+ esac
if is_linux; then
modprobe -n scsi_debug
log_fail "scsi_debug module already installed"
else
log_must modprobe scsi_debug dev_size_mb=$devsize \
- add_host=$hosts num_tgts=$tgts max_luns=$luns
+ add_host=$hosts num_tgts=$tgts max_luns=$luns \
+ sector_size=$sector physblk_exp=$blkexp
block_device_wait
lsscsi | egrep scsi_debug > /dev/null
if (($? == 1)); then
fi
}
+#
+# Unload scsi_debug module, if needed.
+#
+function unload_scsi_debug
+{
+ if lsmod | grep scsi_debug >/dev/null; then
+ log_must modprobe -r scsi_debug
+ fi
+}
+
#
# Get scsi_debug device name.
# Returns basename of scsi_debug device (for example "sdb").
if [[ -f ${ZEDLET_DIR}/zed.pid ]]; then
zedpid=$(cat ${ZEDLET_DIR}/zed.pid)
kill $zedpid
- wait $zedpid
+ while ps -p $zedpid > /dev/null; do
+ sleep 1
+ done
rm -f ${ZEDLET_DIR}/zed.pid
fi
-
return 0
}
+#
+# Drain all zevents
+#
+function zed_events_drain
+{
+ while [ $(zpool events -H | wc -l) -ne 0 ]; do
+ sleep 1
+ zpool events -c >/dev/null
+ done
+}
+
#
# Check is provided device is being active used as a swap device.
#
for SDDEVICE in $(get_debug_device); do
unplug $SDDEVICE
done
- modprobe -r scsi_debug
+ unload_scsi_debug
fi
log_pass
# Create scsi_debug devices for the reopen tests
if is_linux; then
- load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
+ load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
else
log_unsupported "scsi debug module unsupported"
fi
auto_online_001_pos.ksh \
auto_replace_001_pos.ksh \
auto_spare_001_pos.ksh \
- auto_spare_002_pos.ksh
+ auto_spare_002_pos.ksh \
+ auto_spare_ashift.ksh \
+ auto_spare_multiple.ksh
function cleanup
{
- #online last disk before fail
- insert_disk $offline_disk $host
- poolexists $TESTPOOL && destroy_pool $TESTPOOL
+ destroy_pool $TESTPOOL
+ unload_scsi_debug
}
log_assert "Testing automated auto-online FMA test"
# If using the default loop devices, need a scsi_debug device for auto-online
if is_loop_device $DISK1; then
- SD=$(lsscsi | nawk '/scsi_debug/ {print $6; exit}')
- SDDEVICE=$(echo $SD | nawk -F / '{print $3}')
+ load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
+ SDDEVICE=$(get_debug_device)
SDDEVICE_ID=$(get_persistent_disk_name $SDDEVICE)
autoonline_disks="$SDDEVICE"
else
function setup
{
- lsmod | egrep scsi_debug > /dev/null
- if (($? == 1)); then
- load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
- fi
+ load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
+ SD=$(get_debug_device)
+ SDDEVICE_ID=$(get_persistent_disk_name $SD)
# Register vdev_id alias rule for scsi_debug device to create a
# persistent path
- SD=$(lsscsi | nawk '/scsi_debug/ {print $6; exit}' \
- | nawk -F / '{print $3}')
- SDDEVICE_ID=$(get_persistent_disk_name $SD)
log_must eval "echo "alias scsidebug /dev/disk/by-id/$SDDEVICE_ID" \
>> $VDEVID_CONF"
block_device_wait
-
- SDDEVICE=$(udevadm info -q all -n $DEV_DSKDIR/$SD | egrep ID_VDEV \
- | nawk '{print $2; exit}' | nawk -F = '{print $2; exit}')
+ SDDEVICE=$(udevadm info -q all -n $DEV_DSKDIR/$SD \
+ | awk -F'=' '/ID_VDEV=/{print $2; exit}')
[[ -z $SDDEVICE ]] && log_fail "vdev rule was not registered properly"
}
function cleanup
{
- poolexists $TESTPOOL && destroy_pool $TESTPOOL
+ destroy_pool $TESTPOOL
+ unload_scsi_debug
}
log_assert "Testing automated auto-replace FMA test"
# Offline disk
remove_disk $SD
block_device_wait
-log_must modprobe -r scsi_debug
+unload_scsi_debug
# Reimport pool with drive missing
log_must zpool import $TESTPOOL
function cleanup
{
log_must zinject -c all
- poolexists $TESTPOOL && destroy_pool $TESTPOOL
+ destroy_pool $TESTPOOL
rm -f $VDEV_FILES $SPARE_FILE
}
log_onexit cleanup
+# Clear events from previous runs
+zed_events_drain
+
TESTFILE="/$TESTPOOL/$TESTFS/testfile"
for type in "mirror" "raidz" "raidz2"; do
function cleanup
{
log_must zinject -c all
- poolexists $TESTPOOL && destroy_pool $TESTPOOL
+ destroy_pool $TESTPOOL
rm -f $VDEV_FILES $SPARE_FILE
}
log_onexit cleanup
+# Clear events from previous runs
+zed_events_drain
+
TESTFILE="/$TESTPOOL/$TESTFS/testfile"
for type in "mirror" "raidz" "raidz2"; do
log_must dd if=/dev/urandom of=$TESTFILE bs=1M count=16
# 4. Inject CHECKSUM ERRORS on read with a zinject error handler
+ # NOTE: checksum events are ratelimited to max 5 per second, ZED needs
+ # 10 to kick in a spare
log_must zinject -d $FAULT_FILE -e corrupt -f 50 -T read $TESTPOOL
log_must cp $TESTFILE /dev/null
+ log_must sleep 1
+ log_must cp $TESTFILE /dev/null
+ log_must sleep 1
+ log_must cp $TESTFILE /dev/null
# 5. Verify the ZED kicks in a hot spare and expected pool/device status
log_note "Wait for ZED to auto-spare"
--- /dev/null
+#!/bin/ksh -p
+
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Intel Corporation. All rights reserved.
+# Copyright 2017, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/include/math.shlib
+. $STF_SUITE/tests/functional/fault/fault.cfg
+
+#
+# DESCRIPTION:
+# Testing Fault Management Agent ZED Logic - Automated Auto-Spare Test when
+# drive is faulted and a custom ashift value needs to be provided to replace it.
+#
+# STRATEGY:
+# 1. Create a pool from 512b devices and set "ashift" pool property accordingly
+# 2. Add one 512e spare device (4Kn would generate IO errors on replace)
+# 3. Inject IO errors with a zinject error handler
+# 4. Start a scrub
+# 5. Verify the ZED kicks in the hot spare and expected pool/device status
+# 6. Clear the fault
+# 7. Verify the hot spare is available and expected pool/device status
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+ log_must zinject -c all
+ destroy_pool $TESTPOOL
+ unload_scsi_debug
+ rm -f $SAFE_DEVICE $FAIL_DEVICE
+}
+
+log_assert "ZED should replace a device using the configured ashift property"
+log_onexit cleanup
+
+# Clear events from previous runs
+zed_events_drain
+
+SAFE_DEVICE="$TEST_BASE_DIR/safe-dev"
+FAIL_DEVICE="$TEST_BASE_DIR/fail-dev"
+
+# 1. Create a pool from 512b devices and set "ashift" pool property accordingly
+for vdev in $SAFE_DEVICE $FAIL_DEVICE; do
+ truncate -s $SPA_MINDEVSIZE $vdev
+done
+log_must zpool create -f $TESTPOOL mirror $SAFE_DEVICE $FAIL_DEVICE
+# NOTE: file VDEVs should be added as 512b devices, verify this "just in case"
+for vdev in $SAFE_DEVICE $FAIL_DEVICE; do
+ verify_eq "9" "$(zdb -e -l $vdev | awk '/ashift: /{print $2}')" "ashift"
+done
+log_must zpool set ashift=9 $TESTPOOL
+
+# 2. Add one 512e spare device (4Kn would generate IO errors on replace)
+# NOTE: must be larger than the existing 512b devices, add 32m of fudge
+load_scsi_debug $(($SPA_MINDEVSIZE/1024/1024+32)) $SDHOSTS $SDTGTS $SDLUNS '512e'
+SPARE_DEVICE=$(get_debug_device)
+log_must_busy zpool add $TESTPOOL spare $SPARE_DEVICE
+
+# 3. Inject IO errors with a zinject error handler
+log_must zinject -d $FAIL_DEVICE -e io -T all -f 100 $TESTPOOL
+
+# 4. Start a scrub
+log_must zpool scrub $TESTPOOL
+
+# 5. Verify the ZED kicks in a hot spare and expected pool/device status
+log_note "Wait for ZED to auto-spare"
+log_must wait_vdev_state $TESTPOOL $FAIL_DEVICE "FAULTED" 60
+log_must wait_vdev_state $TESTPOOL $SPARE_DEVICE "ONLINE" 60
+log_must wait_hotspare_state $TESTPOOL $SPARE_DEVICE "INUSE"
+log_must check_state $TESTPOOL "" "DEGRADED"
+
+# 6. Clear the fault
+log_must zinject -c all
+log_must zpool clear $TESTPOOL $FAIL_DEVICE
+
+# 7. Verify the hot spare is available and expected pool/device status
+log_must wait_vdev_state $TESTPOOL $FAIL_DEVICE "ONLINE" 60
+log_must wait_hotspare_state $TESTPOOL $SPARE_DEVICE "AVAIL"
+log_must is_pool_resilvered $TESTPOOL
+log_must check_state $TESTPOOL "" "ONLINE"
+
+log_pass "ZED successfully replaces a device using the configured ashift property"
--- /dev/null
+#!/bin/ksh -p
+
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Intel Corporation. All rights reserved.
+# Copyright 2017, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/fault/fault.cfg
+
+#
+# DESCRIPTION:
+# Testing Fault Management Agent ZED Logic - Automated Auto-Spare Test when
+# multiple drives are faulted.
+#
+# STRATEGY:
+# 1. Create a pool with two hot spares
+# 2. Inject IO ERRORS with a zinject error handler on the first device
+# 3. Start a scrub
+# 4. Verify the ZED kicks in a hot spare and expected pool/device status
+# 5. Inject IO ERRORS on a second device
+# 6. Start a scrub
+# 7. Verify the ZED kicks in a second hot spare
+# 8. Clear the fault on both devices
+# 9. Verify the hot spares are available and expected pool/device status
+# 10. Rinse and repeat, this time faulting both devices at the same time
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+ log_must zinject -c all
+ destroy_pool $TESTPOOL
+ rm -f $DATA_DEVS $SPARE_DEVS
+}
+
+log_assert "ZED should be able to handle multiple faulted devices"
+log_onexit cleanup
+
+# Clear events from previous runs
+zed_events_drain
+
+FAULT_DEV1="$TEST_BASE_DIR/fault-dev1"
+FAULT_DEV2="$TEST_BASE_DIR/fault-dev2"
+SAFE_DEV1="$TEST_BASE_DIR/safe-dev1"
+SAFE_DEV2="$TEST_BASE_DIR/safe-dev2"
+DATA_DEVS="$FAULT_DEV1 $FAULT_DEV2 $SAFE_DEV1 $SAFE_DEV2"
+SPARE_DEV1="$TEST_BASE_DIR/spare-dev1"
+SPARE_DEV2="$TEST_BASE_DIR/spare-dev2"
+SPARE_DEVS="$SPARE_DEV1 $SPARE_DEV2"
+
+for type in "mirror" "raidz" "raidz2" "raidz3"; do
+ # 1. Create a pool with two hot spares
+ truncate -s $SPA_MINDEVSIZE $DATA_DEVS $SPARE_DEVS
+ log_must zpool create -f $TESTPOOL $type $DATA_DEVS spare $SPARE_DEVS
+
+ # 2. Inject IO ERRORS with a zinject error handler on the first device
+ log_must zinject -d $FAULT_DEV1 -e io -T all -f 100 $TESTPOOL
+
+ # 3. Start a scrub
+ log_must zpool scrub $TESTPOOL
+
+ # 4. Verify the ZED kicks in a hot spare and expected pool/device status
+ log_note "Wait for ZED to auto-spare"
+ log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "FAULTED" 60
+ log_must wait_vdev_state $TESTPOOL $SPARE_DEV1 "ONLINE" 60
+ log_must wait_hotspare_state $TESTPOOL $SPARE_DEV1 "INUSE"
+ log_must check_state $TESTPOOL "" "DEGRADED"
+
+ # 5. Inject IO ERRORS on a second device
+ log_must zinject -d $FAULT_DEV2 -e io -T all -f 100 $TESTPOOL
+
+ # 6. Start a scrub
+ while is_pool_scrubbing $TESTPOOL || is_pool_resilvering $TESTPOOL; do
+ sleep 1
+ done
+ log_must zpool scrub $TESTPOOL
+
+ # 7. Verify the ZED kicks in a second hot spare
+ log_note "Wait for ZED to auto-spare"
+ log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "FAULTED" 60
+ log_must wait_vdev_state $TESTPOOL $SPARE_DEV2 "ONLINE" 60
+ log_must wait_hotspare_state $TESTPOOL $SPARE_DEV2 "INUSE"
+ log_must check_state $TESTPOOL "" "DEGRADED"
+
+ # 8. Clear the fault on both devices
+ log_must zinject -c all
+ log_must zpool clear $TESTPOOL $FAULT_DEV1
+ log_must zpool clear $TESTPOOL $FAULT_DEV2
+
+ # 9. Verify the hot spares are available and expected pool/device status
+ log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "ONLINE" 60
+ log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "ONLINE" 60
+ log_must wait_hotspare_state $TESTPOOL $SPARE_DEV1 "AVAIL"
+ log_must wait_hotspare_state $TESTPOOL $SPARE_DEV2 "AVAIL"
+ log_must check_state $TESTPOOL "" "ONLINE"
+
+ # Cleanup
+ cleanup
+done
+
+# Rinse and repeat, this time faulting both devices at the same time
+# NOTE: "raidz" is exluded since it cannot survive 2 faulted devices
+# NOTE: "mirror" is a 4-way mirror here and should survive this test
+for type in "mirror" "raidz2" "raidz3"; do
+ # 1. Create a pool with two hot spares
+ truncate -s $SPA_MINDEVSIZE $DATA_DEVS $SPARE_DEVS
+ log_must zpool create -f $TESTPOOL $type $DATA_DEVS spare $SPARE_DEVS
+
+ # 2. Inject IO ERRORS with a zinject error handler on two devices
+ log_must eval "zinject -d $FAULT_DEV1 -e io -T all -f 100 $TESTPOOL &"
+ log_must eval "zinject -d $FAULT_DEV2 -e io -T all -f 100 $TESTPOOL &"
+
+ # 3. Start a scrub
+ log_must zpool scrub $TESTPOOL
+
+ # 4. Verify the ZED kicks in two hot spares and expected pool/device status
+ log_note "Wait for ZED to auto-spare"
+ log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "FAULTED" 60
+ log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "FAULTED" 60
+ log_must wait_vdev_state $TESTPOOL $SPARE_DEV1 "ONLINE" 60
+ log_must wait_vdev_state $TESTPOOL $SPARE_DEV2 "ONLINE" 60
+ log_must wait_hotspare_state $TESTPOOL $SPARE_DEV1 "INUSE"
+ log_must wait_hotspare_state $TESTPOOL $SPARE_DEV2 "INUSE"
+ log_must check_state $TESTPOOL "" "DEGRADED"
+
+ # 5. Clear the fault on both devices
+ log_must zinject -c all
+ log_must zpool clear $TESTPOOL $FAULT_DEV1
+ log_must zpool clear $TESTPOOL $FAULT_DEV2
+
+ # Cleanup
+ cleanup
+done
+
+log_pass "ZED successfully handles multiple faulted devices"
zed_stop
zed_cleanup
-SDDEVICE=$(get_debug_device)
-
-# Offline disk and remove scsi_debug module
-if is_linux; then
- if [ -n "$SDDEVICE" ]; then
- remove_disk $SDDEVICE
- fi
- modprobe -r scsi_debug
-fi
-
log_pass
zed_setup
zed_start
-# Create a scsi_debug device to be used with auto-online (if using loop devices)
-# and auto-replace regardless of other devices
-load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
-
log_pass
projects / mirror_zfs.git / commitdiff