From 76d1dde94ca9cac03fa641b4cf9259d98a706e12 Mon Sep 17 00:00:00 2001 From: Rob N Date: Tue, 9 Apr 2024 04:59:04 +1000 Subject: [PATCH] zinject: inject device errors into ioctls Adds 'ioctl' as a valid IO type for device error injection, so we can simulate a flush error (which OpenZFS currently ignores, but that's by the by). To support this, adding ZIO_STAGE_VDEV_IO_DONE to ZIO_IOCTL_PIPELINE, since that's where device error injection happens. This needs a small exclusion to avoid the vdev_queue, since flushes are not queued, and I'm assuming that the various failure responses are still reasonable for flush failures (probes, media change, etc). This seems reasonable to me, as a flush failure is not unlike a write failure in this regard, however this may be too aggressive or subtle to assume in just this change. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Alexander Motin Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16061 --- cmd/zinject/zinject.c | 6 ++++-- include/sys/zio_impl.h | 5 ++--- man/man8/zinject.8 | 4 +++- man/man8/zpool-events.8 | 2 +- module/zfs/zio.c | 7 +++++-- module/zfs/zio_inject.c | 6 +++--- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cmd/zinject/zinject.c b/cmd/zinject/zinject.c index 8d0cf5d0a..07d3d8af9 100644 --- a/cmd/zinject/zinject.c +++ b/cmd/zinject/zinject.c @@ -265,7 +265,7 @@ usage(void) "\t\tspa_vdev_exit() will trigger a panic.\n" "\n" "\tzinject -d device [-e errno] [-L ] [-F]\n" - "\t\t[-T ] [-f frequency] pool\n\n" + "\t\t[-T ] [-f frequency] pool\n\n" "\t\tInject a fault into a particular device or the device's\n" "\t\tlabel. Label injection can either be 'nvlist', 'uber',\n " "\t\t'pad1', or 'pad2'.\n" @@ -978,12 +978,14 @@ main(int argc, char **argv) io_type = ZIO_TYPE_FREE; } else if (strcasecmp(optarg, "claim") == 0) { io_type = ZIO_TYPE_CLAIM; + } else if (strcasecmp(optarg, "ioctl") == 0) { + io_type = ZIO_TYPE_IOCTL; } else if (strcasecmp(optarg, "all") == 0) { io_type = ZIO_TYPES; } else { (void) fprintf(stderr, "invalid I/O type " "'%s': must be 'read', 'write', 'free', " - "'claim' or 'all'\n", optarg); + "'claim', 'ioctl' or 'all'\n", optarg); usage(); libzfs_fini(g_zfs); return (1); diff --git a/include/sys/zio_impl.h b/include/sys/zio_impl.h index 1c0a44059..4b3726d7e 100644 --- a/include/sys/zio_impl.h +++ b/include/sys/zio_impl.h @@ -153,7 +153,7 @@ enum zio_stage { ZIO_STAGE_READY = 1 << 20, /* RWFCIT */ ZIO_STAGE_VDEV_IO_START = 1 << 21, /* RW--IT */ - ZIO_STAGE_VDEV_IO_DONE = 1 << 22, /* RW---T */ + ZIO_STAGE_VDEV_IO_DONE = 1 << 22, /* RW--IT */ ZIO_STAGE_VDEV_IO_ASSESS = 1 << 23, /* RW--IT */ ZIO_STAGE_CHECKSUM_VERIFY = 1 << 24, /* R----- */ @@ -261,8 +261,7 @@ enum zio_stage { #define ZIO_IOCTL_PIPELINE \ (ZIO_INTERLOCK_STAGES | \ - ZIO_STAGE_VDEV_IO_START | \ - ZIO_STAGE_VDEV_IO_ASSESS) + ZIO_VDEV_IO_STAGES) #define ZIO_TRIM_PIPELINE \ (ZIO_INTERLOCK_STAGES | \ diff --git a/man/man8/zinject.8 b/man/man8/zinject.8 index b692f1213..817dcb7fe 100644 --- a/man/man8/zinject.8 +++ b/man/man8/zinject.8 @@ -19,10 +19,11 @@ .\" CDDL HEADER END .\" .\" Copyright 2013 Darik Horn . All rights reserved. +.\" Copyright (c) 2024, Klara Inc. .\" .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS .\" -.Dd May 26, 2021 +.Dd April 4, 2024 .Dt ZINJECT 8 .Os . @@ -257,6 +258,7 @@ Run for this many seconds before reporting failure. .It Fl T Ar failure Set the failure type to one of .Sy all , +.Sy ioctl , .Sy claim , .Sy free , .Sy read , diff --git a/man/man8/zpool-events.8 b/man/man8/zpool-events.8 index a7a9e3344..12331b7b2 100644 --- a/man/man8/zpool-events.8 +++ b/man/man8/zpool-events.8 @@ -404,7 +404,7 @@ ZIO_STAGE_DVA_CLAIM:0x00080000:---C-- ZIO_STAGE_READY:0x00100000:RWFCIT ZIO_STAGE_VDEV_IO_START:0x00200000:RW--IT -ZIO_STAGE_VDEV_IO_DONE:0x00400000:RW---T +ZIO_STAGE_VDEV_IO_DONE:0x00400000:RW--IT ZIO_STAGE_VDEV_IO_ASSESS:0x00800000:RW--IT ZIO_STAGE_CHECKSUM_VERIFY:0x01000000:R----- diff --git a/module/zfs/zio.c b/module/zfs/zio.c index e96bbda35..08d56eef8 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -4086,14 +4086,17 @@ zio_vdev_io_done(zio_t *zio) } ASSERT(zio->io_type == ZIO_TYPE_READ || - zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_TRIM); + zio->io_type == ZIO_TYPE_WRITE || + zio->io_type == ZIO_TYPE_IOCTL || + zio->io_type == ZIO_TYPE_TRIM); if (zio->io_delay) zio->io_delay = gethrtime() - zio->io_delay; if (vd != NULL && vd->vdev_ops->vdev_op_leaf && vd->vdev_ops != &vdev_draid_spare_ops) { - vdev_queue_io_done(zio); + if (zio->io_type != ZIO_TYPE_IOCTL) + vdev_queue_io_done(zio); if (zio_injection_enabled && zio->io_error == 0) zio->io_error = zio_handle_device_injections(vd, zio, diff --git a/module/zfs/zio_inject.c b/module/zfs/zio_inject.c index 609182f4a..0a4851ecb 100644 --- a/module/zfs/zio_inject.c +++ b/module/zfs/zio_inject.c @@ -364,10 +364,10 @@ zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2) int ret = 0; /* - * We skip over faults in the labels unless it's during - * device open (i.e. zio == NULL). + * We skip over faults in the labels unless it's during device open + * (i.e. zio == NULL) or a device flush (offset is meaningless) */ - if (zio != NULL) { + if (zio != NULL && zio->io_type != ZIO_TYPE_IOCTL) { uint64_t offset = zio->io_offset; if (offset < VDEV_LABEL_START_SIZE || -- 2.39.2