zio_deadman_impl() fix and enhancement

Add the zio_deadman_log_all tunable to print all zios in
zio_deadman_impl().  Also, in all cases, display the depth of the
zio relative to the original parent zio.  This is meant to be used by
developers to gain diagnostic information for hangs which don't involve
fully set-up zio trees or are otherwise stuck or hung in an early stage.

Reviewed-by: Olaf Faaland <faaland1@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: loli10K <ezomori.nozomu@gmail.com>
Signed-off-by: Tim Chase <tim@chase2k.com>
Closes #8362

diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5
index dc9fcc6b4277e4d0722c913b01bc98c4b74b3ebe..bdac6f3a76a22871f97eced256da3b53328d9e47 100644 (file)
--- a/man/man5/zfs-module-parameters.5
+++ b/man/man5/zfs-module-parameters.5
@@ -2638,6 +2638,22 @@ to limit potential SLOG device abuse by single active ZIL writer.
 Default value: \fB786,432\fR.
 .RE
 
+.sp
+.ne 2
+.na
+\fBzio_deadman_log_all\fR (int)
+.ad
+.RS 12n
+If non-zero, the zio deadman will produce debugging messages (see
+\fBzfs_dbgmsg_enable\fR) for all zios, rather than only for leaf
+zios possessing a vdev. This is meant to be used by developers to gain
+diagnostic information for hang conditions which don't involve a mutex
+or other locking primitive; typically conditions in which a thread in
+the zio pipeline is looping indefinitely.
+.sp
+Default value: \fB0\fR.
+.RE
+
 .sp
 .ne 2
 .na

diff --git a/module/zfs/zio.c b/module/zfs/zio.c
index 9ca53490e2c7c37645ad7f53a03e5f3056d50019..0dd63322f0c7b947b9239a5f3e658237e1f0c9d2 100644 (file)
--- a/module/zfs/zio.c
+++ b/module/zfs/zio.c
@@ -62,6 +62,7 @@ const char *zio_type_name[ZIO_TYPES] = {
 };
 
 int zio_dva_throttle_enabled = B_TRUE;
+int zio_deadman_log_all = B_FALSE;
 
 /*
  * ==========================================================================
@@ -1858,30 +1859,30 @@ zio_delay_interrupt(zio_t *zio)
 }
 
 static void
-zio_deadman_impl(zio_t *pio)
+zio_deadman_impl(zio_t *pio, int ziodepth)
 {
        zio_t *cio, *cio_next;
        zio_link_t *zl = NULL;
        vdev_t *vd = pio->io_vd;
 
-       if (vd != NULL && vd->vdev_ops->vdev_op_leaf) {
-               vdev_queue_t *vq = &vd->vdev_queue;
+       if (zio_deadman_log_all || (vd != NULL && vd->vdev_ops->vdev_op_leaf)) {
+               vdev_queue_t *vq = vd ? &vd->vdev_queue : NULL;
                zbookmark_phys_t *zb = &pio->io_bookmark;
                uint64_t delta = gethrtime() - pio->io_timestamp;
                uint64_t failmode = spa_get_deadman_failmode(pio->io_spa);
 
-               zfs_dbgmsg("slow zio: zio=%p timestamp=%llu "
+               zfs_dbgmsg("slow zio[%d]: zio=%p timestamp=%llu "
                    "delta=%llu queued=%llu io=%llu "
                    "path=%s last=%llu "
                    "type=%d priority=%d flags=0x%x "
                    "stage=0x%x pipeline=0x%x pipeline-trace=0x%x "
                    "objset=%llu object=%llu level=%llu blkid=%llu "
                    "offset=%llu size=%llu error=%d",
-                   pio, pio->io_timestamp,
+                   ziodepth, pio, pio->io_timestamp,
                    delta, pio->io_delta, pio->io_delay,
-                   vd->vdev_path, vq->vq_io_complete_ts,
+                   vd ? vd->vdev_path : "NULL", vq ? vq->vq_io_complete_ts : 0,
                    pio->io_type, pio->io_priority, pio->io_flags,
-                   pio->io_state, pio->io_pipeline, pio->io_pipeline_trace,
+                   pio->io_stage, pio->io_pipeline, pio->io_pipeline_trace,
                    zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid,
                    pio->io_offset, pio->io_size, pio->io_error);
                zfs_ereport_post(FM_EREPORT_ZFS_DEADMAN,
@@ -1896,7 +1897,7 @@ zio_deadman_impl(zio_t *pio)
        mutex_enter(&pio->io_lock);
        for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
                cio_next = zio_walk_children(pio, &zl);
-               zio_deadman_impl(cio);
+               zio_deadman_impl(cio, ziodepth + 1);
        }
        mutex_exit(&pio->io_lock);
 }
@@ -1914,7 +1915,7 @@ zio_deadman(zio_t *pio, char *tag)
        if (!zfs_deadman_enabled || spa_suspended(spa))
                return;
 
-       zio_deadman_impl(pio);
+       zio_deadman_impl(pio, 0);
 
        switch (spa_get_deadman_failmode(spa)) {
        case ZIO_FAILURE_MODE_WAIT:
@@ -4865,4 +4866,8 @@ MODULE_PARM_DESC(zfs_sync_pass_rewrite,
 module_param(zio_dva_throttle_enabled, int, 0644);
 MODULE_PARM_DESC(zio_dva_throttle_enabled,
        "Throttle block allocations in the ZIO pipeline");
+
+module_param(zio_deadman_log_all, int, 0644);
+MODULE_PARM_DESC(zio_deadman_log_all,
+       "Log all slow ZIOs, not just those with vdevs");
 #endif