Some ZIO micro-optimizations.

- Pack struct zio_prop by 4 bytes from 84 to 80.
 - Skip new child ZIO locking while linking to parent.  The newly
allocated ZIO is not externally visible yet, so nobody should care.
 - Skip io_bp_copy writes when not used (write && non-debug).

Reviewed-by: Brian Atkinson <batkinson@lanl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by:	Alexander Motin <mav@FreeBSD.org>
Sponsored by:	iXsystems, Inc.
Closes #14985

diff --git a/include/sys/zio.h b/include/sys/zio.h
index 85217b873dc818c66bee7a9e1cf1be59424aaaf5..f4da80783e569bb27b44e513a667bf6848f60b34 100644 (file)
--- a/include/sys/zio.h
+++ b/include/sys/zio.h
@@ -341,9 +341,9 @@ typedef struct zio_prop {
        enum zio_checksum       zp_checksum;
        enum zio_compress       zp_compress;
        uint8_t                 zp_complevel;
-       dmu_object_type_t       zp_type;
        uint8_t                 zp_level;
        uint8_t                 zp_copies;
+       dmu_object_type_t       zp_type;
        boolean_t               zp_dedup;
        boolean_t               zp_dedup_verify;
        boolean_t               zp_nopwrite;
@@ -611,6 +611,7 @@ extern zio_t *zio_walk_parents(zio_t *cio, zio_link_t **);
 extern zio_t *zio_walk_children(zio_t *pio, zio_link_t **);
 extern zio_t *zio_unique_parent(zio_t *cio);
 extern void zio_add_child(zio_t *pio, zio_t *cio);
+extern void zio_add_child_first(zio_t *pio, zio_t *cio);
 
 extern void *zio_buf_alloc(size_t size);
 extern void zio_buf_free(void *buf, size_t size);

diff --git a/module/zfs/zio.c b/module/zfs/zio.c
index fb8164f0aea946db6709d50359124190319b95ed..10279fde89dfbbe537a887cdc7f0299ba439f2dc 100644 (file)
--- a/module/zfs/zio.c
+++ b/module/zfs/zio.c
@@ -626,8 +626,6 @@ zio_unique_parent(zio_t *cio)
 void
 zio_add_child(zio_t *pio, zio_t *cio)
 {
-       zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP);
-
        /*
         * Logical I/Os can have logical, gang, or vdev children.
         * Gang I/Os can have gang or vdev children.
@@ -636,6 +634,7 @@ zio_add_child(zio_t *pio, zio_t *cio)
         */
        ASSERT3S(cio->io_child_type, <=, pio->io_child_type);
 
+       zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP);
        zl->zl_parent = pio;
        zl->zl_child = cio;
 
@@ -644,8 +643,9 @@ zio_add_child(zio_t *pio, zio_t *cio)
 
        ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);
 
+       uint64_t *countp = pio->io_children[cio->io_child_type];
        for (int w = 0; w < ZIO_WAIT_TYPES; w++)
-               pio->io_children[cio->io_child_type][w] += !cio->io_state[w];
+               countp[w] += !cio->io_state[w];
 
        list_insert_head(&pio->io_child_list, zl);
        list_insert_head(&cio->io_parent_list, zl);
@@ -654,6 +654,37 @@ zio_add_child(zio_t *pio, zio_t *cio)
        mutex_exit(&pio->io_lock);
 }
 
+void
+zio_add_child_first(zio_t *pio, zio_t *cio)
+{
+       /*
+        * Logical I/Os can have logical, gang, or vdev children.
+        * Gang I/Os can have gang or vdev children.
+        * Vdev I/Os can only have vdev children.
+        * The following ASSERT captures all of these constraints.
+        */
+       ASSERT3S(cio->io_child_type, <=, pio->io_child_type);
+
+       zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP);
+       zl->zl_parent = pio;
+       zl->zl_child = cio;
+
+       ASSERT(list_is_empty(&cio->io_parent_list));
+       list_insert_head(&cio->io_parent_list, zl);
+
+       mutex_enter(&pio->io_lock);
+
+       ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);
+
+       uint64_t *countp = pio->io_children[cio->io_child_type];
+       for (int w = 0; w < ZIO_WAIT_TYPES; w++)
+               countp[w] += !cio->io_state[w];
+
+       list_insert_head(&pio->io_child_list, zl);
+
+       mutex_exit(&pio->io_lock);
+}
+
 static void
 zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl)
 {
@@ -840,12 +871,14 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
                zio->io_child_type = ZIO_CHILD_LOGICAL;
 
        if (bp != NULL) {
-               zio->io_bp = (blkptr_t *)bp;
-               zio->io_bp_copy = *bp;
-               zio->io_bp_orig = *bp;
                if (type != ZIO_TYPE_WRITE ||
-                   zio->io_child_type == ZIO_CHILD_DDT)
+                   zio->io_child_type == ZIO_CHILD_DDT) {
+                       zio->io_bp_copy = *bp;
                        zio->io_bp = &zio->io_bp_copy;  /* so caller can free */
+               } else {
+                       zio->io_bp = (blkptr_t *)bp;
+               }
+               zio->io_bp_orig = *bp;
                if (zio->io_child_type == ZIO_CHILD_LOGICAL)
                        zio->io_logical = zio;
                if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp))
@@ -880,7 +913,7 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
                        zio->io_logical = pio->io_logical;
                if (zio->io_child_type == ZIO_CHILD_GANG)
                        zio->io_gang_leader = pio->io_gang_leader;
-               zio_add_child(pio, zio);
+               zio_add_child_first(pio, zio);
        }
 
        taskq_init_ent(&zio->io_tqent);
@@ -1601,7 +1634,6 @@ zio_read_bp_init(zio_t *zio)
                abd_return_buf_copy(zio->io_abd, data, psize);
        } else {
                ASSERT(!BP_IS_EMBEDDED(bp));
-               ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);
        }
 
        if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)
@@ -4442,8 +4474,10 @@ zio_ready(zio_t *zio)
                zio->io_ready(zio);
        }
 
+#ifdef ZFS_DEBUG
        if (bp != NULL && bp != &zio->io_bp_copy)
                zio->io_bp_copy = *bp;
+#endif
 
        if (zio->io_error != 0) {
                zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;