Fix stack zio_execute()

Implement zio_execute() as a wrapper around the static function
__zio_execute() so that we can force  __zio_execute() to be inlined.
This reduces stack overhead which is important because __zio_execute()
is called recursively in several zio code paths.  zio_execute() itself
cannot be inlined because it is externally visible.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>

diff --git a/module/zfs/zio.c b/module/zfs/zio.c
index fb60bd8a06a9f07a53be8182eb78857a9d7080c9..c7eb66103952a39c9579f05e6a546e7a2c0efef9 100644 (file)
--- a/module/zfs/zio.c
+++ b/module/zfs/zio.c
@@ -93,6 +93,8 @@ int zio_buf_debug_limit = 16384;
 int zio_buf_debug_limit = 0;
 #endif
 
+static inline void __zio_execute(zio_t *zio);
+
 void
 zio_init(void)
 {
@@ -451,7 +453,7 @@ zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait)
        if (--*countp == 0 && pio->io_stall == countp) {
                pio->io_stall = NULL;
                mutex_exit(&pio->io_lock);
-               zio_execute(pio);
+               __zio_execute(pio);
        } else {
                mutex_exit(&pio->io_lock);
        }
@@ -1122,8 +1124,23 @@ zio_interrupt(zio_t *zio)
  */
 static zio_pipe_stage_t *zio_pipeline[];
 
+/*
+ * zio_execute() is a wrapper around the static function
+ * __zio_execute() so that we can force  __zio_execute() to be
+ * inlined.  This reduces stack overhead which is important
+ * because __zio_execute() is called recursively in several zio
+ * code paths.  zio_execute() itself cannot be inlined because
+ * it is externally visible.
+ */
 void
 zio_execute(zio_t *zio)
+{
+       __zio_execute(zio);
+}
+
+__attribute__((always_inline))
+static inline void
+__zio_execute(zio_t *zio)
 {
        zio->io_executor = curthread;
 
@@ -1169,6 +1186,7 @@ zio_execute(zio_t *zio)
        }
 }
 
+
 /*
  * ==========================================================================
  * Initiate I/O, either sync or async
@@ -1184,7 +1202,7 @@ zio_wait(zio_t *zio)
 
        zio->io_waiter = curthread;
 
-       zio_execute(zio);
+       __zio_execute(zio);
 
        mutex_enter(&zio->io_lock);
        while (zio->io_executor != NULL)
@@ -1214,7 +1232,7 @@ zio_nowait(zio_t *zio)
                zio_add_child(spa->spa_async_zio_root, zio);
        }
 
-       zio_execute(zio);
+       __zio_execute(zio);
 }
 
 /*
@@ -1269,7 +1287,7 @@ zio_reexecute(zio_t *pio)
         * responsibility of the caller to wait on him.
         */
        if (!(pio->io_flags & ZIO_FLAG_GODFATHER))
-               zio_execute(pio);
+               __zio_execute(pio);
 }
 
 void