Use tsd to store tq for taskq_member

To prevent taskq_member holding tq_lock and doing linear search, thus causing
contention. We store the taskq pointer to which the thread belongs in tsd.
This way taskq_member will not need to touch tq_lock, and tsd has per slot
spinlock. So the contention should be reduced greatly.

Signed-off-by: Chunwei Chen <david.chen@osnexus.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #500
Closes #504
Closes #505

diff --git a/include/sys/taskq.h b/include/sys/taskq.h
index 544dbb2bbc3ea9df287c132888aebcaecfd6c966..e7661f7ce8b8f51971052093db0adab334730307 100644 (file)
--- a/include/sys/taskq.h
+++ b/include/sys/taskq.h
@@ -140,7 +140,7 @@ extern void taskq_wait_id(taskq_t *, taskqid_t);
 extern void taskq_wait_outstanding(taskq_t *, taskqid_t);
 extern void taskq_wait(taskq_t *);
 extern int taskq_cancel_id(taskq_t *, taskqid_t);
-extern int taskq_member(taskq_t *, void *);
+extern int taskq_member(taskq_t *, kthread_t *);
 
 #define        taskq_create_proc(name, nthreads, pri, min, max, proc, flags) \
     taskq_create(name, nthreads, pri, min, max, flags)

diff --git a/include/sys/tsd.h b/include/sys/tsd.h
index ebc55b09b92e543c9378aaf606a5cd66048b8644..1894a823231697760a4e9c6d90ce7c52a9395292 100644 (file)
--- a/include/sys/tsd.h
+++ b/include/sys/tsd.h
@@ -35,6 +35,7 @@ typedef void (*dtor_func_t)(void *);
 
 extern int tsd_set(uint_t, void *);
 extern void *tsd_get(uint_t);
+extern void *tsd_get_by_thread(uint_t, kthread_t *);
 extern void tsd_create(uint_t *, dtor_func_t);
 extern void tsd_destroy(uint_t *);
 extern void tsd_exit(void);

diff --git a/module/spl/spl-generic.c b/module/spl/spl-generic.c
index 4d9846cf5702e85eaaa261d93b790d9f04209756..dc3e74aa5042a42fbcc4b4093727308a42503454 100644 (file)
--- a/module/spl/spl-generic.c
+++ b/module/spl/spl-generic.c
@@ -491,29 +491,20 @@ spl_kvmem_init(void)
 
        rc = spl_kmem_init();
        if (rc)
-               goto out1;
+               return (rc);
 
        rc = spl_vmem_init();
-       if (rc)
-               goto out2;
-
-       rc = spl_kmem_cache_init();
-       if (rc)
-               goto out3;
+       if (rc) {
+               spl_kmem_fini();
+               return (rc);
+       }
 
-       return (rc);
-out3:
-       spl_vmem_fini();
-out2:
-       spl_kmem_fini();
-out1:
        return (rc);
 }
 
 static void
 spl_kvmem_fini(void)
 {
-       spl_kmem_cache_fini();
        spl_vmem_fini();
        spl_kmem_fini();
 }
@@ -532,38 +523,43 @@ spl_init(void)
        if ((rc = spl_rw_init()))
                goto out3;
 
-       if ((rc = spl_taskq_init()))
+       if ((rc = spl_tsd_init()))
                goto out4;
 
-       if ((rc = spl_vn_init()))
+       if ((rc = spl_taskq_init()))
                goto out5;
 
-       if ((rc = spl_proc_init()))
+       if ((rc = spl_kmem_cache_init()))
                goto out6;
 
-       if ((rc = spl_kstat_init()))
+       if ((rc = spl_vn_init()))
                goto out7;
 
-       if ((rc = spl_tsd_init()))
+       if ((rc = spl_proc_init()))
                goto out8;
 
-       if ((rc = spl_zlib_init()))
+       if ((rc = spl_kstat_init()))
                goto out9;
 
+       if ((rc = spl_zlib_init()))
+               goto out10;
+
        printk(KERN_NOTICE "SPL: Loaded module v%s-%s%s\n", SPL_META_VERSION,
               SPL_META_RELEASE, SPL_DEBUG_STR);
        return (rc);
 
+out10:
+       spl_kstat_fini();
 out9:
-       spl_tsd_fini();
+       spl_proc_fini();
 out8:
-       spl_kstat_fini();
+       spl_vn_fini();
 out7:
-       spl_proc_fini();
+       spl_kmem_cache_fini();
 out6:
-       spl_vn_fini();
-out5:
        spl_taskq_fini();
+out5:
+       spl_tsd_fini();
 out4:
        spl_rw_fini();
 out3:
@@ -584,11 +580,12 @@ spl_fini(void)
        printk(KERN_NOTICE "SPL: Unloaded module v%s-%s%s\n",
               SPL_META_VERSION, SPL_META_RELEASE, SPL_DEBUG_STR);
        spl_zlib_fini();
-       spl_tsd_fini();
        spl_kstat_fini();
        spl_proc_fini();
        spl_vn_fini();
+       spl_kmem_cache_fini();
        spl_taskq_fini();
+       spl_tsd_fini();
        spl_rw_fini();
        spl_mutex_fini();
        spl_kvmem_fini();

diff --git a/module/spl/spl-taskq.c b/module/spl/spl-taskq.c
index 5f2b725c98a04d5f6c66b7d2fda479d6fdcbd9a1..2b3f3f4bc939f588cffb16f078b578527e29d5da 100644 (file)
--- a/module/spl/spl-taskq.c
+++ b/module/spl/spl-taskq.c
@@ -26,6 +26,7 @@
 
 #include <sys/taskq.h>
 #include <sys/kmem.h>
+#include <sys/tsd.h>
 
 int spl_taskq_thread_bind = 0;
 module_param(spl_taskq_thread_bind, int, 0644);
@@ -57,6 +58,7 @@ static taskq_thread_t *taskq_thread_create(taskq_t *);
 /* List of all taskqs */
 LIST_HEAD(tq_list);
 DECLARE_RWSEM(tq_list_sem);
+static uint_t taskq_tsd;
 
 static int
 task_km_flags(uint_t flags)
@@ -474,38 +476,10 @@ taskq_wait(taskq_t *tq)
 }
 EXPORT_SYMBOL(taskq_wait);
 
-static int
-taskq_member_impl(taskq_t *tq, void *t)
-{
-       struct list_head *l;
-       taskq_thread_t *tqt;
-       int found = 0;
-
-       ASSERT(tq);
-       ASSERT(t);
-       ASSERT(spin_is_locked(&tq->tq_lock));
-
-       list_for_each(l, &tq->tq_thread_list) {
-               tqt = list_entry(l, taskq_thread_t, tqt_thread_list);
-               if (tqt->tqt_thread == (struct task_struct *)t) {
-                       found = 1;
-                       break;
-               }
-       }
-       return (found);
-}
-
 int
-taskq_member(taskq_t *tq, void *t)
+taskq_member(taskq_t *tq, kthread_t *t)
 {
-       int found;
-       unsigned long flags;
-
-       spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
-       found = taskq_member_impl(tq, t);
-       spin_unlock_irqrestore(&tq->tq_lock, flags);
-
-       return (found);
+       return (tq == (taskq_t *)tsd_get_by_thread(taskq_tsd, t));
 }
 EXPORT_SYMBOL(taskq_member);
 
@@ -855,6 +829,7 @@ taskq_thread(void *args)
        sigprocmask(SIG_BLOCK, &blocked, NULL);
        flush_signals(current);
 
+       tsd_set(taskq_tsd, tq);
        spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 
        /* Immediately exit if more threads than allowed were created. */
@@ -959,6 +934,8 @@ error:
        kmem_free(tqt, sizeof (taskq_thread_t));
        spin_unlock_irqrestore(&tq->tq_lock, flags);
 
+       tsd_set(taskq_tsd, NULL);
+
        return (0);
 }
 
@@ -1160,6 +1137,8 @@ EXPORT_SYMBOL(taskq_destroy);
 int
 spl_taskq_init(void)
 {
+       tsd_create(&taskq_tsd, NULL);
+
        system_taskq = taskq_create("spl_system_taskq", MAX(boot_ncpus, 64),
            maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC);
        if (system_taskq == NULL)
@@ -1190,4 +1169,6 @@ spl_taskq_fini(void)
 
        taskq_destroy(system_taskq);
        system_taskq = NULL;
+
+       tsd_destroy(&taskq_tsd);
 }

diff --git a/module/spl/spl-tsd.c b/module/spl/spl-tsd.c
index 4d0800e5a11f5d4b570c79d46dd33ccb3f3de1cf..bf8235063be3d4dfd54871cd84658d1566cabbb2 100644 (file)
--- a/module/spl/spl-tsd.c
+++ b/module/spl/spl-tsd.c
@@ -527,6 +527,33 @@ tsd_get(uint_t key)
 }
 EXPORT_SYMBOL(tsd_get);
 
+/*
+ * tsd_get_by_thread - get thread specific data for specified thread
+ * @key: lookup key
+ * @thread: thread to lookup
+ *
+ * Caller must prevent racing tsd_create() or tsd_destroy().  This
+ * implementation is designed to be fast and scalable, it does not
+ * lock the entire table only a single hash bin.
+ */
+void *
+tsd_get_by_thread(uint_t key, kthread_t *thread)
+{
+       tsd_hash_entry_t *entry;
+
+       ASSERT3P(tsd_hash_table, !=, NULL);
+
+       if ((key == 0) || (key > TSD_KEYS_MAX))
+               return (NULL);
+
+       entry = tsd_hash_search(tsd_hash_table, key, thread->pid);
+       if (entry == NULL)
+               return (NULL);
+
+       return (entry->he_value);
+}
+EXPORT_SYMBOL(tsd_get_by_thread);
+
 /*
  * tsd_create - create thread specific data key
  * @keyp: lookup key address