modules/spl/spl-mutex.c

   1 /*
   2  *  This file is part of the SPL: Solaris Porting Layer.
   3  *
   4  *  Copyright (c) 2008 Lawrence Livermore National Security, LLC.
   5  *  Produced at Lawrence Livermore National Laboratory
   6  *  Written by:
   7  *          Brian Behlendorf <behlendorf1@llnl.gov>,
   8  *          Herb Wartens <wartens2@llnl.gov>,
   9  *          Jim Garlick <garlick@llnl.gov>
  10  *  UCRL-CODE-235197
  11  *
  12  *  This is free software; you can redistribute it and/or modify it
  13  *  under the terms of the GNU General Public License as published by
  14  *  the Free Software Foundation; either version 2 of the License, or
  15  *  (at your option) any later version.
  16  *
  17  *  This is distributed in the hope that it will be useful, but WITHOUT
  18  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  19  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  20  *  for more details.
  21  *
  22  *  You should have received a copy of the GNU General Public License along
  23  *  with this program; if not, write to the Free Software Foundation, Inc.,
  24  *  51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  25  */
  26
  27 #include <sys/mutex.h>
  28
  29 #ifdef DEBUG_SUBSYSTEM
  30 #undef DEBUG_SUBSYSTEM
  31 #endif
  32
  33 #define DEBUG_SUBSYSTEM S_MUTEX
  34
  35 /* Mutex implementation based on those found in Solaris.  This means
  36  * they the MUTEX_DEFAULT type is an adaptive mutex.  When calling
  37  * mutex_enter() your process will spin waiting for the lock if it's
  38  * likely the lock will be free'd shortly.  If it looks like the
  39  * lock will be held for a longer time we schedule and sleep waiting
  40  * for it.  This determination is made by checking if the holder of
  41  * the lock is currently running on cpu or sleeping waiting to be
  42  * scheduled.  If the holder is currently running it's likely the
  43  * lock will be shortly dropped.
  44  *
  45  * XXX: This is basically a rough implementation to see if this
  46  * helps our performance.  If it does a more careful implementation
  47  * should be done, perhaps in assembly.
  48  */
  49
  50 /*  0:         Never spin when trying to aquire lock
  51  * -1:         Spin until aquired or holder yeilds without dropping lock
  52  *  1-MAX_INT: Spin for N attempts before sleeping for lock
  53  */
  54 int mutex_spin_max = 0;
  55
  56 #ifdef DEBUG_MUTEX
  57 int mutex_stats[MUTEX_STATS_SIZE] = { 0 };
  58 spinlock_t mutex_stats_lock;
  59 struct list_head mutex_stats_list;
  60 #endif
  61
  62 void
  63 __spl_mutex_init(kmutex_t *mp, char *name, int type, void *ibc)
  64 {
  65         int flags = KM_SLEEP;
  66
  67         ASSERT(mp);
  68         ASSERT(name);
  69         ASSERT(ibc == NULL);
  70         ASSERT(mp->km_magic != KM_MAGIC); /* Never double init */
  71
  72         mp->km_magic = KM_MAGIC;
  73         mp->km_owner = NULL;
  74         mp->km_name = NULL;
  75         mp->km_name_size = strlen(name) + 1;
  76
  77         switch (type) {
  78                 case MUTEX_DEFAULT:
  79                         mp->km_type = MUTEX_ADAPTIVE;
  80                         break;
  81                 case MUTEX_SPIN:
  82                 case MUTEX_ADAPTIVE:
  83                         mp->km_type = type;
  84                         break;
  85                 default:
  86                         SBUG();
  87         }
  88
  89         /* We may be called when there is a non-zero preempt_count or
  90          * interrupts are disabled is which case we must not sleep.
  91          */
  92         if (current_thread_info()->preempt_count || irqs_disabled())
  93                 flags = KM_NOSLEEP;
  94
  95         /* Semaphore kmem_alloc'ed to keep struct size down (<64b) */
  96         mp->km_sem = kmem_alloc(sizeof(struct semaphore), flags);
  97         if (mp->km_sem == NULL)
  98                 return;
  99
 100         mp->km_name = kmem_alloc(mp->km_name_size, flags);
 101         if (mp->km_name == NULL) {
 102                 kmem_free(mp->km_sem, sizeof(struct semaphore));
 103                 return;
 104         }
 105
 106         sema_init(mp->km_sem, 1);
 107         strncpy(mp->km_name, name, mp->km_name_size);
 108
 109 #ifdef DEBUG_MUTEX
 110         mp->km_stats = kmem_zalloc(sizeof(int) * MUTEX_STATS_SIZE, flags);
 111         if (mp->km_stats == NULL) {
 112                 kmem_free(mp->km_name, mp->km_name_size);
 113                 kmem_free(mp->km_sem, sizeof(struct semaphore));
 114                 return;
 115         }
 116
 117         /* XXX - This appears to be a much more contended lock than I
 118          * would have expected.  To run with this debugging enabled and
 119          * get reasonable performance we may need to be more clever and
 120          * do something like hash the mutex ptr on to one of several
 121          * lists to ease this single point of contention.
 122          */
 123         spin_lock(&mutex_stats_lock);
 124         list_add_tail(&mp->km_list, &mutex_stats_list);
 125         spin_unlock(&mutex_stats_lock);
 126 #endif
 127 }
 128 EXPORT_SYMBOL(__spl_mutex_init);
 129
 130 void
 131 __spl_mutex_destroy(kmutex_t *mp)
 132 {
 133         ASSERT(mp);
 134         ASSERT(mp->km_magic == KM_MAGIC);
 135
 136 #ifdef DEBUG_MUTEX
 137         spin_lock(&mutex_stats_lock);
 138         list_del_init(&mp->km_list);
 139         spin_unlock(&mutex_stats_lock);
 140
 141         kmem_free(mp->km_stats, sizeof(int) * MUTEX_STATS_SIZE);
 142 #endif
 143         kmem_free(mp->km_name, mp->km_name_size);
 144         kmem_free(mp->km_sem, sizeof(struct semaphore));
 145
 146         memset(mp, KM_POISON, sizeof(*mp));
 147 }
 148 EXPORT_SYMBOL(__spl_mutex_destroy);
 149
 150 /* Return 1 if we acquired the mutex, else zero.  */
 151 int
 152 __mutex_tryenter(kmutex_t *mp)
 153 {
 154         int rc;
 155         ENTRY;
 156
 157         ASSERT(mp);
 158         ASSERT(mp->km_magic == KM_MAGIC);
 159         MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_TOTAL);
 160         MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_TOTAL);
 161
 162         rc = down_trylock(mp->km_sem);
 163         if (rc == 0) {
 164                 ASSERT(mp->km_owner == NULL);
 165                 mp->km_owner = current;
 166                 MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_NOT_HELD);
 167                 MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_NOT_HELD);
 168         }
 169
 170         RETURN(!rc);
 171 }
 172 EXPORT_SYMBOL(__mutex_tryenter);
 173
 174 static void
 175 mutex_enter_adaptive(kmutex_t *mp)
 176 {
 177         struct task_struct *owner;
 178         int count = 0;
 179
 180         /* Lock is not held so we expect to aquire the lock */
 181         if ((owner = mp->km_owner) == NULL) {
 182                 down(mp->km_sem);
 183                 MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_NOT_HELD);
 184                 MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_NOT_HELD);
 185         } else {
 186                 /* The lock is held by a currently running task which
 187                  * we expect will drop the lock before leaving the
 188                  * head of the runqueue.  So the ideal thing to do
 189                  * is spin until we aquire the lock and avoid a
 190                  * context switch.  However it is also possible the
 191                  * task holding the lock yields the processor with
 192                  * out dropping lock.  In which case, we know it's
 193                  * going to be a while so we stop spinning and go
 194                  * to sleep waiting for the lock to be available.
 195                  * This should strike the optimum balance between
 196                  * spinning and sleeping waiting for a lock.
 197                  */
 198                 while (task_curr(owner) && (count <= mutex_spin_max)) {
 199                         if (down_trylock(mp->km_sem) == 0) {
 200                                 MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
 201                                 MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
 202                                 GOTO(out, count);
 203                         }
 204                         count++;
 205                 }
 206
 207                 /* The lock is held by a sleeping task so it's going to
 208                  * cost us minimally one context switch.  We might as
 209                  * well sleep and yield the processor to other tasks.
 210                  */
 211                 down(mp->km_sem);
 212                 MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SLEEP);
 213                 MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SLEEP);
 214         }
 215 out:
 216         MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_TOTAL);
 217         MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_TOTAL);
 218 }
 219
 220 void
 221 __mutex_enter(kmutex_t *mp)
 222 {
 223         ENTRY;
 224         ASSERT(mp);
 225         ASSERT(mp->km_magic == KM_MAGIC);
 226
 227         switch (mp->km_type) {
 228                 case MUTEX_SPIN:
 229                         while (down_trylock(mp->km_sem));
 230                         MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
 231                         MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
 232                         break;
 233                 case MUTEX_ADAPTIVE:
 234                         mutex_enter_adaptive(mp);
 235                         break;
 236         }
 237
 238         ASSERT(mp->km_owner == NULL);
 239         mp->km_owner = current;
 240
 241         EXIT;
 242 }
 243 EXPORT_SYMBOL(__mutex_enter);
 244
 245 void
 246 __mutex_exit(kmutex_t *mp)
 247 {
 248         ENTRY;
 249         ASSERT(mp);
 250         ASSERT(mp->km_magic == KM_MAGIC);
 251         ASSERT(mp->km_owner == current);
 252         mp->km_owner = NULL;
 253         up(mp->km_sem);
 254         EXIT;
 255 }
 256 EXPORT_SYMBOL(__mutex_exit);
 257
 258 /* Return 1 if mutex is held by current process, else zero.  */
 259 int
 260 __mutex_owned(kmutex_t *mp)
 261 {
 262         ENTRY;
 263         ASSERT(mp);
 264         ASSERT(mp->km_magic == KM_MAGIC);
 265         RETURN(mp->km_owner == current);
 266 }
 267 EXPORT_SYMBOL(__mutex_owned);
 268
 269 /* Return owner if mutex is owned, else NULL.  */
 270 kthread_t *
 271 __spl_mutex_owner(kmutex_t *mp)
 272 {
 273         ENTRY;
 274         ASSERT(mp);
 275         ASSERT(mp->km_magic == KM_MAGIC);
 276         RETURN(mp->km_owner);
 277 }
 278 EXPORT_SYMBOL(__spl_mutex_owner);
 279
 280 int
 281 spl_mutex_init(void)
 282 {
 283         ENTRY;
 284 #ifdef DEBUG_MUTEX
 285         spin_lock_init(&mutex_stats_lock);
 286         INIT_LIST_HEAD(&mutex_stats_list);
 287 #endif
 288         RETURN(0);
 289 }
 290
 291 void
 292 spl_mutex_fini(void)
 293 {
 294         ENTRY;
 295 #ifdef DEBUG_MUTEX
 296         ASSERT(list_empty(&mutex_stats_list));
 297 #endif
 298         EXIT;
 299 }
 300
 301 module_param(mutex_spin_max, int, 0644);
 302 MODULE_PARM_DESC(mutex_spin_max, "Spin a maximum of N times to aquire lock");