kernel/locking/qrwlock.c

   1 /*
   2  * Queue read/write lock
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
  15  *
  16  * Authors: Waiman Long <waiman.long@hp.com>
  17  */
  18 #include <linux/smp.h>
  19 #include <linux/bug.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/percpu.h>
  22 #include <linux/hardirq.h>
  23 #include <asm/qrwlock.h>
  24
  25 /**
  26  * rspin_until_writer_unlock - inc reader count & spin until writer is gone
  27  * @lock  : Pointer to queue rwlock structure
  28  * @writer: Current queue rwlock writer status byte
  29  *
  30  * In interrupt context or at the head of the queue, the reader will just
  31  * increment the reader count & wait until the writer releases the lock.
  32  */
  33 static __always_inline void
  34 rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts)
  35 {
  36         while ((cnts & _QW_WMASK) == _QW_LOCKED) {
  37                 cpu_relax_lowlatency();
  38                 cnts = smp_load_acquire((u32 *)&lock->cnts);
  39         }
  40 }
  41
  42 /**
  43  * queue_read_lock_slowpath - acquire read lock of a queue rwlock
  44  * @lock: Pointer to queue rwlock structure
  45  */
  46 void queue_read_lock_slowpath(struct qrwlock *lock)
  47 {
  48         u32 cnts;
  49
  50         /*
  51          * Readers come here when they cannot get the lock without waiting
  52          */
  53         if (unlikely(in_interrupt())) {
  54                 /*
  55                  * Readers in interrupt context will spin until the lock is
  56                  * available without waiting in the queue.
  57                  */
  58                 cnts = smp_load_acquire((u32 *)&lock->cnts);
  59                 rspin_until_writer_unlock(lock, cnts);
  60                 return;
  61         }
  62         atomic_sub(_QR_BIAS, &lock->cnts);
  63
  64         /*
  65          * Put the reader into the wait queue
  66          */
  67         arch_spin_lock(&lock->lock);
  68
  69         /*
  70          * At the head of the wait queue now, wait until the writer state
  71          * goes to 0 and then try to increment the reader count and get
  72          * the lock. It is possible that an incoming writer may steal the
  73          * lock in the interim, so it is necessary to check the writer byte
  74          * to make sure that the write lock isn't taken.
  75          */
  76         while (atomic_read(&lock->cnts) & _QW_WMASK)
  77                 cpu_relax_lowlatency();
  78
  79         cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS;
  80         rspin_until_writer_unlock(lock, cnts);
  81
  82         /*
  83          * Signal the next one in queue to become queue head
  84          */
  85         arch_spin_unlock(&lock->lock);
  86 }
  87 EXPORT_SYMBOL(queue_read_lock_slowpath);
  88
  89 /**
  90  * queue_write_lock_slowpath - acquire write lock of a queue rwlock
  91  * @lock : Pointer to queue rwlock structure
  92  */
  93 void queue_write_lock_slowpath(struct qrwlock *lock)
  94 {
  95         u32 cnts;
  96
  97         /* Put the writer into the wait queue */
  98         arch_spin_lock(&lock->lock);
  99
 100         /* Try to acquire the lock directly if no reader is present */
 101         if (!atomic_read(&lock->cnts) &&
 102             (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0))
 103                 goto unlock;
 104
 105         /*
 106          * Set the waiting flag to notify readers that a writer is pending,
 107          * or wait for a previous writer to go away.
 108          */
 109         for (;;) {
 110                 cnts = atomic_read(&lock->cnts);
 111                 if (!(cnts & _QW_WMASK) &&
 112                     (atomic_cmpxchg(&lock->cnts, cnts,
 113                                     cnts | _QW_WAITING) == cnts))
 114                         break;
 115
 116                 cpu_relax_lowlatency();
 117         }
 118
 119         /* When no more readers, set the locked flag */
 120         for (;;) {
 121                 cnts = atomic_read(&lock->cnts);
 122                 if ((cnts == _QW_WAITING) &&
 123                     (atomic_cmpxchg(&lock->cnts, _QW_WAITING,
 124                                     _QW_LOCKED) == _QW_WAITING))
 125                         break;
 126
 127                 cpu_relax_lowlatency();
 128         }
 129 unlock:
 130         arch_spin_unlock(&lock->lock);
 131 }
 132 EXPORT_SYMBOL(queue_write_lock_slowpath);