rwsem: shorter spinlocked section in rwsem_down_failed_common()

[mirror_ubuntu-jammy-kernel.git] / lib / rwsem.c

diff --git a/lib/rwsem.c b/lib/rwsem.c
index ad5e0df16ab4a8b375d4db948a36f168cd40c05c..40636454cf3cf7215969db7985affbae5edd2f38 100644 (file)
--- a/lib/rwsem.c
+++ b/lib/rwsem.c
@@ -30,12 +30,15 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
 
 EXPORT_SYMBOL(__init_rwsem);
 
+enum rwsem_waiter_type {
+       RWSEM_WAITING_FOR_WRITE,
+       RWSEM_WAITING_FOR_READ
+};
+
 struct rwsem_waiter {
        struct list_head list;
        struct task_struct *task;
-       unsigned int flags;
-#define RWSEM_WAITING_FOR_READ 0x00000001
-#define RWSEM_WAITING_FOR_WRITE        0x00000002
+       enum rwsem_waiter_type type;
 };
 
 /* Wake types for __rwsem_do_wake().  Note that RWSEM_WAKE_NO_ACTIVE and
@@ -65,7 +68,7 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wake_type)
        signed long woken, loop, adjustment;
 
        waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
-       if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE))
+       if (waiter->type != RWSEM_WAITING_FOR_WRITE)
                goto readers_only;
 
        if (wake_type == RWSEM_WAKE_READ_OWNED)
@@ -112,10 +115,10 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wake_type)
                waiter = list_entry(waiter->list.next,
                                        struct rwsem_waiter, list);
 
-       } while (waiter->flags & RWSEM_WAITING_FOR_READ);
+       } while (waiter->type != RWSEM_WAITING_FOR_WRITE);
 
        adjustment = woken * RWSEM_ACTIVE_READ_BIAS;
-       if (waiter->flags & RWSEM_WAITING_FOR_READ)
+       if (waiter->type != RWSEM_WAITING_FOR_WRITE)
                /* hit end of list above */
                adjustment -= RWSEM_WAITING_BIAS;
 
@@ -148,7 +151,7 @@ static int try_get_writer_sem(struct rw_semaphore *sem,
 
        /* only steal when first waiter is writing */
        fwaiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
-       if (!(fwaiter->flags & RWSEM_WAITING_FOR_WRITE))
+       if (fwaiter->type != RWSEM_WAITING_FOR_WRITE)
                return 0;
 
        adjustment = RWSEM_ACTIVE_WRITE_BIAS;
@@ -179,20 +182,18 @@ try_again_write:
  */
 static struct rw_semaphore __sched *
 rwsem_down_failed_common(struct rw_semaphore *sem,
-                        unsigned int flags, signed long adjustment)
+                        enum rwsem_waiter_type type, signed long adjustment)
 {
        struct rwsem_waiter waiter;
        struct task_struct *tsk = current;
        signed long count;
 
-       set_task_state(tsk, TASK_UNINTERRUPTIBLE);
-
        /* set up my own style of waitqueue */
-       raw_spin_lock_irq(&sem->wait_lock);
        waiter.task = tsk;
-       waiter.flags = flags;
+       waiter.type = type;
        get_task_struct(tsk);
 
+       raw_spin_lock_irq(&sem->wait_lock);
        if (list_empty(&sem->wait_list))
                adjustment += RWSEM_WAITING_BIAS;
        list_add_tail(&waiter.list, &sem->wait_list);
@@ -215,20 +216,20 @@ rwsem_down_failed_common(struct rw_semaphore *sem,
        raw_spin_unlock_irq(&sem->wait_lock);
 
        /* wait to be given the lock */
-       for (;;) {
+       while (true) {
+               set_task_state(tsk, TASK_UNINTERRUPTIBLE);
                if (!waiter.task)
                        break;
 
                raw_spin_lock_irq(&sem->wait_lock);
                /* Try to get the writer sem, may steal from the head writer: */
-               if (flags == RWSEM_WAITING_FOR_WRITE)
+               if (type == RWSEM_WAITING_FOR_WRITE)
                        if (try_get_writer_sem(sem, &waiter)) {
                                raw_spin_unlock_irq(&sem->wait_lock);
                                return sem;
                        }
                raw_spin_unlock_irq(&sem->wait_lock);
                schedule();
-               set_task_state(tsk, TASK_UNINTERRUPTIBLE);
        }
 
        tsk->state = TASK_RUNNING;