locking/mutex: clear MUTEX_FLAGS if wait_list is empty due to signal

BugLink: https://bugs.launchpad.net/bugs/1931896
[ Upstream commit 3a010c493271f04578b133de977e0e5dd2848cea ]

When a interruptible mutex locker is interrupted by a signal
without acquiring this lock and removed from the wait queue.
if the mutex isn't contended enough to have a waiter
put into the wait queue again, the setting of the WAITER
bit will force mutex locker to go into the slowpath to
acquire the lock every time, so if the wait queue is empty,
the WAITER bit need to be clear.

Fixes: 040a0a371005 ("mutex: Add support for wound/wait style locks")
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Zqiang <qiang.zhang@windriver.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20210517034005.30828-1-qiang.zhang@windriver.com
Signed-off-by: Sasha Levin <sashal@kernel.org>
Signed-off-by: Kamal Mostafa <kamal@canonical.com>
Signed-off-by: Kelsey Skunberg <kelsey.skunberg@canonical.com>

diff --git a/kernel/locking/mutex-debug.c b/kernel/locking/mutex-debug.c
index a7276aaf2abc0b75ee5cf15738b5a2d7e0e46f5c..db9301591e3fcca7bfba67c11942a07ae2cf3de6 100644 (file)
--- a/kernel/locking/mutex-debug.c
+++ b/kernel/locking/mutex-debug.c
@@ -57,7 +57,7 @@ void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
        task->blocked_on = waiter;
 }
 
-void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
+void debug_mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
                         struct task_struct *task)
 {
        DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
@@ -65,7 +65,7 @@ void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
        DEBUG_LOCKS_WARN_ON(task->blocked_on != waiter);
        task->blocked_on = NULL;
 
-       list_del_init(&waiter->list);
+       INIT_LIST_HEAD(&waiter->list);
        waiter->task = NULL;
 }
 

diff --git a/kernel/locking/mutex-debug.h b/kernel/locking/mutex-debug.h
index 1edd3f45a4ecbad7979f26944a9173b3de028959..53e631e1d76da7858839a9aa92d47bff9603c9c2 100644 (file)
--- a/kernel/locking/mutex-debug.h
+++ b/kernel/locking/mutex-debug.h
@@ -22,7 +22,7 @@ extern void debug_mutex_free_waiter(struct mutex_waiter *waiter);
 extern void debug_mutex_add_waiter(struct mutex *lock,
                                   struct mutex_waiter *waiter,
                                   struct task_struct *task);
-extern void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
+extern void debug_mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
                                struct task_struct *task);
 extern void debug_mutex_unlock(struct mutex *lock);
 extern void debug_mutex_init(struct mutex *lock, const char *name,

diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 2c25b830203cd7be860bdef5ceb856ca065aaceb..15ac7c4bb111726f0f7ef31d4e4ffd5e0f7e4e29 100644 (file)
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -204,7 +204,7 @@ static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_wait
  * Add @waiter to a given location in the lock wait_list and set the
  * FLAG_WAITERS flag if it's the first waiter.
  */
-static void __sched
+static void
 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
                   struct list_head *list)
 {
@@ -215,6 +215,16 @@ __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
                __mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
 }
 
+static void
+__mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
+{
+       list_del(&waiter->list);
+       if (likely(list_empty(&lock->wait_list)))
+               __mutex_clear_flag(lock, MUTEX_FLAGS);
+
+       debug_mutex_remove_waiter(lock, waiter, current);
+}
+
 /*
  * Give up ownership to a specific task, when @task = NULL, this is equivalent
  * to a regular unlock. Sets PICKUP on a handoff, clears HANDOF, preserves
@@ -1071,9 +1081,7 @@ acquired:
                        __ww_mutex_check_waiters(lock, ww_ctx);
        }
 
-       mutex_remove_waiter(lock, &waiter, current);
-       if (likely(list_empty(&lock->wait_list)))
-               __mutex_clear_flag(lock, MUTEX_FLAGS);
+       __mutex_remove_waiter(lock, &waiter);
 
        debug_mutex_free_waiter(&waiter);
 
@@ -1090,7 +1098,7 @@ skip_wait:
 
 err:
        __set_current_state(TASK_RUNNING);
-       mutex_remove_waiter(lock, &waiter, current);
+       __mutex_remove_waiter(lock, &waiter);
 err_early_kill:
        spin_unlock(&lock->wait_lock);
        debug_mutex_free_waiter(&waiter);

diff --git a/kernel/locking/mutex.h b/kernel/locking/mutex.h
index 1c2287d3fa7191589c6d8f0d5f4e67c3282f8dfe..f0c710b1d1927131c2526dcfc7b96010ecaa45af 100644 (file)
--- a/kernel/locking/mutex.h
+++ b/kernel/locking/mutex.h
@@ -10,12 +10,10 @@
  * !CONFIG_DEBUG_MUTEXES case. Most of them are NOPs:
  */
 
-#define mutex_remove_waiter(lock, waiter, task) \
-               __list_del((waiter)->list.prev, (waiter)->list.next)
-
 #define debug_mutex_wake_waiter(lock, waiter)          do { } while (0)
 #define debug_mutex_free_waiter(waiter)                        do { } while (0)
 #define debug_mutex_add_waiter(lock, waiter, ti)       do { } while (0)
+#define debug_mutex_remove_waiter(lock, waiter, ti)     do { } while (0)
 #define debug_mutex_unlock(lock)                       do { } while (0)
 #define debug_mutex_init(lock, name, key)              do { } while (0)