]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
19c5d690 | 2 | /* |
925b9cd1 WL |
3 | * The least significant 2 bits of the owner value has the following |
4 | * meanings when set. | |
5 | * - RWSEM_READER_OWNED (bit 0): The rwsem is owned by readers | |
6 | * - RWSEM_ANONYMOUSLY_OWNED (bit 1): The rwsem is anonymously owned, | |
7 | * i.e. the owner(s) cannot be readily determined. It can be reader | |
8 | * owned or the owning writer is indeterminate. | |
19c5d690 | 9 | * |
925b9cd1 WL |
10 | * When a writer acquires a rwsem, it puts its task_struct pointer |
11 | * into the owner field. It is cleared after an unlock. | |
12 | * | |
13 | * When a reader acquires a rwsem, it will also puts its task_struct | |
14 | * pointer into the owner field with both the RWSEM_READER_OWNED and | |
15 | * RWSEM_ANONYMOUSLY_OWNED bits set. On unlock, the owner field will | |
16 | * largely be left untouched. So for a free or reader-owned rwsem, | |
17 | * the owner value may contain information about the last reader that | |
18 | * acquires the rwsem. The anonymous bit is set because that particular | |
19 | * reader may or may not still own the lock. | |
20 | * | |
21 | * That information may be helpful in debugging cases where the system | |
22 | * seems to hang on a reader owned rwsem especially if only one reader | |
23 | * is involved. Ideally we would like to track all the readers that own | |
24 | * a rwsem, but the overhead is simply too big. | |
19c5d690 | 25 | */ |
925b9cd1 WL |
26 | #define RWSEM_READER_OWNED (1UL << 0) |
27 | #define RWSEM_ANONYMOUSLY_OWNED (1UL << 1) | |
19c5d690 | 28 | |
5149cbac WL |
29 | #ifdef CONFIG_DEBUG_RWSEMS |
30 | # define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c) | |
31 | #else | |
32 | # define DEBUG_RWSEMS_WARN_ON(c) | |
33 | #endif | |
34 | ||
7a215f89 | 35 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
fb6a44f3 WL |
36 | /* |
37 | * All writes to owner are protected by WRITE_ONCE() to make sure that | |
38 | * store tearing can't happen as optimistic spinners may read and use | |
39 | * the owner value concurrently without lock. Read from owner, however, | |
40 | * may not need READ_ONCE() as long as the pointer value is only used | |
41 | * for comparison and isn't being dereferenced. | |
42 | */ | |
7a215f89 DB |
43 | static inline void rwsem_set_owner(struct rw_semaphore *sem) |
44 | { | |
fb6a44f3 | 45 | WRITE_ONCE(sem->owner, current); |
7a215f89 DB |
46 | } |
47 | ||
48 | static inline void rwsem_clear_owner(struct rw_semaphore *sem) | |
49 | { | |
fb6a44f3 | 50 | WRITE_ONCE(sem->owner, NULL); |
7a215f89 DB |
51 | } |
52 | ||
925b9cd1 WL |
53 | /* |
54 | * The task_struct pointer of the last owning reader will be left in | |
55 | * the owner field. | |
56 | * | |
57 | * Note that the owner value just indicates the task has owned the rwsem | |
58 | * previously, it may not be the real owner or one of the real owners | |
59 | * anymore when that field is examined, so take it with a grain of salt. | |
60 | */ | |
61 | static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, | |
62 | struct task_struct *owner) | |
63 | { | |
64 | unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED | |
65 | | RWSEM_ANONYMOUSLY_OWNED; | |
66 | ||
67 | WRITE_ONCE(sem->owner, (struct task_struct *)val); | |
68 | } | |
69 | ||
19c5d690 WL |
70 | static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) |
71 | { | |
925b9cd1 | 72 | __rwsem_set_reader_owned(sem, current); |
19c5d690 WL |
73 | } |
74 | ||
d7d760ef WL |
75 | /* |
76 | * Return true if the a rwsem waiter can spin on the rwsem's owner | |
77 | * and steal the lock, i.e. the lock is not anonymously owned. | |
78 | * N.B. !owner is considered spinnable. | |
79 | */ | |
80 | static inline bool is_rwsem_owner_spinnable(struct task_struct *owner) | |
19c5d690 | 81 | { |
d7d760ef | 82 | return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED); |
19c5d690 WL |
83 | } |
84 | ||
d7d760ef WL |
85 | /* |
86 | * Return true if rwsem is owned by an anonymous writer or readers. | |
87 | */ | |
88 | static inline bool rwsem_has_anonymous_owner(struct task_struct *owner) | |
19c5d690 | 89 | { |
d7d760ef | 90 | return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED; |
19c5d690 | 91 | } |
925b9cd1 WL |
92 | |
93 | #ifdef CONFIG_DEBUG_RWSEMS | |
94 | /* | |
95 | * With CONFIG_DEBUG_RWSEMS configured, it will make sure that if there | |
96 | * is a task pointer in owner of a reader-owned rwsem, it will be the | |
97 | * real owner or one of the real owners. The only exception is when the | |
98 | * unlock is done by up_read_non_owner(). | |
99 | */ | |
100 | #define rwsem_clear_reader_owned rwsem_clear_reader_owned | |
101 | static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) | |
102 | { | |
103 | unsigned long val = (unsigned long)current | RWSEM_READER_OWNED | |
104 | | RWSEM_ANONYMOUSLY_OWNED; | |
105 | if (READ_ONCE(sem->owner) == (struct task_struct *)val) | |
106 | cmpxchg_relaxed((unsigned long *)&sem->owner, val, | |
107 | RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED); | |
108 | } | |
109 | #endif | |
110 | ||
7a215f89 DB |
111 | #else |
112 | static inline void rwsem_set_owner(struct rw_semaphore *sem) | |
113 | { | |
114 | } | |
115 | ||
116 | static inline void rwsem_clear_owner(struct rw_semaphore *sem) | |
117 | { | |
118 | } | |
19c5d690 | 119 | |
925b9cd1 WL |
120 | static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, |
121 | struct task_struct *owner) | |
122 | { | |
123 | } | |
124 | ||
19c5d690 WL |
125 | static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) |
126 | { | |
127 | } | |
7a215f89 | 128 | #endif |
925b9cd1 WL |
129 | |
130 | #ifndef rwsem_clear_reader_owned | |
131 | static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) | |
132 | { | |
133 | } | |
134 | #endif |