]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+ |
2 | * | |
3 | * Written by David Howells (dhowells@redhat.com). | |
4 | * | |
99122a3f | 5 | * Derived from asm-x86/semaphore.h |
1da177e4 LT |
6 | * |
7 | * | |
8 | * The MSW of the count is the negated number of active writers and waiting | |
9 | * lockers, and the LSW is the total number of active locks | |
10 | * | |
11 | * The lock count is initialized to 0 (no active and no waiting lockers). | |
12 | * | |
13 | * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an | |
14 | * uncontended lock. This can be determined because XADD returns the old value. | |
15 | * Readers increment by 1 and see a positive value when uncontended, negative | |
16 | * if there are writers (and maybe) readers waiting (in which case it goes to | |
17 | * sleep). | |
18 | * | |
19 | * The value of WAITING_BIAS supports up to 32766 waiting processes. This can | |
20 | * be extended to 65534 by manually checking the whole MSW rather than relying | |
21 | * on the S flag. | |
22 | * | |
23 | * The value of ACTIVE_BIAS supports up to 65535 active processes. | |
24 | * | |
25 | * This should be totally fair - if anything is waiting, a process that wants a | |
26 | * lock will go to the back of the queue. When the currently active lock is | |
27 | * released, if there's a writer at the front of the queue, then that and only | |
28 | * that will be woken up; if there's a bunch of consequtive readers at the | |
29 | * front, then they'll all be woken up, but no other readers will be. | |
30 | */ | |
31 | ||
1965aae3 PA |
32 | #ifndef _ASM_X86_RWSEM_H |
33 | #define _ASM_X86_RWSEM_H | |
1da177e4 LT |
34 | |
35 | #ifndef _LINUX_RWSEM_H | |
36 | #error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead" | |
37 | #endif | |
38 | ||
39 | #ifdef __KERNEL__ | |
40 | ||
41 | #include <linux/list.h> | |
42 | #include <linux/spinlock.h> | |
4ea2176d | 43 | #include <linux/lockdep.h> |
1838ef1d | 44 | #include <asm/asm.h> |
1da177e4 LT |
45 | |
46 | struct rwsem_waiter; | |
47 | ||
d50efc6c IM |
48 | extern asmregparm struct rw_semaphore * |
49 | rwsem_down_read_failed(struct rw_semaphore *sem); | |
50 | extern asmregparm struct rw_semaphore * | |
51 | rwsem_down_write_failed(struct rw_semaphore *sem); | |
52 | extern asmregparm struct rw_semaphore * | |
53 | rwsem_wake(struct rw_semaphore *); | |
54 | extern asmregparm struct rw_semaphore * | |
55 | rwsem_downgrade_wake(struct rw_semaphore *sem); | |
1da177e4 LT |
56 | |
57 | /* | |
58 | * the semaphore definition | |
5d0b7235 | 59 | * |
1838ef1d PA |
60 | * The bias values and the counter type limits the number of |
61 | * potential readers/writers to 32767 for 32 bits and 2147483647 | |
62 | * for 64 bits. | |
1da177e4 | 63 | */ |
6e5609a9 | 64 | |
1838ef1d PA |
65 | #ifdef CONFIG_X86_64 |
66 | # define RWSEM_ACTIVE_MASK 0xffffffffL | |
67 | #else | |
68 | # define RWSEM_ACTIVE_MASK 0x0000ffffL | |
69 | #endif | |
70 | ||
71 | #define RWSEM_UNLOCKED_VALUE 0x00000000L | |
72 | #define RWSEM_ACTIVE_BIAS 0x00000001L | |
73 | #define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) | |
1da177e4 LT |
74 | #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS |
75 | #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) | |
6e5609a9 | 76 | |
1838ef1d | 77 | typedef signed long rwsem_count_t; |
5d0b7235 | 78 | |
6e5609a9 | 79 | struct rw_semaphore { |
5d0b7235 | 80 | rwsem_count_t count; |
1da177e4 LT |
81 | spinlock_t wait_lock; |
82 | struct list_head wait_list; | |
4ea2176d IM |
83 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
84 | struct lockdep_map dep_map; | |
85 | #endif | |
1da177e4 LT |
86 | }; |
87 | ||
4ea2176d IM |
88 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
89 | # define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname } | |
90 | #else | |
91 | # define __RWSEM_DEP_MAP_INIT(lockname) | |
92 | #endif | |
93 | ||
94 | ||
6e5609a9 JP |
95 | #define __RWSEM_INITIALIZER(name) \ |
96 | { \ | |
97 | RWSEM_UNLOCKED_VALUE, __SPIN_LOCK_UNLOCKED((name).wait_lock), \ | |
98 | LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) \ | |
99 | } | |
1da177e4 | 100 | |
6e5609a9 | 101 | #define DECLARE_RWSEM(name) \ |
1da177e4 LT |
102 | struct rw_semaphore name = __RWSEM_INITIALIZER(name) |
103 | ||
4ea2176d IM |
104 | extern void __init_rwsem(struct rw_semaphore *sem, const char *name, |
105 | struct lock_class_key *key); | |
106 | ||
107 | #define init_rwsem(sem) \ | |
108 | do { \ | |
109 | static struct lock_class_key __key; \ | |
110 | \ | |
111 | __init_rwsem((sem), #sem, &__key); \ | |
112 | } while (0) | |
1da177e4 LT |
113 | |
114 | /* | |
115 | * lock for reading | |
116 | */ | |
117 | static inline void __down_read(struct rw_semaphore *sem) | |
118 | { | |
6e5609a9 | 119 | asm volatile("# beginning down_read\n\t" |
1838ef1d | 120 | LOCK_PREFIX _ASM_INC "(%1)\n\t" |
6e5609a9 JP |
121 | /* adds 0x00000001, returns the old value */ |
122 | " jns 1f\n" | |
123 | " call call_rwsem_down_read_failed\n" | |
124 | "1:\n\t" | |
125 | "# ending down_read\n\t" | |
126 | : "+m" (sem->count) | |
127 | : "a" (sem) | |
128 | : "memory", "cc"); | |
1da177e4 LT |
129 | } |
130 | ||
131 | /* | |
132 | * trylock for reading -- returns 1 if successful, 0 if contention | |
133 | */ | |
134 | static inline int __down_read_trylock(struct rw_semaphore *sem) | |
135 | { | |
5d0b7235 | 136 | rwsem_count_t result, tmp; |
6e5609a9 | 137 | asm volatile("# beginning __down_read_trylock\n\t" |
59c33fa7 | 138 | " mov %0,%1\n\t" |
6e5609a9 | 139 | "1:\n\t" |
59c33fa7 LT |
140 | " mov %1,%2\n\t" |
141 | " add %3,%2\n\t" | |
6e5609a9 | 142 | " jle 2f\n\t" |
59c33fa7 | 143 | LOCK_PREFIX " cmpxchg %2,%0\n\t" |
6e5609a9 JP |
144 | " jnz 1b\n\t" |
145 | "2:\n\t" | |
146 | "# ending __down_read_trylock\n\t" | |
147 | : "+m" (sem->count), "=&a" (result), "=&r" (tmp) | |
148 | : "i" (RWSEM_ACTIVE_READ_BIAS) | |
149 | : "memory", "cc"); | |
150 | return result >= 0 ? 1 : 0; | |
1da177e4 LT |
151 | } |
152 | ||
153 | /* | |
154 | * lock for writing | |
155 | */ | |
4ea2176d | 156 | static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) |
1da177e4 | 157 | { |
5d0b7235 | 158 | rwsem_count_t tmp; |
1da177e4 LT |
159 | |
160 | tmp = RWSEM_ACTIVE_WRITE_BIAS; | |
6e5609a9 | 161 | asm volatile("# beginning down_write\n\t" |
59c33fa7 | 162 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
6e5609a9 | 163 | /* subtract 0x0000ffff, returns the old value */ |
59c33fa7 | 164 | " test %1,%1\n\t" |
6e5609a9 JP |
165 | /* was the count 0 before? */ |
166 | " jz 1f\n" | |
167 | " call call_rwsem_down_write_failed\n" | |
168 | "1:\n" | |
169 | "# ending down_write" | |
170 | : "+m" (sem->count), "=d" (tmp) | |
171 | : "a" (sem), "1" (tmp) | |
172 | : "memory", "cc"); | |
1da177e4 LT |
173 | } |
174 | ||
4ea2176d IM |
175 | static inline void __down_write(struct rw_semaphore *sem) |
176 | { | |
177 | __down_write_nested(sem, 0); | |
178 | } | |
179 | ||
1da177e4 LT |
180 | /* |
181 | * trylock for writing -- returns 1 if successful, 0 if contention | |
182 | */ | |
183 | static inline int __down_write_trylock(struct rw_semaphore *sem) | |
184 | { | |
5d0b7235 LT |
185 | rwsem_count_t ret = cmpxchg(&sem->count, |
186 | RWSEM_UNLOCKED_VALUE, | |
187 | RWSEM_ACTIVE_WRITE_BIAS); | |
1da177e4 LT |
188 | if (ret == RWSEM_UNLOCKED_VALUE) |
189 | return 1; | |
190 | return 0; | |
191 | } | |
192 | ||
193 | /* | |
194 | * unlock after reading | |
195 | */ | |
196 | static inline void __up_read(struct rw_semaphore *sem) | |
197 | { | |
5d0b7235 | 198 | rwsem_count_t tmp = -RWSEM_ACTIVE_READ_BIAS; |
6e5609a9 | 199 | asm volatile("# beginning __up_read\n\t" |
59c33fa7 | 200 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
6e5609a9 JP |
201 | /* subtracts 1, returns the old value */ |
202 | " jns 1f\n\t" | |
203 | " call call_rwsem_wake\n" | |
204 | "1:\n" | |
205 | "# ending __up_read\n" | |
206 | : "+m" (sem->count), "=d" (tmp) | |
207 | : "a" (sem), "1" (tmp) | |
208 | : "memory", "cc"); | |
1da177e4 LT |
209 | } |
210 | ||
211 | /* | |
212 | * unlock after writing | |
213 | */ | |
214 | static inline void __up_write(struct rw_semaphore *sem) | |
215 | { | |
5d0b7235 | 216 | rwsem_count_t tmp; |
6e5609a9 | 217 | asm volatile("# beginning __up_write\n\t" |
59c33fa7 | 218 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
6e5609a9 JP |
219 | /* tries to transition |
220 | 0xffff0001 -> 0x00000000 */ | |
221 | " jz 1f\n" | |
222 | " call call_rwsem_wake\n" | |
223 | "1:\n\t" | |
224 | "# ending __up_write\n" | |
59c33fa7 LT |
225 | : "+m" (sem->count), "=d" (tmp) |
226 | : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS) | |
227 | : "memory", "cc"); | |
1da177e4 LT |
228 | } |
229 | ||
230 | /* | |
231 | * downgrade write lock to read lock | |
232 | */ | |
233 | static inline void __downgrade_write(struct rw_semaphore *sem) | |
234 | { | |
6e5609a9 | 235 | asm volatile("# beginning __downgrade_write\n\t" |
1838ef1d | 236 | LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t" |
0d1622d7 AK |
237 | /* |
238 | * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386) | |
239 | * 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64) | |
240 | */ | |
6e5609a9 JP |
241 | " jns 1f\n\t" |
242 | " call call_rwsem_downgrade_wake\n" | |
243 | "1:\n\t" | |
244 | "# ending __downgrade_write\n" | |
245 | : "+m" (sem->count) | |
0d1622d7 | 246 | : "a" (sem), "er" (-RWSEM_WAITING_BIAS) |
6e5609a9 | 247 | : "memory", "cc"); |
1da177e4 LT |
248 | } |
249 | ||
250 | /* | |
251 | * implement atomic add functionality | |
252 | */ | |
1838ef1d PA |
253 | static inline void rwsem_atomic_add(rwsem_count_t delta, |
254 | struct rw_semaphore *sem) | |
1da177e4 | 255 | { |
1838ef1d | 256 | asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0" |
6e5609a9 | 257 | : "+m" (sem->count) |
1838ef1d | 258 | : "er" (delta)); |
1da177e4 LT |
259 | } |
260 | ||
261 | /* | |
262 | * implement exchange and add functionality | |
263 | */ | |
1838ef1d PA |
264 | static inline rwsem_count_t rwsem_atomic_update(rwsem_count_t delta, |
265 | struct rw_semaphore *sem) | |
1da177e4 | 266 | { |
5d0b7235 | 267 | rwsem_count_t tmp = delta; |
1da177e4 | 268 | |
6e5609a9 JP |
269 | asm volatile(LOCK_PREFIX "xadd %0,%1" |
270 | : "+r" (tmp), "+m" (sem->count) | |
271 | : : "memory"); | |
1da177e4 | 272 | |
6e5609a9 | 273 | return tmp + delta; |
1da177e4 LT |
274 | } |
275 | ||
eb92f4ef RVR |
276 | static inline int rwsem_is_locked(struct rw_semaphore *sem) |
277 | { | |
278 | return (sem->count != 0); | |
279 | } | |
280 | ||
1da177e4 | 281 | #endif /* __KERNEL__ */ |
1965aae3 | 282 | #endif /* _ASM_X86_RWSEM_H */ |