]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - kernel/spinlock.c
[PATCH] Char: stallion, implement fail paths
[mirror_ubuntu-bionic-kernel.git] / kernel / spinlock.c
CommitLineData
1da177e4
LT
1/*
2 * Copyright (2004) Linus Torvalds
3 *
4 * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
5 *
fb1c8f93
IM
6 * Copyright (2004, 2005) Ingo Molnar
7 *
8 * This file contains the spinlock/rwlock implementations for the
9 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
0cb91a22
AK
10 *
11 * Note that some architectures have special knowledge about the
12 * stack frames of these functions in their profile_pc. If you
13 * change anything significant here that could change the stack
14 * frame contact the architecture maintainers.
1da177e4
LT
15 */
16
1da177e4
LT
17#include <linux/linkage.h>
18#include <linux/preempt.h>
19#include <linux/spinlock.h>
20#include <linux/interrupt.h>
8a25d5de 21#include <linux/debug_locks.h>
1da177e4
LT
22#include <linux/module.h>
23
1da177e4
LT
24int __lockfunc _spin_trylock(spinlock_t *lock)
25{
26 preempt_disable();
8a25d5de
IM
27 if (_raw_spin_trylock(lock)) {
28 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4 29 return 1;
8a25d5de 30 }
1da177e4
LT
31
32 preempt_enable();
33 return 0;
34}
35EXPORT_SYMBOL(_spin_trylock);
36
37int __lockfunc _read_trylock(rwlock_t *lock)
38{
39 preempt_disable();
8a25d5de
IM
40 if (_raw_read_trylock(lock)) {
41 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4 42 return 1;
8a25d5de 43 }
1da177e4
LT
44
45 preempt_enable();
46 return 0;
47}
48EXPORT_SYMBOL(_read_trylock);
49
50int __lockfunc _write_trylock(rwlock_t *lock)
51{
52 preempt_disable();
8a25d5de
IM
53 if (_raw_write_trylock(lock)) {
54 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4 55 return 1;
8a25d5de 56 }
1da177e4
LT
57
58 preempt_enable();
59 return 0;
60}
61EXPORT_SYMBOL(_write_trylock);
62
8a25d5de
IM
63/*
64 * If lockdep is enabled then we use the non-preemption spin-ops
65 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
66 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
67 */
68#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
fc47e7b5 69 defined(CONFIG_DEBUG_LOCK_ALLOC)
1da177e4
LT
70
71void __lockfunc _read_lock(rwlock_t *lock)
72{
73 preempt_disable();
8a25d5de 74 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
75 _raw_read_lock(lock);
76}
77EXPORT_SYMBOL(_read_lock);
78
79unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
80{
81 unsigned long flags;
82
83 local_irq_save(flags);
84 preempt_disable();
8a25d5de
IM
85 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
86 /*
87 * On lockdep we dont want the hand-coded irq-enable of
88 * _raw_spin_lock_flags() code, because lockdep assumes
89 * that interrupts are not re-enabled during lock-acquire:
90 */
91#ifdef CONFIG_PROVE_LOCKING
92 _raw_spin_lock(lock);
93#else
fb1c8f93 94 _raw_spin_lock_flags(lock, &flags);
8a25d5de 95#endif
1da177e4
LT
96 return flags;
97}
98EXPORT_SYMBOL(_spin_lock_irqsave);
99
100void __lockfunc _spin_lock_irq(spinlock_t *lock)
101{
102 local_irq_disable();
103 preempt_disable();
8a25d5de 104 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
105 _raw_spin_lock(lock);
106}
107EXPORT_SYMBOL(_spin_lock_irq);
108
109void __lockfunc _spin_lock_bh(spinlock_t *lock)
110{
111 local_bh_disable();
112 preempt_disable();
8a25d5de 113 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
114 _raw_spin_lock(lock);
115}
116EXPORT_SYMBOL(_spin_lock_bh);
117
118unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
119{
120 unsigned long flags;
121
122 local_irq_save(flags);
123 preempt_disable();
8a25d5de 124 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
125 _raw_read_lock(lock);
126 return flags;
127}
128EXPORT_SYMBOL(_read_lock_irqsave);
129
130void __lockfunc _read_lock_irq(rwlock_t *lock)
131{
132 local_irq_disable();
133 preempt_disable();
8a25d5de 134 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
135 _raw_read_lock(lock);
136}
137EXPORT_SYMBOL(_read_lock_irq);
138
139void __lockfunc _read_lock_bh(rwlock_t *lock)
140{
141 local_bh_disable();
142 preempt_disable();
8a25d5de 143 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
144 _raw_read_lock(lock);
145}
146EXPORT_SYMBOL(_read_lock_bh);
147
148unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
149{
150 unsigned long flags;
151
152 local_irq_save(flags);
153 preempt_disable();
8a25d5de 154 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
155 _raw_write_lock(lock);
156 return flags;
157}
158EXPORT_SYMBOL(_write_lock_irqsave);
159
160void __lockfunc _write_lock_irq(rwlock_t *lock)
161{
162 local_irq_disable();
163 preempt_disable();
8a25d5de 164 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
165 _raw_write_lock(lock);
166}
167EXPORT_SYMBOL(_write_lock_irq);
168
169void __lockfunc _write_lock_bh(rwlock_t *lock)
170{
171 local_bh_disable();
172 preempt_disable();
8a25d5de 173 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
174 _raw_write_lock(lock);
175}
176EXPORT_SYMBOL(_write_lock_bh);
177
178void __lockfunc _spin_lock(spinlock_t *lock)
179{
180 preempt_disable();
8a25d5de 181 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
182 _raw_spin_lock(lock);
183}
184
185EXPORT_SYMBOL(_spin_lock);
186
187void __lockfunc _write_lock(rwlock_t *lock)
188{
189 preempt_disable();
8a25d5de 190 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
191 _raw_write_lock(lock);
192}
193
194EXPORT_SYMBOL(_write_lock);
195
196#else /* CONFIG_PREEMPT: */
197
198/*
199 * This could be a long-held lock. We both prepare to spin for a long
200 * time (making _this_ CPU preemptable if possible), and we also signal
201 * towards that other CPU that it should break the lock ASAP.
202 *
203 * (We do this in a function because inlining it would be excessive.)
204 */
205
206#define BUILD_LOCK_OPS(op, locktype) \
207void __lockfunc _##op##_lock(locktype##_t *lock) \
208{ \
1da177e4 209 for (;;) { \
ee25e96f 210 preempt_disable(); \
1da177e4
LT
211 if (likely(_raw_##op##_trylock(lock))) \
212 break; \
213 preempt_enable(); \
ee25e96f 214 \
1da177e4
LT
215 if (!(lock)->break_lock) \
216 (lock)->break_lock = 1; \
217 while (!op##_can_lock(lock) && (lock)->break_lock) \
ef6edc97 218 _raw_##op##_relax(&lock->raw_lock); \
1da177e4
LT
219 } \
220 (lock)->break_lock = 0; \
221} \
222 \
223EXPORT_SYMBOL(_##op##_lock); \
224 \
225unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
226{ \
227 unsigned long flags; \
228 \
1da177e4 229 for (;;) { \
ee25e96f 230 preempt_disable(); \
1da177e4
LT
231 local_irq_save(flags); \
232 if (likely(_raw_##op##_trylock(lock))) \
233 break; \
234 local_irq_restore(flags); \
1da177e4 235 preempt_enable(); \
ee25e96f 236 \
1da177e4
LT
237 if (!(lock)->break_lock) \
238 (lock)->break_lock = 1; \
239 while (!op##_can_lock(lock) && (lock)->break_lock) \
ef6edc97 240 _raw_##op##_relax(&lock->raw_lock); \
1da177e4
LT
241 } \
242 (lock)->break_lock = 0; \
243 return flags; \
244} \
245 \
246EXPORT_SYMBOL(_##op##_lock_irqsave); \
247 \
248void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
249{ \
250 _##op##_lock_irqsave(lock); \
251} \
252 \
253EXPORT_SYMBOL(_##op##_lock_irq); \
254 \
255void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
256{ \
257 unsigned long flags; \
258 \
259 /* */ \
260 /* Careful: we must exclude softirqs too, hence the */ \
261 /* irq-disabling. We use the generic preemption-aware */ \
262 /* function: */ \
263 /**/ \
264 flags = _##op##_lock_irqsave(lock); \
265 local_bh_disable(); \
266 local_irq_restore(flags); \
267} \
268 \
269EXPORT_SYMBOL(_##op##_lock_bh)
270
271/*
272 * Build preemption-friendly versions of the following
273 * lock-spinning functions:
274 *
275 * _[spin|read|write]_lock()
276 * _[spin|read|write]_lock_irq()
277 * _[spin|read|write]_lock_irqsave()
278 * _[spin|read|write]_lock_bh()
279 */
280BUILD_LOCK_OPS(spin, spinlock);
281BUILD_LOCK_OPS(read, rwlock);
282BUILD_LOCK_OPS(write, rwlock);
283
284#endif /* CONFIG_PREEMPT */
285
8a25d5de
IM
286#ifdef CONFIG_DEBUG_LOCK_ALLOC
287
288void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
289{
290 preempt_disable();
291 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
292 _raw_spin_lock(lock);
293}
294
295EXPORT_SYMBOL(_spin_lock_nested);
cfd3ef23
AV
296unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
297{
298 unsigned long flags;
299
300 local_irq_save(flags);
301 preempt_disable();
302 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
303 /*
304 * On lockdep we dont want the hand-coded irq-enable of
305 * _raw_spin_lock_flags() code, because lockdep assumes
306 * that interrupts are not re-enabled during lock-acquire:
307 */
308#ifdef CONFIG_PROVE_SPIN_LOCKING
309 _raw_spin_lock(lock);
310#else
311 _raw_spin_lock_flags(lock, &flags);
312#endif
313 return flags;
314}
315
316EXPORT_SYMBOL(_spin_lock_irqsave_nested);
8a25d5de
IM
317
318#endif
319
1da177e4
LT
320void __lockfunc _spin_unlock(spinlock_t *lock)
321{
8a25d5de 322 spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
323 _raw_spin_unlock(lock);
324 preempt_enable();
325}
326EXPORT_SYMBOL(_spin_unlock);
327
328void __lockfunc _write_unlock(rwlock_t *lock)
329{
8a25d5de 330 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
331 _raw_write_unlock(lock);
332 preempt_enable();
333}
334EXPORT_SYMBOL(_write_unlock);
335
336void __lockfunc _read_unlock(rwlock_t *lock)
337{
8a25d5de 338 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
339 _raw_read_unlock(lock);
340 preempt_enable();
341}
342EXPORT_SYMBOL(_read_unlock);
343
344void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
345{
8a25d5de 346 spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
347 _raw_spin_unlock(lock);
348 local_irq_restore(flags);
349 preempt_enable();
350}
351EXPORT_SYMBOL(_spin_unlock_irqrestore);
352
353void __lockfunc _spin_unlock_irq(spinlock_t *lock)
354{
8a25d5de 355 spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
356 _raw_spin_unlock(lock);
357 local_irq_enable();
358 preempt_enable();
359}
360EXPORT_SYMBOL(_spin_unlock_irq);
361
362void __lockfunc _spin_unlock_bh(spinlock_t *lock)
363{
8a25d5de 364 spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 365 _raw_spin_unlock(lock);
10f02d1c 366 preempt_enable_no_resched();
8a25d5de 367 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4
LT
368}
369EXPORT_SYMBOL(_spin_unlock_bh);
370
371void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
372{
8a25d5de 373 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
374 _raw_read_unlock(lock);
375 local_irq_restore(flags);
376 preempt_enable();
377}
378EXPORT_SYMBOL(_read_unlock_irqrestore);
379
380void __lockfunc _read_unlock_irq(rwlock_t *lock)
381{
8a25d5de 382 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
383 _raw_read_unlock(lock);
384 local_irq_enable();
385 preempt_enable();
386}
387EXPORT_SYMBOL(_read_unlock_irq);
388
389void __lockfunc _read_unlock_bh(rwlock_t *lock)
390{
8a25d5de 391 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 392 _raw_read_unlock(lock);
10f02d1c 393 preempt_enable_no_resched();
8a25d5de 394 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4
LT
395}
396EXPORT_SYMBOL(_read_unlock_bh);
397
398void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
399{
8a25d5de 400 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
401 _raw_write_unlock(lock);
402 local_irq_restore(flags);
403 preempt_enable();
404}
405EXPORT_SYMBOL(_write_unlock_irqrestore);
406
407void __lockfunc _write_unlock_irq(rwlock_t *lock)
408{
8a25d5de 409 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
410 _raw_write_unlock(lock);
411 local_irq_enable();
412 preempt_enable();
413}
414EXPORT_SYMBOL(_write_unlock_irq);
415
416void __lockfunc _write_unlock_bh(rwlock_t *lock)
417{
8a25d5de 418 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 419 _raw_write_unlock(lock);
10f02d1c 420 preempt_enable_no_resched();
8a25d5de 421 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4
LT
422}
423EXPORT_SYMBOL(_write_unlock_bh);
424
425int __lockfunc _spin_trylock_bh(spinlock_t *lock)
426{
427 local_bh_disable();
428 preempt_disable();
8a25d5de
IM
429 if (_raw_spin_trylock(lock)) {
430 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4 431 return 1;
8a25d5de 432 }
1da177e4 433
10f02d1c 434 preempt_enable_no_resched();
8a25d5de 435 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4
LT
436 return 0;
437}
438EXPORT_SYMBOL(_spin_trylock_bh);
439
440int in_lock_functions(unsigned long addr)
441{
442 /* Linker adds these: start and end of __lockfunc functions */
443 extern char __lock_text_start[], __lock_text_end[];
444
445 return addr >= (unsigned long)__lock_text_start
446 && addr < (unsigned long)__lock_text_end;
447}
448EXPORT_SYMBOL(in_lock_functions);