]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - kernel/spinlock.c
[PATCH] BLOCK: Stop fallback_migrate_page() from using page_has_buffers() [try #6]
[mirror_ubuntu-artful-kernel.git] / kernel / spinlock.c
CommitLineData
1da177e4
LT
1/*
2 * Copyright (2004) Linus Torvalds
3 *
4 * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
5 *
fb1c8f93
IM
6 * Copyright (2004, 2005) Ingo Molnar
7 *
8 * This file contains the spinlock/rwlock implementations for the
9 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
0cb91a22
AK
10 *
11 * Note that some architectures have special knowledge about the
12 * stack frames of these functions in their profile_pc. If you
13 * change anything significant here that could change the stack
14 * frame contact the architecture maintainers.
1da177e4
LT
15 */
16
1da177e4
LT
17#include <linux/linkage.h>
18#include <linux/preempt.h>
19#include <linux/spinlock.h>
20#include <linux/interrupt.h>
8a25d5de 21#include <linux/debug_locks.h>
1da177e4
LT
22#include <linux/module.h>
23
1da177e4
LT
24int __lockfunc _spin_trylock(spinlock_t *lock)
25{
26 preempt_disable();
8a25d5de
IM
27 if (_raw_spin_trylock(lock)) {
28 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4 29 return 1;
8a25d5de 30 }
1da177e4
LT
31
32 preempt_enable();
33 return 0;
34}
35EXPORT_SYMBOL(_spin_trylock);
36
37int __lockfunc _read_trylock(rwlock_t *lock)
38{
39 preempt_disable();
8a25d5de
IM
40 if (_raw_read_trylock(lock)) {
41 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4 42 return 1;
8a25d5de 43 }
1da177e4
LT
44
45 preempt_enable();
46 return 0;
47}
48EXPORT_SYMBOL(_read_trylock);
49
50int __lockfunc _write_trylock(rwlock_t *lock)
51{
52 preempt_disable();
8a25d5de
IM
53 if (_raw_write_trylock(lock)) {
54 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4 55 return 1;
8a25d5de 56 }
1da177e4
LT
57
58 preempt_enable();
59 return 0;
60}
61EXPORT_SYMBOL(_write_trylock);
62
8a25d5de
IM
63/*
64 * If lockdep is enabled then we use the non-preemption spin-ops
65 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
66 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
67 */
68#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
fc47e7b5 69 defined(CONFIG_DEBUG_LOCK_ALLOC)
1da177e4
LT
70
71void __lockfunc _read_lock(rwlock_t *lock)
72{
73 preempt_disable();
8a25d5de 74 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
75 _raw_read_lock(lock);
76}
77EXPORT_SYMBOL(_read_lock);
78
79unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
80{
81 unsigned long flags;
82
83 local_irq_save(flags);
84 preempt_disable();
8a25d5de
IM
85 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
86 /*
87 * On lockdep we dont want the hand-coded irq-enable of
88 * _raw_spin_lock_flags() code, because lockdep assumes
89 * that interrupts are not re-enabled during lock-acquire:
90 */
91#ifdef CONFIG_PROVE_LOCKING
92 _raw_spin_lock(lock);
93#else
fb1c8f93 94 _raw_spin_lock_flags(lock, &flags);
8a25d5de 95#endif
1da177e4
LT
96 return flags;
97}
98EXPORT_SYMBOL(_spin_lock_irqsave);
99
100void __lockfunc _spin_lock_irq(spinlock_t *lock)
101{
102 local_irq_disable();
103 preempt_disable();
8a25d5de 104 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
105 _raw_spin_lock(lock);
106}
107EXPORT_SYMBOL(_spin_lock_irq);
108
109void __lockfunc _spin_lock_bh(spinlock_t *lock)
110{
111 local_bh_disable();
112 preempt_disable();
8a25d5de 113 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
114 _raw_spin_lock(lock);
115}
116EXPORT_SYMBOL(_spin_lock_bh);
117
118unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
119{
120 unsigned long flags;
121
122 local_irq_save(flags);
123 preempt_disable();
8a25d5de 124 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
125 _raw_read_lock(lock);
126 return flags;
127}
128EXPORT_SYMBOL(_read_lock_irqsave);
129
130void __lockfunc _read_lock_irq(rwlock_t *lock)
131{
132 local_irq_disable();
133 preempt_disable();
8a25d5de 134 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
135 _raw_read_lock(lock);
136}
137EXPORT_SYMBOL(_read_lock_irq);
138
139void __lockfunc _read_lock_bh(rwlock_t *lock)
140{
141 local_bh_disable();
142 preempt_disable();
8a25d5de 143 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
144 _raw_read_lock(lock);
145}
146EXPORT_SYMBOL(_read_lock_bh);
147
148unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
149{
150 unsigned long flags;
151
152 local_irq_save(flags);
153 preempt_disable();
8a25d5de 154 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
155 _raw_write_lock(lock);
156 return flags;
157}
158EXPORT_SYMBOL(_write_lock_irqsave);
159
160void __lockfunc _write_lock_irq(rwlock_t *lock)
161{
162 local_irq_disable();
163 preempt_disable();
8a25d5de 164 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
165 _raw_write_lock(lock);
166}
167EXPORT_SYMBOL(_write_lock_irq);
168
169void __lockfunc _write_lock_bh(rwlock_t *lock)
170{
171 local_bh_disable();
172 preempt_disable();
8a25d5de 173 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
174 _raw_write_lock(lock);
175}
176EXPORT_SYMBOL(_write_lock_bh);
177
178void __lockfunc _spin_lock(spinlock_t *lock)
179{
180 preempt_disable();
8a25d5de 181 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
182 _raw_spin_lock(lock);
183}
184
185EXPORT_SYMBOL(_spin_lock);
186
187void __lockfunc _write_lock(rwlock_t *lock)
188{
189 preempt_disable();
8a25d5de 190 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4
LT
191 _raw_write_lock(lock);
192}
193
194EXPORT_SYMBOL(_write_lock);
195
196#else /* CONFIG_PREEMPT: */
197
198/*
199 * This could be a long-held lock. We both prepare to spin for a long
200 * time (making _this_ CPU preemptable if possible), and we also signal
201 * towards that other CPU that it should break the lock ASAP.
202 *
203 * (We do this in a function because inlining it would be excessive.)
204 */
205
206#define BUILD_LOCK_OPS(op, locktype) \
207void __lockfunc _##op##_lock(locktype##_t *lock) \
208{ \
1da177e4 209 for (;;) { \
ee25e96f 210 preempt_disable(); \
1da177e4
LT
211 if (likely(_raw_##op##_trylock(lock))) \
212 break; \
213 preempt_enable(); \
ee25e96f 214 \
1da177e4
LT
215 if (!(lock)->break_lock) \
216 (lock)->break_lock = 1; \
217 while (!op##_can_lock(lock) && (lock)->break_lock) \
218 cpu_relax(); \
1da177e4
LT
219 } \
220 (lock)->break_lock = 0; \
221} \
222 \
223EXPORT_SYMBOL(_##op##_lock); \
224 \
225unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
226{ \
227 unsigned long flags; \
228 \
1da177e4 229 for (;;) { \
ee25e96f 230 preempt_disable(); \
1da177e4
LT
231 local_irq_save(flags); \
232 if (likely(_raw_##op##_trylock(lock))) \
233 break; \
234 local_irq_restore(flags); \
1da177e4 235 preempt_enable(); \
ee25e96f 236 \
1da177e4
LT
237 if (!(lock)->break_lock) \
238 (lock)->break_lock = 1; \
239 while (!op##_can_lock(lock) && (lock)->break_lock) \
240 cpu_relax(); \
1da177e4
LT
241 } \
242 (lock)->break_lock = 0; \
243 return flags; \
244} \
245 \
246EXPORT_SYMBOL(_##op##_lock_irqsave); \
247 \
248void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
249{ \
250 _##op##_lock_irqsave(lock); \
251} \
252 \
253EXPORT_SYMBOL(_##op##_lock_irq); \
254 \
255void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
256{ \
257 unsigned long flags; \
258 \
259 /* */ \
260 /* Careful: we must exclude softirqs too, hence the */ \
261 /* irq-disabling. We use the generic preemption-aware */ \
262 /* function: */ \
263 /**/ \
264 flags = _##op##_lock_irqsave(lock); \
265 local_bh_disable(); \
266 local_irq_restore(flags); \
267} \
268 \
269EXPORT_SYMBOL(_##op##_lock_bh)
270
271/*
272 * Build preemption-friendly versions of the following
273 * lock-spinning functions:
274 *
275 * _[spin|read|write]_lock()
276 * _[spin|read|write]_lock_irq()
277 * _[spin|read|write]_lock_irqsave()
278 * _[spin|read|write]_lock_bh()
279 */
280BUILD_LOCK_OPS(spin, spinlock);
281BUILD_LOCK_OPS(read, rwlock);
282BUILD_LOCK_OPS(write, rwlock);
283
284#endif /* CONFIG_PREEMPT */
285
8a25d5de
IM
286#ifdef CONFIG_DEBUG_LOCK_ALLOC
287
288void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
289{
290 preempt_disable();
291 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
292 _raw_spin_lock(lock);
293}
294
295EXPORT_SYMBOL(_spin_lock_nested);
296
297#endif
298
1da177e4
LT
299void __lockfunc _spin_unlock(spinlock_t *lock)
300{
8a25d5de 301 spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
302 _raw_spin_unlock(lock);
303 preempt_enable();
304}
305EXPORT_SYMBOL(_spin_unlock);
306
307void __lockfunc _write_unlock(rwlock_t *lock)
308{
8a25d5de 309 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
310 _raw_write_unlock(lock);
311 preempt_enable();
312}
313EXPORT_SYMBOL(_write_unlock);
314
315void __lockfunc _read_unlock(rwlock_t *lock)
316{
8a25d5de 317 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
318 _raw_read_unlock(lock);
319 preempt_enable();
320}
321EXPORT_SYMBOL(_read_unlock);
322
323void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
324{
8a25d5de 325 spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
326 _raw_spin_unlock(lock);
327 local_irq_restore(flags);
328 preempt_enable();
329}
330EXPORT_SYMBOL(_spin_unlock_irqrestore);
331
332void __lockfunc _spin_unlock_irq(spinlock_t *lock)
333{
8a25d5de 334 spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
335 _raw_spin_unlock(lock);
336 local_irq_enable();
337 preempt_enable();
338}
339EXPORT_SYMBOL(_spin_unlock_irq);
340
341void __lockfunc _spin_unlock_bh(spinlock_t *lock)
342{
8a25d5de 343 spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 344 _raw_spin_unlock(lock);
10f02d1c 345 preempt_enable_no_resched();
8a25d5de 346 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4
LT
347}
348EXPORT_SYMBOL(_spin_unlock_bh);
349
350void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
351{
8a25d5de 352 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
353 _raw_read_unlock(lock);
354 local_irq_restore(flags);
355 preempt_enable();
356}
357EXPORT_SYMBOL(_read_unlock_irqrestore);
358
359void __lockfunc _read_unlock_irq(rwlock_t *lock)
360{
8a25d5de 361 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
362 _raw_read_unlock(lock);
363 local_irq_enable();
364 preempt_enable();
365}
366EXPORT_SYMBOL(_read_unlock_irq);
367
368void __lockfunc _read_unlock_bh(rwlock_t *lock)
369{
8a25d5de 370 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 371 _raw_read_unlock(lock);
10f02d1c 372 preempt_enable_no_resched();
8a25d5de 373 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4
LT
374}
375EXPORT_SYMBOL(_read_unlock_bh);
376
377void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
378{
8a25d5de 379 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
380 _raw_write_unlock(lock);
381 local_irq_restore(flags);
382 preempt_enable();
383}
384EXPORT_SYMBOL(_write_unlock_irqrestore);
385
386void __lockfunc _write_unlock_irq(rwlock_t *lock)
387{
8a25d5de 388 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4
LT
389 _raw_write_unlock(lock);
390 local_irq_enable();
391 preempt_enable();
392}
393EXPORT_SYMBOL(_write_unlock_irq);
394
395void __lockfunc _write_unlock_bh(rwlock_t *lock)
396{
8a25d5de 397 rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 398 _raw_write_unlock(lock);
10f02d1c 399 preempt_enable_no_resched();
8a25d5de 400 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4
LT
401}
402EXPORT_SYMBOL(_write_unlock_bh);
403
404int __lockfunc _spin_trylock_bh(spinlock_t *lock)
405{
406 local_bh_disable();
407 preempt_disable();
8a25d5de
IM
408 if (_raw_spin_trylock(lock)) {
409 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4 410 return 1;
8a25d5de 411 }
1da177e4 412
10f02d1c 413 preempt_enable_no_resched();
8a25d5de 414 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4
LT
415 return 0;
416}
417EXPORT_SYMBOL(_spin_trylock_bh);
418
419int in_lock_functions(unsigned long addr)
420{
421 /* Linker adds these: start and end of __lockfunc functions */
422 extern char __lock_text_start[], __lock_text_end[];
423
424 return addr >= (unsigned long)__lock_text_start
425 && addr < (unsigned long)__lock_text_end;
426}
427EXPORT_SYMBOL(in_lock_functions);