[mirror_ubuntu-bionic-kernel.git] / kernel / spinlock.c

/*
 * Copyright (2004) Linus Torvalds
 *
 * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
 *
 * Copyright (2004, 2005) Ingo Molnar
 *
 * This file contains the spinlock/rwlock implementations for the
 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
 *
 * Note that some architectures have special knowledge about the
 * stack frames of these functions in their profile_pc. If you
 * change anything significant here that could change the stack
 * frame contact the architecture maintainers.
 */

#include <linux/linkage.h>
#include <linux/preempt.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/debug_locks.h>
#include <linux/module.h>

int __lockfunc _spin_trylock(spinlock_t *lock)
{
	preempt_disable();
	if (_raw_spin_trylock(lock)) {
		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}
	
	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_spin_trylock);

int __lockfunc _read_trylock(rwlock_t *lock)
{
	preempt_disable();
	if (_raw_read_trylock(lock)) {
		rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_read_trylock);

int __lockfunc _write_trylock(rwlock_t *lock)
{
	preempt_disable();
	if (_raw_write_trylock(lock)) {
		rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_write_trylock);

/*
 * If lockdep is enabled then we use the non-preemption spin-ops
 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
 */
#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
	defined(CONFIG_DEBUG_LOCK_ALLOC)

void __lockfunc _read_lock(rwlock_t *lock)
{
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock);

unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	/*
	 * On lockdep we dont want the hand-coded irq-enable of
	 * _raw_spin_lock_flags() code, because lockdep assumes
	 * that interrupts are not re-enabled during lock-acquire:
	 */
#ifdef CONFIG_PROVE_LOCKING
	_raw_spin_lock(lock);
#else
	_raw_spin_lock_flags(lock, &flags);
#endif
	return flags;
}
EXPORT_SYMBOL(_spin_lock_irqsave);

void __lockfunc _spin_lock_irq(spinlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_spin_lock(lock);
}
EXPORT_SYMBOL(_spin_lock_irq);

void __lockfunc _spin_lock_bh(spinlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_spin_lock(lock);
}
EXPORT_SYMBOL(_spin_lock_bh);

unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_read_lock(lock);
	return flags;
}
EXPORT_SYMBOL(_read_lock_irqsave);

void __lockfunc _read_lock_irq(rwlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock_irq);

void __lockfunc _read_lock_bh(rwlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock_bh);

unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_write_lock(lock);
	return flags;
}
EXPORT_SYMBOL(_write_lock_irqsave);

void __lockfunc _write_lock_irq(rwlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_write_lock(lock);
}
EXPORT_SYMBOL(_write_lock_irq);

void __lockfunc _write_lock_bh(rwlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_write_lock(lock);
}
EXPORT_SYMBOL(_write_lock_bh);

void __lockfunc _spin_lock(spinlock_t *lock)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_spin_lock(lock);
}

EXPORT_SYMBOL(_spin_lock);

void __lockfunc _write_lock(rwlock_t *lock)
{
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_write_lock(lock);
}

EXPORT_SYMBOL(_write_lock);

#else /* CONFIG_PREEMPT: */

/*
 * This could be a long-held lock. We both prepare to spin for a long
 * time (making _this_ CPU preemptable if possible), and we also signal
 * towards that other CPU that it should break the lock ASAP.
 *
 * (We do this in a function because inlining it would be excessive.)
 */

#define BUILD_LOCK_OPS(op, locktype)					\
void __lockfunc _##op##_lock(locktype##_t *lock)			\
{									\
	for (;;) {							\
		preempt_disable();					\
		if (likely(_raw_##op##_trylock(lock)))			\
			break;						\
		preempt_enable();					\
									\
		if (!(lock)->break_lock)				\
			(lock)->break_lock = 1;				\
		while (!op##_can_lock(lock) && (lock)->break_lock)	\
			_raw_##op##_relax(&lock->raw_lock);		\
	}								\
	(lock)->break_lock = 0;						\
}									\
									\
EXPORT_SYMBOL(_##op##_lock);						\
									\
unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)	\
{									\
	unsigned long flags;						\
									\
	for (;;) {							\
		preempt_disable();					\
		local_irq_save(flags);					\
		if (likely(_raw_##op##_trylock(lock)))			\
			break;						\
		local_irq_restore(flags);				\
		preempt_enable();					\
									\
		if (!(lock)->break_lock)				\
			(lock)->break_lock = 1;				\
		while (!op##_can_lock(lock) && (lock)->break_lock)	\
			_raw_##op##_relax(&lock->raw_lock);		\
	}								\
	(lock)->break_lock = 0;						\
	return flags;							\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_irqsave);					\
									\
void __lockfunc _##op##_lock_irq(locktype##_t *lock)			\
{									\
	_##op##_lock_irqsave(lock);					\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_irq);					\
									\
void __lockfunc _##op##_lock_bh(locktype##_t *lock)			\
{									\
	unsigned long flags;						\
									\
	/*							*/	\
	/* Careful: we must exclude softirqs too, hence the	*/	\
	/* irq-disabling. We use the generic preemption-aware	*/	\
	/* function:						*/	\
	/**/								\
	flags = _##op##_lock_irqsave(lock);				\
	local_bh_disable();						\
	local_irq_restore(flags);					\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_bh)

/*
 * Build preemption-friendly versions of the following
 * lock-spinning functions:
 *
 *         _[spin|read|write]_lock()
 *         _[spin|read|write]_lock_irq()
 *         _[spin|read|write]_lock_irqsave()
 *         _[spin|read|write]_lock_bh()
 */
BUILD_LOCK_OPS(spin, spinlock);
BUILD_LOCK_OPS(read, rwlock);
BUILD_LOCK_OPS(write, rwlock);

#endif /* CONFIG_PREEMPT */

#ifdef CONFIG_DEBUG_LOCK_ALLOC

void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	_raw_spin_lock(lock);
}

EXPORT_SYMBOL(_spin_lock_nested);
unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	/*
	 * On lockdep we dont want the hand-coded irq-enable of
	 * _raw_spin_lock_flags() code, because lockdep assumes
	 * that interrupts are not re-enabled during lock-acquire:
	 */
#ifdef CONFIG_PROVE_SPIN_LOCKING
	_raw_spin_lock(lock);
#else
	_raw_spin_lock_flags(lock, &flags);
#endif
	return flags;
}

EXPORT_SYMBOL(_spin_lock_irqsave_nested);

#endif

void __lockfunc _spin_unlock(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock);

void __lockfunc _write_unlock(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock);

void __lockfunc _read_unlock(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock);

void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irqrestore);

void __lockfunc _spin_unlock_irq(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irq);

void __lockfunc _spin_unlock_bh(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_spin_unlock_bh);

void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irqrestore);

void __lockfunc _read_unlock_irq(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irq);

void __lockfunc _read_unlock_bh(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_read_unlock_bh);

void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irqrestore);

void __lockfunc _write_unlock_irq(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irq);

void __lockfunc _write_unlock_bh(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_write_unlock_bh);

int __lockfunc _spin_trylock_bh(spinlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	if (_raw_spin_trylock(lock)) {
		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	return 0;
}
EXPORT_SYMBOL(_spin_trylock_bh);

int in_lock_functions(unsigned long addr)
{
	/* Linker adds these: start and end of __lockfunc functions */
	extern char __lock_text_start[], __lock_text_end[];

	return addr >= (unsigned long)__lock_text_start
	&& addr < (unsigned long)__lock_text_end;
}
EXPORT_SYMBOL(in_lock_functions);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Copyright (2004) Linus Torvalds
	3	*
	4	* Author: Zwane Mwaikambo <zwane@fsmlabs.com>
	5	*
fb1c8f93 IM	6	* Copyright (2004, 2005) Ingo Molnar
	7	*
	8	* This file contains the spinlock/rwlock implementations for the
	9	* SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
0cb91a22 AK	10	*
	11	* Note that some architectures have special knowledge about the
	12	* stack frames of these functions in their profile_pc. If you
	13	* change anything significant here that could change the stack
	14	* frame contact the architecture maintainers.
1da177e4 LT	15	*/
1da177e4 LT	16
1da177e4 LT	17	#include <linux/linkage.h>
	18	#include <linux/preempt.h>
	19	#include <linux/spinlock.h>
	20	#include <linux/interrupt.h>
8a25d5de	21	#include <linux/debug_locks.h>
1da177e4 LT	22	#include <linux/module.h>
1da177e4 LT	23
1da177e4 LT	24	int __lockfunc _spin_trylock(spinlock_t *lock)
	25	{
	26	preempt_disable();
8a25d5de IM	27	if (_raw_spin_trylock(lock)) {
8a25d5de IM	28	spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	29	return 1;
8a25d5de	30	}
1da177e4 LT	31
	32	preempt_enable();
	33	return 0;
	34	}
	35	EXPORT_SYMBOL(_spin_trylock);
	36
	37	int __lockfunc _read_trylock(rwlock_t *lock)
	38	{
	39	preempt_disable();
8a25d5de IM	40	if (_raw_read_trylock(lock)) {
8a25d5de IM	41	rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	42	return 1;
8a25d5de	43	}
1da177e4 LT	44
	45	preempt_enable();
	46	return 0;
	47	}
	48	EXPORT_SYMBOL(_read_trylock);
	49
	50	int __lockfunc _write_trylock(rwlock_t *lock)
	51	{
	52	preempt_disable();
8a25d5de IM	53	if (_raw_write_trylock(lock)) {
8a25d5de IM	54	rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	55	return 1;
8a25d5de	56	}
1da177e4 LT	57
	58	preempt_enable();
	59	return 0;
	60	}
	61	EXPORT_SYMBOL(_write_trylock);
	62
8a25d5de IM	63	/*
	64	* If lockdep is enabled then we use the non-preemption spin-ops
	65	* even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
	66	* not re-enabled during lock-acquire (which the preempt-spin-ops do):
	67	*/
	68	#if !defined(CONFIG_PREEMPT) \|\| !defined(CONFIG_SMP) \|\| \
fc47e7b5	69	defined(CONFIG_DEBUG_LOCK_ALLOC)
1da177e4 LT	70
	71	void __lockfunc _read_lock(rwlock_t *lock)
	72	{
	73	preempt_disable();
8a25d5de	74	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	75	_raw_read_lock(lock);
	76	}
	77	EXPORT_SYMBOL(_read_lock);
	78
	79	unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
	80	{
	81	unsigned long flags;
	82
	83	local_irq_save(flags);
	84	preempt_disable();
8a25d5de IM	85	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	86	/*
	87	* On lockdep we dont want the hand-coded irq-enable of
	88	* _raw_spin_lock_flags() code, because lockdep assumes
	89	* that interrupts are not re-enabled during lock-acquire:
	90	*/
	91	#ifdef CONFIG_PROVE_LOCKING
	92	_raw_spin_lock(lock);
	93	#else
fb1c8f93	94	_raw_spin_lock_flags(lock, &flags);
8a25d5de	95	#endif
1da177e4 LT	96	return flags;
	97	}
	98	EXPORT_SYMBOL(_spin_lock_irqsave);
	99
	100	void __lockfunc _spin_lock_irq(spinlock_t *lock)
	101	{
	102	local_irq_disable();
	103	preempt_disable();
8a25d5de	104	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	105	_raw_spin_lock(lock);
	106	}
	107	EXPORT_SYMBOL(_spin_lock_irq);
	108
	109	void __lockfunc _spin_lock_bh(spinlock_t *lock)
	110	{
	111	local_bh_disable();
	112	preempt_disable();
8a25d5de	113	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	114	_raw_spin_lock(lock);
	115	}
	116	EXPORT_SYMBOL(_spin_lock_bh);
	117
	118	unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
	119	{
	120	unsigned long flags;
	121
	122	local_irq_save(flags);
	123	preempt_disable();
8a25d5de	124	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	125	_raw_read_lock(lock);
	126	return flags;
	127	}
	128	EXPORT_SYMBOL(_read_lock_irqsave);
	129
	130	void __lockfunc _read_lock_irq(rwlock_t *lock)
	131	{
	132	local_irq_disable();
	133	preempt_disable();
8a25d5de	134	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	135	_raw_read_lock(lock);
	136	}
	137	EXPORT_SYMBOL(_read_lock_irq);
	138
	139	void __lockfunc _read_lock_bh(rwlock_t *lock)
	140	{
	141	local_bh_disable();
	142	preempt_disable();
8a25d5de	143	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	144	_raw_read_lock(lock);
	145	}
	146	EXPORT_SYMBOL(_read_lock_bh);
	147
	148	unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
	149	{
	150	unsigned long flags;
	151
	152	local_irq_save(flags);
	153	preempt_disable();
8a25d5de	154	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	155	_raw_write_lock(lock);
	156	return flags;
	157	}
	158	EXPORT_SYMBOL(_write_lock_irqsave);
	159
	160	void __lockfunc _write_lock_irq(rwlock_t *lock)
	161	{
	162	local_irq_disable();
	163	preempt_disable();
8a25d5de	164	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	165	_raw_write_lock(lock);
	166	}
	167	EXPORT_SYMBOL(_write_lock_irq);
	168
	169	void __lockfunc _write_lock_bh(rwlock_t *lock)
	170	{
	171	local_bh_disable();
	172	preempt_disable();
8a25d5de	173	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	174	_raw_write_lock(lock);
	175	}
	176	EXPORT_SYMBOL(_write_lock_bh);
	177
	178	void __lockfunc _spin_lock(spinlock_t *lock)
	179	{
	180	preempt_disable();
8a25d5de	181	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	182	_raw_spin_lock(lock);
	183	}
	184
	185	EXPORT_SYMBOL(_spin_lock);
	186
	187	void __lockfunc _write_lock(rwlock_t *lock)
	188	{
	189	preempt_disable();
8a25d5de	190	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	191	_raw_write_lock(lock);
	192	}
	193
	194	EXPORT_SYMBOL(_write_lock);
	195
	196	#else /* CONFIG_PREEMPT: */
	197
	198	/*
	199	* This could be a long-held lock. We both prepare to spin for a long
	200	* time (making _this_ CPU preemptable if possible), and we also signal
	201	* towards that other CPU that it should break the lock ASAP.
	202	*
	203	* (We do this in a function because inlining it would be excessive.)
	204	*/
	205
	206	#define BUILD_LOCK_OPS(op, locktype) \
	207	void __lockfunc _##op##_lock(locktype##_t *lock) \
	208	{ \
1da177e4	209	for (;;) { \
ee25e96f	210	preempt_disable(); \
1da177e4 LT	211	if (likely(_raw_##op##_trylock(lock))) \
	212	break; \
	213	preempt_enable(); \
ee25e96f	214	\
1da177e4 LT	215	if (!(lock)->break_lock) \
	216	(lock)->break_lock = 1; \
	217	while (!op##_can_lock(lock) && (lock)->break_lock) \
ef6edc97	218	_raw_##op##_relax(&lock->raw_lock); \
1da177e4 LT	219	} \
	220	(lock)->break_lock = 0; \
	221	} \
	222	\
	223	EXPORT_SYMBOL(_##op##_lock); \
	224	\
	225	unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
	226	{ \
	227	unsigned long flags; \
	228	\
1da177e4	229	for (;;) { \
ee25e96f	230	preempt_disable(); \
1da177e4 LT	231	local_irq_save(flags); \
	232	if (likely(_raw_##op##_trylock(lock))) \
	233	break; \
	234	local_irq_restore(flags); \
1da177e4	235	preempt_enable(); \
ee25e96f	236	\
1da177e4 LT	237	if (!(lock)->break_lock) \
	238	(lock)->break_lock = 1; \
	239	while (!op##_can_lock(lock) && (lock)->break_lock) \
ef6edc97	240	_raw_##op##_relax(&lock->raw_lock); \
1da177e4 LT	241	} \
	242	(lock)->break_lock = 0; \
	243	return flags; \
	244	} \
	245	\
	246	EXPORT_SYMBOL(_##op##_lock_irqsave); \
	247	\
	248	void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
	249	{ \
	250	_##op##_lock_irqsave(lock); \
	251	} \
	252	\
	253	EXPORT_SYMBOL(_##op##_lock_irq); \
	254	\
	255	void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
	256	{ \
	257	unsigned long flags; \
	258	\
	259	/* */ \
	260	/* Careful: we must exclude softirqs too, hence the */ \
	261	/* irq-disabling. We use the generic preemption-aware */ \
	262	/* function: */ \
	263	/**/ \
	264	flags = _##op##_lock_irqsave(lock); \
	265	local_bh_disable(); \
	266	local_irq_restore(flags); \
	267	} \
	268	\
	269	EXPORT_SYMBOL(_##op##_lock_bh)
	270
	271	/*
	272	* Build preemption-friendly versions of the following
	273	* lock-spinning functions:
	274	*
	275	* _[spin\|read\|write]_lock()
	276	* _[spin\|read\|write]_lock_irq()
	277	* _[spin\|read\|write]_lock_irqsave()
	278	* _[spin\|read\|write]_lock_bh()
	279	*/
	280	BUILD_LOCK_OPS(spin, spinlock);
	281	BUILD_LOCK_OPS(read, rwlock);
	282	BUILD_LOCK_OPS(write, rwlock);
	283
	284	#endif /* CONFIG_PREEMPT */
	285
8a25d5de IM	286	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	287
	288	void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
	289	{
	290	preempt_disable();
	291	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	292	_raw_spin_lock(lock);
	293	}
	294
	295	EXPORT_SYMBOL(_spin_lock_nested);
cfd3ef23 AV	296	unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
	297	{
	298	unsigned long flags;
	299
	300	local_irq_save(flags);
	301	preempt_disable();
	302	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	303	/*
	304	* On lockdep we dont want the hand-coded irq-enable of
	305	* _raw_spin_lock_flags() code, because lockdep assumes
	306	* that interrupts are not re-enabled during lock-acquire:
	307	*/
	308	#ifdef CONFIG_PROVE_SPIN_LOCKING
	309	_raw_spin_lock(lock);
	310	#else
	311	_raw_spin_lock_flags(lock, &flags);
	312	#endif
	313	return flags;
	314	}
	315
	316	EXPORT_SYMBOL(_spin_lock_irqsave_nested);
8a25d5de IM	317
	318	#endif
	319
1da177e4 LT	320	void __lockfunc _spin_unlock(spinlock_t *lock)
1da177e4 LT	321	{
8a25d5de	322	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	323	_raw_spin_unlock(lock);
	324	preempt_enable();
	325	}
	326	EXPORT_SYMBOL(_spin_unlock);
	327
	328	void __lockfunc _write_unlock(rwlock_t *lock)
	329	{
8a25d5de	330	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	331	_raw_write_unlock(lock);
	332	preempt_enable();
	333	}
	334	EXPORT_SYMBOL(_write_unlock);
	335
	336	void __lockfunc _read_unlock(rwlock_t *lock)
	337	{
8a25d5de	338	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	339	_raw_read_unlock(lock);
	340	preempt_enable();
	341	}
	342	EXPORT_SYMBOL(_read_unlock);
	343
	344	void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
	345	{
8a25d5de	346	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	347	_raw_spin_unlock(lock);
	348	local_irq_restore(flags);
	349	preempt_enable();
	350	}
	351	EXPORT_SYMBOL(_spin_unlock_irqrestore);
	352
	353	void __lockfunc _spin_unlock_irq(spinlock_t *lock)
	354	{
8a25d5de	355	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	356	_raw_spin_unlock(lock);
	357	local_irq_enable();
	358	preempt_enable();
	359	}
	360	EXPORT_SYMBOL(_spin_unlock_irq);
	361
	362	void __lockfunc _spin_unlock_bh(spinlock_t *lock)
	363	{
8a25d5de	364	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	365	_raw_spin_unlock(lock);
10f02d1c	366	preempt_enable_no_resched();
8a25d5de	367	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	368	}
	369	EXPORT_SYMBOL(_spin_unlock_bh);
	370
	371	void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
	372	{
8a25d5de	373	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	374	_raw_read_unlock(lock);
	375	local_irq_restore(flags);
	376	preempt_enable();
	377	}
	378	EXPORT_SYMBOL(_read_unlock_irqrestore);
	379
	380	void __lockfunc _read_unlock_irq(rwlock_t *lock)
	381	{
8a25d5de	382	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	383	_raw_read_unlock(lock);
	384	local_irq_enable();
	385	preempt_enable();
	386	}
	387	EXPORT_SYMBOL(_read_unlock_irq);
	388
	389	void __lockfunc _read_unlock_bh(rwlock_t *lock)
	390	{
8a25d5de	391	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	392	_raw_read_unlock(lock);
10f02d1c	393	preempt_enable_no_resched();
8a25d5de	394	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	395	}
	396	EXPORT_SYMBOL(_read_unlock_bh);
	397
	398	void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
	399	{
8a25d5de	400	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	401	_raw_write_unlock(lock);
	402	local_irq_restore(flags);
	403	preempt_enable();
	404	}
	405	EXPORT_SYMBOL(_write_unlock_irqrestore);
	406
	407	void __lockfunc _write_unlock_irq(rwlock_t *lock)
	408	{
8a25d5de	409	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	410	_raw_write_unlock(lock);
	411	local_irq_enable();
	412	preempt_enable();
	413	}
	414	EXPORT_SYMBOL(_write_unlock_irq);
	415
	416	void __lockfunc _write_unlock_bh(rwlock_t *lock)
	417	{
8a25d5de	418	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	419	_raw_write_unlock(lock);
10f02d1c	420	preempt_enable_no_resched();
8a25d5de	421	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	422	}
	423	EXPORT_SYMBOL(_write_unlock_bh);
	424
	425	int __lockfunc _spin_trylock_bh(spinlock_t *lock)
	426	{
	427	local_bh_disable();
	428	preempt_disable();
8a25d5de IM	429	if (_raw_spin_trylock(lock)) {
8a25d5de IM	430	spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	431	return 1;
8a25d5de	432	}
1da177e4	433
10f02d1c	434	preempt_enable_no_resched();
8a25d5de	435	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	436	return 0;
	437	}
	438	EXPORT_SYMBOL(_spin_trylock_bh);
	439
	440	int in_lock_functions(unsigned long addr)
	441	{
	442	/* Linker adds these: start and end of __lockfunc functions */
	443	extern char __lock_text_start[], __lock_text_end[];
	444
	445	return addr >= (unsigned long)__lock_text_start
	446	&& addr < (unsigned long)__lock_text_end;
	447	}
	448	EXPORT_SYMBOL(in_lock_functions);