[mirror_ubuntu-focal-kernel.git] / arch / mips / include / asm / spinlock.h

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 */
#ifndef _ASM_SPINLOCK_H
#define _ASM_SPINLOCK_H

#include <linux/compiler.h>

#include <asm/barrier.h>
#include <asm/compiler.h>
#include <asm/war.h>

/*
 * Your basic SMP spinlocks, allowing only a single CPU anywhere
 *
 * Simple spin lock operations.	 There are two variants, one clears IRQ's
 * on the local processor, one does not.
 *
 * These are fair FIFO ticket locks
 *
 * (the type definitions are in asm/spinlock_types.h)
 */


/*
 * Ticket locks are conceptually two parts, one indicating the current head of
 * the queue, and the other indicating the current tail. The lock is acquired
 * by atomically noting the tail and incrementing it by one (thus adding
 * ourself to the queue and noting our position), then waiting until the head
 * becomes equal to the the initial value of the tail.
 */

static inline int arch_spin_is_locked(arch_spinlock_t *lock)
{
	u32 counters = ACCESS_ONCE(lock->lock);

	return ((counters >> 16) ^ counters) & 0xffff;
}

#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
#define arch_spin_unlock_wait(x) \
	while (arch_spin_is_locked(x)) { cpu_relax(); }

static inline int arch_spin_is_contended(arch_spinlock_t *lock)
{
	u32 counters = ACCESS_ONCE(lock->lock);

	return (((counters >> 16) - counters) & 0xffff) > 1;
}
#define arch_spin_is_contended	arch_spin_is_contended

static inline void arch_spin_lock(arch_spinlock_t *lock)
{
	int my_ticket;
	int tmp;
	int inc = 0x10000;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_lock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
		"	sc	%[my_ticket], %[ticket_ptr]		\n"
		"	beqzl	%[my_ticket], 1b			\n"
		"	 nop						\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[ticket], %[ticket], 0xffff		\n"
		"	bne	%[ticket], %[my_ticket], 4f		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"2:							\n"
		"	.subsection 2					\n"
		"4:	andi	%[ticket], %[ticket], 0xffff		\n"
		"	sll	%[ticket], 5				\n"
		"							\n"
		"6:	bnez	%[ticket], 6b				\n"
		"	 subu	%[ticket], 1				\n"
		"							\n"
		"	lhu	%[ticket], %[serving_now_ptr]		\n"
		"	beq	%[ticket], %[my_ticket], 2b		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"	b	4b					\n"
		"	 subu	%[ticket], %[ticket], 1			\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
		  [serving_now_ptr] "+m" (lock->h.serving_now),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (my_ticket)
		: [inc] "r" (inc));
	} else {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_lock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
		"	sc	%[my_ticket], %[ticket_ptr]		\n"
		"	beqz	%[my_ticket], 1b			\n"
		"	 srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[ticket], %[ticket], 0xffff		\n"
		"	bne	%[ticket], %[my_ticket], 4f		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"2:							\n"
		"	.subsection 2					\n"
		"4:	andi	%[ticket], %[ticket], 0x1fff		\n"
		"	sll	%[ticket], 5				\n"
		"							\n"
		"6:	bnez	%[ticket], 6b				\n"
		"	 subu	%[ticket], 1				\n"
		"							\n"
		"	lhu	%[ticket], %[serving_now_ptr]		\n"
		"	beq	%[ticket], %[my_ticket], 2b		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"	b	4b					\n"
		"	 subu	%[ticket], %[ticket], 1			\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
		  [serving_now_ptr] "+m" (lock->h.serving_now),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (my_ticket)
		: [inc] "r" (inc));
	}

	smp_llsc_mb();
}

static inline void arch_spin_unlock(arch_spinlock_t *lock)
{
	unsigned int serving_now = lock->h.serving_now + 1;
	wmb();
	lock->h.serving_now = (u16)serving_now;
	nudge_writes();
}

static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
{
	int tmp, tmp2, tmp3;
	int inc = 0x10000;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_trylock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[now_serving], %[ticket], 0xffff	\n"
		"	bne	%[my_ticket], %[now_serving], 3f	\n"
		"	 addu	%[ticket], %[ticket], %[inc]		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqzl	%[ticket], 1b				\n"
		"	 li	%[ticket], 1				\n"
		"2:							\n"
		"	.subsection 2					\n"
		"3:	b	2b					\n"
		"	 li	%[ticket], 0				\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (tmp2),
		  [now_serving] "=&r" (tmp3)
		: [inc] "r" (inc));
	} else {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_trylock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[now_serving], %[ticket], 0xffff	\n"
		"	bne	%[my_ticket], %[now_serving], 3f	\n"
		"	 addu	%[ticket], %[ticket], %[inc]		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqz	%[ticket], 1b				\n"
		"	 li	%[ticket], 1				\n"
		"2:							\n"
		"	.subsection 2					\n"
		"3:	b	2b					\n"
		"	 li	%[ticket], 0				\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (tmp2),
		  [now_serving] "=&r" (tmp3)
		: [inc] "r" (inc));
	}

	smp_llsc_mb();

	return tmp;
}

/*
 * Read-write spinlocks, allowing multiple readers but only one writer.
 *
 * NOTE! it is quite common to have readers in interrupts but no interrupt
 * writers. For those circumstances we can "mix" irq-safe locks - any writer
 * needs to get a irq-safe write-lock, but readers can get non-irqsafe
 * read-locks.
 */

/*
 * read_can_lock - would read_trylock() succeed?
 * @lock: the rwlock in question.
 */
#define arch_read_can_lock(rw)	((rw)->lock >= 0)

/*
 * write_can_lock - would write_trylock() succeed?
 * @lock: the rwlock in question.
 */
#define arch_write_can_lock(rw) (!(rw)->lock)

static inline void arch_read_lock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_lock	\n"
		"1:	ll	%1, %2					\n"
		"	bltz	%1, 1b					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
		: GCC_OFF12_ASM() (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_read_lock	\n"
			"	bltz	%1, 1b				\n"
			"	 addu	%1, 1				\n"
			"2:	sc	%1, %0				\n"
			: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
			: GCC_OFF12_ASM() (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}

	smp_llsc_mb();
}

/* Note the use of sub, not subu which will make the kernel die with an
   overflow exception if we ever try to unlock an rwlock that is already
   unlocked or is being held by a writer.  */
static inline void arch_read_unlock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	smp_mb__before_llsc();

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"1:	ll	%1, %2		# arch_read_unlock	\n"
		"	sub	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
		: GCC_OFF12_ASM() (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_read_unlock	\n"
			"	sub	%1, 1				\n"
			"	sc	%1, %0				\n"
			: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
			: GCC_OFF12_ASM() (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}
}

static inline void arch_write_lock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_write_lock	\n"
		"1:	ll	%1, %2					\n"
		"	bnez	%1, 1b					\n"
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
		: GCC_OFF12_ASM() (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_write_lock	\n"
			"	bnez	%1, 1b				\n"
			"	 lui	%1, 0x8000			\n"
			"2:	sc	%1, %0				\n"
			: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
			: GCC_OFF12_ASM() (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}

	smp_llsc_mb();
}

static inline void arch_write_unlock(arch_rwlock_t *rw)
{
	smp_mb();

	__asm__ __volatile__(
	"				# arch_write_unlock	\n"
	"	sw	$0, %0					\n"
	: "=m" (rw->lock)
	: "m" (rw->lock)
	: "memory");
}

static inline int arch_read_trylock(arch_rwlock_t *rw)
{
	unsigned int tmp;
	int ret;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bltz	%1, 2f					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	.set	reorder					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"2:							\n"
		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
		: GCC_OFF12_ASM() (rw->lock)
		: "memory");
	} else {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bltz	%1, 2f					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqz	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"2:							\n"
		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
		: GCC_OFF12_ASM() (rw->lock)
		: "memory");
	}

	return ret;
}

static inline int arch_write_trylock(arch_rwlock_t *rw)
{
	unsigned int tmp;
	int ret;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_write_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bnez	%1, 2f					\n"
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"	.set	reorder					\n"
		"2:							\n"
		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
		: GCC_OFF12_ASM() (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"	ll	%1, %3	# arch_write_trylock	\n"
			"	li	%2, 0				\n"
			"	bnez	%1, 2f				\n"
			"	lui	%1, 0x8000			\n"
			"	sc	%1, %0				\n"
			"	li	%2, 1				\n"
			"2:						\n"
			: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp),
			  "=&r" (ret)
			: GCC_OFF12_ASM() (rw->lock)
			: "memory");
		} while (unlikely(!tmp));

		smp_llsc_mb();
	}

	return ret;
}

#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)

#define arch_spin_relax(lock)	cpu_relax()
#define arch_read_relax(lock)	cpu_relax()
#define arch_write_relax(lock)	cpu_relax()

#endif /* _ASM_SPINLOCK_H */
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
f65e4fa8	6	* Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
1da177e4 LT	7	* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
	8	*/
	9	#ifndef _ASM_SPINLOCK_H
	10	#define _ASM_SPINLOCK_H
	11
2a31b033 RB	12	#include <linux/compiler.h>
2a31b033 RB	13
0004a9df	14	#include <asm/barrier.h>
b0984c43	15	#include <asm/compiler.h>
1da177e4 LT	16	#include <asm/war.h>
	17
	18	/*
	19	* Your basic SMP spinlocks, allowing only a single CPU anywhere
2a31b033	20	*
70342287	21	* Simple spin lock operations. There are two variants, one clears IRQ's
2a31b033 RB	22	* on the local processor, one does not.
	23	*
	24	* These are fair FIFO ticket locks
	25	*
	26	* (the type definitions are in asm/spinlock_types.h)
1da177e4 LT	27	*/
1da177e4 LT	28
1da177e4 LT	29
1da177e4 LT	30	/*
2a31b033 RB	31	* Ticket locks are conceptually two parts, one indicating the current head of
	32	* the queue, and the other indicating the current tail. The lock is acquired
	33	* by atomically noting the tail and incrementing it by one (thus adding
	34	* ourself to the queue and noting our position), then waiting until the head
	35	* becomes equal to the the initial value of the tail.
1da177e4 LT	36	*/
1da177e4 LT	37
0199c4e6	38	static inline int arch_spin_is_locked(arch_spinlock_t *lock)
2a31b033	39	{
500c2e1f	40	u32 counters = ACCESS_ONCE(lock->lock);
2a31b033	41
500c2e1f	42	return ((counters >> 16) ^ counters) & 0xffff;
2a31b033 RB	43	}
2a31b033 RB	44
0199c4e6 TG	45	#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
	46	#define arch_spin_unlock_wait(x) \
	47	while (arch_spin_is_locked(x)) { cpu_relax(); }
2a31b033	48
0199c4e6	49	static inline int arch_spin_is_contended(arch_spinlock_t *lock)
2a31b033	50	{
500c2e1f	51	u32 counters = ACCESS_ONCE(lock->lock);
2a31b033	52
500c2e1f	53	return (((counters >> 16) - counters) & 0xffff) > 1;
2a31b033	54	}
0199c4e6	55	#define arch_spin_is_contended arch_spin_is_contended
2a31b033	56
0199c4e6	57	static inline void arch_spin_lock(arch_spinlock_t *lock)
1da177e4	58	{
2a31b033 RB	59	int my_ticket;
2a31b033 RB	60	int tmp;
500c2e1f	61	int inc = 0x10000;
1da177e4 LT	62
1da177e4 LT	63	if (R10000_LLSC_WAR) {
2a31b033	64	__asm__ __volatile__ (
0199c4e6	65	" .set push # arch_spin_lock \n"
2a31b033 RB	66	" .set noreorder \n"
	67	" \n"
	68	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f	69	" addu %[my_ticket], %[ticket], %[inc] \n"
2a31b033 RB	70	" sc %[my_ticket], %[ticket_ptr] \n"
2a31b033 RB	71	" beqzl %[my_ticket], 1b \n"
1da177e4	72	" nop \n"
500c2e1f DD	73	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f DD	74	" andi %[ticket], %[ticket], 0xffff \n"
2a31b033 RB	75	" bne %[ticket], %[my_ticket], 4f \n"
	76	" subu %[ticket], %[my_ticket], %[ticket] \n"
	77	"2: \n"
	78	" .subsection 2 \n"
500c2e1f	79	"4: andi %[ticket], %[ticket], 0xffff \n"
0e6826c7	80	" sll %[ticket], 5 \n"
2a31b033 RB	81	" \n"
	82	"6: bnez %[ticket], 6b \n"
	83	" subu %[ticket], 1 \n"
	84	" \n"
500c2e1f	85	" lhu %[ticket], %[serving_now_ptr] \n"
2a31b033 RB	86	" beq %[ticket], %[my_ticket], 2b \n"
2a31b033 RB	87	" subu %[ticket], %[my_ticket], %[ticket] \n"
0e6826c7	88	" b 4b \n"
2a31b033 RB	89	" subu %[ticket], %[ticket], 1 \n"
	90	" .previous \n"
	91	" .set pop \n"
b0984c43	92	: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
500c2e1f	93	[serving_now_ptr] "+m" (lock->h.serving_now),
2a31b033	94	[ticket] "=&r" (tmp),
500c2e1f DD	95	[my_ticket] "=&r" (my_ticket)
500c2e1f DD	96	: [inc] "r" (inc));
1da177e4	97	} else {
2a31b033	98	__asm__ __volatile__ (
0199c4e6	99	" .set push # arch_spin_lock \n"
2a31b033 RB	100	" .set noreorder \n"
2a31b033 RB	101	" \n"
500c2e1f DD	102	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f DD	103	" addu %[my_ticket], %[ticket], %[inc] \n"
2a31b033	104	" sc %[my_ticket], %[ticket_ptr] \n"
500c2e1f DD	105	" beqz %[my_ticket], 1b \n"
	106	" srl %[my_ticket], %[ticket], 16 \n"
	107	" andi %[ticket], %[ticket], 0xffff \n"
2a31b033 RB	108	" bne %[ticket], %[my_ticket], 4f \n"
	109	" subu %[ticket], %[my_ticket], %[ticket] \n"
	110	"2: \n"
f65e4fa8	111	" .subsection 2 \n"
2a31b033	112	"4: andi %[ticket], %[ticket], 0x1fff \n"
0e6826c7	113	" sll %[ticket], 5 \n"
2a31b033 RB	114	" \n"
	115	"6: bnez %[ticket], 6b \n"
	116	" subu %[ticket], 1 \n"
	117	" \n"
500c2e1f	118	" lhu %[ticket], %[serving_now_ptr] \n"
2a31b033 RB	119	" beq %[ticket], %[my_ticket], 2b \n"
2a31b033 RB	120	" subu %[ticket], %[my_ticket], %[ticket] \n"
0e6826c7	121	" b 4b \n"
2a31b033	122	" subu %[ticket], %[ticket], 1 \n"
f65e4fa8	123	" .previous \n"
2a31b033	124	" .set pop \n"
b0984c43	125	: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
500c2e1f	126	[serving_now_ptr] "+m" (lock->h.serving_now),
2a31b033	127	[ticket] "=&r" (tmp),
500c2e1f DD	128	[my_ticket] "=&r" (my_ticket)
500c2e1f DD	129	: [inc] "r" (inc));
1da177e4	130	}
0004a9df	131
17099b11	132	smp_llsc_mb();
1da177e4 LT	133	}
1da177e4 LT	134
0199c4e6	135	static inline void arch_spin_unlock(arch_spinlock_t *lock)
1da177e4	136	{
500c2e1f DD	137	unsigned int serving_now = lock->h.serving_now + 1;
	138	wmb();
	139	lock->h.serving_now = (u16)serving_now;
	140	nudge_writes();
1da177e4 LT	141	}
1da177e4 LT	142
0199c4e6	143	static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
1da177e4	144	{
2a31b033	145	int tmp, tmp2, tmp3;
500c2e1f	146	int inc = 0x10000;
1da177e4 LT	147
1da177e4 LT	148	if (R10000_LLSC_WAR) {
2a31b033	149	__asm__ __volatile__ (
0199c4e6	150	" .set push # arch_spin_trylock \n"
2a31b033 RB	151	" .set noreorder \n"
	152	" \n"
	153	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f	154	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f	155	" andi %[now_serving], %[ticket], 0xffff \n"
2a31b033	156	" bne %[my_ticket], %[now_serving], 3f \n"
500c2e1f	157	" addu %[ticket], %[ticket], %[inc] \n"
2a31b033 RB	158	" sc %[ticket], %[ticket_ptr] \n"
	159	" beqzl %[ticket], 1b \n"
	160	" li %[ticket], 1 \n"
	161	"2: \n"
	162	" .subsection 2 \n"
	163	"3: b 2b \n"
	164	" li %[ticket], 0 \n"
	165	" .previous \n"
	166	" .set pop \n"
b0984c43	167	: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
2a31b033 RB	168	[ticket] "=&r" (tmp),
2a31b033 RB	169	[my_ticket] "=&r" (tmp2),
500c2e1f DD	170	[now_serving] "=&r" (tmp3)
500c2e1f DD	171	: [inc] "r" (inc));
1da177e4	172	} else {
2a31b033	173	__asm__ __volatile__ (
0199c4e6	174	" .set push # arch_spin_trylock \n"
2a31b033 RB	175	" .set noreorder \n"
2a31b033 RB	176	" \n"
500c2e1f DD	177	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f DD	178	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f	179	" andi %[now_serving], %[ticket], 0xffff \n"
2a31b033	180	" bne %[my_ticket], %[now_serving], 3f \n"
500c2e1f	181	" addu %[ticket], %[ticket], %[inc] \n"
2a31b033	182	" sc %[ticket], %[ticket_ptr] \n"
500c2e1f	183	" beqz %[ticket], 1b \n"
2a31b033 RB	184	" li %[ticket], 1 \n"
2a31b033 RB	185	"2: \n"
f65e4fa8	186	" .subsection 2 \n"
2a31b033 RB	187	"3: b 2b \n"
2a31b033 RB	188	" li %[ticket], 0 \n"
f65e4fa8	189	" .previous \n"
2a31b033	190	" .set pop \n"
b0984c43	191	: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
2a31b033 RB	192	[ticket] "=&r" (tmp),
2a31b033 RB	193	[my_ticket] "=&r" (tmp2),
500c2e1f DD	194	[now_serving] "=&r" (tmp3)
500c2e1f DD	195	: [inc] "r" (inc));
1da177e4 LT	196	}
1da177e4 LT	197
17099b11	198	smp_llsc_mb();
0004a9df	199
2a31b033	200	return tmp;
1da177e4 LT	201	}
	202
	203	/*
	204	* Read-write spinlocks, allowing multiple readers but only one writer.
	205	*
	206	* NOTE! it is quite common to have readers in interrupts but no interrupt
	207	* writers. For those circumstances we can "mix" irq-safe locks - any writer
	208	* needs to get a irq-safe write-lock, but readers can get non-irqsafe
	209	* read-locks.
	210	*/
	211
e3c48078 RB	212	/*
	213	* read_can_lock - would read_trylock() succeed?
	214	* @lock: the rwlock in question.
	215	*/
e5931943	216	#define arch_read_can_lock(rw) ((rw)->lock >= 0)
e3c48078 RB	217
	218	/*
	219	* write_can_lock - would write_trylock() succeed?
	220	* @lock: the rwlock in question.
	221	*/
70342287	222	#define arch_write_can_lock(rw) (!(rw)->lock)
e3c48078	223
e5931943	224	static inline void arch_read_lock(arch_rwlock_t *rw)
1da177e4 LT	225	{
	226	unsigned int tmp;
	227
	228	if (R10000_LLSC_WAR) {
	229	__asm__ __volatile__(
e5931943	230	" .set noreorder # arch_read_lock \n"
1da177e4 LT	231	"1: ll %1, %2 \n"
	232	" bltz %1, 1b \n"
	233	" addu %1, 1 \n"
	234	" sc %1, %0 \n"
	235	" beqzl %1, 1b \n"
	236	" nop \n"
1da177e4	237	" .set reorder \n"
b0984c43 MR	238	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
b0984c43 MR	239	: GCC_OFF12_ASM() (rw->lock)
1da177e4 LT	240	: "memory");
1da177e4 LT	241	} else {
e01961ce RB	242	do {
	243	__asm__ __volatile__(
	244	"1: ll %1, %2 # arch_read_lock \n"
	245	" bltz %1, 1b \n"
	246	" addu %1, 1 \n"
	247	"2: sc %1, %0 \n"
b0984c43 MR	248	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
b0984c43 MR	249	: GCC_OFF12_ASM() (rw->lock)
e01961ce RB	250	: "memory");
e01961ce RB	251	} while (unlikely(!tmp));
1da177e4	252	}
0004a9df	253
17099b11	254	smp_llsc_mb();
1da177e4 LT	255	}
	256
	257	/* Note the use of sub, not subu which will make the kernel die with an
	258	overflow exception if we ever try to unlock an rwlock that is already
	259	unlocked or is being held by a writer. */
e5931943	260	static inline void arch_read_unlock(arch_rwlock_t *rw)
1da177e4 LT	261	{
	262	unsigned int tmp;
	263
f252ffd5	264	smp_mb__before_llsc();
0004a9df	265
1da177e4 LT	266	if (R10000_LLSC_WAR) {
1da177e4 LT	267	__asm__ __volatile__(
e5931943	268	"1: ll %1, %2 # arch_read_unlock \n"
1da177e4 LT	269	" sub %1, 1 \n"
	270	" sc %1, %0 \n"
	271	" beqzl %1, 1b \n"
b0984c43 MR	272	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
b0984c43 MR	273	: GCC_OFF12_ASM() (rw->lock)
1da177e4 LT	274	: "memory");
1da177e4 LT	275	} else {
e01961ce RB	276	do {
	277	__asm__ __volatile__(
	278	"1: ll %1, %2 # arch_read_unlock \n"
	279	" sub %1, 1 \n"
	280	" sc %1, %0 \n"
b0984c43 MR	281	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
b0984c43 MR	282	: GCC_OFF12_ASM() (rw->lock)
e01961ce RB	283	: "memory");
e01961ce RB	284	} while (unlikely(!tmp));
1da177e4 LT	285	}
	286	}
	287
e5931943	288	static inline void arch_write_lock(arch_rwlock_t *rw)
1da177e4 LT	289	{
	290	unsigned int tmp;
	291
	292	if (R10000_LLSC_WAR) {
	293	__asm__ __volatile__(
e5931943	294	" .set noreorder # arch_write_lock \n"
1da177e4 LT	295	"1: ll %1, %2 \n"
	296	" bnez %1, 1b \n"
	297	" lui %1, 0x8000 \n"
	298	" sc %1, %0 \n"
	299	" beqzl %1, 1b \n"
0004a9df	300	" nop \n"
1da177e4	301	" .set reorder \n"
b0984c43 MR	302	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
b0984c43 MR	303	: GCC_OFF12_ASM() (rw->lock)
1da177e4 LT	304	: "memory");
1da177e4 LT	305	} else {
e01961ce RB	306	do {
	307	__asm__ __volatile__(
	308	"1: ll %1, %2 # arch_write_lock \n"
	309	" bnez %1, 1b \n"
	310	" lui %1, 0x8000 \n"
	311	"2: sc %1, %0 \n"
b0984c43 MR	312	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
b0984c43 MR	313	: GCC_OFF12_ASM() (rw->lock)
e01961ce RB	314	: "memory");
e01961ce RB	315	} while (unlikely(!tmp));
1da177e4	316	}
0004a9df	317
17099b11	318	smp_llsc_mb();
1da177e4 LT	319	}
1da177e4 LT	320
e5931943	321	static inline void arch_write_unlock(arch_rwlock_t *rw)
1da177e4	322	{
0004a9df RB	323	smp_mb();
0004a9df RB	324
1da177e4	325	__asm__ __volatile__(
e5931943	326	" # arch_write_unlock \n"
1da177e4 LT	327	" sw $0, %0 \n"
	328	: "=m" (rw->lock)
	329	: "m" (rw->lock)
	330	: "memory");
	331	}
	332
e5931943	333	static inline int arch_read_trylock(arch_rwlock_t *rw)
65316fd1 RB	334	{
	335	unsigned int tmp;
	336	int ret;
	337
	338	if (R10000_LLSC_WAR) {
	339	__asm__ __volatile__(
e5931943	340	" .set noreorder # arch_read_trylock \n"
65316fd1 RB	341	" li %2, 0 \n"
65316fd1 RB	342	"1: ll %1, %3 \n"
d52c2d5a	343	" bltz %1, 2f \n"
65316fd1 RB	344	" addu %1, 1 \n"
65316fd1 RB	345	" sc %1, %0 \n"
65316fd1	346	" .set reorder \n"
0004a9df RB	347	" beqzl %1, 1b \n"
0004a9df RB	348	" nop \n"
17099b11	349	__WEAK_LLSC_MB
65316fd1 RB	350	" li %2, 1 \n"
65316fd1 RB	351	"2: \n"
b0984c43 MR	352	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
b0984c43 MR	353	: GCC_OFF12_ASM() (rw->lock)
65316fd1 RB	354	: "memory");
	355	} else {
	356	__asm__ __volatile__(
e5931943	357	" .set noreorder # arch_read_trylock \n"
65316fd1 RB	358	" li %2, 0 \n"
65316fd1 RB	359	"1: ll %1, %3 \n"
d52c2d5a	360	" bltz %1, 2f \n"
65316fd1 RB	361	" addu %1, 1 \n"
	362	" sc %1, %0 \n"
	363	" beqz %1, 1b \n"
0004a9df	364	" nop \n"
65316fd1	365	" .set reorder \n"
17099b11	366	__WEAK_LLSC_MB
65316fd1 RB	367	" li %2, 1 \n"
65316fd1 RB	368	"2: \n"
b0984c43 MR	369	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
b0984c43 MR	370	: GCC_OFF12_ASM() (rw->lock)
65316fd1 RB	371	: "memory");
	372	}
	373
	374	return ret;
	375	}
1da177e4	376
e5931943	377	static inline int arch_write_trylock(arch_rwlock_t *rw)
1da177e4 LT	378	{
	379	unsigned int tmp;
	380	int ret;
	381
	382	if (R10000_LLSC_WAR) {
	383	__asm__ __volatile__(
e5931943	384	" .set noreorder # arch_write_trylock \n"
1da177e4 LT	385	" li %2, 0 \n"
	386	"1: ll %1, %3 \n"
	387	" bnez %1, 2f \n"
	388	" lui %1, 0x8000 \n"
	389	" sc %1, %0 \n"
	390	" beqzl %1, 1b \n"
0004a9df	391	" nop \n"
17099b11	392	__WEAK_LLSC_MB
1da177e4 LT	393	" li %2, 1 \n"
	394	" .set reorder \n"
	395	"2: \n"
b0984c43 MR	396	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
b0984c43 MR	397	: GCC_OFF12_ASM() (rw->lock)
1da177e4 LT	398	: "memory");
1da177e4 LT	399	} else {
e01961ce RB	400	do {
	401	__asm__ __volatile__(
	402	" ll %1, %3 # arch_write_trylock \n"
	403	" li %2, 0 \n"
	404	" bnez %1, 2f \n"
	405	" lui %1, 0x8000 \n"
	406	" sc %1, %0 \n"
	407	" li %2, 1 \n"
	408	"2: \n"
b0984c43 MR	409	: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp),
	410	"=&r" (ret)
	411	: GCC_OFF12_ASM() (rw->lock)
e01961ce RB	412	: "memory");
	413	} while (unlikely(!tmp));
	414
	415	smp_llsc_mb();
1da177e4 LT	416	}
	417
	418	return ret;
	419	}
	420
e5931943 TG	421	#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
e5931943 TG	422	#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
65316fd1	423
0199c4e6 TG	424	#define arch_spin_relax(lock) cpu_relax()
	425	#define arch_read_relax(lock) cpu_relax()
	426	#define arch_write_relax(lock) cpu_relax()
ef6edc97	427
1da177e4	428	#endif /* _ASM_SPINLOCK_H */