[mirror_ubuntu-bionic-kernel.git] / arch / s390 / lib / spinlock.c

/*
 *    Out of line spinlock code.
 *
 *    Copyright IBM Corp. 2004, 2006
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
 */

#include <linux/types.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/jiffies.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/percpu.h>
#include <asm/io.h>

int spin_retry = -1;

static int __init spin_retry_init(void)
{
	if (spin_retry < 0)
		spin_retry = 1000;
	return 0;
}
early_initcall(spin_retry_init);

/**
 * spin_retry= parameter
 */
static int __init spin_retry_setup(char *str)
{
	spin_retry = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("spin_retry=", spin_retry_setup);

struct spin_wait {
	struct spin_wait *next, *prev;
	int node_id;
} __aligned(32);

static DEFINE_PER_CPU_ALIGNED(struct spin_wait, spin_wait[4]);

#define _Q_LOCK_CPU_OFFSET	0
#define _Q_LOCK_STEAL_OFFSET	16
#define _Q_TAIL_IDX_OFFSET	18
#define _Q_TAIL_CPU_OFFSET	20

#define _Q_LOCK_CPU_MASK	0x0000ffff
#define _Q_LOCK_STEAL_ADD	0x00010000
#define _Q_LOCK_STEAL_MASK	0x00030000
#define _Q_TAIL_IDX_MASK	0x000c0000
#define _Q_TAIL_CPU_MASK	0xfff00000

#define _Q_LOCK_MASK		(_Q_LOCK_CPU_MASK | _Q_LOCK_STEAL_MASK)
#define _Q_TAIL_MASK		(_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)

void arch_spin_lock_setup(int cpu)
{
	struct spin_wait *node;
	int ix;

	node = per_cpu_ptr(&spin_wait[0], cpu);
	for (ix = 0; ix < 4; ix++, node++) {
		memset(node, 0, sizeof(*node));
		node->node_id = ((cpu + 1) << _Q_TAIL_CPU_OFFSET) +
			(ix << _Q_TAIL_IDX_OFFSET);
	}
}

static inline int arch_load_niai4(int *lock)
{
	int owner;

	asm volatile(
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
		"	.long	0xb2fa0040\n"	/* NIAI 4 */
#endif
		"	l	%0,%1\n"
		: "=d" (owner) : "Q" (*lock) : "memory");
       return owner;
}

static inline int arch_cmpxchg_niai8(int *lock, int old, int new)
{
	int expected = old;

	asm volatile(
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
		"	.long	0xb2fa0080\n"	/* NIAI 8 */
#endif
		"	cs	%0,%3,%1\n"
		: "=d" (old), "=Q" (*lock)
		: "0" (old), "d" (new), "Q" (*lock)
		: "cc", "memory");
	return expected == old;
}

static inline struct spin_wait *arch_spin_decode_tail(int lock)
{
	int ix, cpu;

	ix = (lock & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
	cpu = (lock & _Q_TAIL_CPU_MASK) >> _Q_TAIL_CPU_OFFSET;
	return per_cpu_ptr(&spin_wait[ix], cpu - 1);
}

static inline int arch_spin_yield_target(int lock, struct spin_wait *node)
{
	if (lock & _Q_LOCK_CPU_MASK)
		return lock & _Q_LOCK_CPU_MASK;
	if (node == NULL || node->prev == NULL)
		return 0;	/* 0 -> no target cpu */
	while (node->prev)
		node = node->prev;
	return node->node_id >> _Q_TAIL_CPU_OFFSET;
}

static inline void arch_spin_lock_queued(arch_spinlock_t *lp)
{
	struct spin_wait *node, *next;
	int lockval, ix, node_id, tail_id, old, new, owner, count;

	ix = S390_lowcore.spinlock_index++;
	barrier();
	lockval = SPINLOCK_LOCKVAL;	/* cpu + 1 */
	node = this_cpu_ptr(&spin_wait[ix]);
	node->prev = node->next = NULL;
	node_id = node->node_id;

	/* Enqueue the node for this CPU in the spinlock wait queue */
	while (1) {
		old = READ_ONCE(lp->lock);
		if ((old & _Q_LOCK_CPU_MASK) == 0 &&
		    (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) {
			/*
			 * The lock is free but there may be waiters.
			 * With no waiters simply take the lock, if there
			 * are waiters try to steal the lock. The lock may
			 * be stolen three times before the next queued
			 * waiter will get the lock.
			 */
			new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval;
			if (__atomic_cmpxchg_bool(&lp->lock, old, new))
				/* Got the lock */
				goto out;
			/* lock passing in progress */
			continue;
		}
		/* Make the node of this CPU the new tail. */
		new = node_id | (old & _Q_LOCK_MASK);
		if (__atomic_cmpxchg_bool(&lp->lock, old, new))
			break;
	}
	/* Set the 'next' pointer of the tail node in the queue */
	tail_id = old & _Q_TAIL_MASK;
	if (tail_id != 0) {
		node->prev = arch_spin_decode_tail(tail_id);
		WRITE_ONCE(node->prev->next, node);
	}

	/* Pass the virtual CPU to the lock holder if it is not running */
	owner = arch_spin_yield_target(old, node);
	if (owner && arch_vcpu_is_preempted(owner - 1))
		smp_yield_cpu(owner - 1);

	/* Spin on the CPU local node->prev pointer */
	if (tail_id != 0) {
		count = spin_retry;
		while (READ_ONCE(node->prev) != NULL) {
			if (count-- >= 0)
				continue;
			count = spin_retry;
			/* Query running state of lock holder again. */
			owner = arch_spin_yield_target(old, node);
			if (owner && arch_vcpu_is_preempted(owner - 1))
				smp_yield_cpu(owner - 1);
		}
	}

	/* Spin on the lock value in the spinlock_t */
	count = spin_retry;
	while (1) {
		old = READ_ONCE(lp->lock);
		owner = old & _Q_LOCK_CPU_MASK;
		if (!owner) {
			tail_id = old & _Q_TAIL_MASK;
			new = ((tail_id != node_id) ? tail_id : 0) | lockval;
			if (__atomic_cmpxchg_bool(&lp->lock, old, new))
				/* Got the lock */
				break;
			continue;
		}
		if (count-- >= 0)
			continue;
		count = spin_retry;
		if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(owner - 1))
			smp_yield_cpu(owner - 1);
	}

	/* Pass lock_spin job to next CPU in the queue */
	if (node_id && tail_id != node_id) {
		/* Wait until the next CPU has set up the 'next' pointer */
		while ((next = READ_ONCE(node->next)) == NULL)
			;
		next->prev = NULL;
	}

 out:
	S390_lowcore.spinlock_index--;
}

static inline void arch_spin_lock_classic(arch_spinlock_t *lp)
{
	int lockval, old, new, owner, count;

	lockval = SPINLOCK_LOCKVAL;	/* cpu + 1 */

	/* Pass the virtual CPU to the lock holder if it is not running */
	owner = arch_spin_yield_target(ACCESS_ONCE(lp->lock), NULL);
	if (owner && arch_vcpu_is_preempted(owner - 1))
		smp_yield_cpu(owner - 1);

	count = spin_retry;
	while (1) {
		old = arch_load_niai4(&lp->lock);
		owner = old & _Q_LOCK_CPU_MASK;
		/* Try to get the lock if it is free. */
		if (!owner) {
			new = (old & _Q_TAIL_MASK) | lockval;
			if (arch_cmpxchg_niai8(&lp->lock, old, new))
				/* Got the lock */
			       return;
			continue;
		}
		if (count-- >= 0)
			continue;
		count = spin_retry;
		if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(owner - 1))
			smp_yield_cpu(owner - 1);
	}
}

void arch_spin_lock_wait(arch_spinlock_t *lp)
{
	/* Use classic spinlocks + niai if the steal time is >= 10% */
	if (test_cpu_flag(CIF_DEDICATED_CPU))
		arch_spin_lock_queued(lp);
	else
		arch_spin_lock_classic(lp);
}
EXPORT_SYMBOL(arch_spin_lock_wait);

int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
	int cpu = SPINLOCK_LOCKVAL;
	int owner, count;

	for (count = spin_retry; count > 0; count--) {
		owner = READ_ONCE(lp->lock);
		/* Try to get the lock if it is free. */
		if (!owner) {
			if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
				return 1;
		}
	}
	return 0;
}
EXPORT_SYMBOL(arch_spin_trylock_retry);

void arch_read_lock_wait(arch_rwlock_t *rw)
{
	if (unlikely(in_interrupt())) {
		while (READ_ONCE(rw->cnts) & 0x10000)
			barrier();
		return;
	}

	/* Remove this reader again to allow recursive read locking */
	__atomic_add_const(-1, &rw->cnts);
	/* Put the reader into the wait queue */
	arch_spin_lock(&rw->wait);
	/* Now add this reader to the count value again */
	__atomic_add_const(1, &rw->cnts);
	/* Loop until the writer is done */
	while (READ_ONCE(rw->cnts) & 0x10000)
		barrier();
	arch_spin_unlock(&rw->wait);
}
EXPORT_SYMBOL(arch_read_lock_wait);

void arch_write_lock_wait(arch_rwlock_t *rw)
{
	int old;

	/* Add this CPU to the write waiters */
	__atomic_add(0x20000, &rw->cnts);

	/* Put the writer into the wait queue */
	arch_spin_lock(&rw->wait);

	while (1) {
		old = READ_ONCE(rw->cnts);
		if ((old & 0x1ffff) == 0 &&
		    __atomic_cmpxchg_bool(&rw->cnts, old, old | 0x10000))
			/* Got the lock */
			break;
		barrier();
	}

	arch_spin_unlock(&rw->wait);
}
EXPORT_SYMBOL(arch_write_lock_wait);

void arch_spin_relax(arch_spinlock_t *lp)
{
	int cpu;

	cpu = READ_ONCE(lp->lock) & _Q_LOCK_CPU_MASK;
	if (!cpu)
		return;
	if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted(cpu - 1))
		return;
	smp_yield_cpu(cpu - 1);
}
EXPORT_SYMBOL(arch_spin_relax);
Commit	Line	Data
951f22d5	1	/*
951f22d5 MS	2	* Out of line spinlock code.
951f22d5 MS	3	*
a53c8fab	4	* Copyright IBM Corp. 2004, 2006
951f22d5 MS	5	* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
	6	*/
	7
	8	#include <linux/types.h>
d3217967	9	#include <linux/export.h>
951f22d5	10	#include <linux/spinlock.h>
b96f7d88	11	#include <linux/jiffies.h>
951f22d5	12	#include <linux/init.h>
8b646bd7	13	#include <linux/smp.h>
b96f7d88	14	#include <linux/percpu.h>
951f22d5 MS	15	#include <asm/io.h>
951f22d5 MS	16
2c72a44e MS	17	int spin_retry = -1;
	18
	19	static int __init spin_retry_init(void)
	20	{
	21	if (spin_retry < 0)
b13de4b7	22	spin_retry = 1000;
2c72a44e MS	23	return 0;
	24	}
	25	early_initcall(spin_retry_init);
951f22d5 MS	26
	27	/**
	28	* spin_retry= parameter
	29	*/
	30	static int __init spin_retry_setup(char *str)
	31	{
	32	spin_retry = simple_strtoul(str, &str, 0);
	33	return 1;
	34	}
	35	__setup("spin_retry=", spin_retry_setup);
	36
b96f7d88 MS	37	struct spin_wait {
	38	struct spin_wait next, prev;
	39	int node_id;
	40	} __aligned(32);
	41
	42	static DEFINE_PER_CPU_ALIGNED(struct spin_wait, spin_wait[4]);
	43
	44	#define _Q_LOCK_CPU_OFFSET 0
	45	#define _Q_LOCK_STEAL_OFFSET 16
	46	#define _Q_TAIL_IDX_OFFSET 18
	47	#define _Q_TAIL_CPU_OFFSET 20
	48
	49	#define _Q_LOCK_CPU_MASK 0x0000ffff
	50	#define _Q_LOCK_STEAL_ADD 0x00010000
	51	#define _Q_LOCK_STEAL_MASK 0x00030000
	52	#define _Q_TAIL_IDX_MASK 0x000c0000
	53	#define _Q_TAIL_CPU_MASK 0xfff00000
	54
	55	#define _Q_LOCK_MASK (_Q_LOCK_CPU_MASK \| _Q_LOCK_STEAL_MASK)
	56	#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK \| _Q_TAIL_CPU_MASK)
	57
	58	void arch_spin_lock_setup(int cpu)
	59	{
	60	struct spin_wait *node;
	61	int ix;
	62
	63	node = per_cpu_ptr(&spin_wait[0], cpu);
	64	for (ix = 0; ix < 4; ix++, node++) {
	65	memset(node, 0, sizeof(*node));
	66	node->node_id = ((cpu + 1) << _Q_TAIL_CPU_OFFSET) +
	67	(ix << _Q_TAIL_IDX_OFFSET);
	68	}
	69	}
	70
7f7e6e28 MS	71	static inline int arch_load_niai4(int *lock)
	72	{
	73	int owner;
	74
	75	asm volatile(
	76	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	77	" .long 0xb2fa0040\n" /* NIAI 4 */
	78	#endif
	79	" l %0,%1\n"
	80	: "=d" (owner) : "Q" (*lock) : "memory");
	81	return owner;
	82	}
	83
	84	static inline int arch_cmpxchg_niai8(int *lock, int old, int new)
	85	{
	86	int expected = old;
	87
	88	asm volatile(
	89	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	90	" .long 0xb2fa0080\n" /* NIAI 8 */
	91	#endif
	92	" cs %0,%3,%1\n"
	93	: "=d" (old), "=Q" (*lock)
	94	: "0" (old), "d" (new), "Q" (*lock)
	95	: "cc", "memory");
	96	return expected == old;
	97	}
	98
b96f7d88	99	static inline struct spin_wait *arch_spin_decode_tail(int lock)
951f22d5	100	{
b96f7d88 MS	101	int ix, cpu;
	102
	103	ix = (lock & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
	104	cpu = (lock & _Q_TAIL_CPU_MASK) >> _Q_TAIL_CPU_OFFSET;
	105	return per_cpu_ptr(&spin_wait[ix], cpu - 1);
	106	}
	107
	108	static inline int arch_spin_yield_target(int lock, struct spin_wait *node)
	109	{
	110	if (lock & _Q_LOCK_CPU_MASK)
	111	return lock & _Q_LOCK_CPU_MASK;
	112	if (node == NULL \|\| node->prev == NULL)
	113	return 0; /* 0 -> no target cpu */
	114	while (node->prev)
	115	node = node->prev;
	116	return node->node_id >> _Q_TAIL_CPU_OFFSET;
	117	}
	118
	119	static inline void arch_spin_lock_queued(arch_spinlock_t *lp)
	120	{
	121	struct spin_wait node, next;
	122	int lockval, ix, node_id, tail_id, old, new, owner, count;
	123
	124	ix = S390_lowcore.spinlock_index++;
	125	barrier();
	126	lockval = SPINLOCK_LOCKVAL; /* cpu + 1 */
	127	node = this_cpu_ptr(&spin_wait[ix]);
	128	node->prev = node->next = NULL;
	129	node_id = node->node_id;
	130
	131	/* Enqueue the node for this CPU in the spinlock wait queue */
	132	while (1) {
	133	old = READ_ONCE(lp->lock);
	134	if ((old & _Q_LOCK_CPU_MASK) == 0 &&
	135	(old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) {
	136	/*
	137	* The lock is free but there may be waiters.
	138	* With no waiters simply take the lock, if there
	139	* are waiters try to steal the lock. The lock may
	140	* be stolen three times before the next queued
	141	* waiter will get the lock.
	142	*/
	143	new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) \| lockval;
	144	if (__atomic_cmpxchg_bool(&lp->lock, old, new))
	145	/* Got the lock */
	146	goto out;
	147	/* lock passing in progress */
	148	continue;
	149	}
	150	/* Make the node of this CPU the new tail. */
	151	new = node_id \| (old & _Q_LOCK_MASK);
	152	if (__atomic_cmpxchg_bool(&lp->lock, old, new))
	153	break;
	154	}
	155	/* Set the 'next' pointer of the tail node in the queue */
	156	tail_id = old & _Q_TAIL_MASK;
	157	if (tail_id != 0) {
	158	node->prev = arch_spin_decode_tail(tail_id);
	159	WRITE_ONCE(node->prev->next, node);
	160	}
7f7e6e28 MS	161
7f7e6e28 MS	162	/* Pass the virtual CPU to the lock holder if it is not running */
b96f7d88	163	owner = arch_spin_yield_target(old, node);
81533803 MS	164	if (owner && arch_vcpu_is_preempted(owner - 1))
81533803 MS	165	smp_yield_cpu(owner - 1);
951f22d5	166
b96f7d88 MS	167	/* Spin on the CPU local node->prev pointer */
	168	if (tail_id != 0) {
	169	count = spin_retry;
	170	while (READ_ONCE(node->prev) != NULL) {
	171	if (count-- >= 0)
	172	continue;
	173	count = spin_retry;
	174	/* Query running state of lock holder again. */
	175	owner = arch_spin_yield_target(old, node);
	176	if (owner && arch_vcpu_is_preempted(owner - 1))
	177	smp_yield_cpu(owner - 1);
	178	}
	179	}
	180
	181	/* Spin on the lock value in the spinlock_t */
7f7e6e28	182	count = spin_retry;
951f22d5	183	while (1) {
b96f7d88 MS	184	old = READ_ONCE(lp->lock);
b96f7d88 MS	185	owner = old & _Q_LOCK_CPU_MASK;
470ada6b	186	if (!owner) {
b96f7d88 MS	187	tail_id = old & _Q_TAIL_MASK;
	188	new = ((tail_id != node_id) ? tail_id : 0) \| lockval;
	189	if (__atomic_cmpxchg_bool(&lp->lock, old, new))
	190	/* Got the lock */
	191	break;
470ada6b	192	continue;
951f22d5	193	}
7f7e6e28	194	if (count-- >= 0)
470ada6b	195	continue;
470ada6b	196	count = spin_retry;
81533803 MS	197	if (!MACHINE_IS_LPAR \|\| arch_vcpu_is_preempted(owner - 1))
81533803 MS	198	smp_yield_cpu(owner - 1);
951f22d5	199	}
b96f7d88 MS	200
	201	/* Pass lock_spin job to next CPU in the queue */
	202	if (node_id && tail_id != node_id) {
	203	/* Wait until the next CPU has set up the 'next' pointer */
	204	while ((next = READ_ONCE(node->next)) == NULL)
	205	;
	206	next->prev = NULL;
	207	}
	208
	209	out:
	210	S390_lowcore.spinlock_index--;
951f22d5	211	}
951f22d5	212
b96f7d88	213	static inline void arch_spin_lock_classic(arch_spinlock_t *lp)
894cdde2	214	{
b96f7d88	215	int lockval, old, new, owner, count;
894cdde2	216
b96f7d88	217	lockval = SPINLOCK_LOCKVAL; /* cpu + 1 */
7f7e6e28 MS	218
7f7e6e28 MS	219	/* Pass the virtual CPU to the lock holder if it is not running */
b96f7d88	220	owner = arch_spin_yield_target(ACCESS_ONCE(lp->lock), NULL);
81533803 MS	221	if (owner && arch_vcpu_is_preempted(owner - 1))
81533803 MS	222	smp_yield_cpu(owner - 1);
7f7e6e28 MS	223
7f7e6e28 MS	224	count = spin_retry;
894cdde2	225	while (1) {
b96f7d88 MS	226	old = arch_load_niai4(&lp->lock);
b96f7d88 MS	227	owner = old & _Q_LOCK_CPU_MASK;
470ada6b MS	228	/* Try to get the lock if it is free. */
470ada6b MS	229	if (!owner) {
b96f7d88 MS	230	new = (old & _Q_TAIL_MASK) \| lockval;
	231	if (arch_cmpxchg_niai8(&lp->lock, old, new))
	232	/* Got the lock */
	233	return;
84976952	234	continue;
470ada6b	235	}
7f7e6e28	236	if (count-- >= 0)
470ada6b	237	continue;
470ada6b	238	count = spin_retry;
81533803 MS	239	if (!MACHINE_IS_LPAR \|\| arch_vcpu_is_preempted(owner - 1))
81533803 MS	240	smp_yield_cpu(owner - 1);
894cdde2 HH	241	}
894cdde2 HH	242	}
b96f7d88 MS	243
	244	void arch_spin_lock_wait(arch_spinlock_t *lp)
	245	{
	246	/* Use classic spinlocks + niai if the steal time is >= 10% */
	247	if (test_cpu_flag(CIF_DEDICATED_CPU))
	248	arch_spin_lock_queued(lp);
	249	else
	250	arch_spin_lock_classic(lp);
	251	}
	252	EXPORT_SYMBOL(arch_spin_lock_wait);
894cdde2	253
0199c4e6	254	int arch_spin_trylock_retry(arch_spinlock_t *lp)
951f22d5	255	{
02c503ff MS	256	int cpu = SPINLOCK_LOCKVAL;
02c503ff MS	257	int owner, count;
951f22d5	258
2c72a44e	259	for (count = spin_retry; count > 0; count--) {
187b5f41	260	owner = READ_ONCE(lp->lock);
2c72a44e MS	261	/* Try to get the lock if it is free. */
2c72a44e MS	262	if (!owner) {
02c503ff	263	if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
2c72a44e	264	return 1;
b13de4b7	265	}
2c72a44e	266	}
951f22d5 MS	267	return 0;
951f22d5 MS	268	}
0199c4e6	269	EXPORT_SYMBOL(arch_spin_trylock_retry);
951f22d5	270
eb3b7b84	271	void arch_read_lock_wait(arch_rwlock_t *rw)
951f22d5	272	{
eb3b7b84 MS	273	if (unlikely(in_interrupt())) {
	274	while (READ_ONCE(rw->cnts) & 0x10000)
	275	barrier();
	276	return;
951f22d5	277	}
951f22d5	278
eb3b7b84 MS	279	/* Remove this reader again to allow recursive read locking */
	280	__atomic_add_const(-1, &rw->cnts);
	281	/* Put the reader into the wait queue */
	282	arch_spin_lock(&rw->wait);
	283	/* Now add this reader to the count value again */
	284	__atomic_add_const(1, &rw->cnts);
	285	/* Loop until the writer is done */
	286	while (READ_ONCE(rw->cnts) & 0x10000)
	287	barrier();
	288	arch_spin_unlock(&rw->wait);
951f22d5	289	}
eb3b7b84	290	EXPORT_SYMBOL(arch_read_lock_wait);
951f22d5	291
eb3b7b84	292	void arch_write_lock_wait(arch_rwlock_t *rw)
bbae71bf	293	{
eb3b7b84	294	int old;
bbae71bf	295
eb3b7b84 MS	296	/* Add this CPU to the write waiters */
eb3b7b84 MS	297	__atomic_add(0x20000, &rw->cnts);
bbae71bf	298
eb3b7b84 MS	299	/* Put the writer into the wait queue */
eb3b7b84 MS	300	arch_spin_lock(&rw->wait);
951f22d5 MS	301
951f22d5 MS	302	while (1) {
eb3b7b84 MS	303	old = READ_ONCE(rw->cnts);
	304	if ((old & 0x1ffff) == 0 &&
	305	__atomic_cmpxchg_bool(&rw->cnts, old, old \| 0x10000))
	306	/* Got the lock */
94232a43	307	break;
eb3b7b84	308	barrier();
951f22d5	309	}
d59b93da	310
eb3b7b84	311	arch_spin_unlock(&rw->wait);
d59b93da	312	}
eb3b7b84	313	EXPORT_SYMBOL(arch_write_lock_wait);
b96f7d88 MS	314
	315	void arch_spin_relax(arch_spinlock_t *lp)
	316	{
	317	int cpu;
	318
	319	cpu = READ_ONCE(lp->lock) & _Q_LOCK_CPU_MASK;
	320	if (!cpu)
	321	return;
	322	if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted(cpu - 1))
	323	return;
	324	smp_yield_cpu(cpu - 1);
	325	}
	326	EXPORT_SYMBOL(arch_spin_relax);