[mirror_ubuntu-artful-kernel.git] / arch / s390 / lib / spinlock.c

/*
 *    Out of line spinlock code.
 *
 *    Copyright IBM Corp. 2004, 2006
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <asm/io.h>

int spin_retry = 1000;

/**
 * spin_retry= parameter
 */
static int __init spin_retry_setup(char *str)
{
	spin_retry = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("spin_retry=", spin_retry_setup);

void arch_spin_lock_wait(arch_spinlock_t *lp)
{
	int count = spin_retry;
	unsigned int cpu = ~smp_processor_id();
	unsigned int owner;

	while (1) {
		owner = lp->owner_cpu;
		if (!owner || smp_vcpu_scheduled(~owner)) {
			for (count = spin_retry; count > 0; count--) {
				if (arch_spin_is_locked(lp))
					continue;
				if (_raw_compare_and_swap(&lp->owner_cpu, 0,
							  cpu) == 0)
					return;
			}
			if (MACHINE_IS_LPAR)
				continue;
		}
		owner = lp->owner_cpu;
		if (owner)
			smp_yield_cpu(~owner);
		if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
			return;
	}
}
EXPORT_SYMBOL(arch_spin_lock_wait);

void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
{
	int count = spin_retry;
	unsigned int cpu = ~smp_processor_id();
	unsigned int owner;

	local_irq_restore(flags);
	while (1) {
		owner = lp->owner_cpu;
		if (!owner || smp_vcpu_scheduled(~owner)) {
			for (count = spin_retry; count > 0; count--) {
				if (arch_spin_is_locked(lp))
					continue;
				local_irq_disable();
				if (_raw_compare_and_swap(&lp->owner_cpu, 0,
							  cpu) == 0)
					return;
				local_irq_restore(flags);
			}
			if (MACHINE_IS_LPAR)
				continue;
		}
		owner = lp->owner_cpu;
		if (owner)
			smp_yield_cpu(~owner);
		local_irq_disable();
		if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
			return;
		local_irq_restore(flags);
	}
}
EXPORT_SYMBOL(arch_spin_lock_wait_flags);

int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
	unsigned int cpu = ~smp_processor_id();
	int count;

	for (count = spin_retry; count > 0; count--) {
		if (arch_spin_is_locked(lp))
			continue;
		if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(arch_spin_trylock_retry);

void arch_spin_relax(arch_spinlock_t *lock)
{
	unsigned int cpu = lock->owner_cpu;
	if (cpu != 0) {
		if (MACHINE_IS_VM || MACHINE_IS_KVM ||
		    !smp_vcpu_scheduled(~cpu))
			smp_yield_cpu(~cpu);
	}
}
EXPORT_SYMBOL(arch_spin_relax);

void _raw_read_lock_wait(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		if (!arch_read_can_lock(rw))
			continue;
		old = rw->lock & 0x7fffffffU;
		if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
			return;
	}
}
EXPORT_SYMBOL(_raw_read_lock_wait);

void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
{
	unsigned int old;
	int count = spin_retry;

	local_irq_restore(flags);
	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		if (!arch_read_can_lock(rw))
			continue;
		old = rw->lock & 0x7fffffffU;
		local_irq_disable();
		if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
			return;
	}
}
EXPORT_SYMBOL(_raw_read_lock_wait_flags);

int _raw_read_trylock_retry(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (count-- > 0) {
		if (!arch_read_can_lock(rw))
			continue;
		old = rw->lock & 0x7fffffffU;
		if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(_raw_read_trylock_retry);

void _raw_write_lock_wait(arch_rwlock_t *rw)
{
	int count = spin_retry;

	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		if (!arch_write_can_lock(rw))
			continue;
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
			return;
	}
}
EXPORT_SYMBOL(_raw_write_lock_wait);

void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
{
	int count = spin_retry;

	local_irq_restore(flags);
	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		if (!arch_write_can_lock(rw))
			continue;
		local_irq_disable();
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
			return;
	}
}
EXPORT_SYMBOL(_raw_write_lock_wait_flags);

int _raw_write_trylock_retry(arch_rwlock_t *rw)
{
	int count = spin_retry;

	while (count-- > 0) {
		if (!arch_write_can_lock(rw))
			continue;
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(_raw_write_trylock_retry);
Commit	Line	Data
951f22d5	1	/*
951f22d5 MS	2	* Out of line spinlock code.
951f22d5 MS	3	*
a53c8fab	4	* Copyright IBM Corp. 2004, 2006
951f22d5 MS	5	* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
	6	*/
	7
	8	#include <linux/types.h>
	9	#include <linux/module.h>
	10	#include <linux/spinlock.h>
	11	#include <linux/init.h>
8b646bd7	12	#include <linux/smp.h>
951f22d5 MS	13	#include <asm/io.h>
951f22d5 MS	14
951f22d5 MS	15	int spin_retry = 1000;
	16
	17	/**
	18	* spin_retry= parameter
	19	*/
	20	static int __init spin_retry_setup(char *str)
	21	{
	22	spin_retry = simple_strtoul(str, &str, 0);
	23	return 1;
	24	}
	25	__setup("spin_retry=", spin_retry_setup);
	26
0199c4e6	27	void arch_spin_lock_wait(arch_spinlock_t *lp)
951f22d5 MS	28	{
951f22d5 MS	29	int count = spin_retry;
3c1fcfe2	30	unsigned int cpu = ~smp_processor_id();
59b69787	31	unsigned int owner;
951f22d5 MS	32
951f22d5 MS	33	while (1) {
59b69787 GS	34	owner = lp->owner_cpu;
	35	if (!owner \|\| smp_vcpu_scheduled(~owner)) {
	36	for (count = spin_retry; count > 0; count--) {
	37	if (arch_spin_is_locked(lp))
	38	continue;
	39	if (_raw_compare_and_swap(&lp->owner_cpu, 0,
	40	cpu) == 0)
	41	return;
	42	}
	43	if (MACHINE_IS_LPAR)
	44	continue;
951f22d5	45	}
59b69787 GS	46	owner = lp->owner_cpu;
59b69787 GS	47	if (owner)
8b646bd7	48	smp_yield_cpu(~owner);
3b4beb31	49	if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
951f22d5 MS	50	return;
	51	}
	52	}
0199c4e6	53	EXPORT_SYMBOL(arch_spin_lock_wait);
951f22d5	54
0199c4e6	55	void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
894cdde2 HH	56	{
	57	int count = spin_retry;
	58	unsigned int cpu = ~smp_processor_id();
59b69787	59	unsigned int owner;
894cdde2 HH	60
	61	local_irq_restore(flags);
	62	while (1) {
59b69787 GS	63	owner = lp->owner_cpu;
	64	if (!owner \|\| smp_vcpu_scheduled(~owner)) {
	65	for (count = spin_retry; count > 0; count--) {
	66	if (arch_spin_is_locked(lp))
	67	continue;
	68	local_irq_disable();
	69	if (_raw_compare_and_swap(&lp->owner_cpu, 0,
	70	cpu) == 0)
	71	return;
	72	local_irq_restore(flags);
	73	}
	74	if (MACHINE_IS_LPAR)
	75	continue;
894cdde2	76	}
59b69787 GS	77	owner = lp->owner_cpu;
59b69787 GS	78	if (owner)
8b646bd7	79	smp_yield_cpu(~owner);
894cdde2 HH	80	local_irq_disable();
	81	if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
	82	return;
	83	local_irq_restore(flags);
	84	}
	85	}
0199c4e6	86	EXPORT_SYMBOL(arch_spin_lock_wait_flags);
894cdde2	87
0199c4e6	88	int arch_spin_trylock_retry(arch_spinlock_t *lp)
951f22d5	89	{
3c1fcfe2 MS	90	unsigned int cpu = ~smp_processor_id();
3c1fcfe2 MS	91	int count;
951f22d5	92
3c1fcfe2	93	for (count = spin_retry; count > 0; count--) {
0199c4e6	94	if (arch_spin_is_locked(lp))
96567161	95	continue;
3b4beb31	96	if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
951f22d5 MS	97	return 1;
	98	}
	99	return 0;
	100	}
0199c4e6	101	EXPORT_SYMBOL(arch_spin_trylock_retry);
951f22d5	102
0199c4e6	103	void arch_spin_relax(arch_spinlock_t *lock)
3c1fcfe2 MS	104	{
3c1fcfe2 MS	105	unsigned int cpu = lock->owner_cpu;
59b69787 GS	106	if (cpu != 0) {
	107	if (MACHINE_IS_VM \|\| MACHINE_IS_KVM \|\|
	108	!smp_vcpu_scheduled(~cpu))
8b646bd7	109	smp_yield_cpu(~cpu);
59b69787	110	}
3c1fcfe2	111	}
0199c4e6	112	EXPORT_SYMBOL(arch_spin_relax);
3c1fcfe2	113
fb3a6bbc	114	void _raw_read_lock_wait(arch_rwlock_t *rw)
951f22d5 MS	115	{
	116	unsigned int old;
	117	int count = spin_retry;
	118
	119	while (1) {
	120	if (count-- <= 0) {
8b646bd7	121	smp_yield();
951f22d5 MS	122	count = spin_retry;
951f22d5 MS	123	}
e5931943	124	if (!arch_read_can_lock(rw))
96567161	125	continue;
951f22d5 MS	126	old = rw->lock & 0x7fffffffU;
	127	if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
	128	return;
	129	}
	130	}
	131	EXPORT_SYMBOL(_raw_read_lock_wait);
	132
fb3a6bbc	133	void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
ce58ae6f HC	134	{
	135	unsigned int old;
	136	int count = spin_retry;
	137
	138	local_irq_restore(flags);
	139	while (1) {
	140	if (count-- <= 0) {
8b646bd7	141	smp_yield();
ce58ae6f HC	142	count = spin_retry;
ce58ae6f HC	143	}
e5931943	144	if (!arch_read_can_lock(rw))
ce58ae6f HC	145	continue;
	146	old = rw->lock & 0x7fffffffU;
	147	local_irq_disable();
	148	if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
	149	return;
	150	}
	151	}
	152	EXPORT_SYMBOL(_raw_read_lock_wait_flags);
	153
fb3a6bbc	154	int _raw_read_trylock_retry(arch_rwlock_t *rw)
951f22d5 MS	155	{
	156	unsigned int old;
	157	int count = spin_retry;
	158
	159	while (count-- > 0) {
e5931943	160	if (!arch_read_can_lock(rw))
96567161	161	continue;
951f22d5 MS	162	old = rw->lock & 0x7fffffffU;
	163	if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
	164	return 1;
	165	}
	166	return 0;
	167	}
	168	EXPORT_SYMBOL(_raw_read_trylock_retry);
	169
fb3a6bbc	170	void _raw_write_lock_wait(arch_rwlock_t *rw)
951f22d5 MS	171	{
	172	int count = spin_retry;
	173
	174	while (1) {
	175	if (count-- <= 0) {
8b646bd7	176	smp_yield();
951f22d5 MS	177	count = spin_retry;
951f22d5 MS	178	}
e5931943	179	if (!arch_write_can_lock(rw))
96567161	180	continue;
951f22d5 MS	181	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
	182	return;
	183	}
	184	}
	185	EXPORT_SYMBOL(_raw_write_lock_wait);
	186
fb3a6bbc	187	void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
ce58ae6f HC	188	{
	189	int count = spin_retry;
	190
	191	local_irq_restore(flags);
	192	while (1) {
	193	if (count-- <= 0) {
8b646bd7	194	smp_yield();
ce58ae6f HC	195	count = spin_retry;
ce58ae6f HC	196	}
e5931943	197	if (!arch_write_can_lock(rw))
ce58ae6f HC	198	continue;
	199	local_irq_disable();
	200	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
	201	return;
	202	}
	203	}
	204	EXPORT_SYMBOL(_raw_write_lock_wait_flags);
	205
fb3a6bbc	206	int _raw_write_trylock_retry(arch_rwlock_t *rw)
951f22d5 MS	207	{
	208	int count = spin_retry;
	209
	210	while (count-- > 0) {
e5931943	211	if (!arch_write_can_lock(rw))
96567161	212	continue;
951f22d5 MS	213	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
	214	return 1;
	215	}
	216	return 0;
	217	}
	218	EXPORT_SYMBOL(_raw_write_trylock_retry);