[mirror_zfs.git] / lib / libspl / asm-generic / atomic.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2009 by Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <atomic.h>
#include <assert.h>
#include <pthread.h>

/*
 * All operations are implemented by serializing them through a global
 * pthread mutex.  This provides a correct generic implementation.
 * However all supported architectures are encouraged to provide a
 * native implementation is assembly for performance reasons.
 */
pthread_mutex_t atomic_lock = PTHREAD_MUTEX_INITIALIZER;

/*
 * Theses are the void returning variants
 */

#define	ATOMIC_INC(name, type) \
	void atomic_inc_##name(volatile type *target)			\
	{								\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		(*target)++;						\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
	}

ATOMIC_INC(long, unsigned long)
ATOMIC_INC(8, uint8_t)
ATOMIC_INC(uchar, uchar_t)
ATOMIC_INC(16, uint16_t)
ATOMIC_INC(ushort, ushort_t)
ATOMIC_INC(32, uint32_t)
ATOMIC_INC(uint, uint_t)
ATOMIC_INC(ulong, ulong_t)
ATOMIC_INC(64, uint64_t)


#define	ATOMIC_DEC(name, type) \
	void atomic_dec_##name(volatile type *target)			\
	{								\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		(*target)--;						\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
	}

ATOMIC_DEC(long, unsigned long)
ATOMIC_DEC(8, uint8_t)
ATOMIC_DEC(uchar, uchar_t)
ATOMIC_DEC(16, uint16_t)
ATOMIC_DEC(ushort, ushort_t)
ATOMIC_DEC(32, uint32_t)
ATOMIC_DEC(uint, uint_t)
ATOMIC_DEC(ulong, ulong_t)
ATOMIC_DEC(64, uint64_t)


#define	ATOMIC_ADD(name, type1, type2) \
	void atomic_add_##name(volatile type1 *target, type2 bits)	\
	{								\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		*target += bits;					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
	}

ATOMIC_ADD(8, uint8_t, int8_t)
ATOMIC_ADD(char, uchar_t, signed char)
ATOMIC_ADD(16, uint16_t, int16_t)
ATOMIC_ADD(short, ushort_t, short)
ATOMIC_ADD(32, uint32_t, int32_t)
ATOMIC_ADD(int, uint_t, int)
ATOMIC_ADD(long, ulong_t, long)
ATOMIC_ADD(64, uint64_t, int64_t)

void
atomic_add_ptr(volatile void *target, ssize_t bits)
{
	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	*(caddr_t *)target += bits;
	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
}


#define	ATOMIC_SUB(name, type1, type2) \
	void atomic_sub_##name(volatile type1 *target, type2 bits)	\
	{								\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		*target -= bits;					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
	}

ATOMIC_SUB(8, uint8_t, int8_t)
ATOMIC_SUB(char, uchar_t, signed char)
ATOMIC_SUB(16, uint16_t, int16_t)
ATOMIC_SUB(short, ushort_t, short)
ATOMIC_SUB(32, uint32_t, int32_t)
ATOMIC_SUB(int, uint_t, int)
ATOMIC_SUB(long, ulong_t, long)
ATOMIC_SUB(64, uint64_t, int64_t)

void
atomic_sub_ptr(volatile void *target, ssize_t bits)
{
	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	*(caddr_t *)target -= bits;
	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
}


#define	ATOMIC_OR(name, type) \
	void atomic_or_##name(volatile type *target, type bits)		\
	{								\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		*target |= bits;					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
	}

ATOMIC_OR(8, uint8_t)
ATOMIC_OR(uchar, uchar_t)
ATOMIC_OR(16, uint16_t)
ATOMIC_OR(ushort, ushort_t)
ATOMIC_OR(32, uint32_t)
ATOMIC_OR(uint, uint_t)
ATOMIC_OR(ulong, ulong_t)
ATOMIC_OR(64, uint64_t)


#define	ATOMIC_AND(name, type) \
	void atomic_and_##name(volatile type *target, type bits)	\
	{								\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		*target &= bits;					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
	}

ATOMIC_AND(8, uint8_t)
ATOMIC_AND(uchar, uchar_t)
ATOMIC_AND(16, uint16_t)
ATOMIC_AND(ushort, ushort_t)
ATOMIC_AND(32, uint32_t)
ATOMIC_AND(uint, uint_t)
ATOMIC_AND(ulong, ulong_t)
ATOMIC_AND(64, uint64_t)


/*
 * New value returning variants
 */

#define	ATOMIC_INC_NV(name, type) \
	type atomic_inc_##name##_nv(volatile type *target)		\
	{								\
		type rc;						\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		rc = (++(*target));					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
		return (rc);						\
	}

ATOMIC_INC_NV(long, unsigned long)
ATOMIC_INC_NV(8, uint8_t)
ATOMIC_INC_NV(uchar, uchar_t)
ATOMIC_INC_NV(16, uint16_t)
ATOMIC_INC_NV(ushort, ushort_t)
ATOMIC_INC_NV(32, uint32_t)
ATOMIC_INC_NV(uint, uint_t)
ATOMIC_INC_NV(ulong, ulong_t)
ATOMIC_INC_NV(64, uint64_t)


#define	ATOMIC_DEC_NV(name, type) \
	type atomic_dec_##name##_nv(volatile type *target)		\
	{								\
		type rc;						\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		rc = (--(*target));					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
		return (rc);						\
	}

ATOMIC_DEC_NV(long, unsigned long)
ATOMIC_DEC_NV(8, uint8_t)
ATOMIC_DEC_NV(uchar, uchar_t)
ATOMIC_DEC_NV(16, uint16_t)
ATOMIC_DEC_NV(ushort, ushort_t)
ATOMIC_DEC_NV(32, uint32_t)
ATOMIC_DEC_NV(uint, uint_t)
ATOMIC_DEC_NV(ulong, ulong_t)
ATOMIC_DEC_NV(64, uint64_t)


#define	ATOMIC_ADD_NV(name, type1, type2) \
	type1 atomic_add_##name##_nv(volatile type1 *target, type2 bits)\
	{								\
		type1 rc;						\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		rc = (*target += bits);					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
		return (rc);						\
	}

ATOMIC_ADD_NV(8, uint8_t, int8_t)
ATOMIC_ADD_NV(char, uchar_t, signed char)
ATOMIC_ADD_NV(16, uint16_t, int16_t)
ATOMIC_ADD_NV(short, ushort_t, short)
ATOMIC_ADD_NV(32, uint32_t, int32_t)
ATOMIC_ADD_NV(int, uint_t, int)
ATOMIC_ADD_NV(long, ulong_t, long)
ATOMIC_ADD_NV(64, uint64_t, int64_t)

void *
atomic_add_ptr_nv(volatile void *target, ssize_t bits)
{
	void *ptr;

	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	ptr = (*(caddr_t *)target += bits);
	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);

	return (ptr);
}


#define	ATOMIC_SUB_NV(name, type1, type2) \
	type1 atomic_sub_##name##_nv(volatile type1 *target, type2 bits)\
	{								\
		type1 rc;						\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		rc = (*target -= bits);					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
		return (rc);						\
	}

ATOMIC_SUB_NV(8, uint8_t, int8_t)
ATOMIC_SUB_NV(char, uchar_t, signed char)
ATOMIC_SUB_NV(16, uint16_t, int16_t)
ATOMIC_SUB_NV(short, ushort_t, short)
ATOMIC_SUB_NV(32, uint32_t, int32_t)
ATOMIC_SUB_NV(int, uint_t, int)
ATOMIC_SUB_NV(long, ulong_t, long)
ATOMIC_SUB_NV(64, uint64_t, int64_t)

void *
atomic_sub_ptr_nv(volatile void *target, ssize_t bits)
{
	void *ptr;

	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	ptr = (*(caddr_t *)target -= bits);
	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);

	return (ptr);
}


#define	ATOMIC_OR_NV(name, type) \
	type atomic_or_##name##_nv(volatile type *target, type bits)	\
	{								\
		type rc;						\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		rc = (*target |= bits);					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
		return (rc);						\
	}

ATOMIC_OR_NV(long, unsigned long)
ATOMIC_OR_NV(8, uint8_t)
ATOMIC_OR_NV(uchar, uchar_t)
ATOMIC_OR_NV(16, uint16_t)
ATOMIC_OR_NV(ushort, ushort_t)
ATOMIC_OR_NV(32, uint32_t)
ATOMIC_OR_NV(uint, uint_t)
ATOMIC_OR_NV(ulong, ulong_t)
ATOMIC_OR_NV(64, uint64_t)


#define	ATOMIC_AND_NV(name, type) \
	type atomic_and_##name##_nv(volatile type *target, type bits)	\
	{								\
		type rc;						\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		rc = (*target &= bits);					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
		return (rc);						\
	}

ATOMIC_AND_NV(long, unsigned long)
ATOMIC_AND_NV(8, uint8_t)
ATOMIC_AND_NV(uchar, uchar_t)
ATOMIC_AND_NV(16, uint16_t)
ATOMIC_AND_NV(ushort, ushort_t)
ATOMIC_AND_NV(32, uint32_t)
ATOMIC_AND_NV(uint, uint_t)
ATOMIC_AND_NV(ulong, ulong_t)
ATOMIC_AND_NV(64, uint64_t)


/*
 *  If *arg1 == arg2, set *arg1 = arg3; return old value
 */

#define	ATOMIC_CAS(name, type) \
	type atomic_cas_##name(volatile type *target, type arg1, type arg2) \
	{								\
		type old;						\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		old = *target;						\
		if (old == arg1)					\
			*target = arg2;					\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
		return (old);						\
	}

ATOMIC_CAS(8, uint8_t)
ATOMIC_CAS(uchar, uchar_t)
ATOMIC_CAS(16, uint16_t)
ATOMIC_CAS(ushort, ushort_t)
ATOMIC_CAS(32, uint32_t)
ATOMIC_CAS(uint, uint_t)
ATOMIC_CAS(ulong, ulong_t)
ATOMIC_CAS(64, uint64_t)

void *
atomic_cas_ptr(volatile void *target, void *arg1, void *arg2)
{
	void *old;

	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	old = *(void **)target;
	if (old == arg1)
		*(void **)target = arg2;
	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);

	return (old);
}


/*
 * Swap target and return old value
 */

#define	ATOMIC_SWAP(name, type) \
	type atomic_swap_##name(volatile type *target, type bits)	\
	{								\
		type old;						\
		VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);	\
		old = *target;						\
		*target = bits;						\
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);	\
		return (old);						\
	}

ATOMIC_SWAP(8, uint8_t)
ATOMIC_SWAP(uchar, uchar_t)
ATOMIC_SWAP(16, uint16_t)
ATOMIC_SWAP(ushort, ushort_t)
ATOMIC_SWAP(32, uint32_t)
ATOMIC_SWAP(uint, uint_t)
ATOMIC_SWAP(ulong, ulong_t)
ATOMIC_SWAP(64, uint64_t)

void *
atomic_swap_ptr(volatile void *target, void *bits)
{
	void *old;

	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	old = *(void **)target;
	*(void **)target = bits;
	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);

	return (old);
}


int
atomic_set_long_excl(volatile ulong_t *target, uint_t value)
{
	ulong_t bit;

	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	bit = (1UL << value);
	if ((*target & bit) != 0) {
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
		return (-1);
	}
	*target |= bit;
	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);

	return (0);
}

int
atomic_clear_long_excl(volatile ulong_t *target, uint_t value)
{
	ulong_t bit;

	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	bit = (1UL << value);
	if ((*target & bit) != 0) {
		VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
		return (-1);
	}
	*target &= ~bit;
	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);

	return (0);
}

void
membar_enter(void)
{
	/* XXX - Implement me */
}

void
membar_exit(void)
{
	/* XXX - Implement me */
}

void
membar_producer(void)
{
	/* XXX - Implement me */
}

void
membar_consumer(void)
{
	/* XXX - Implement me */
}

/* Legacy kernel interfaces; they will go away (eventually). */

uint8_t
cas8(uint8_t *target, uint8_t arg1, uint8_t arg2)
{
	return (atomic_cas_8(target, arg1, arg2));
}

uint32_t
cas32(uint32_t *target, uint32_t arg1, uint32_t arg2)
{
	return (atomic_cas_32(target, arg1, arg2));
}

uint64_t
cas64(uint64_t *target, uint64_t arg1, uint64_t arg2)
{
	return (atomic_cas_64(target, arg1, arg2));
}

ulong_t
caslong(ulong_t *target, ulong_t arg1, ulong_t arg2)
{
	return (atomic_cas_ulong(target, arg1, arg2));
}

void *
casptr(void *target, void *arg1, void *arg2)
{
	return (atomic_cas_ptr(target, arg1, arg2));
}

void
atomic_and_long(ulong_t *target, ulong_t bits)
{
	return (atomic_and_ulong(target, bits));
}

void
atomic_or_long(ulong_t *target, ulong_t bits)
{
	return (atomic_or_ulong(target, bits));
}
Commit	Line	Data
a26baf28 BB	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License, Version 1.0 only
	6	* (the "License"). You may not use this file except in compliance
	7	* with the License.
	8	*
	9	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	10	* or http://www.opensolaris.org/os/licensing.
	11	* See the License for the specific language governing permissions
	12	* and limitations under the License.
	13	*
	14	* When distributing Covered Code, include this CDDL HEADER in each
	15	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	16	* If applicable, add the following below this CDDL HEADER, with the
	17	* fields enclosed by brackets "[]" replaced with your own identifying
	18	* information: Portions Copyright [yyyy] [name of copyright owner]
	19	*
	20	* CDDL HEADER END
	21	*/
	22	/*
	23	* Copyright (c) 2009 by Sun Microsystems, Inc. All rights reserved.
	24	* Use is subject to license terms.
	25	*/
	26
	27	#include <atomic.h>
	28	#include <assert.h>
	29	#include <pthread.h>
	30
	31	/*
	32	* All operations are implemented by serializing them through a global
	33	* pthread mutex. This provides a correct generic implementation.
	34	* However all supported architectures are encouraged to provide a
	35	* native implementation is assembly for performance reasons.
	36	*/
	37	pthread_mutex_t atomic_lock = PTHREAD_MUTEX_INITIALIZER;
	38
	39	/*
	40	* Theses are the void returning variants
	41	*/
	42
d1d7e268	43	#define ATOMIC_INC(name, type) \
a26baf28 BB	44	void atomic_inc_##name(volatile type *target) \
	45	{ \
	46	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	47	(*target)++; \
	48	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
	49	}
	50
	51	ATOMIC_INC(long, unsigned long)
	52	ATOMIC_INC(8, uint8_t)
	53	ATOMIC_INC(uchar, uchar_t)
	54	ATOMIC_INC(16, uint16_t)
	55	ATOMIC_INC(ushort, ushort_t)
	56	ATOMIC_INC(32, uint32_t)
	57	ATOMIC_INC(uint, uint_t)
	58	ATOMIC_INC(ulong, ulong_t)
	59	ATOMIC_INC(64, uint64_t)
	60
	61
d1d7e268	62	#define ATOMIC_DEC(name, type) \
a26baf28 BB	63	void atomic_dec_##name(volatile type *target) \
	64	{ \
	65	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	66	(*target)--; \
	67	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
	68	}
	69
	70	ATOMIC_DEC(long, unsigned long)
	71	ATOMIC_DEC(8, uint8_t)
	72	ATOMIC_DEC(uchar, uchar_t)
	73	ATOMIC_DEC(16, uint16_t)
	74	ATOMIC_DEC(ushort, ushort_t)
	75	ATOMIC_DEC(32, uint32_t)
	76	ATOMIC_DEC(uint, uint_t)
	77	ATOMIC_DEC(ulong, ulong_t)
	78	ATOMIC_DEC(64, uint64_t)
	79
	80
d1d7e268	81	#define ATOMIC_ADD(name, type1, type2) \
a26baf28 BB	82	void atomic_add_##name(volatile type1 *target, type2 bits) \
	83	{ \
	84	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	85	*target += bits; \
	86	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
	87	}
	88
	89	ATOMIC_ADD(8, uint8_t, int8_t)
	90	ATOMIC_ADD(char, uchar_t, signed char)
	91	ATOMIC_ADD(16, uint16_t, int16_t)
	92	ATOMIC_ADD(short, ushort_t, short)
	93	ATOMIC_ADD(32, uint32_t, int32_t)
	94	ATOMIC_ADD(int, uint_t, int)
	95	ATOMIC_ADD(long, ulong_t, long)
	96	ATOMIC_ADD(64, uint64_t, int64_t)
	97
d1d7e268 MK	98	void
d1d7e268 MK	99	atomic_add_ptr(volatile void *target, ssize_t bits)
a26baf28 BB	100	{
	101	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	102	(caddr_t )target += bits;
	103	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
	104	}
	105
	106
d1d7e268	107	#define ATOMIC_SUB(name, type1, type2) \
142e6dd1 ED	108	void atomic_sub_##name(volatile type1 *target, type2 bits) \
	109	{ \
	110	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	111	*target -= bits; \
	112	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
	113	}
	114
	115	ATOMIC_SUB(8, uint8_t, int8_t)
	116	ATOMIC_SUB(char, uchar_t, signed char)
	117	ATOMIC_SUB(16, uint16_t, int16_t)
	118	ATOMIC_SUB(short, ushort_t, short)
	119	ATOMIC_SUB(32, uint32_t, int32_t)
	120	ATOMIC_SUB(int, uint_t, int)
	121	ATOMIC_SUB(long, ulong_t, long)
	122	ATOMIC_SUB(64, uint64_t, int64_t)
	123
d1d7e268 MK	124	void
d1d7e268 MK	125	atomic_sub_ptr(volatile void *target, ssize_t bits)
142e6dd1 ED	126	{
	127	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	128	(caddr_t )target -= bits;
	129	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
	130	}
	131
	132
d1d7e268	133	#define ATOMIC_OR(name, type) \
a26baf28 BB	134	void atomic_or_##name(volatile type *target, type bits) \
	135	{ \
	136	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	137	*target \|= bits; \
	138	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
	139	}
	140
	141	ATOMIC_OR(8, uint8_t)
	142	ATOMIC_OR(uchar, uchar_t)
	143	ATOMIC_OR(16, uint16_t)
	144	ATOMIC_OR(ushort, ushort_t)
	145	ATOMIC_OR(32, uint32_t)
	146	ATOMIC_OR(uint, uint_t)
	147	ATOMIC_OR(ulong, ulong_t)
	148	ATOMIC_OR(64, uint64_t)
	149
	150
d1d7e268	151	#define ATOMIC_AND(name, type) \
a26baf28 BB	152	void atomic_and_##name(volatile type *target, type bits) \
	153	{ \
	154	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	155	*target &= bits; \
	156	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
	157	}
	158
	159	ATOMIC_AND(8, uint8_t)
	160	ATOMIC_AND(uchar, uchar_t)
	161	ATOMIC_AND(16, uint16_t)
	162	ATOMIC_AND(ushort, ushort_t)
	163	ATOMIC_AND(32, uint32_t)
	164	ATOMIC_AND(uint, uint_t)
	165	ATOMIC_AND(ulong, ulong_t)
	166	ATOMIC_AND(64, uint64_t)
	167
	168
	169	/*
	170	* New value returning variants
	171	*/
	172
d1d7e268	173	#define ATOMIC_INC_NV(name, type) \
a26baf28 BB	174	type atomic_inc_##name##_nv(volatile type *target) \
	175	{ \
	176	type rc; \
	177	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	178	rc = (++(*target)); \
	179	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
d1d7e268	180	return (rc); \
a26baf28 BB	181	}
	182
	183	ATOMIC_INC_NV(long, unsigned long)
	184	ATOMIC_INC_NV(8, uint8_t)
	185	ATOMIC_INC_NV(uchar, uchar_t)
	186	ATOMIC_INC_NV(16, uint16_t)
	187	ATOMIC_INC_NV(ushort, ushort_t)
	188	ATOMIC_INC_NV(32, uint32_t)
	189	ATOMIC_INC_NV(uint, uint_t)
	190	ATOMIC_INC_NV(ulong, ulong_t)
	191	ATOMIC_INC_NV(64, uint64_t)
	192
	193
d1d7e268	194	#define ATOMIC_DEC_NV(name, type) \
a26baf28 BB	195	type atomic_dec_##name##_nv(volatile type *target) \
	196	{ \
	197	type rc; \
	198	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	199	rc = (--(*target)); \
	200	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
d1d7e268	201	return (rc); \
a26baf28 BB	202	}
	203
	204	ATOMIC_DEC_NV(long, unsigned long)
	205	ATOMIC_DEC_NV(8, uint8_t)
	206	ATOMIC_DEC_NV(uchar, uchar_t)
	207	ATOMIC_DEC_NV(16, uint16_t)
	208	ATOMIC_DEC_NV(ushort, ushort_t)
	209	ATOMIC_DEC_NV(32, uint32_t)
	210	ATOMIC_DEC_NV(uint, uint_t)
	211	ATOMIC_DEC_NV(ulong, ulong_t)
	212	ATOMIC_DEC_NV(64, uint64_t)
	213
	214
d1d7e268	215	#define ATOMIC_ADD_NV(name, type1, type2) \
a26baf28 BB	216	type1 atomic_add_##name##_nv(volatile type1 *target, type2 bits)\
	217	{ \
	218	type1 rc; \
	219	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	220	rc = (*target += bits); \
	221	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
d1d7e268	222	return (rc); \
a26baf28 BB	223	}
	224
	225	ATOMIC_ADD_NV(8, uint8_t, int8_t)
	226	ATOMIC_ADD_NV(char, uchar_t, signed char)
	227	ATOMIC_ADD_NV(16, uint16_t, int16_t)
	228	ATOMIC_ADD_NV(short, ushort_t, short)
	229	ATOMIC_ADD_NV(32, uint32_t, int32_t)
	230	ATOMIC_ADD_NV(int, uint_t, int)
	231	ATOMIC_ADD_NV(long, ulong_t, long)
	232	ATOMIC_ADD_NV(64, uint64_t, int64_t)
	233
d1d7e268 MK	234	void *
d1d7e268 MK	235	atomic_add_ptr_nv(volatile void *target, ssize_t bits)
a26baf28 BB	236	{
	237	void *ptr;
	238
	239	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	240	ptr = ((caddr_t )target += bits);
	241	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
	242
d1d7e268	243	return (ptr);
a26baf28 BB	244	}
	245
	246
d1d7e268	247	#define ATOMIC_SUB_NV(name, type1, type2) \
142e6dd1 ED	248	type1 atomic_sub_##name##_nv(volatile type1 *target, type2 bits)\
	249	{ \
	250	type1 rc; \
	251	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	252	rc = (*target -= bits); \
	253	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
d1d7e268	254	return (rc); \
142e6dd1 ED	255	}
	256
	257	ATOMIC_SUB_NV(8, uint8_t, int8_t)
	258	ATOMIC_SUB_NV(char, uchar_t, signed char)
	259	ATOMIC_SUB_NV(16, uint16_t, int16_t)
	260	ATOMIC_SUB_NV(short, ushort_t, short)
	261	ATOMIC_SUB_NV(32, uint32_t, int32_t)
	262	ATOMIC_SUB_NV(int, uint_t, int)
	263	ATOMIC_SUB_NV(long, ulong_t, long)
	264	ATOMIC_SUB_NV(64, uint64_t, int64_t)
	265
d1d7e268 MK	266	void *
d1d7e268 MK	267	atomic_sub_ptr_nv(volatile void *target, ssize_t bits)
142e6dd1 ED	268	{
	269	void *ptr;
	270
	271	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	272	ptr = ((caddr_t )target -= bits);
	273	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
	274
d1d7e268	275	return (ptr);
142e6dd1 ED	276	}
	277
	278
d1d7e268	279	#define ATOMIC_OR_NV(name, type) \
a26baf28 BB	280	type atomic_or_##name##_nv(volatile type *target, type bits) \
	281	{ \
	282	type rc; \
	283	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	284	rc = (*target \|= bits); \
	285	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
d1d7e268	286	return (rc); \
a26baf28 BB	287	}
	288
	289	ATOMIC_OR_NV(long, unsigned long)
	290	ATOMIC_OR_NV(8, uint8_t)
	291	ATOMIC_OR_NV(uchar, uchar_t)
	292	ATOMIC_OR_NV(16, uint16_t)
	293	ATOMIC_OR_NV(ushort, ushort_t)
	294	ATOMIC_OR_NV(32, uint32_t)
	295	ATOMIC_OR_NV(uint, uint_t)
	296	ATOMIC_OR_NV(ulong, ulong_t)
	297	ATOMIC_OR_NV(64, uint64_t)
	298
	299
d1d7e268	300	#define ATOMIC_AND_NV(name, type) \
a26baf28 BB	301	type atomic_and_##name##_nv(volatile type *target, type bits) \
	302	{ \
	303	type rc; \
	304	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	305	rc = (*target &= bits); \
	306	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
d1d7e268	307	return (rc); \
a26baf28 BB	308	}
	309
	310	ATOMIC_AND_NV(long, unsigned long)
	311	ATOMIC_AND_NV(8, uint8_t)
	312	ATOMIC_AND_NV(uchar, uchar_t)
	313	ATOMIC_AND_NV(16, uint16_t)
	314	ATOMIC_AND_NV(ushort, ushort_t)
	315	ATOMIC_AND_NV(32, uint32_t)
	316	ATOMIC_AND_NV(uint, uint_t)
	317	ATOMIC_AND_NV(ulong, ulong_t)
	318	ATOMIC_AND_NV(64, uint64_t)
	319
	320
	321	/*
	322	* If arg1 == arg2, set arg1 = arg3; return old value
	323	*/
	324
d1d7e268	325	#define ATOMIC_CAS(name, type) \
a26baf28 BB	326	type atomic_cas_##name(volatile type *target, type arg1, type arg2) \
	327	{ \
	328	type old; \
	329	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	330	old = *target; \
	331	if (old == arg1) \
	332	*target = arg2; \
	333	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
d1d7e268	334	return (old); \
a26baf28 BB	335	}
	336
	337	ATOMIC_CAS(8, uint8_t)
	338	ATOMIC_CAS(uchar, uchar_t)
	339	ATOMIC_CAS(16, uint16_t)
	340	ATOMIC_CAS(ushort, ushort_t)
	341	ATOMIC_CAS(32, uint32_t)
	342	ATOMIC_CAS(uint, uint_t)
	343	ATOMIC_CAS(ulong, ulong_t)
	344	ATOMIC_CAS(64, uint64_t)
	345
d1d7e268 MK	346	void *
d1d7e268 MK	347	atomic_cas_ptr(volatile void target, void arg1, void *arg2)
a26baf28 BB	348	{
	349	void *old;
	350
	351	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	352	old = (void *)target;
d1d7e268 MK	353	if (old == arg1)
d1d7e268 MK	354	(void *)target = arg2;
a26baf28 BB	355	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
a26baf28 BB	356
d1d7e268	357	return (old);
a26baf28 BB	358	}
	359
	360
	361	/*
	362	* Swap target and return old value
	363	*/
	364
d1d7e268	365	#define ATOMIC_SWAP(name, type) \
a26baf28 BB	366	type atomic_swap_##name(volatile type *target, type bits) \
	367	{ \
	368	type old; \
	369	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
	370	old = *target; \
	371	*target = bits; \
	372	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
d1d7e268	373	return (old); \
a26baf28 BB	374	}
	375
	376	ATOMIC_SWAP(8, uint8_t)
	377	ATOMIC_SWAP(uchar, uchar_t)
	378	ATOMIC_SWAP(16, uint16_t)
	379	ATOMIC_SWAP(ushort, ushort_t)
	380	ATOMIC_SWAP(32, uint32_t)
	381	ATOMIC_SWAP(uint, uint_t)
	382	ATOMIC_SWAP(ulong, ulong_t)
	383	ATOMIC_SWAP(64, uint64_t)
	384
d1d7e268 MK	385	void *
d1d7e268 MK	386	atomic_swap_ptr(volatile void target, void bits)
a26baf28 BB	387	{
	388	void *old;
	389
	390	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	391	old = (void *)target;
	392	(void *)target = bits;
	393	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
	394
d1d7e268	395	return (old);
a26baf28 BB	396	}
	397
	398
d1d7e268 MK	399	int
d1d7e268 MK	400	atomic_set_long_excl(volatile ulong_t *target, uint_t value)
a26baf28 BB	401	{
	402	ulong_t bit;
	403
	404	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	405	bit = (1UL << value);
	406	if ((*target & bit) != 0) {
	407	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
d1d7e268	408	return (-1);
a26baf28 BB	409	}
	410	*target \|= bit;
	411	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
	412
d1d7e268	413	return (0);
a26baf28 BB	414	}
a26baf28 BB	415
d1d7e268 MK	416	int
d1d7e268 MK	417	atomic_clear_long_excl(volatile ulong_t *target, uint_t value)
a26baf28 BB	418	{
	419	ulong_t bit;
	420
	421	VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
	422	bit = (1UL << value);
	423	if ((*target & bit) != 0) {
	424	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
d1d7e268	425	return (-1);
a26baf28 BB	426	}
	427	*target &= ~bit;
	428	VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
	429
d1d7e268	430	return (0);
a26baf28 BB	431	}
a26baf28 BB	432
d1d7e268 MK	433	void
d1d7e268 MK	434	membar_enter(void)
a26baf28 BB	435	{
	436	/* XXX - Implement me */
	437	}
	438
d1d7e268 MK	439	void
d1d7e268 MK	440	membar_exit(void)
a26baf28 BB	441	{
	442	/* XXX - Implement me */
	443	}
	444
d1d7e268 MK	445	void
d1d7e268 MK	446	membar_producer(void)
a26baf28 BB	447	{
	448	/* XXX - Implement me */
	449	}
	450
d1d7e268 MK	451	void
d1d7e268 MK	452	membar_consumer(void)
a26baf28 BB	453	{
	454	/* XXX - Implement me */
	455	}
	456
	457	/* Legacy kernel interfaces; they will go away (eventually). */
	458
d1d7e268 MK	459	uint8_t
d1d7e268 MK	460	cas8(uint8_t *target, uint8_t arg1, uint8_t arg2)
a26baf28	461	{
d1d7e268	462	return (atomic_cas_8(target, arg1, arg2));
a26baf28 BB	463	}
a26baf28 BB	464
d1d7e268 MK	465	uint32_t
d1d7e268 MK	466	cas32(uint32_t *target, uint32_t arg1, uint32_t arg2)
a26baf28	467	{
d1d7e268	468	return (atomic_cas_32(target, arg1, arg2));
a26baf28 BB	469	}
a26baf28 BB	470
d1d7e268 MK	471	uint64_t
d1d7e268 MK	472	cas64(uint64_t *target, uint64_t arg1, uint64_t arg2)
a26baf28	473	{
d1d7e268	474	return (atomic_cas_64(target, arg1, arg2));
a26baf28 BB	475	}
a26baf28 BB	476
d1d7e268 MK	477	ulong_t
d1d7e268 MK	478	caslong(ulong_t *target, ulong_t arg1, ulong_t arg2)
a26baf28	479	{
d1d7e268	480	return (atomic_cas_ulong(target, arg1, arg2));
a26baf28 BB	481	}
a26baf28 BB	482
d1d7e268 MK	483	void *
d1d7e268 MK	484	casptr(void target, void arg1, void *arg2)
a26baf28	485	{
d1d7e268	486	return (atomic_cas_ptr(target, arg1, arg2));
a26baf28 BB	487	}
a26baf28 BB	488
d1d7e268 MK	489	void
d1d7e268 MK	490	atomic_and_long(ulong_t *target, ulong_t bits)
a26baf28	491	{
d1d7e268	492	return (atomic_and_ulong(target, bits));
a26baf28 BB	493	}
a26baf28 BB	494
d1d7e268 MK	495	void
d1d7e268 MK	496	atomic_or_long(ulong_t *target, ulong_t bits)
a26baf28	497	{
d1d7e268	498	return (atomic_or_ulong(target, bits));
a26baf28	499	}