[mirror_qemu.git] / accel / tcg / atomic_template.h

/*
 * Atomic helper templates
 * Included from tcg-runtime.c and cputlb.c.
 *
 * Copyright (c) 2016 Red Hat, Inc
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#include "trace/mem.h"

#if DATA_SIZE == 16
# define SUFFIX     o
# define DATA_TYPE  Int128
# define BSWAP      bswap128
# define SHIFT      4
#elif DATA_SIZE == 8
# define SUFFIX     q
# define DATA_TYPE  uint64_t
# define SDATA_TYPE int64_t
# define BSWAP      bswap64
# define SHIFT      3
#elif DATA_SIZE == 4
# define SUFFIX     l
# define DATA_TYPE  uint32_t
# define SDATA_TYPE int32_t
# define BSWAP      bswap32
# define SHIFT      2
#elif DATA_SIZE == 2
# define SUFFIX     w
# define DATA_TYPE  uint16_t
# define SDATA_TYPE int16_t
# define BSWAP      bswap16
# define SHIFT      1
#elif DATA_SIZE == 1
# define SUFFIX     b
# define DATA_TYPE  uint8_t
# define SDATA_TYPE int8_t
# define BSWAP
# define SHIFT      0
#else
# error unsupported data size
#endif

#if DATA_SIZE >= 4
# define ABI_TYPE  DATA_TYPE
#else
# define ABI_TYPE  uint32_t
#endif

#define ATOMIC_TRACE_RMW do {                                           \
        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
                                                                        \
        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr,             \
                                    info | TRACE_MEM_ST);               \
    } while (0)

#define ATOMIC_TRACE_LD do {                                            \
        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
                                                                        \
        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
    } while (0)

# define ATOMIC_TRACE_ST do {                                           \
        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
                                                                        \
        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
    } while (0)

/* Define host-endian atomic operations.  Note that END is used within
   the ATOMIC_NAME macro, and redefined below.  */
#if DATA_SIZE == 1
# define END
# define MEND _be /* either le or be would be fine */
#elif defined(HOST_WORDS_BIGENDIAN)
# define END  _be
# define MEND _be
#else
# define END  _le
# define MEND _le
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;

    ATOMIC_TRACE_RMW;
    ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
    ATOMIC_MMU_CLEANUP;
    return ret;
}

#if DATA_SIZE >= 16
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;

    ATOMIC_TRACE_LD;
    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
    ATOMIC_MMU_CLEANUP;
    return val;
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;

    ATOMIC_TRACE_ST;
    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
    ATOMIC_MMU_CLEANUP;
}
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;

    ATOMIC_TRACE_RMW;
    ret = atomic_xchg__nocheck(haddr, val);
    ATOMIC_MMU_CLEANUP;
    return ret;
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                 ABI_TYPE val EXTRA_ARGS)                           \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
                                                                    \
    ATOMIC_TRACE_RMW;                                               \
    ret = atomic_##X(haddr, val);                                   \
    ATOMIC_MMU_CLEANUP;                                             \
    return ret;                                                     \
}

GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(add_fetch)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* These helpers are, as a whole, full barriers.  Within the helper,
 * the leading barrier is explicit and the trailing barrier is within
 * cmpxchg primitive.
 *
 * Trace this load + RMW loop as a single RMW op. This way, regardless
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE xval EXTRA_ARGS)                   \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE cmp, old, new, val = xval;                           \
                                                                    \
    ATOMIC_TRACE_RMW;                                               \
    smp_mb();                                                       \
    cmp = atomic_read__nocheck(haddr);                              \
    do {                                                            \
        old = cmp; new = FN(old, val);                              \
        cmp = atomic_cmpxchg__nocheck(haddr, old, new);             \
    } while (cmp != old);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
    return RET;                                                     \
}

GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)

GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA SIZE >= 16 */

#undef END
#undef MEND

#if DATA_SIZE > 1

/* Define reverse-host-endian atomic operations.  Note that END is used
   within the ATOMIC_NAME macro.  */
#ifdef HOST_WORDS_BIGENDIAN
# define END  _le
# define MEND _le
#else
# define END  _be
# define MEND _be
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;

    ATOMIC_TRACE_RMW;
    ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
    ATOMIC_MMU_CLEANUP;
    return BSWAP(ret);
}

#if DATA_SIZE >= 16
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;

    ATOMIC_TRACE_LD;
    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
    ATOMIC_MMU_CLEANUP;
    return BSWAP(val);
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;

    ATOMIC_TRACE_ST;
    val = BSWAP(val);
    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
    ATOMIC_MMU_CLEANUP;
}
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    ABI_TYPE ret;

    ATOMIC_TRACE_RMW;
    ret = atomic_xchg__nocheck(haddr, BSWAP(val));
    ATOMIC_MMU_CLEANUP;
    return BSWAP(ret);
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                 ABI_TYPE val EXTRA_ARGS)                           \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
                                                                    \
    ATOMIC_TRACE_RMW;                                               \
    ret = atomic_##X(haddr, BSWAP(val));                            \
    ATOMIC_MMU_CLEANUP;                                             \
    return BSWAP(ret);                                              \
}

GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* These helpers are, as a whole, full barriers.  Within the helper,
 * the leading barrier is explicit and the trailing barrier is within
 * cmpxchg primitive.
 *
 * Trace this load + RMW loop as a single RMW op. This way, regardless
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE xval EXTRA_ARGS)                   \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
                                                                    \
    ATOMIC_TRACE_RMW;                                               \
    smp_mb();                                                       \
    ldn = atomic_read__nocheck(haddr);                              \
    do {                                                            \
        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
        ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));      \
    } while (ldo != ldn);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
    return RET;                                                     \
}

GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)

GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

/* Note that for addition, we need to use a separate cmpxchg loop instead
   of bswaps for the reverse-host-endian helpers.  */
#define ADD(X, Y)   (X + Y)
GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
#undef ADD

#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA_SIZE >= 16 */

#undef END
#undef MEND
#endif /* DATA_SIZE > 1 */

#undef ATOMIC_TRACE_ST
#undef ATOMIC_TRACE_LD
#undef ATOMIC_TRACE_RMW

#undef BSWAP
#undef ABI_TYPE
#undef DATA_TYPE
#undef SDATA_TYPE
#undef SUFFIX
#undef DATA_SIZE
#undef SHIFT
Commit	Line	Data
c482cb11 RH	1	/*
	2	* Atomic helper templates
	3	* Included from tcg-runtime.c and cputlb.c.
	4	*
	5	* Copyright (c) 2016 Red Hat, Inc
	6	*
	7	* This library is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU Lesser General Public
	9	* License as published by the Free Software Foundation; either
	10	* version 2 of the License, or (at your option) any later version.
	11	*
	12	* This library is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	15	* Lesser General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU Lesser General Public
	18	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
d071f4cd EC	21	#include "trace/mem.h"
d071f4cd EC	22
7ebee43e RH	23	#if DATA_SIZE == 16
	24	# define SUFFIX o
	25	# define DATA_TYPE Int128
	26	# define BSWAP bswap128
d071f4cd	27	# define SHIFT 4
7ebee43e	28	#elif DATA_SIZE == 8
c482cb11 RH	29	# define SUFFIX q
c482cb11 RH	30	# define DATA_TYPE uint64_t
5507c2bf	31	# define SDATA_TYPE int64_t
c482cb11	32	# define BSWAP bswap64
d071f4cd	33	# define SHIFT 3
c482cb11 RH	34	#elif DATA_SIZE == 4
	35	# define SUFFIX l
	36	# define DATA_TYPE uint32_t
5507c2bf	37	# define SDATA_TYPE int32_t
c482cb11	38	# define BSWAP bswap32
d071f4cd	39	# define SHIFT 2
c482cb11 RH	40	#elif DATA_SIZE == 2
	41	# define SUFFIX w
	42	# define DATA_TYPE uint16_t
5507c2bf	43	# define SDATA_TYPE int16_t
c482cb11	44	# define BSWAP bswap16
d071f4cd	45	# define SHIFT 1
c482cb11 RH	46	#elif DATA_SIZE == 1
	47	# define SUFFIX b
	48	# define DATA_TYPE uint8_t
5507c2bf	49	# define SDATA_TYPE int8_t
c482cb11	50	# define BSWAP
d071f4cd	51	# define SHIFT 0
c482cb11 RH	52	#else
	53	# error unsupported data size
	54	#endif
	55
	56	#if DATA_SIZE >= 4
	57	# define ABI_TYPE DATA_TYPE
	58	#else
	59	# define ABI_TYPE uint32_t
	60	#endif
	61
d071f4cd EC	62	#define ATOMIC_TRACE_RMW do { \
	63	uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
	64	\
	65	trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
	66	trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, \
	67	info \| TRACE_MEM_ST); \
	68	} while (0)
	69
	70	#define ATOMIC_TRACE_LD do { \
	71	uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
	72	\
	73	trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
	74	} while (0)
	75
	76	# define ATOMIC_TRACE_ST do { \
	77	uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
	78	\
	79	trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
	80	} while (0)
	81
c482cb11 RH	82	/* Define host-endian atomic operations. Note that END is used within
	83	the ATOMIC_NAME macro, and redefined below. */
	84	#if DATA_SIZE == 1
	85	# define END
d071f4cd	86	# define MEND _be /* either le or be would be fine */
c482cb11 RH	87	#elif defined(HOST_WORDS_BIGENDIAN)
c482cb11 RH	88	# define END _be
d071f4cd	89	# define MEND _be
c482cb11 RH	90	#else
c482cb11 RH	91	# define END _le
d071f4cd	92	# define MEND _le
c482cb11 RH	93	#endif
	94
	95	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	96	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	97	{
34d49937	98	ATOMIC_MMU_DECLS;
c482cb11	99	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd EC	100	DATA_TYPE ret;
	101
	102	ATOMIC_TRACE_RMW;
	103	ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
ec603b55 RH	104	ATOMIC_MMU_CLEANUP;
ec603b55 RH	105	return ret;
c482cb11 RH	106	}
c482cb11 RH	107
7ebee43e RH	108	#if DATA_SIZE >= 16
	109	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
	110	{
34d49937	111	ATOMIC_MMU_DECLS;
7ebee43e	112	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd EC	113
d071f4cd EC	114	ATOMIC_TRACE_LD;
7ebee43e	115	__atomic_load(haddr, &val, __ATOMIC_RELAXED);
ec603b55	116	ATOMIC_MMU_CLEANUP;
7ebee43e RH	117	return val;
	118	}
	119
	120	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	121	ABI_TYPE val EXTRA_ARGS)
	122	{
34d49937	123	ATOMIC_MMU_DECLS;
7ebee43e	124	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd EC	125
d071f4cd EC	126	ATOMIC_TRACE_ST;
7ebee43e	127	__atomic_store(haddr, &val, __ATOMIC_RELAXED);
ec603b55	128	ATOMIC_MMU_CLEANUP;
7ebee43e RH	129	}
7ebee43e RH	130	#else
c482cb11 RH	131	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	132	ABI_TYPE val EXTRA_ARGS)
	133	{
34d49937	134	ATOMIC_MMU_DECLS;
c482cb11	135	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd EC	136	DATA_TYPE ret;
	137
	138	ATOMIC_TRACE_RMW;
	139	ret = atomic_xchg__nocheck(haddr, val);
ec603b55 RH	140	ATOMIC_MMU_CLEANUP;
ec603b55 RH	141	return ret;
c482cb11 RH	142	}
	143
	144	#define GEN_ATOMIC_HELPER(X) \
	145	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	146	ABI_TYPE val EXTRA_ARGS) \
	147	{ \
34d49937	148	ATOMIC_MMU_DECLS; \
c482cb11	149	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
d071f4cd EC	150	DATA_TYPE ret; \
	151	\
	152	ATOMIC_TRACE_RMW; \
	153	ret = atomic_##X(haddr, val); \
ec603b55 RH	154	ATOMIC_MMU_CLEANUP; \
	155	return ret; \
	156	}
c482cb11 RH	157
	158	GEN_ATOMIC_HELPER(fetch_add)
	159	GEN_ATOMIC_HELPER(fetch_and)
	160	GEN_ATOMIC_HELPER(fetch_or)
	161	GEN_ATOMIC_HELPER(fetch_xor)
	162	GEN_ATOMIC_HELPER(add_fetch)
	163	GEN_ATOMIC_HELPER(and_fetch)
	164	GEN_ATOMIC_HELPER(or_fetch)
	165	GEN_ATOMIC_HELPER(xor_fetch)
	166
	167	#undef GEN_ATOMIC_HELPER
5507c2bf RH	168
	169	/* These helpers are, as a whole, full barriers. Within the helper,
	170	* the leading barrier is explicit and the trailing barrier is within
	171	* cmpxchg primitive.
d071f4cd EC	172	*
	173	* Trace this load + RMW loop as a single RMW op. This way, regardless
	174	* of CF_PARALLEL's value, we'll trace just a read and a write.
5507c2bf RH	175	*/
	176	#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
	177	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	178	ABI_TYPE xval EXTRA_ARGS) \
	179	{ \
	180	ATOMIC_MMU_DECLS; \
	181	XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	182	XDATA_TYPE cmp, old, new, val = xval; \
d071f4cd EC	183	\
d071f4cd EC	184	ATOMIC_TRACE_RMW; \
5507c2bf RH	185	smp_mb(); \
	186	cmp = atomic_read__nocheck(haddr); \
	187	do { \
	188	old = cmp; new = FN(old, val); \
	189	cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
	190	} while (cmp != old); \
	191	ATOMIC_MMU_CLEANUP; \
	192	return RET; \
	193	}
	194
	195	GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
	196	GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
	197	GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
	198	GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
	199
	200	GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
	201	GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
	202	GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
	203	GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
	204
	205	#undef GEN_ATOMIC_HELPER_FN
7ebee43e RH	206	#endif /* DATA SIZE >= 16 */
7ebee43e RH	207
c482cb11	208	#undef END
d071f4cd	209	#undef MEND
c482cb11 RH	210
	211	#if DATA_SIZE > 1
	212
	213	/* Define reverse-host-endian atomic operations. Note that END is used
	214	within the ATOMIC_NAME macro. */
	215	#ifdef HOST_WORDS_BIGENDIAN
	216	# define END _le
d071f4cd	217	# define MEND _le
c482cb11 RH	218	#else
c482cb11 RH	219	# define END _be
d071f4cd	220	# define MEND _be
c482cb11 RH	221	#endif
	222
	223	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	224	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	225	{
34d49937	226	ATOMIC_MMU_DECLS;
c482cb11	227	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd EC	228	DATA_TYPE ret;
	229
	230	ATOMIC_TRACE_RMW;
	231	ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
ec603b55 RH	232	ATOMIC_MMU_CLEANUP;
ec603b55 RH	233	return BSWAP(ret);
c482cb11 RH	234	}
c482cb11 RH	235
7ebee43e RH	236	#if DATA_SIZE >= 16
	237	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
	238	{
34d49937	239	ATOMIC_MMU_DECLS;
7ebee43e	240	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd EC	241
d071f4cd EC	242	ATOMIC_TRACE_LD;
7ebee43e	243	__atomic_load(haddr, &val, __ATOMIC_RELAXED);
ec603b55	244	ATOMIC_MMU_CLEANUP;
7ebee43e RH	245	return BSWAP(val);
	246	}
	247
	248	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	249	ABI_TYPE val EXTRA_ARGS)
	250	{
34d49937	251	ATOMIC_MMU_DECLS;
7ebee43e	252	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd EC	253
d071f4cd EC	254	ATOMIC_TRACE_ST;
7ebee43e RH	255	val = BSWAP(val);
7ebee43e RH	256	__atomic_store(haddr, &val, __ATOMIC_RELAXED);
ec603b55	257	ATOMIC_MMU_CLEANUP;
7ebee43e RH	258	}
7ebee43e RH	259	#else
c482cb11 RH	260	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	261	ABI_TYPE val EXTRA_ARGS)
	262	{
34d49937	263	ATOMIC_MMU_DECLS;
c482cb11	264	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd EC	265	ABI_TYPE ret;
	266
	267	ATOMIC_TRACE_RMW;
	268	ret = atomic_xchg__nocheck(haddr, BSWAP(val));
ec603b55 RH	269	ATOMIC_MMU_CLEANUP;
ec603b55 RH	270	return BSWAP(ret);
c482cb11 RH	271	}
	272
	273	#define GEN_ATOMIC_HELPER(X) \
	274	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	275	ABI_TYPE val EXTRA_ARGS) \
	276	{ \
34d49937	277	ATOMIC_MMU_DECLS; \
c482cb11	278	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
d071f4cd EC	279	DATA_TYPE ret; \
	280	\
	281	ATOMIC_TRACE_RMW; \
	282	ret = atomic_##X(haddr, BSWAP(val)); \
ec603b55 RH	283	ATOMIC_MMU_CLEANUP; \
ec603b55 RH	284	return BSWAP(ret); \
c482cb11 RH	285	}
	286
	287	GEN_ATOMIC_HELPER(fetch_and)
	288	GEN_ATOMIC_HELPER(fetch_or)
	289	GEN_ATOMIC_HELPER(fetch_xor)
	290	GEN_ATOMIC_HELPER(and_fetch)
	291	GEN_ATOMIC_HELPER(or_fetch)
	292	GEN_ATOMIC_HELPER(xor_fetch)
	293
	294	#undef GEN_ATOMIC_HELPER
	295
5507c2bf RH	296	/* These helpers are, as a whole, full barriers. Within the helper,
	297	* the leading barrier is explicit and the trailing barrier is within
	298	* cmpxchg primitive.
d071f4cd EC	299	*
	300	* Trace this load + RMW loop as a single RMW op. This way, regardless
	301	* of CF_PARALLEL's value, we'll trace just a read and a write.
5507c2bf RH	302	*/
	303	#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
	304	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	305	ABI_TYPE xval EXTRA_ARGS) \
	306	{ \
	307	ATOMIC_MMU_DECLS; \
	308	XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	309	XDATA_TYPE ldo, ldn, old, new, val = xval; \
d071f4cd EC	310	\
d071f4cd EC	311	ATOMIC_TRACE_RMW; \
5507c2bf RH	312	smp_mb(); \
	313	ldn = atomic_read__nocheck(haddr); \
	314	do { \
	315	ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
	316	ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
	317	} while (ldo != ldn); \
	318	ATOMIC_MMU_CLEANUP; \
	319	return RET; \
	320	}
	321
	322	GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
	323	GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
	324	GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
	325	GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
	326
	327	GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
	328	GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
	329	GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
	330	GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
	331
58edf9ee RH	332	/* Note that for addition, we need to use a separate cmpxchg loop instead
	333	of bswaps for the reverse-host-endian helpers. */
	334	#define ADD(X, Y) (X + Y)
	335	GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
	336	GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
	337	#undef ADD
	338
5507c2bf	339	#undef GEN_ATOMIC_HELPER_FN
7ebee43e	340	#endif /* DATA_SIZE >= 16 */
c482cb11 RH	341
c482cb11 RH	342	#undef END
d071f4cd	343	#undef MEND
c482cb11 RH	344	#endif /* DATA_SIZE > 1 */
c482cb11 RH	345
d071f4cd EC	346	#undef ATOMIC_TRACE_ST
	347	#undef ATOMIC_TRACE_LD
	348	#undef ATOMIC_TRACE_RMW
	349
c482cb11 RH	350	#undef BSWAP
	351	#undef ABI_TYPE
	352	#undef DATA_TYPE
5507c2bf	353	#undef SDATA_TYPE
c482cb11 RH	354	#undef SUFFIX
c482cb11 RH	355	#undef DATA_SIZE
d071f4cd	356	#undef SHIFT