[mirror_qemu.git] / accel / tcg / atomic_template.h

/*
 * Atomic helper templates
 * Included from tcg-runtime.c and cputlb.c.
 *
 * Copyright (c) 2016 Red Hat, Inc
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#include "qemu/plugin.h"
#include "trace/mem.h"

#if DATA_SIZE == 16
# define SUFFIX     o
# define DATA_TYPE  Int128
# define BSWAP      bswap128
# define SHIFT      4
#elif DATA_SIZE == 8
# define SUFFIX     q
# define DATA_TYPE  uint64_t
# define SDATA_TYPE int64_t
# define BSWAP      bswap64
# define SHIFT      3
#elif DATA_SIZE == 4
# define SUFFIX     l
# define DATA_TYPE  uint32_t
# define SDATA_TYPE int32_t
# define BSWAP      bswap32
# define SHIFT      2
#elif DATA_SIZE == 2
# define SUFFIX     w
# define DATA_TYPE  uint16_t
# define SDATA_TYPE int16_t
# define BSWAP      bswap16
# define SHIFT      1
#elif DATA_SIZE == 1
# define SUFFIX     b
# define DATA_TYPE  uint8_t
# define SDATA_TYPE int8_t
# define BSWAP
# define SHIFT      0
#else
# error unsupported data size
#endif

#if DATA_SIZE >= 4
# define ABI_TYPE  DATA_TYPE
#else
# define ABI_TYPE  uint32_t
#endif

/* Define host-endian atomic operations.  Note that END is used within
   the ATOMIC_NAME macro, and redefined below.  */
#if DATA_SIZE == 1
# define END
#elif defined(HOST_WORDS_BIGENDIAN)
# define END  _be
#else
# define END  _le
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
#if DATA_SIZE == 16
    ret = atomic16_cmpxchg(haddr, cmpv, newv);
#else
    ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
#endif
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return ret;
}

#if DATA_SIZE >= 16
#if HAVE_ATOMIC128
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_ld_pre(env, addr, info);
    val = atomic16_read(haddr);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_ld_post(env, addr, info);
    return val;
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, true,
                                         ATOMIC_MMU_IDX);

    atomic_trace_st_pre(env, addr, info);
    atomic16_set(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_st_post(env, addr, info);
}
#endif
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
    ret = atomic_xchg__nocheck(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return ret;
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE val EXTRA_ARGS)                    \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,    \
                                         ATOMIC_MMU_IDX);           \
    atomic_trace_rmw_pre(env, addr, info);                          \
    ret = atomic_##X(haddr, val);                                   \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return ret;                                                     \
}

GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(add_fetch)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* These helpers are, as a whole, full barriers.  Within the helper,
 * the leading barrier is explicit and the trailing barrier is within
 * cmpxchg primitive.
 *
 * Trace this load + RMW loop as a single RMW op. This way, regardless
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE xval EXTRA_ARGS)                   \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE cmp, old, new, val = xval;                           \
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,    \
                                         ATOMIC_MMU_IDX);           \
    atomic_trace_rmw_pre(env, addr, info);                          \
    smp_mb();                                                       \
    cmp = atomic_read__nocheck(haddr);                              \
    do {                                                            \
        old = cmp; new = FN(old, val);                              \
        cmp = atomic_cmpxchg__nocheck(haddr, old, new);             \
    } while (cmp != old);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return RET;                                                     \
}

GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)

GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA SIZE >= 16 */

#undef END

#if DATA_SIZE > 1

/* Define reverse-host-endian atomic operations.  Note that END is used
   within the ATOMIC_NAME macro.  */
#ifdef HOST_WORDS_BIGENDIAN
# define END  _le
#else
# define END  _be
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
#if DATA_SIZE == 16
    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
#else
    ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
#endif
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return BSWAP(ret);
}

#if DATA_SIZE >= 16
#if HAVE_ATOMIC128
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_ld_pre(env, addr, info);
    val = atomic16_read(haddr);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_ld_post(env, addr, info);
    return BSWAP(val);
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true,
                                         ATOMIC_MMU_IDX);

    val = BSWAP(val);
    atomic_trace_st_pre(env, addr, info);
    val = BSWAP(val);
    atomic16_set(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_st_post(env, addr, info);
}
#endif
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    ABI_TYPE ret;
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
    ret = atomic_xchg__nocheck(haddr, BSWAP(val));
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return BSWAP(ret);
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE val EXTRA_ARGS)                    \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP,    \
                                         false, ATOMIC_MMU_IDX);    \
    atomic_trace_rmw_pre(env, addr, info);                          \
    ret = atomic_##X(haddr, BSWAP(val));                            \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return BSWAP(ret);                                              \
}

GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* These helpers are, as a whole, full barriers.  Within the helper,
 * the leading barrier is explicit and the trailing barrier is within
 * cmpxchg primitive.
 *
 * Trace this load + RMW loop as a single RMW op. This way, regardless
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE xval EXTRA_ARGS)                   \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP,    \
                                         false, ATOMIC_MMU_IDX);    \
    atomic_trace_rmw_pre(env, addr, info);                          \
    smp_mb();                                                       \
    ldn = atomic_read__nocheck(haddr);                              \
    do {                                                            \
        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
        ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));      \
    } while (ldo != ldn);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return RET;                                                     \
}

GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)

GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

/* Note that for addition, we need to use a separate cmpxchg loop instead
   of bswaps for the reverse-host-endian helpers.  */
#define ADD(X, Y)   (X + Y)
GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
#undef ADD

#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA_SIZE >= 16 */

#undef END
#endif /* DATA_SIZE > 1 */

#undef BSWAP
#undef ABI_TYPE
#undef DATA_TYPE
#undef SDATA_TYPE
#undef SUFFIX
#undef DATA_SIZE
#undef SHIFT
Commit	Line	Data
	1	/*
	2	* Atomic helper templates
	3	* Included from tcg-runtime.c and cputlb.c.
	4	*
	5	* Copyright (c) 2016 Red Hat, Inc
	6	*
	7	* This library is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU Lesser General Public
	9	* License as published by the Free Software Foundation; either
	10	* version 2.1 of the License, or (at your option) any later version.
	11	*
	12	* This library is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	15	* Lesser General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU Lesser General Public
	18	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
	21	#include "qemu/plugin.h"
	22	#include "trace/mem.h"
	23
	24	#if DATA_SIZE == 16
	25	# define SUFFIX o
	26	# define DATA_TYPE Int128
	27	# define BSWAP bswap128
	28	# define SHIFT 4
	29	#elif DATA_SIZE == 8
	30	# define SUFFIX q
	31	# define DATA_TYPE uint64_t
	32	# define SDATA_TYPE int64_t
	33	# define BSWAP bswap64
	34	# define SHIFT 3
	35	#elif DATA_SIZE == 4
	36	# define SUFFIX l
	37	# define DATA_TYPE uint32_t
	38	# define SDATA_TYPE int32_t
	39	# define BSWAP bswap32
	40	# define SHIFT 2
	41	#elif DATA_SIZE == 2
	42	# define SUFFIX w
	43	# define DATA_TYPE uint16_t
	44	# define SDATA_TYPE int16_t
	45	# define BSWAP bswap16
	46	# define SHIFT 1
	47	#elif DATA_SIZE == 1
	48	# define SUFFIX b
	49	# define DATA_TYPE uint8_t
	50	# define SDATA_TYPE int8_t
	51	# define BSWAP
	52	# define SHIFT 0
	53	#else
	54	# error unsupported data size
	55	#endif
	56
	57	#if DATA_SIZE >= 4
	58	# define ABI_TYPE DATA_TYPE
	59	#else
	60	# define ABI_TYPE uint32_t
	61	#endif
	62
	63	/* Define host-endian atomic operations. Note that END is used within
	64	the ATOMIC_NAME macro, and redefined below. */
	65	#if DATA_SIZE == 1
	66	# define END
	67	#elif defined(HOST_WORDS_BIGENDIAN)
	68	# define END _be
	69	#else
	70	# define END _le
	71	#endif
	72
	73	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	74	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	75	{
	76	ATOMIC_MMU_DECLS;
	77	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	78	DATA_TYPE ret;
	79	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
	80	ATOMIC_MMU_IDX);
	81
	82	atomic_trace_rmw_pre(env, addr, info);
	83	#if DATA_SIZE == 16
	84	ret = atomic16_cmpxchg(haddr, cmpv, newv);
	85	#else
	86	ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
	87	#endif
	88	ATOMIC_MMU_CLEANUP;
	89	atomic_trace_rmw_post(env, addr, info);
	90	return ret;
	91	}
	92
	93	#if DATA_SIZE >= 16
	94	#if HAVE_ATOMIC128
	95	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
	96	{
	97	ATOMIC_MMU_DECLS;
	98	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
	99	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
	100	ATOMIC_MMU_IDX);
	101
	102	atomic_trace_ld_pre(env, addr, info);
	103	val = atomic16_read(haddr);
	104	ATOMIC_MMU_CLEANUP;
	105	atomic_trace_ld_post(env, addr, info);
	106	return val;
	107	}
	108
	109	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	110	ABI_TYPE val EXTRA_ARGS)
	111	{
	112	ATOMIC_MMU_DECLS;
	113	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	114	uint16_t info = trace_mem_build_info(SHIFT, false, 0, true,
	115	ATOMIC_MMU_IDX);
	116
	117	atomic_trace_st_pre(env, addr, info);
	118	atomic16_set(haddr, val);
	119	ATOMIC_MMU_CLEANUP;
	120	atomic_trace_st_post(env, addr, info);
	121	}
	122	#endif
	123	#else
	124	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	125	ABI_TYPE val EXTRA_ARGS)
	126	{
	127	ATOMIC_MMU_DECLS;
	128	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	129	DATA_TYPE ret;
	130	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
	131	ATOMIC_MMU_IDX);
	132
	133	atomic_trace_rmw_pre(env, addr, info);
	134	ret = atomic_xchg__nocheck(haddr, val);
	135	ATOMIC_MMU_CLEANUP;
	136	atomic_trace_rmw_post(env, addr, info);
	137	return ret;
	138	}
	139
	140	#define GEN_ATOMIC_HELPER(X) \
	141	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	142	ABI_TYPE val EXTRA_ARGS) \
	143	{ \
	144	ATOMIC_MMU_DECLS; \
	145	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	146	DATA_TYPE ret; \
	147	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
	148	ATOMIC_MMU_IDX); \
	149	atomic_trace_rmw_pre(env, addr, info); \
	150	ret = atomic_##X(haddr, val); \
	151	ATOMIC_MMU_CLEANUP; \
	152	atomic_trace_rmw_post(env, addr, info); \
	153	return ret; \
	154	}
	155
	156	GEN_ATOMIC_HELPER(fetch_add)
	157	GEN_ATOMIC_HELPER(fetch_and)
	158	GEN_ATOMIC_HELPER(fetch_or)
	159	GEN_ATOMIC_HELPER(fetch_xor)
	160	GEN_ATOMIC_HELPER(add_fetch)
	161	GEN_ATOMIC_HELPER(and_fetch)
	162	GEN_ATOMIC_HELPER(or_fetch)
	163	GEN_ATOMIC_HELPER(xor_fetch)
	164
	165	#undef GEN_ATOMIC_HELPER
	166
	167	/* These helpers are, as a whole, full barriers. Within the helper,
	168	* the leading barrier is explicit and the trailing barrier is within
	169	* cmpxchg primitive.
	170	*
	171	* Trace this load + RMW loop as a single RMW op. This way, regardless
	172	* of CF_PARALLEL's value, we'll trace just a read and a write.
	173	*/
	174	#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
	175	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	176	ABI_TYPE xval EXTRA_ARGS) \
	177	{ \
	178	ATOMIC_MMU_DECLS; \
	179	XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	180	XDATA_TYPE cmp, old, new, val = xval; \
	181	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
	182	ATOMIC_MMU_IDX); \
	183	atomic_trace_rmw_pre(env, addr, info); \
	184	smp_mb(); \
	185	cmp = atomic_read__nocheck(haddr); \
	186	do { \
	187	old = cmp; new = FN(old, val); \
	188	cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
	189	} while (cmp != old); \
	190	ATOMIC_MMU_CLEANUP; \
	191	atomic_trace_rmw_post(env, addr, info); \
	192	return RET; \
	193	}
	194
	195	GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
	196	GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
	197	GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
	198	GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
	199
	200	GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
	201	GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
	202	GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
	203	GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
	204
	205	#undef GEN_ATOMIC_HELPER_FN
	206	#endif /* DATA SIZE >= 16 */
	207
	208	#undef END
	209
	210	#if DATA_SIZE > 1
	211
	212	/* Define reverse-host-endian atomic operations. Note that END is used
	213	within the ATOMIC_NAME macro. */
	214	#ifdef HOST_WORDS_BIGENDIAN
	215	# define END _le
	216	#else
	217	# define END _be
	218	#endif
	219
	220	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	221	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	222	{
	223	ATOMIC_MMU_DECLS;
	224	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	225	DATA_TYPE ret;
	226	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
	227	ATOMIC_MMU_IDX);
	228
	229	atomic_trace_rmw_pre(env, addr, info);
	230	#if DATA_SIZE == 16
	231	ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
	232	#else
	233	ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
	234	#endif
	235	ATOMIC_MMU_CLEANUP;
	236	atomic_trace_rmw_post(env, addr, info);
	237	return BSWAP(ret);
	238	}
	239
	240	#if DATA_SIZE >= 16
	241	#if HAVE_ATOMIC128
	242	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
	243	{
	244	ATOMIC_MMU_DECLS;
	245	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
	246	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
	247	ATOMIC_MMU_IDX);
	248
	249	atomic_trace_ld_pre(env, addr, info);
	250	val = atomic16_read(haddr);
	251	ATOMIC_MMU_CLEANUP;
	252	atomic_trace_ld_post(env, addr, info);
	253	return BSWAP(val);
	254	}
	255
	256	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	257	ABI_TYPE val EXTRA_ARGS)
	258	{
	259	ATOMIC_MMU_DECLS;
	260	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	261	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true,
	262	ATOMIC_MMU_IDX);
	263
	264	val = BSWAP(val);
	265	atomic_trace_st_pre(env, addr, info);
	266	val = BSWAP(val);
	267	atomic16_set(haddr, val);
	268	ATOMIC_MMU_CLEANUP;
	269	atomic_trace_st_post(env, addr, info);
	270	}
	271	#endif
	272	#else
	273	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	274	ABI_TYPE val EXTRA_ARGS)
	275	{
	276	ATOMIC_MMU_DECLS;
	277	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	278	ABI_TYPE ret;
	279	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
	280	ATOMIC_MMU_IDX);
	281
	282	atomic_trace_rmw_pre(env, addr, info);
	283	ret = atomic_xchg__nocheck(haddr, BSWAP(val));
	284	ATOMIC_MMU_CLEANUP;
	285	atomic_trace_rmw_post(env, addr, info);
	286	return BSWAP(ret);
	287	}
	288
	289	#define GEN_ATOMIC_HELPER(X) \
	290	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	291	ABI_TYPE val EXTRA_ARGS) \
	292	{ \
	293	ATOMIC_MMU_DECLS; \
	294	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	295	DATA_TYPE ret; \
	296	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
	297	false, ATOMIC_MMU_IDX); \
	298	atomic_trace_rmw_pre(env, addr, info); \
	299	ret = atomic_##X(haddr, BSWAP(val)); \
	300	ATOMIC_MMU_CLEANUP; \
	301	atomic_trace_rmw_post(env, addr, info); \
	302	return BSWAP(ret); \
	303	}
	304
	305	GEN_ATOMIC_HELPER(fetch_and)
	306	GEN_ATOMIC_HELPER(fetch_or)
	307	GEN_ATOMIC_HELPER(fetch_xor)
	308	GEN_ATOMIC_HELPER(and_fetch)
	309	GEN_ATOMIC_HELPER(or_fetch)
	310	GEN_ATOMIC_HELPER(xor_fetch)
	311
	312	#undef GEN_ATOMIC_HELPER
	313
	314	/* These helpers are, as a whole, full barriers. Within the helper,
	315	* the leading barrier is explicit and the trailing barrier is within
	316	* cmpxchg primitive.
	317	*
	318	* Trace this load + RMW loop as a single RMW op. This way, regardless
	319	* of CF_PARALLEL's value, we'll trace just a read and a write.
	320	*/
	321	#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
	322	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	323	ABI_TYPE xval EXTRA_ARGS) \
	324	{ \
	325	ATOMIC_MMU_DECLS; \
	326	XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	327	XDATA_TYPE ldo, ldn, old, new, val = xval; \
	328	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
	329	false, ATOMIC_MMU_IDX); \
	330	atomic_trace_rmw_pre(env, addr, info); \
	331	smp_mb(); \
	332	ldn = atomic_read__nocheck(haddr); \
	333	do { \
	334	ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
	335	ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
	336	} while (ldo != ldn); \
	337	ATOMIC_MMU_CLEANUP; \
	338	atomic_trace_rmw_post(env, addr, info); \
	339	return RET; \
	340	}
	341
	342	GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
	343	GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
	344	GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
	345	GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
	346
	347	GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
	348	GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
	349	GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
	350	GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
	351
	352	/* Note that for addition, we need to use a separate cmpxchg loop instead
	353	of bswaps for the reverse-host-endian helpers. */
	354	#define ADD(X, Y) (X + Y)
	355	GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
	356	GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
	357	#undef ADD
	358
	359	#undef GEN_ATOMIC_HELPER_FN
	360	#endif /* DATA_SIZE >= 16 */
	361
	362	#undef END
	363	#endif /* DATA_SIZE > 1 */
	364
	365	#undef BSWAP
	366	#undef ABI_TYPE
	367	#undef DATA_TYPE
	368	#undef SDATA_TYPE
	369	#undef SUFFIX
	370	#undef DATA_SIZE
	371	#undef SHIFT