[mirror_qemu.git] / accel / tcg / atomic_template.h

/*
 * Atomic helper templates
 * Included from tcg-runtime.c and cputlb.c.
 *
 * Copyright (c) 2016 Red Hat, Inc
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#include "qemu/plugin.h"
#include "trace/mem.h"

#if DATA_SIZE == 16
# define SUFFIX     o
# define DATA_TYPE  Int128
# define BSWAP      bswap128
# define SHIFT      4
#elif DATA_SIZE == 8
# define SUFFIX     q
# define DATA_TYPE  uint64_t
# define SDATA_TYPE int64_t
# define BSWAP      bswap64
# define SHIFT      3
#elif DATA_SIZE == 4
# define SUFFIX     l
# define DATA_TYPE  uint32_t
# define SDATA_TYPE int32_t
# define BSWAP      bswap32
# define SHIFT      2
#elif DATA_SIZE == 2
# define SUFFIX     w
# define DATA_TYPE  uint16_t
# define SDATA_TYPE int16_t
# define BSWAP      bswap16
# define SHIFT      1
#elif DATA_SIZE == 1
# define SUFFIX     b
# define DATA_TYPE  uint8_t
# define SDATA_TYPE int8_t
# define BSWAP
# define SHIFT      0
#else
# error unsupported data size
#endif

#if DATA_SIZE >= 4
# define ABI_TYPE  DATA_TYPE
#else
# define ABI_TYPE  uint32_t
#endif

/* Define host-endian atomic operations.  Note that END is used within
   the ATOMIC_NAME macro, and redefined below.  */
#if DATA_SIZE == 1
# define END
#elif defined(HOST_WORDS_BIGENDIAN)
# define END  _be
#else
# define END  _le
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
#if DATA_SIZE == 16
    ret = atomic16_cmpxchg(haddr, cmpv, newv);
#else
    ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
#endif
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return ret;
}

#if DATA_SIZE >= 16
#if HAVE_ATOMIC128
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_ld_pre(env, addr, info);
    val = atomic16_read(haddr);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_ld_post(env, addr, info);
    return val;
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, true,
                                         ATOMIC_MMU_IDX);

    atomic_trace_st_pre(env, addr, info);
    atomic16_set(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_st_post(env, addr, info);
}
#endif
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
    ret = atomic_xchg__nocheck(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return ret;
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE val EXTRA_ARGS)                    \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,    \
                                         ATOMIC_MMU_IDX);           \
    atomic_trace_rmw_pre(env, addr, info);                          \
    ret = atomic_##X(haddr, val);                                   \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return ret;                                                     \
}

GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(add_fetch)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* These helpers are, as a whole, full barriers.  Within the helper,
 * the leading barrier is explicit and the trailing barrier is within
 * cmpxchg primitive.
 *
 * Trace this load + RMW loop as a single RMW op. This way, regardless
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE xval EXTRA_ARGS)                   \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE cmp, old, new, val = xval;                           \
    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,    \
                                         ATOMIC_MMU_IDX);           \
    atomic_trace_rmw_pre(env, addr, info);                          \
    smp_mb();                                                       \
    cmp = atomic_read__nocheck(haddr);                              \
    do {                                                            \
        old = cmp; new = FN(old, val);                              \
        cmp = atomic_cmpxchg__nocheck(haddr, old, new);             \
    } while (cmp != old);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return RET;                                                     \
}

GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)

GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA SIZE >= 16 */

#undef END

#if DATA_SIZE > 1

/* Define reverse-host-endian atomic operations.  Note that END is used
   within the ATOMIC_NAME macro.  */
#ifdef HOST_WORDS_BIGENDIAN
# define END  _le
#else
# define END  _be
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
#if DATA_SIZE == 16
    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
#else
    ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
#endif
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return BSWAP(ret);
}

#if DATA_SIZE >= 16
#if HAVE_ATOMIC128
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_ld_pre(env, addr, info);
    val = atomic16_read(haddr);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_ld_post(env, addr, info);
    return BSWAP(val);
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true,
                                         ATOMIC_MMU_IDX);

    val = BSWAP(val);
    atomic_trace_st_pre(env, addr, info);
    val = BSWAP(val);
    atomic16_set(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_st_post(env, addr, info);
}
#endif
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    ABI_TYPE ret;
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
                                         ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
    ret = atomic_xchg__nocheck(haddr, BSWAP(val));
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return BSWAP(ret);
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE val EXTRA_ARGS)                    \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP,    \
                                         false, ATOMIC_MMU_IDX);    \
    atomic_trace_rmw_pre(env, addr, info);                          \
    ret = atomic_##X(haddr, BSWAP(val));                            \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return BSWAP(ret);                                              \
}

GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* These helpers are, as a whole, full barriers.  Within the helper,
 * the leading barrier is explicit and the trailing barrier is within
 * cmpxchg primitive.
 *
 * Trace this load + RMW loop as a single RMW op. This way, regardless
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE xval EXTRA_ARGS)                   \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP,    \
                                         false, ATOMIC_MMU_IDX);    \
    atomic_trace_rmw_pre(env, addr, info);                          \
    smp_mb();                                                       \
    ldn = atomic_read__nocheck(haddr);                              \
    do {                                                            \
        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
        ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));      \
    } while (ldo != ldn);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return RET;                                                     \
}

GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)

GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

/* Note that for addition, we need to use a separate cmpxchg loop instead
   of bswaps for the reverse-host-endian helpers.  */
#define ADD(X, Y)   (X + Y)
GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
#undef ADD

#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA_SIZE >= 16 */

#undef END
#endif /* DATA_SIZE > 1 */

#undef BSWAP
#undef ABI_TYPE
#undef DATA_TYPE
#undef SDATA_TYPE
#undef SUFFIX
#undef DATA_SIZE
#undef SHIFT
Commit	Line	Data
c482cb11 RH	1	/*
	2	* Atomic helper templates
	3	* Included from tcg-runtime.c and cputlb.c.
	4	*
	5	* Copyright (c) 2016 Red Hat, Inc
	6	*
	7	* This library is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU Lesser General Public
	9	* License as published by the Free Software Foundation; either
fb0343d5	10	* version 2.1 of the License, or (at your option) any later version.
c482cb11 RH	11	*
	12	* This library is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	15	* Lesser General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU Lesser General Public
	18	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
e6d86bed	21	#include "qemu/plugin.h"
d071f4cd EC	22	#include "trace/mem.h"
d071f4cd EC	23
7ebee43e RH	24	#if DATA_SIZE == 16
	25	# define SUFFIX o
	26	# define DATA_TYPE Int128
	27	# define BSWAP bswap128
d071f4cd	28	# define SHIFT 4
7ebee43e	29	#elif DATA_SIZE == 8
c482cb11 RH	30	# define SUFFIX q
c482cb11 RH	31	# define DATA_TYPE uint64_t
5507c2bf	32	# define SDATA_TYPE int64_t
c482cb11	33	# define BSWAP bswap64
d071f4cd	34	# define SHIFT 3
c482cb11 RH	35	#elif DATA_SIZE == 4
	36	# define SUFFIX l
	37	# define DATA_TYPE uint32_t
5507c2bf	38	# define SDATA_TYPE int32_t
c482cb11	39	# define BSWAP bswap32
d071f4cd	40	# define SHIFT 2
c482cb11 RH	41	#elif DATA_SIZE == 2
	42	# define SUFFIX w
	43	# define DATA_TYPE uint16_t
5507c2bf	44	# define SDATA_TYPE int16_t
c482cb11	45	# define BSWAP bswap16
d071f4cd	46	# define SHIFT 1
c482cb11 RH	47	#elif DATA_SIZE == 1
	48	# define SUFFIX b
	49	# define DATA_TYPE uint8_t
5507c2bf	50	# define SDATA_TYPE int8_t
c482cb11	51	# define BSWAP
d071f4cd	52	# define SHIFT 0
c482cb11 RH	53	#else
	54	# error unsupported data size
	55	#endif
	56
	57	#if DATA_SIZE >= 4
	58	# define ABI_TYPE DATA_TYPE
	59	#else
	60	# define ABI_TYPE uint32_t
	61	#endif
	62
	63	/* Define host-endian atomic operations. Note that END is used within
	64	the ATOMIC_NAME macro, and redefined below. */
	65	#if DATA_SIZE == 1
	66	# define END
	67	#elif defined(HOST_WORDS_BIGENDIAN)
	68	# define END _be
	69	#else
	70	# define END _le
	71	#endif
	72
	73	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	74	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	75	{
34d49937	76	ATOMIC_MMU_DECLS;
c482cb11	77	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd	78	DATA_TYPE ret;
4e6b1384 RH	79	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
4e6b1384 RH	80	ATOMIC_MMU_IDX);
d071f4cd	81
cfec3885	82	atomic_trace_rmw_pre(env, addr, info);
e6cd4bb5 RH	83	#if DATA_SIZE == 16
	84	ret = atomic16_cmpxchg(haddr, cmpv, newv);
	85	#else
d071f4cd	86	ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
e6cd4bb5	87	#endif
ec603b55	88	ATOMIC_MMU_CLEANUP;
cfec3885	89	atomic_trace_rmw_post(env, addr, info);
ec603b55	90	return ret;
c482cb11 RH	91	}
c482cb11 RH	92
7ebee43e	93	#if DATA_SIZE >= 16
e6cd4bb5	94	#if HAVE_ATOMIC128
7ebee43e RH	95	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
7ebee43e RH	96	{
34d49937	97	ATOMIC_MMU_DECLS;
7ebee43e	98	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
4e6b1384 RH	99	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
4e6b1384 RH	100	ATOMIC_MMU_IDX);
d071f4cd	101
cfec3885	102	atomic_trace_ld_pre(env, addr, info);
e6cd4bb5	103	val = atomic16_read(haddr);
ec603b55	104	ATOMIC_MMU_CLEANUP;
cfec3885	105	atomic_trace_ld_post(env, addr, info);
7ebee43e RH	106	return val;
	107	}
	108
	109	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	110	ABI_TYPE val EXTRA_ARGS)
	111	{
34d49937	112	ATOMIC_MMU_DECLS;
7ebee43e	113	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
4e6b1384 RH	114	uint16_t info = trace_mem_build_info(SHIFT, false, 0, true,
4e6b1384 RH	115	ATOMIC_MMU_IDX);
d071f4cd	116
cfec3885	117	atomic_trace_st_pre(env, addr, info);
e6cd4bb5	118	atomic16_set(haddr, val);
ec603b55	119	ATOMIC_MMU_CLEANUP;
cfec3885	120	atomic_trace_st_post(env, addr, info);
7ebee43e	121	}
e6cd4bb5	122	#endif
7ebee43e	123	#else
c482cb11 RH	124	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	125	ABI_TYPE val EXTRA_ARGS)
	126	{
34d49937	127	ATOMIC_MMU_DECLS;
c482cb11	128	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd	129	DATA_TYPE ret;
4e6b1384 RH	130	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
4e6b1384 RH	131	ATOMIC_MMU_IDX);
d071f4cd	132
cfec3885	133	atomic_trace_rmw_pre(env, addr, info);
d071f4cd	134	ret = atomic_xchg__nocheck(haddr, val);
ec603b55	135	ATOMIC_MMU_CLEANUP;
cfec3885	136	atomic_trace_rmw_post(env, addr, info);
ec603b55	137	return ret;
c482cb11 RH	138	}
	139
	140	#define GEN_ATOMIC_HELPER(X) \
	141	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
d2fac5f6	142	ABI_TYPE val EXTRA_ARGS) \
c482cb11	143	{ \
34d49937	144	ATOMIC_MMU_DECLS; \
c482cb11	145	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
d071f4cd	146	DATA_TYPE ret; \
4e6b1384 RH	147	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
4e6b1384 RH	148	ATOMIC_MMU_IDX); \
cfec3885	149	atomic_trace_rmw_pre(env, addr, info); \
d071f4cd	150	ret = atomic_##X(haddr, val); \
ec603b55	151	ATOMIC_MMU_CLEANUP; \
cfec3885	152	atomic_trace_rmw_post(env, addr, info); \
ec603b55 RH	153	return ret; \
ec603b55 RH	154	}
c482cb11 RH	155
	156	GEN_ATOMIC_HELPER(fetch_add)
	157	GEN_ATOMIC_HELPER(fetch_and)
	158	GEN_ATOMIC_HELPER(fetch_or)
	159	GEN_ATOMIC_HELPER(fetch_xor)
	160	GEN_ATOMIC_HELPER(add_fetch)
	161	GEN_ATOMIC_HELPER(and_fetch)
	162	GEN_ATOMIC_HELPER(or_fetch)
	163	GEN_ATOMIC_HELPER(xor_fetch)
	164
	165	#undef GEN_ATOMIC_HELPER
5507c2bf RH	166
	167	/* These helpers are, as a whole, full barriers. Within the helper,
	168	* the leading barrier is explicit and the trailing barrier is within
	169	* cmpxchg primitive.
d071f4cd EC	170	*
	171	* Trace this load + RMW loop as a single RMW op. This way, regardless
	172	* of CF_PARALLEL's value, we'll trace just a read and a write.
5507c2bf RH	173	*/
	174	#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
	175	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	176	ABI_TYPE xval EXTRA_ARGS) \
	177	{ \
	178	ATOMIC_MMU_DECLS; \
	179	XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	180	XDATA_TYPE cmp, old, new, val = xval; \
4e6b1384 RH	181	uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
4e6b1384 RH	182	ATOMIC_MMU_IDX); \
cfec3885	183	atomic_trace_rmw_pre(env, addr, info); \
5507c2bf RH	184	smp_mb(); \
	185	cmp = atomic_read__nocheck(haddr); \
	186	do { \
	187	old = cmp; new = FN(old, val); \
	188	cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
	189	} while (cmp != old); \
	190	ATOMIC_MMU_CLEANUP; \
cfec3885	191	atomic_trace_rmw_post(env, addr, info); \
5507c2bf RH	192	return RET; \
	193	}
	194
	195	GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
	196	GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
	197	GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
	198	GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
	199
	200	GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
	201	GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
	202	GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
	203	GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
	204
	205	#undef GEN_ATOMIC_HELPER_FN
7ebee43e RH	206	#endif /* DATA SIZE >= 16 */
7ebee43e RH	207
c482cb11 RH	208	#undef END
	209
	210	#if DATA_SIZE > 1
	211
	212	/* Define reverse-host-endian atomic operations. Note that END is used
	213	within the ATOMIC_NAME macro. */
	214	#ifdef HOST_WORDS_BIGENDIAN
	215	# define END _le
	216	#else
	217	# define END _be
	218	#endif
	219
	220	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	221	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	222	{
34d49937	223	ATOMIC_MMU_DECLS;
c482cb11	224	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd	225	DATA_TYPE ret;
4e6b1384 RH	226	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
4e6b1384 RH	227	ATOMIC_MMU_IDX);
d071f4cd	228
cfec3885	229	atomic_trace_rmw_pre(env, addr, info);
e6cd4bb5 RH	230	#if DATA_SIZE == 16
	231	ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
	232	#else
d071f4cd	233	ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
e6cd4bb5	234	#endif
ec603b55	235	ATOMIC_MMU_CLEANUP;
cfec3885	236	atomic_trace_rmw_post(env, addr, info);
ec603b55	237	return BSWAP(ret);
c482cb11 RH	238	}
c482cb11 RH	239
7ebee43e	240	#if DATA_SIZE >= 16
e6cd4bb5	241	#if HAVE_ATOMIC128
7ebee43e RH	242	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
7ebee43e RH	243	{
34d49937	244	ATOMIC_MMU_DECLS;
7ebee43e	245	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
4e6b1384 RH	246	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
4e6b1384 RH	247	ATOMIC_MMU_IDX);
d071f4cd	248
cfec3885	249	atomic_trace_ld_pre(env, addr, info);
e6cd4bb5	250	val = atomic16_read(haddr);
ec603b55	251	ATOMIC_MMU_CLEANUP;
cfec3885	252	atomic_trace_ld_post(env, addr, info);
7ebee43e RH	253	return BSWAP(val);
	254	}
	255
	256	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	257	ABI_TYPE val EXTRA_ARGS)
	258	{
34d49937	259	ATOMIC_MMU_DECLS;
7ebee43e	260	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
4e6b1384 RH	261	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true,
4e6b1384 RH	262	ATOMIC_MMU_IDX);
d071f4cd	263
cfec3885 EC	264	val = BSWAP(val);
cfec3885 EC	265	atomic_trace_st_pre(env, addr, info);
7ebee43e	266	val = BSWAP(val);
e6cd4bb5	267	atomic16_set(haddr, val);
ec603b55	268	ATOMIC_MMU_CLEANUP;
cfec3885	269	atomic_trace_st_post(env, addr, info);
7ebee43e	270	}
e6cd4bb5	271	#endif
7ebee43e	272	#else
c482cb11 RH	273	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	274	ABI_TYPE val EXTRA_ARGS)
	275	{
34d49937	276	ATOMIC_MMU_DECLS;
c482cb11	277	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd	278	ABI_TYPE ret;
4e6b1384 RH	279	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
4e6b1384 RH	280	ATOMIC_MMU_IDX);
d071f4cd	281
cfec3885	282	atomic_trace_rmw_pre(env, addr, info);
d071f4cd	283	ret = atomic_xchg__nocheck(haddr, BSWAP(val));
ec603b55	284	ATOMIC_MMU_CLEANUP;
cfec3885	285	atomic_trace_rmw_post(env, addr, info);
ec603b55	286	return BSWAP(ret);
c482cb11 RH	287	}
	288
	289	#define GEN_ATOMIC_HELPER(X) \
	290	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
358f6348	291	ABI_TYPE val EXTRA_ARGS) \
c482cb11	292	{ \
34d49937	293	ATOMIC_MMU_DECLS; \
c482cb11	294	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
d071f4cd	295	DATA_TYPE ret; \
4e6b1384 RH	296	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
4e6b1384 RH	297	false, ATOMIC_MMU_IDX); \
cfec3885	298	atomic_trace_rmw_pre(env, addr, info); \
d071f4cd	299	ret = atomic_##X(haddr, BSWAP(val)); \
ec603b55	300	ATOMIC_MMU_CLEANUP; \
cfec3885	301	atomic_trace_rmw_post(env, addr, info); \
ec603b55	302	return BSWAP(ret); \
c482cb11 RH	303	}
	304
	305	GEN_ATOMIC_HELPER(fetch_and)
	306	GEN_ATOMIC_HELPER(fetch_or)
	307	GEN_ATOMIC_HELPER(fetch_xor)
	308	GEN_ATOMIC_HELPER(and_fetch)
	309	GEN_ATOMIC_HELPER(or_fetch)
	310	GEN_ATOMIC_HELPER(xor_fetch)
	311
	312	#undef GEN_ATOMIC_HELPER
	313
5507c2bf RH	314	/* These helpers are, as a whole, full barriers. Within the helper,
	315	* the leading barrier is explicit and the trailing barrier is within
	316	* cmpxchg primitive.
d071f4cd EC	317	*
	318	* Trace this load + RMW loop as a single RMW op. This way, regardless
	319	* of CF_PARALLEL's value, we'll trace just a read and a write.
5507c2bf RH	320	*/
	321	#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
	322	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	323	ABI_TYPE xval EXTRA_ARGS) \
	324	{ \
	325	ATOMIC_MMU_DECLS; \
	326	XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	327	XDATA_TYPE ldo, ldn, old, new, val = xval; \
4e6b1384 RH	328	uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
4e6b1384 RH	329	false, ATOMIC_MMU_IDX); \
cfec3885	330	atomic_trace_rmw_pre(env, addr, info); \
5507c2bf RH	331	smp_mb(); \
	332	ldn = atomic_read__nocheck(haddr); \
	333	do { \
	334	ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
	335	ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
	336	} while (ldo != ldn); \
	337	ATOMIC_MMU_CLEANUP; \
cfec3885	338	atomic_trace_rmw_post(env, addr, info); \
5507c2bf RH	339	return RET; \
	340	}
	341
	342	GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
	343	GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
	344	GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
	345	GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
	346
	347	GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
	348	GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
	349	GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
	350	GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
	351
58edf9ee RH	352	/* Note that for addition, we need to use a separate cmpxchg loop instead
	353	of bswaps for the reverse-host-endian helpers. */
	354	#define ADD(X, Y) (X + Y)
	355	GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
	356	GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
	357	#undef ADD
	358
5507c2bf	359	#undef GEN_ATOMIC_HELPER_FN
7ebee43e	360	#endif /* DATA_SIZE >= 16 */
c482cb11 RH	361
	362	#undef END
	363	#endif /* DATA_SIZE > 1 */
	364
	365	#undef BSWAP
	366	#undef ABI_TYPE
	367	#undef DATA_TYPE
5507c2bf	368	#undef SDATA_TYPE
c482cb11 RH	369	#undef SUFFIX
c482cb11 RH	370	#undef DATA_SIZE
d071f4cd	371	#undef SHIFT