[mirror_qemu.git] / atomic_template.h

/*
 * Atomic helper templates
 * Included from tcg-runtime.c and cputlb.c.
 *
 * Copyright (c) 2016 Red Hat, Inc
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#if DATA_SIZE == 16
# define SUFFIX     o
# define DATA_TYPE  Int128
# define BSWAP      bswap128
#elif DATA_SIZE == 8
# define SUFFIX     q
# define DATA_TYPE  uint64_t
# define BSWAP      bswap64
#elif DATA_SIZE == 4
# define SUFFIX     l
# define DATA_TYPE  uint32_t
# define BSWAP      bswap32
#elif DATA_SIZE == 2
# define SUFFIX     w
# define DATA_TYPE  uint16_t
# define BSWAP      bswap16
#elif DATA_SIZE == 1
# define SUFFIX     b
# define DATA_TYPE  uint8_t
# define BSWAP
#else
# error unsupported data size
#endif

#if DATA_SIZE >= 4
# define ABI_TYPE  DATA_TYPE
#else
# define ABI_TYPE  uint32_t
#endif

/* Define host-endian atomic operations.  Note that END is used within
   the ATOMIC_NAME macro, and redefined below.  */
#if DATA_SIZE == 1
# define END
#elif defined(HOST_WORDS_BIGENDIAN)
# define END  _be
#else
# define END  _le
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    return atomic_cmpxchg__nocheck(haddr, cmpv, newv);
}

#if DATA_SIZE >= 16
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
    return val;
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
}
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    return atomic_xchg__nocheck(haddr, val);
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                 ABI_TYPE val EXTRA_ARGS)                           \
{                                                                   \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    return atomic_##X(haddr, val);                                  \
}                                                                   \

GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(add_fetch)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER
#endif /* DATA SIZE >= 16 */

#undef END

#if DATA_SIZE > 1

/* Define reverse-host-endian atomic operations.  Note that END is used
   within the ATOMIC_NAME macro.  */
#ifdef HOST_WORDS_BIGENDIAN
# define END  _le
#else
# define END  _be
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    return BSWAP(atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)));
}

#if DATA_SIZE >= 16
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
    return BSWAP(val);
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    val = BSWAP(val);
    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
}
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    return BSWAP(atomic_xchg__nocheck(haddr, BSWAP(val)));
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                 ABI_TYPE val EXTRA_ARGS)                           \
{                                                                   \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    return BSWAP(atomic_##X(haddr, BSWAP(val)));                    \
}

GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* Note that for addition, we need to use a separate cmpxchg loop instead
   of bswaps for the reverse-host-endian helpers.  */
ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr,
                         ABI_TYPE val EXTRA_ARGS)
{
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ldo, ldn, ret, sto;

    ldo = atomic_read__nocheck(haddr);
    while (1) {
        ret = BSWAP(ldo);
        sto = BSWAP(ret + val);
        ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
        if (ldn == ldo) {
            return ret;
        }
        ldo = ldn;
    }
}

ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr,
                         ABI_TYPE val EXTRA_ARGS)
{
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ldo, ldn, ret, sto;

    ldo = atomic_read__nocheck(haddr);
    while (1) {
        ret = BSWAP(ldo) + val;
        sto = BSWAP(ret);
        ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
        if (ldn == ldo) {
            return ret;
        }
        ldo = ldn;
    }
}
#endif /* DATA_SIZE >= 16 */

#undef END
#endif /* DATA_SIZE > 1 */

#undef BSWAP
#undef ABI_TYPE
#undef DATA_TYPE
#undef SUFFIX
#undef DATA_SIZE
Commit	Line	Data
c482cb11 RH	1	/*
	2	* Atomic helper templates
	3	* Included from tcg-runtime.c and cputlb.c.
	4	*
	5	* Copyright (c) 2016 Red Hat, Inc
	6	*
	7	* This library is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU Lesser General Public
	9	* License as published by the Free Software Foundation; either
	10	* version 2 of the License, or (at your option) any later version.
	11	*
	12	* This library is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	15	* Lesser General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU Lesser General Public
	18	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
7ebee43e RH	21	#if DATA_SIZE == 16
	22	# define SUFFIX o
	23	# define DATA_TYPE Int128
	24	# define BSWAP bswap128
	25	#elif DATA_SIZE == 8
c482cb11 RH	26	# define SUFFIX q
	27	# define DATA_TYPE uint64_t
	28	# define BSWAP bswap64
	29	#elif DATA_SIZE == 4
	30	# define SUFFIX l
	31	# define DATA_TYPE uint32_t
	32	# define BSWAP bswap32
	33	#elif DATA_SIZE == 2
	34	# define SUFFIX w
	35	# define DATA_TYPE uint16_t
	36	# define BSWAP bswap16
	37	#elif DATA_SIZE == 1
	38	# define SUFFIX b
	39	# define DATA_TYPE uint8_t
	40	# define BSWAP
	41	#else
	42	# error unsupported data size
	43	#endif
	44
	45	#if DATA_SIZE >= 4
	46	# define ABI_TYPE DATA_TYPE
	47	#else
	48	# define ABI_TYPE uint32_t
	49	#endif
	50
	51	/* Define host-endian atomic operations. Note that END is used within
	52	the ATOMIC_NAME macro, and redefined below. */
	53	#if DATA_SIZE == 1
	54	# define END
	55	#elif defined(HOST_WORDS_BIGENDIAN)
	56	# define END _be
	57	#else
	58	# define END _le
	59	#endif
	60
	61	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	62	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	63	{
	64	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	65	return atomic_cmpxchg__nocheck(haddr, cmpv, newv);
	66	}
	67
7ebee43e RH	68	#if DATA_SIZE >= 16
	69	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
	70	{
	71	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
	72	__atomic_load(haddr, &val, __ATOMIC_RELAXED);
	73	return val;
	74	}
	75
	76	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	77	ABI_TYPE val EXTRA_ARGS)
	78	{
	79	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	80	__atomic_store(haddr, &val, __ATOMIC_RELAXED);
	81	}
	82	#else
c482cb11 RH	83	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	84	ABI_TYPE val EXTRA_ARGS)
	85	{
	86	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	87	return atomic_xchg__nocheck(haddr, val);
	88	}
	89
	90	#define GEN_ATOMIC_HELPER(X) \
	91	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	92	ABI_TYPE val EXTRA_ARGS) \
	93	{ \
	94	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	95	return atomic_##X(haddr, val); \
	96	} \
	97
	98	GEN_ATOMIC_HELPER(fetch_add)
	99	GEN_ATOMIC_HELPER(fetch_and)
	100	GEN_ATOMIC_HELPER(fetch_or)
	101	GEN_ATOMIC_HELPER(fetch_xor)
	102	GEN_ATOMIC_HELPER(add_fetch)
	103	GEN_ATOMIC_HELPER(and_fetch)
	104	GEN_ATOMIC_HELPER(or_fetch)
	105	GEN_ATOMIC_HELPER(xor_fetch)
	106
	107	#undef GEN_ATOMIC_HELPER
7ebee43e RH	108	#endif /* DATA SIZE >= 16 */
7ebee43e RH	109
c482cb11 RH	110	#undef END
	111
	112	#if DATA_SIZE > 1
	113
	114	/* Define reverse-host-endian atomic operations. Note that END is used
	115	within the ATOMIC_NAME macro. */
	116	#ifdef HOST_WORDS_BIGENDIAN
	117	# define END _le
	118	#else
	119	# define END _be
	120	#endif
	121
	122	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	123	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	124	{
	125	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	126	return BSWAP(atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)));
	127	}
	128
7ebee43e RH	129	#if DATA_SIZE >= 16
	130	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
	131	{
	132	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
	133	__atomic_load(haddr, &val, __ATOMIC_RELAXED);
	134	return BSWAP(val);
	135	}
	136
	137	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	138	ABI_TYPE val EXTRA_ARGS)
	139	{
	140	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	141	val = BSWAP(val);
	142	__atomic_store(haddr, &val, __ATOMIC_RELAXED);
	143	}
	144	#else
c482cb11 RH	145	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	146	ABI_TYPE val EXTRA_ARGS)
	147	{
	148	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	149	return BSWAP(atomic_xchg__nocheck(haddr, BSWAP(val)));
	150	}
	151
	152	#define GEN_ATOMIC_HELPER(X) \
	153	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	154	ABI_TYPE val EXTRA_ARGS) \
	155	{ \
	156	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	157	return BSWAP(atomic_##X(haddr, BSWAP(val))); \
	158	}
	159
	160	GEN_ATOMIC_HELPER(fetch_and)
	161	GEN_ATOMIC_HELPER(fetch_or)
	162	GEN_ATOMIC_HELPER(fetch_xor)
	163	GEN_ATOMIC_HELPER(and_fetch)
	164	GEN_ATOMIC_HELPER(or_fetch)
	165	GEN_ATOMIC_HELPER(xor_fetch)
	166
	167	#undef GEN_ATOMIC_HELPER
	168
	169	/* Note that for addition, we need to use a separate cmpxchg loop instead
	170	of bswaps for the reverse-host-endian helpers. */
	171	ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr,
	172	ABI_TYPE val EXTRA_ARGS)
	173	{
	174	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	175	DATA_TYPE ldo, ldn, ret, sto;
	176
	177	ldo = atomic_read__nocheck(haddr);
	178	while (1) {
	179	ret = BSWAP(ldo);
	180	sto = BSWAP(ret + val);
	181	ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
	182	if (ldn == ldo) {
	183	return ret;
	184	}
	185	ldo = ldn;
	186	}
	187	}
	188
	189	ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr,
	190	ABI_TYPE val EXTRA_ARGS)
	191	{
	192	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
	193	DATA_TYPE ldo, ldn, ret, sto;
	194
	195	ldo = atomic_read__nocheck(haddr);
	196	while (1) {
	197	ret = BSWAP(ldo) + val;
	198	sto = BSWAP(ret);
	199	ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
	200	if (ldn == ldo) {
	201	return ret;
	202	}
	203	ldo = ldn;
	204	}
	205	}
7ebee43e	206	#endif /* DATA_SIZE >= 16 */
c482cb11 RH	207
	208	#undef END
	209	#endif /* DATA_SIZE > 1 */
	210
	211	#undef BSWAP
	212	#undef ABI_TYPE
	213	#undef DATA_TYPE
	214	#undef SUFFIX
	215	#undef DATA_SIZE