* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
+ * version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
+#include "qemu/plugin.h"
+#include "trace/mem.h"
+
#if DATA_SIZE == 16
# define SUFFIX o
# define DATA_TYPE Int128
# define BSWAP bswap128
+# define SHIFT 4
#elif DATA_SIZE == 8
# define SUFFIX q
# define DATA_TYPE uint64_t
+# define SDATA_TYPE int64_t
# define BSWAP bswap64
+# define SHIFT 3
#elif DATA_SIZE == 4
# define SUFFIX l
# define DATA_TYPE uint32_t
+# define SDATA_TYPE int32_t
# define BSWAP bswap32
+# define SHIFT 2
#elif DATA_SIZE == 2
# define SUFFIX w
# define DATA_TYPE uint16_t
+# define SDATA_TYPE int16_t
# define BSWAP bswap16
+# define SHIFT 1
#elif DATA_SIZE == 1
# define SUFFIX b
# define DATA_TYPE uint8_t
+# define SDATA_TYPE int8_t
# define BSWAP
+# define SHIFT 0
#else
# error unsupported data size
#endif
ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
+ ATOMIC_MMU_DECLS;
DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
- return atomic_cmpxchg__nocheck(haddr, cmpv, newv);
+ DATA_TYPE ret;
+ uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
+ ATOMIC_MMU_IDX);
+
+ atomic_trace_rmw_pre(env, addr, info);
+#if DATA_SIZE == 16
+ ret = atomic16_cmpxchg(haddr, cmpv, newv);
+#else
+ ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
+#endif
+ ATOMIC_MMU_CLEANUP;
+ atomic_trace_rmw_post(env, addr, info);
+ return ret;
}
#if DATA_SIZE >= 16
+#if HAVE_ATOMIC128
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
+ ATOMIC_MMU_DECLS;
DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
- __atomic_load(haddr, &val, __ATOMIC_RELAXED);
+ uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
+ ATOMIC_MMU_IDX);
+
+ atomic_trace_ld_pre(env, addr, info);
+ val = atomic16_read(haddr);
+ ATOMIC_MMU_CLEANUP;
+ atomic_trace_ld_post(env, addr, info);
return val;
}
void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
ABI_TYPE val EXTRA_ARGS)
{
+ ATOMIC_MMU_DECLS;
DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
- __atomic_store(haddr, &val, __ATOMIC_RELAXED);
+ uint16_t info = trace_mem_build_info(SHIFT, false, 0, true,
+ ATOMIC_MMU_IDX);
+
+ atomic_trace_st_pre(env, addr, info);
+ atomic16_set(haddr, val);
+ ATOMIC_MMU_CLEANUP;
+ atomic_trace_st_post(env, addr, info);
}
+#endif
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
ABI_TYPE val EXTRA_ARGS)
{
+ ATOMIC_MMU_DECLS;
DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
- return atomic_xchg__nocheck(haddr, val);
+ DATA_TYPE ret;
+ uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
+ ATOMIC_MMU_IDX);
+
+ atomic_trace_rmw_pre(env, addr, info);
+ ret = qatomic_xchg__nocheck(haddr, val);
+ ATOMIC_MMU_CLEANUP;
+ atomic_trace_rmw_post(env, addr, info);
+ return ret;
}
#define GEN_ATOMIC_HELPER(X) \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
- ABI_TYPE val EXTRA_ARGS) \
+ ABI_TYPE val EXTRA_ARGS) \
{ \
+ ATOMIC_MMU_DECLS; \
DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
- return atomic_##X(haddr, val); \
-} \
+ DATA_TYPE ret; \
+ uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
+ ATOMIC_MMU_IDX); \
+ atomic_trace_rmw_pre(env, addr, info); \
+ ret = qatomic_##X(haddr, val); \
+ ATOMIC_MMU_CLEANUP; \
+ atomic_trace_rmw_post(env, addr, info); \
+ return ret; \
+}
GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(xor_fetch)
#undef GEN_ATOMIC_HELPER
+
+/* These helpers are, as a whole, full barriers. Within the helper,
+ * the leading barrier is explicit and the trailing barrier is within
+ * cmpxchg primitive.
+ *
+ * Trace this load + RMW loop as a single RMW op. This way, regardless
+ * of CF_PARALLEL's value, we'll trace just a read and a write.
+ */
+#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
+ ABI_TYPE xval EXTRA_ARGS) \
+{ \
+ ATOMIC_MMU_DECLS; \
+ XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
+ XDATA_TYPE cmp, old, new, val = xval; \
+ uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
+ ATOMIC_MMU_IDX); \
+ atomic_trace_rmw_pre(env, addr, info); \
+ smp_mb(); \
+ cmp = qatomic_read__nocheck(haddr); \
+ do { \
+ old = cmp; new = FN(old, val); \
+ cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \
+ } while (cmp != old); \
+ ATOMIC_MMU_CLEANUP; \
+ atomic_trace_rmw_post(env, addr, info); \
+ return RET; \
+}
+
+GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
+GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
+GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
+GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
+
+GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
+GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
+GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
+GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
+
+#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA SIZE >= 16 */
#undef END
ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
+ ATOMIC_MMU_DECLS;
DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
- return BSWAP(atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)));
+ DATA_TYPE ret;
+ uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
+ ATOMIC_MMU_IDX);
+
+ atomic_trace_rmw_pre(env, addr, info);
+#if DATA_SIZE == 16
+ ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
+#else
+ ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
+#endif
+ ATOMIC_MMU_CLEANUP;
+ atomic_trace_rmw_post(env, addr, info);
+ return BSWAP(ret);
}
#if DATA_SIZE >= 16
+#if HAVE_ATOMIC128
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
+ ATOMIC_MMU_DECLS;
DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
- __atomic_load(haddr, &val, __ATOMIC_RELAXED);
+ uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
+ ATOMIC_MMU_IDX);
+
+ atomic_trace_ld_pre(env, addr, info);
+ val = atomic16_read(haddr);
+ ATOMIC_MMU_CLEANUP;
+ atomic_trace_ld_post(env, addr, info);
return BSWAP(val);
}
void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
ABI_TYPE val EXTRA_ARGS)
{
+ ATOMIC_MMU_DECLS;
DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
+ uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true,
+ ATOMIC_MMU_IDX);
+
+ val = BSWAP(val);
+ atomic_trace_st_pre(env, addr, info);
val = BSWAP(val);
- __atomic_store(haddr, &val, __ATOMIC_RELAXED);
+ atomic16_set(haddr, val);
+ ATOMIC_MMU_CLEANUP;
+ atomic_trace_st_post(env, addr, info);
}
+#endif
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
ABI_TYPE val EXTRA_ARGS)
{
+ ATOMIC_MMU_DECLS;
DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
- return BSWAP(atomic_xchg__nocheck(haddr, BSWAP(val)));
+ ABI_TYPE ret;
+ uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
+ ATOMIC_MMU_IDX);
+
+ atomic_trace_rmw_pre(env, addr, info);
+ ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
+ ATOMIC_MMU_CLEANUP;
+ atomic_trace_rmw_post(env, addr, info);
+ return BSWAP(ret);
}
#define GEN_ATOMIC_HELPER(X) \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
- ABI_TYPE val EXTRA_ARGS) \
+ ABI_TYPE val EXTRA_ARGS) \
{ \
+ ATOMIC_MMU_DECLS; \
DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
- return BSWAP(atomic_##X(haddr, BSWAP(val))); \
+ DATA_TYPE ret; \
+ uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
+ false, ATOMIC_MMU_IDX); \
+ atomic_trace_rmw_pre(env, addr, info); \
+ ret = qatomic_##X(haddr, BSWAP(val)); \
+ ATOMIC_MMU_CLEANUP; \
+ atomic_trace_rmw_post(env, addr, info); \
+ return BSWAP(ret); \
}
GEN_ATOMIC_HELPER(fetch_and)
#undef GEN_ATOMIC_HELPER
+/* These helpers are, as a whole, full barriers. Within the helper,
+ * the leading barrier is explicit and the trailing barrier is within
+ * cmpxchg primitive.
+ *
+ * Trace this load + RMW loop as a single RMW op. This way, regardless
+ * of CF_PARALLEL's value, we'll trace just a read and a write.
+ */
+#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
+ ABI_TYPE xval EXTRA_ARGS) \
+{ \
+ ATOMIC_MMU_DECLS; \
+ XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
+ XDATA_TYPE ldo, ldn, old, new, val = xval; \
+ uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
+ false, ATOMIC_MMU_IDX); \
+ atomic_trace_rmw_pre(env, addr, info); \
+ smp_mb(); \
+ ldn = qatomic_read__nocheck(haddr); \
+ do { \
+ ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
+ ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
+ } while (ldo != ldn); \
+ ATOMIC_MMU_CLEANUP; \
+ atomic_trace_rmw_post(env, addr, info); \
+ return RET; \
+}
+
+GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
+GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
+GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
+GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
+
+GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
+GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
+GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
+GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
+
/* Note that for addition, we need to use a separate cmpxchg loop instead
of bswaps for the reverse-host-endian helpers. */
-ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr,
- ABI_TYPE val EXTRA_ARGS)
-{
- DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
- DATA_TYPE ldo, ldn, ret, sto;
-
- ldo = atomic_read__nocheck(haddr);
- while (1) {
- ret = BSWAP(ldo);
- sto = BSWAP(ret + val);
- ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
- if (ldn == ldo) {
- return ret;
- }
- ldo = ldn;
- }
-}
+#define ADD(X, Y) (X + Y)
+GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
+GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
+#undef ADD
-ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr,
- ABI_TYPE val EXTRA_ARGS)
-{
- DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
- DATA_TYPE ldo, ldn, ret, sto;
-
- ldo = atomic_read__nocheck(haddr);
- while (1) {
- ret = BSWAP(ldo) + val;
- sto = BSWAP(ret);
- ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
- if (ldn == ldo) {
- return ret;
- }
- ldo = ldn;
- }
-}
+#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA_SIZE >= 16 */
#undef END
#undef BSWAP
#undef ABI_TYPE
#undef DATA_TYPE
+#undef SDATA_TYPE
#undef SUFFIX
#undef DATA_SIZE
+#undef SHIFT
projects / mirror_qemu.git / blobdiff