accel/tcg/atomic_template.h

   1 /*
   2  * Atomic helper templates
   3  * Included from tcg-runtime.c and cputlb.c.
   4  *
   5  * Copyright (c) 2016 Red Hat, Inc
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #include "qemu/plugin.h"
  22
  23 #if DATA_SIZE == 16
  24 # define SUFFIX     o
  25 # define DATA_TYPE  Int128
  26 # define BSWAP      bswap128
  27 # define SHIFT      4
  28 #elif DATA_SIZE == 8
  29 # define SUFFIX     q
  30 # define DATA_TYPE  aligned_uint64_t
  31 # define SDATA_TYPE aligned_int64_t
  32 # define BSWAP      bswap64
  33 # define SHIFT      3
  34 #elif DATA_SIZE == 4
  35 # define SUFFIX     l
  36 # define DATA_TYPE  uint32_t
  37 # define SDATA_TYPE int32_t
  38 # define BSWAP      bswap32
  39 # define SHIFT      2
  40 #elif DATA_SIZE == 2
  41 # define SUFFIX     w
  42 # define DATA_TYPE  uint16_t
  43 # define SDATA_TYPE int16_t
  44 # define BSWAP      bswap16
  45 # define SHIFT      1
  46 #elif DATA_SIZE == 1
  47 # define SUFFIX     b
  48 # define DATA_TYPE  uint8_t
  49 # define SDATA_TYPE int8_t
  50 # define BSWAP
  51 # define SHIFT      0
  52 #else
  53 # error unsupported data size
  54 #endif
  55
  56 #if DATA_SIZE >= 4
  57 # define ABI_TYPE  DATA_TYPE
  58 #else
  59 # define ABI_TYPE  uint32_t
  60 #endif
  61
  62 /* Define host-endian atomic operations.  Note that END is used within
  63    the ATOMIC_NAME macro, and redefined below.  */
  64 #if DATA_SIZE == 1
  65 # define END
  66 #elif HOST_BIG_ENDIAN
  67 # define END  _be
  68 #else
  69 # define END  _le
  70 #endif
  71
  72 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
  73                               ABI_TYPE cmpv, ABI_TYPE newv,
  74                               MemOpIdx oi, uintptr_t retaddr)
  75 {
  76     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);
  77     DATA_TYPE ret;
  78
  79 #if DATA_SIZE == 16
  80     ret = atomic16_cmpxchg(haddr, cmpv, newv);
  81 #else
  82     ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
  83 #endif
  84     ATOMIC_MMU_CLEANUP;
  85     atomic_trace_rmw_post(env, addr, oi);
  86     return ret;
  87 }
  88
  89 #if DATA_SIZE < 16
  90 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
  91                            MemOpIdx oi, uintptr_t retaddr)
  92 {
  93     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);
  94     DATA_TYPE ret;
  95
  96     ret = qatomic_xchg__nocheck(haddr, val);
  97     ATOMIC_MMU_CLEANUP;
  98     atomic_trace_rmw_post(env, addr, oi);
  99     return ret;
 100 }
 101
 102 #define GEN_ATOMIC_HELPER(X)                                        \
 103 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
 104                         ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 105 {                                                                   \
 106     DATA_TYPE *haddr, ret;                                          \
 107     haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);   \
 108     ret = qatomic_##X(haddr, val);                                  \
 109     ATOMIC_MMU_CLEANUP;                                             \
 110     atomic_trace_rmw_post(env, addr, oi);                           \
 111     return ret;                                                     \
 112 }
 113
 114 GEN_ATOMIC_HELPER(fetch_add)
 115 GEN_ATOMIC_HELPER(fetch_and)
 116 GEN_ATOMIC_HELPER(fetch_or)
 117 GEN_ATOMIC_HELPER(fetch_xor)
 118 GEN_ATOMIC_HELPER(add_fetch)
 119 GEN_ATOMIC_HELPER(and_fetch)
 120 GEN_ATOMIC_HELPER(or_fetch)
 121 GEN_ATOMIC_HELPER(xor_fetch)
 122
 123 #undef GEN_ATOMIC_HELPER
 124
 125 /*
 126  * These helpers are, as a whole, full barriers.  Within the helper,
 127  * the leading barrier is explicit and the trailing barrier is within
 128  * cmpxchg primitive.
 129  *
 130  * Trace this load + RMW loop as a single RMW op. This way, regardless
 131  * of CF_PARALLEL's value, we'll trace just a read and a write.
 132  */
 133 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 134 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
 135                         ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 136 {                                                                   \
 137     XDATA_TYPE *haddr, cmp, old, new, val = xval;                   \
 138     haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);   \
 139     smp_mb();                                                       \
 140     cmp = qatomic_read__nocheck(haddr);                             \
 141     do {                                                            \
 142         old = cmp; new = FN(old, val);                              \
 143         cmp = qatomic_cmpxchg__nocheck(haddr, old, new);            \
 144     } while (cmp != old);                                           \
 145     ATOMIC_MMU_CLEANUP;                                             \
 146     atomic_trace_rmw_post(env, addr, oi);                           \
 147     return RET;                                                     \
 148 }
 149
 150 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 151 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 152 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 153 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 154
 155 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 156 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 157 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 158 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 159
 160 #undef GEN_ATOMIC_HELPER_FN
 161 #endif /* DATA SIZE < 16 */
 162
 163 #undef END
 164
 165 #if DATA_SIZE > 1
 166
 167 /* Define reverse-host-endian atomic operations.  Note that END is used
 168    within the ATOMIC_NAME macro.  */
 169 #if HOST_BIG_ENDIAN
 170 # define END  _le
 171 #else
 172 # define END  _be
 173 #endif
 174
 175 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
 176                               ABI_TYPE cmpv, ABI_TYPE newv,
 177                               MemOpIdx oi, uintptr_t retaddr)
 178 {
 179     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);
 180     DATA_TYPE ret;
 181
 182 #if DATA_SIZE == 16
 183     ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
 184 #else
 185     ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 186 #endif
 187     ATOMIC_MMU_CLEANUP;
 188     atomic_trace_rmw_post(env, addr, oi);
 189     return BSWAP(ret);
 190 }
 191
 192 #if DATA_SIZE < 16
 193 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
 194                            MemOpIdx oi, uintptr_t retaddr)
 195 {
 196     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);
 197     ABI_TYPE ret;
 198
 199     ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
 200     ATOMIC_MMU_CLEANUP;
 201     atomic_trace_rmw_post(env, addr, oi);
 202     return BSWAP(ret);
 203 }
 204
 205 #define GEN_ATOMIC_HELPER(X)                                        \
 206 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
 207                         ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 208 {                                                                   \
 209     DATA_TYPE *haddr, ret;                                          \
 210     haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);   \
 211     ret = qatomic_##X(haddr, BSWAP(val));                           \
 212     ATOMIC_MMU_CLEANUP;                                             \
 213     atomic_trace_rmw_post(env, addr, oi);                           \
 214     return BSWAP(ret);                                              \
 215 }
 216
 217 GEN_ATOMIC_HELPER(fetch_and)
 218 GEN_ATOMIC_HELPER(fetch_or)
 219 GEN_ATOMIC_HELPER(fetch_xor)
 220 GEN_ATOMIC_HELPER(and_fetch)
 221 GEN_ATOMIC_HELPER(or_fetch)
 222 GEN_ATOMIC_HELPER(xor_fetch)
 223
 224 #undef GEN_ATOMIC_HELPER
 225
 226 /* These helpers are, as a whole, full barriers.  Within the helper,
 227  * the leading barrier is explicit and the trailing barrier is within
 228  * cmpxchg primitive.
 229  *
 230  * Trace this load + RMW loop as a single RMW op. This way, regardless
 231  * of CF_PARALLEL's value, we'll trace just a read and a write.
 232  */
 233 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 234 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
 235                         ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 236 {                                                                   \
 237     XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval;              \
 238     haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);   \
 239     smp_mb();                                                       \
 240     ldn = qatomic_read__nocheck(haddr);                             \
 241     do {                                                            \
 242         ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
 243         ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));     \
 244     } while (ldo != ldn);                                           \
 245     ATOMIC_MMU_CLEANUP;                                             \
 246     atomic_trace_rmw_post(env, addr, oi);                           \
 247     return RET;                                                     \
 248 }
 249
 250 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 251 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 252 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 253 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 254
 255 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 256 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 257 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 258 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 259
 260 /* Note that for addition, we need to use a separate cmpxchg loop instead
 261    of bswaps for the reverse-host-endian helpers.  */
 262 #define ADD(X, Y)   (X + Y)
 263 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
 264 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 265 #undef ADD
 266
 267 #undef GEN_ATOMIC_HELPER_FN
 268 #endif /* DATA_SIZE < 16 */
 269
 270 #undef END
 271 #endif /* DATA_SIZE > 1 */
 272
 273 #undef BSWAP
 274 #undef ABI_TYPE
 275 #undef DATA_TYPE
 276 #undef SDATA_TYPE
 277 #undef SUFFIX
 278 #undef DATA_SIZE
 279 #undef SHIFT