accel/tcg/atomic_template.h

   1 /*
   2  * Atomic helper templates
   3  * Included from tcg-runtime.c and cputlb.c.
   4  *
   5  * Copyright (c) 2016 Red Hat, Inc
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #include "qemu/plugin.h"
  22
  23 #if DATA_SIZE == 16
  24 # define SUFFIX     o
  25 # define DATA_TYPE  Int128
  26 # define BSWAP      bswap128
  27 # define SHIFT      4
  28 #elif DATA_SIZE == 8
  29 # define SUFFIX     q
  30 # define DATA_TYPE  aligned_uint64_t
  31 # define SDATA_TYPE aligned_int64_t
  32 # define BSWAP      bswap64
  33 # define SHIFT      3
  34 #elif DATA_SIZE == 4
  35 # define SUFFIX     l
  36 # define DATA_TYPE  uint32_t
  37 # define SDATA_TYPE int32_t
  38 # define BSWAP      bswap32
  39 # define SHIFT      2
  40 #elif DATA_SIZE == 2
  41 # define SUFFIX     w
  42 # define DATA_TYPE  uint16_t
  43 # define SDATA_TYPE int16_t
  44 # define BSWAP      bswap16
  45 # define SHIFT      1
  46 #elif DATA_SIZE == 1
  47 # define SUFFIX     b
  48 # define DATA_TYPE  uint8_t
  49 # define SDATA_TYPE int8_t
  50 # define BSWAP
  51 # define SHIFT      0
  52 #else
  53 # error unsupported data size
  54 #endif
  55
  56 #if DATA_SIZE >= 4
  57 # define ABI_TYPE  DATA_TYPE
  58 #else
  59 # define ABI_TYPE  uint32_t
  60 #endif
  61
  62 /* Define host-endian atomic operations.  Note that END is used within
  63    the ATOMIC_NAME macro, and redefined below.  */
  64 #if DATA_SIZE == 1
  65 # define END
  66 #elif defined(HOST_WORDS_BIGENDIAN)
  67 # define END  _be
  68 #else
  69 # define END  _le
  70 #endif
  71
  72 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  73                               ABI_TYPE cmpv, ABI_TYPE newv,
  74                               MemOpIdx oi, uintptr_t retaddr)
  75 {
  76     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
  77                                          PAGE_READ | PAGE_WRITE, retaddr);
  78     DATA_TYPE ret;
  79
  80     atomic_trace_rmw_pre(env, addr, oi);
  81 #if DATA_SIZE == 16
  82     ret = atomic16_cmpxchg(haddr, cmpv, newv);
  83 #else
  84     ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
  85 #endif
  86     ATOMIC_MMU_CLEANUP;
  87     atomic_trace_rmw_post(env, addr, oi);
  88     return ret;
  89 }
  90
  91 #if DATA_SIZE >= 16
  92 #if HAVE_ATOMIC128
  93 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr,
  94                          MemOpIdx oi, uintptr_t retaddr)
  95 {
  96     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
  97                                          PAGE_READ, retaddr);
  98     DATA_TYPE val;
  99
 100     atomic_trace_ld_pre(env, addr, oi);
 101     val = atomic16_read(haddr);
 102     ATOMIC_MMU_CLEANUP;
 103     atomic_trace_ld_post(env, addr, oi);
 104     return val;
 105 }
 106
 107 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 108                      MemOpIdx oi, uintptr_t retaddr)
 109 {
 110     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 111                                          PAGE_WRITE, retaddr);
 112
 113     atomic_trace_st_pre(env, addr, oi);
 114     atomic16_set(haddr, val);
 115     ATOMIC_MMU_CLEANUP;
 116     atomic_trace_st_post(env, addr, oi);
 117 }
 118 #endif
 119 #else
 120 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 121                            MemOpIdx oi, uintptr_t retaddr)
 122 {
 123     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 124                                          PAGE_READ | PAGE_WRITE, retaddr);
 125     DATA_TYPE ret;
 126
 127     atomic_trace_rmw_pre(env, addr, oi);
 128     ret = qatomic_xchg__nocheck(haddr, val);
 129     ATOMIC_MMU_CLEANUP;
 130     atomic_trace_rmw_post(env, addr, oi);
 131     return ret;
 132 }
 133
 134 #define GEN_ATOMIC_HELPER(X)                                        \
 135 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 136                         ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 137 {                                                                   \
 138     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,  \
 139                                          PAGE_READ | PAGE_WRITE, retaddr); \
 140     DATA_TYPE ret;                                                  \
 141     atomic_trace_rmw_pre(env, addr, oi);                            \
 142     ret = qatomic_##X(haddr, val);                                  \
 143     ATOMIC_MMU_CLEANUP;                                             \
 144     atomic_trace_rmw_post(env, addr, oi);                           \
 145     return ret;                                                     \
 146 }
 147
 148 GEN_ATOMIC_HELPER(fetch_add)
 149 GEN_ATOMIC_HELPER(fetch_and)
 150 GEN_ATOMIC_HELPER(fetch_or)
 151 GEN_ATOMIC_HELPER(fetch_xor)
 152 GEN_ATOMIC_HELPER(add_fetch)
 153 GEN_ATOMIC_HELPER(and_fetch)
 154 GEN_ATOMIC_HELPER(or_fetch)
 155 GEN_ATOMIC_HELPER(xor_fetch)
 156
 157 #undef GEN_ATOMIC_HELPER
 158
 159 /*
 160  * These helpers are, as a whole, full barriers.  Within the helper,
 161  * the leading barrier is explicit and the trailing barrier is within
 162  * cmpxchg primitive.
 163  *
 164  * Trace this load + RMW loop as a single RMW op. This way, regardless
 165  * of CF_PARALLEL's value, we'll trace just a read and a write.
 166  */
 167 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 168 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 169                         ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 170 {                                                                   \
 171     XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
 172                                           PAGE_READ | PAGE_WRITE, retaddr); \
 173     XDATA_TYPE cmp, old, new, val = xval;                           \
 174     atomic_trace_rmw_pre(env, addr, oi);                            \
 175     smp_mb();                                                       \
 176     cmp = qatomic_read__nocheck(haddr);                             \
 177     do {                                                            \
 178         old = cmp; new = FN(old, val);                              \
 179         cmp = qatomic_cmpxchg__nocheck(haddr, old, new);            \
 180     } while (cmp != old);                                           \
 181     ATOMIC_MMU_CLEANUP;                                             \
 182     atomic_trace_rmw_post(env, addr, oi);                           \
 183     return RET;                                                     \
 184 }
 185
 186 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 187 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 188 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 189 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 190
 191 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 192 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 193 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 194 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 195
 196 #undef GEN_ATOMIC_HELPER_FN
 197 #endif /* DATA SIZE >= 16 */
 198
 199 #undef END
 200
 201 #if DATA_SIZE > 1
 202
 203 /* Define reverse-host-endian atomic operations.  Note that END is used
 204    within the ATOMIC_NAME macro.  */
 205 #ifdef HOST_WORDS_BIGENDIAN
 206 # define END  _le
 207 #else
 208 # define END  _be
 209 #endif
 210
 211 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 212                               ABI_TYPE cmpv, ABI_TYPE newv,
 213                               MemOpIdx oi, uintptr_t retaddr)
 214 {
 215     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 216                                          PAGE_READ | PAGE_WRITE, retaddr);
 217     DATA_TYPE ret;
 218
 219     atomic_trace_rmw_pre(env, addr, oi);
 220 #if DATA_SIZE == 16
 221     ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
 222 #else
 223     ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 224 #endif
 225     ATOMIC_MMU_CLEANUP;
 226     atomic_trace_rmw_post(env, addr, oi);
 227     return BSWAP(ret);
 228 }
 229
 230 #if DATA_SIZE >= 16
 231 #if HAVE_ATOMIC128
 232 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr,
 233                          MemOpIdx oi, uintptr_t retaddr)
 234 {
 235     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 236                                          PAGE_READ, retaddr);
 237     DATA_TYPE val;
 238
 239     atomic_trace_ld_pre(env, addr, oi);
 240     val = atomic16_read(haddr);
 241     ATOMIC_MMU_CLEANUP;
 242     atomic_trace_ld_post(env, addr, oi);
 243     return BSWAP(val);
 244 }
 245
 246 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 247                      MemOpIdx oi, uintptr_t retaddr)
 248 {
 249     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 250                                          PAGE_WRITE, retaddr);
 251
 252     atomic_trace_st_pre(env, addr, oi);
 253     val = BSWAP(val);
 254     atomic16_set(haddr, val);
 255     ATOMIC_MMU_CLEANUP;
 256     atomic_trace_st_post(env, addr, oi);
 257 }
 258 #endif
 259 #else
 260 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 261                            MemOpIdx oi, uintptr_t retaddr)
 262 {
 263     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 264                                          PAGE_READ | PAGE_WRITE, retaddr);
 265     ABI_TYPE ret;
 266
 267     atomic_trace_rmw_pre(env, addr, oi);
 268     ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
 269     ATOMIC_MMU_CLEANUP;
 270     atomic_trace_rmw_post(env, addr, oi);
 271     return BSWAP(ret);
 272 }
 273
 274 #define GEN_ATOMIC_HELPER(X)                                        \
 275 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 276                         ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 277 {                                                                   \
 278     DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,  \
 279                                          PAGE_READ | PAGE_WRITE, retaddr); \
 280     DATA_TYPE ret;                                                  \
 281     atomic_trace_rmw_pre(env, addr, oi);                            \
 282     ret = qatomic_##X(haddr, BSWAP(val));                           \
 283     ATOMIC_MMU_CLEANUP;                                             \
 284     atomic_trace_rmw_post(env, addr, oi);                           \
 285     return BSWAP(ret);                                              \
 286 }
 287
 288 GEN_ATOMIC_HELPER(fetch_and)
 289 GEN_ATOMIC_HELPER(fetch_or)
 290 GEN_ATOMIC_HELPER(fetch_xor)
 291 GEN_ATOMIC_HELPER(and_fetch)
 292 GEN_ATOMIC_HELPER(or_fetch)
 293 GEN_ATOMIC_HELPER(xor_fetch)
 294
 295 #undef GEN_ATOMIC_HELPER
 296
 297 /* These helpers are, as a whole, full barriers.  Within the helper,
 298  * the leading barrier is explicit and the trailing barrier is within
 299  * cmpxchg primitive.
 300  *
 301  * Trace this load + RMW loop as a single RMW op. This way, regardless
 302  * of CF_PARALLEL's value, we'll trace just a read and a write.
 303  */
 304 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 305 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 306                         ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 307 {                                                                   \
 308     XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
 309                                           PAGE_READ | PAGE_WRITE, retaddr); \
 310     XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
 311     atomic_trace_rmw_pre(env, addr, oi);                            \
 312     smp_mb();                                                       \
 313     ldn = qatomic_read__nocheck(haddr);                             \
 314     do {                                                            \
 315         ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
 316         ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));     \
 317     } while (ldo != ldn);                                           \
 318     ATOMIC_MMU_CLEANUP;                                             \
 319     atomic_trace_rmw_post(env, addr, oi);                           \
 320     return RET;                                                     \
 321 }
 322
 323 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 324 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 325 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 326 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 327
 328 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 329 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 330 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 331 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 332
 333 /* Note that for addition, we need to use a separate cmpxchg loop instead
 334    of bswaps for the reverse-host-endian helpers.  */
 335 #define ADD(X, Y)   (X + Y)
 336 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
 337 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 338 #undef ADD
 339
 340 #undef GEN_ATOMIC_HELPER_FN
 341 #endif /* DATA_SIZE >= 16 */
 342
 343 #undef END
 344 #endif /* DATA_SIZE > 1 */
 345
 346 #undef BSWAP
 347 #undef ABI_TYPE
 348 #undef DATA_TYPE
 349 #undef SDATA_TYPE
 350 #undef SUFFIX
 351 #undef DATA_SIZE
 352 #undef SHIFT