include/qemu/atomic128.h

   1 /*
   2  * Simple interface for 128-bit atomic operations.
   3  *
   4  * Copyright (C) 2018 Linaro, Ltd.
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  * See docs/devel/atomics.txt for discussion about the guarantees each
  10  * atomic primitive is meant to provide.
  11  */
  12
  13 #ifndef QEMU_ATOMIC128_H
  14 #define QEMU_ATOMIC128_H
  15
  16 #include "qemu/int128.h"
  17
  18 /*
  19  * GCC is a house divided about supporting large atomic operations.
  20  *
  21  * For hosts that only have large compare-and-swap, a legalistic reading
  22  * of the C++ standard means that one cannot implement __atomic_read on
  23  * read-only memory, and thus all atomic operations must synchronize
  24  * through libatomic.
  25  *
  26  * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878
  27  *
  28  * This interpretation is not especially helpful for QEMU.
  29  * For softmmu, all RAM is always read/write from the hypervisor.
  30  * For user-only, if the guest doesn't implement such an __atomic_read
  31  * then the host need not worry about it either.
  32  *
  33  * Moreover, using libatomic is not an option, because its interface is
  34  * built for std::atomic<T>, and requires that *all* accesses to such an
  35  * object go through the library.  In our case we do not have an object
  36  * in the C/C++ sense, but a view of memory as seen by the guest.
  37  * The guest may issue a large atomic operation and then access those
  38  * pieces using word-sized accesses.  From the hypervisor, we have no
  39  * way to connect those two actions.
  40  *
  41  * Therefore, special case each platform.
  42  */
  43
  44 #if defined(CONFIG_ATOMIC128)
  45 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
  46 {
  47     return atomic_cmpxchg__nocheck(ptr, cmp, new);
  48 }
  49 # define HAVE_CMPXCHG128 1
  50 #elif defined(CONFIG_CMPXCHG128)
  51 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
  52 {
  53     return __sync_val_compare_and_swap_16(ptr, cmp, new);
  54 }
  55 # define HAVE_CMPXCHG128 1
  56 #elif defined(__aarch64__)
  57 /* Through gcc 8, aarch64 has no support for 128-bit at all.  */
  58 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
  59 {
  60     uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
  61     uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
  62     uint64_t oldl, oldh;
  63     uint32_t tmp;
  64
  65     asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
  66         "cmp %[oldl], %[cmpl]\n\t"
  67         "ccmp %[oldh], %[cmph], #0, eq\n\t"
  68         "b.ne 1f\n\t"
  69         "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
  70         "cbnz %w[tmp], 0b\n"
  71         "1:"
  72         : [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
  73           [oldl] "=&r"(oldl), [oldh] "=&r"(oldh)
  74         : [cmpl] "r"(cmpl), [cmph] "r"(cmph),
  75           [newl] "r"(newl), [newh] "r"(newh)
  76         : "memory", "cc");
  77
  78     return int128_make128(oldl, oldh);
  79 }
  80 # define HAVE_CMPXCHG128 1
  81 #else
  82 /* Fallback definition that must be optimized away, or error.  */
  83 Int128 QEMU_ERROR("unsupported atomic")
  84     atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
  85 # define HAVE_CMPXCHG128 0
  86 #endif /* Some definition for HAVE_CMPXCHG128 */
  87
  88
  89 #if defined(CONFIG_ATOMIC128)
  90 static inline Int128 atomic16_read(Int128 *ptr)
  91 {
  92     return atomic_read__nocheck(ptr);
  93 }
  94
  95 static inline void atomic16_set(Int128 *ptr, Int128 val)
  96 {
  97     atomic_set__nocheck(ptr, val);
  98 }
  99
 100 # define HAVE_ATOMIC128 1
 101 #elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
 102 /* We can do better than cmpxchg for AArch64.  */
 103 static inline Int128 atomic16_read(Int128 *ptr)
 104 {
 105     uint64_t l, h;
 106     uint32_t tmp;
 107
 108     /* The load must be paired with the store to guarantee not tearing.  */
 109     asm("0: ldxp %[l], %[h], %[mem]\n\t"
 110         "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
 111         "cbnz %w[tmp], 0b"
 112         : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
 113
 114     return int128_make128(l, h);
 115 }
 116
 117 static inline void atomic16_set(Int128 *ptr, Int128 val)
 118 {
 119     uint64_t l = int128_getlo(val), h = int128_gethi(val);
 120     uint64_t t1, t2;
 121
 122     /* Load into temporaries to acquire the exclusive access lock.  */
 123     asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
 124         "stxp %w[t1], %[l], %[h], %[mem]\n\t"
 125         "cbnz %w[t1], 0b"
 126         : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
 127         : [l] "r"(l), [h] "r"(h));
 128 }
 129
 130 # define HAVE_ATOMIC128 1
 131 #elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
 132 static inline Int128 atomic16_read(Int128 *ptr)
 133 {
 134     /* Maybe replace 0 with 0, returning the old value.  */
 135     return atomic16_cmpxchg(ptr, 0, 0);
 136 }
 137
 138 static inline void atomic16_set(Int128 *ptr, Int128 val)
 139 {
 140     Int128 old = *ptr, cmp;
 141     do {
 142         cmp = old;
 143         old = atomic16_cmpxchg(ptr, cmp, val);
 144     } while (old != cmp);
 145 }
 146
 147 # define HAVE_ATOMIC128 1
 148 #else
 149 /* Fallback definitions that must be optimized away, or error.  */
 150 Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr);
 151 void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
 152 # define HAVE_ATOMIC128 0
 153 #endif /* Some definition for HAVE_ATOMIC128 */
 154
 155 #endif /* QEMU_ATOMIC128_H */