atomic_template.h

   1 /*
   2  * Atomic helper templates
   3  * Included from tcg-runtime.c and cputlb.c.
   4  *
   5  * Copyright (c) 2016 Red Hat, Inc
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #if DATA_SIZE == 16
  22 # define SUFFIX     o
  23 # define DATA_TYPE  Int128
  24 # define BSWAP      bswap128
  25 #elif DATA_SIZE == 8
  26 # define SUFFIX     q
  27 # define DATA_TYPE  uint64_t
  28 # define BSWAP      bswap64
  29 #elif DATA_SIZE == 4
  30 # define SUFFIX     l
  31 # define DATA_TYPE  uint32_t
  32 # define BSWAP      bswap32
  33 #elif DATA_SIZE == 2
  34 # define SUFFIX     w
  35 # define DATA_TYPE  uint16_t
  36 # define BSWAP      bswap16
  37 #elif DATA_SIZE == 1
  38 # define SUFFIX     b
  39 # define DATA_TYPE  uint8_t
  40 # define BSWAP
  41 #else
  42 # error unsupported data size
  43 #endif
  44
  45 #if DATA_SIZE >= 4
  46 # define ABI_TYPE  DATA_TYPE
  47 #else
  48 # define ABI_TYPE  uint32_t
  49 #endif
  50
  51 /* Define host-endian atomic operations.  Note that END is used within
  52    the ATOMIC_NAME macro, and redefined below.  */
  53 #if DATA_SIZE == 1
  54 # define END
  55 #elif defined(HOST_WORDS_BIGENDIAN)
  56 # define END  _be
  57 #else
  58 # define END  _le
  59 #endif
  60
  61 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  62                               ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
  63 {
  64     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  65     return atomic_cmpxchg__nocheck(haddr, cmpv, newv);
  66 }
  67
  68 #if DATA_SIZE >= 16
  69 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
  70 {
  71     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
  72     __atomic_load(haddr, &val, __ATOMIC_RELAXED);
  73     return val;
  74 }
  75
  76 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
  77                      ABI_TYPE val EXTRA_ARGS)
  78 {
  79     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  80     __atomic_store(haddr, &val, __ATOMIC_RELAXED);
  81 }
  82 #else
  83 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
  84                            ABI_TYPE val EXTRA_ARGS)
  85 {
  86     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  87     return atomic_xchg__nocheck(haddr, val);
  88 }
  89
  90 #define GEN_ATOMIC_HELPER(X)                                        \
  91 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
  92                  ABI_TYPE val EXTRA_ARGS)                           \
  93 {                                                                   \
  94     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
  95     return atomic_##X(haddr, val);                                  \
  96 }                                                                   \
  97
  98 GEN_ATOMIC_HELPER(fetch_add)
  99 GEN_ATOMIC_HELPER(fetch_and)
 100 GEN_ATOMIC_HELPER(fetch_or)
 101 GEN_ATOMIC_HELPER(fetch_xor)
 102 GEN_ATOMIC_HELPER(add_fetch)
 103 GEN_ATOMIC_HELPER(and_fetch)
 104 GEN_ATOMIC_HELPER(or_fetch)
 105 GEN_ATOMIC_HELPER(xor_fetch)
 106
 107 #undef GEN_ATOMIC_HELPER
 108 #endif /* DATA SIZE >= 16 */
 109
 110 #undef END
 111
 112 #if DATA_SIZE > 1
 113
 114 /* Define reverse-host-endian atomic operations.  Note that END is used
 115    within the ATOMIC_NAME macro.  */
 116 #ifdef HOST_WORDS_BIGENDIAN
 117 # define END  _le
 118 #else
 119 # define END  _be
 120 #endif
 121
 122 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 123                               ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
 124 {
 125     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 126     return BSWAP(atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)));
 127 }
 128
 129 #if DATA_SIZE >= 16
 130 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 131 {
 132     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 133     __atomic_load(haddr, &val, __ATOMIC_RELAXED);
 134     return BSWAP(val);
 135 }
 136
 137 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 138                      ABI_TYPE val EXTRA_ARGS)
 139 {
 140     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 141     val = BSWAP(val);
 142     __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 143 }
 144 #else
 145 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 146                            ABI_TYPE val EXTRA_ARGS)
 147 {
 148     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 149     return BSWAP(atomic_xchg__nocheck(haddr, BSWAP(val)));
 150 }
 151
 152 #define GEN_ATOMIC_HELPER(X)                                        \
 153 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 154                  ABI_TYPE val EXTRA_ARGS)                           \
 155 {                                                                   \
 156     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 157     return BSWAP(atomic_##X(haddr, BSWAP(val)));                    \
 158 }
 159
 160 GEN_ATOMIC_HELPER(fetch_and)
 161 GEN_ATOMIC_HELPER(fetch_or)
 162 GEN_ATOMIC_HELPER(fetch_xor)
 163 GEN_ATOMIC_HELPER(and_fetch)
 164 GEN_ATOMIC_HELPER(or_fetch)
 165 GEN_ATOMIC_HELPER(xor_fetch)
 166
 167 #undef GEN_ATOMIC_HELPER
 168
 169 /* Note that for addition, we need to use a separate cmpxchg loop instead
 170    of bswaps for the reverse-host-endian helpers.  */
 171 ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr,
 172                          ABI_TYPE val EXTRA_ARGS)
 173 {
 174     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 175     DATA_TYPE ldo, ldn, ret, sto;
 176
 177     ldo = atomic_read__nocheck(haddr);
 178     while (1) {
 179         ret = BSWAP(ldo);
 180         sto = BSWAP(ret + val);
 181         ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
 182         if (ldn == ldo) {
 183             return ret;
 184         }
 185         ldo = ldn;
 186     }
 187 }
 188
 189 ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr,
 190                          ABI_TYPE val EXTRA_ARGS)
 191 {
 192     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 193     DATA_TYPE ldo, ldn, ret, sto;
 194
 195     ldo = atomic_read__nocheck(haddr);
 196     while (1) {
 197         ret = BSWAP(ldo) + val;
 198         sto = BSWAP(ret);
 199         ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
 200         if (ldn == ldo) {
 201             return ret;
 202         }
 203         ldo = ldn;
 204     }
 205 }
 206 #endif /* DATA_SIZE >= 16 */
 207
 208 #undef END
 209 #endif /* DATA_SIZE > 1 */
 210
 211 #undef BSWAP
 212 #undef ABI_TYPE
 213 #undef DATA_TYPE
 214 #undef SUFFIX
 215 #undef DATA_SIZE