accel/tcg/atomic_template.h

   1 /*
   2  * Atomic helper templates
   3  * Included from tcg-runtime.c and cputlb.c.
   4  *
   5  * Copyright (c) 2016 Red Hat, Inc
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #if DATA_SIZE == 16
  22 # define SUFFIX     o
  23 # define DATA_TYPE  Int128
  24 # define BSWAP      bswap128
  25 #elif DATA_SIZE == 8
  26 # define SUFFIX     q
  27 # define DATA_TYPE  uint64_t
  28 # define BSWAP      bswap64
  29 #elif DATA_SIZE == 4
  30 # define SUFFIX     l
  31 # define DATA_TYPE  uint32_t
  32 # define BSWAP      bswap32
  33 #elif DATA_SIZE == 2
  34 # define SUFFIX     w
  35 # define DATA_TYPE  uint16_t
  36 # define BSWAP      bswap16
  37 #elif DATA_SIZE == 1
  38 # define SUFFIX     b
  39 # define DATA_TYPE  uint8_t
  40 # define BSWAP
  41 #else
  42 # error unsupported data size
  43 #endif
  44
  45 #if DATA_SIZE >= 4
  46 # define ABI_TYPE  DATA_TYPE
  47 #else
  48 # define ABI_TYPE  uint32_t
  49 #endif
  50
  51 /* Define host-endian atomic operations.  Note that END is used within
  52    the ATOMIC_NAME macro, and redefined below.  */
  53 #if DATA_SIZE == 1
  54 # define END
  55 #elif defined(HOST_WORDS_BIGENDIAN)
  56 # define END  _be
  57 #else
  58 # define END  _le
  59 #endif
  60
  61 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  62                               ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
  63 {
  64     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  65     DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
  66     ATOMIC_MMU_CLEANUP;
  67     return ret;
  68 }
  69
  70 #if DATA_SIZE >= 16
  71 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
  72 {
  73     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
  74     __atomic_load(haddr, &val, __ATOMIC_RELAXED);
  75     ATOMIC_MMU_CLEANUP;
  76     return val;
  77 }
  78
  79 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
  80                      ABI_TYPE val EXTRA_ARGS)
  81 {
  82     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  83     __atomic_store(haddr, &val, __ATOMIC_RELAXED);
  84     ATOMIC_MMU_CLEANUP;
  85 }
  86 #else
  87 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
  88                            ABI_TYPE val EXTRA_ARGS)
  89 {
  90     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  91     DATA_TYPE ret = atomic_xchg__nocheck(haddr, val);
  92     ATOMIC_MMU_CLEANUP;
  93     return ret;
  94 }
  95
  96 #define GEN_ATOMIC_HELPER(X)                                        \
  97 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
  98                  ABI_TYPE val EXTRA_ARGS)                           \
  99 {                                                                   \
 100     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 101     DATA_TYPE ret = atomic_##X(haddr, val);                         \
 102     ATOMIC_MMU_CLEANUP;                                             \
 103     return ret;                                                     \
 104 }
 105
 106 GEN_ATOMIC_HELPER(fetch_add)
 107 GEN_ATOMIC_HELPER(fetch_and)
 108 GEN_ATOMIC_HELPER(fetch_or)
 109 GEN_ATOMIC_HELPER(fetch_xor)
 110 GEN_ATOMIC_HELPER(add_fetch)
 111 GEN_ATOMIC_HELPER(and_fetch)
 112 GEN_ATOMIC_HELPER(or_fetch)
 113 GEN_ATOMIC_HELPER(xor_fetch)
 114
 115 #undef GEN_ATOMIC_HELPER
 116 #endif /* DATA SIZE >= 16 */
 117
 118 #undef END
 119
 120 #if DATA_SIZE > 1
 121
 122 /* Define reverse-host-endian atomic operations.  Note that END is used
 123    within the ATOMIC_NAME macro.  */
 124 #ifdef HOST_WORDS_BIGENDIAN
 125 # define END  _le
 126 #else
 127 # define END  _be
 128 #endif
 129
 130 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 131                               ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
 132 {
 133     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 134     DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 135     ATOMIC_MMU_CLEANUP;
 136     return BSWAP(ret);
 137 }
 138
 139 #if DATA_SIZE >= 16
 140 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 141 {
 142     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 143     __atomic_load(haddr, &val, __ATOMIC_RELAXED);
 144     ATOMIC_MMU_CLEANUP;
 145     return BSWAP(val);
 146 }
 147
 148 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 149                      ABI_TYPE val EXTRA_ARGS)
 150 {
 151     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 152     val = BSWAP(val);
 153     __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 154     ATOMIC_MMU_CLEANUP;
 155 }
 156 #else
 157 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 158                            ABI_TYPE val EXTRA_ARGS)
 159 {
 160     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 161     ABI_TYPE ret = atomic_xchg__nocheck(haddr, BSWAP(val));
 162     ATOMIC_MMU_CLEANUP;
 163     return BSWAP(ret);
 164 }
 165
 166 #define GEN_ATOMIC_HELPER(X)                                        \
 167 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 168                  ABI_TYPE val EXTRA_ARGS)                           \
 169 {                                                                   \
 170     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 171     DATA_TYPE ret = atomic_##X(haddr, BSWAP(val));                  \
 172     ATOMIC_MMU_CLEANUP;                                             \
 173     return BSWAP(ret);                                              \
 174 }
 175
 176 GEN_ATOMIC_HELPER(fetch_and)
 177 GEN_ATOMIC_HELPER(fetch_or)
 178 GEN_ATOMIC_HELPER(fetch_xor)
 179 GEN_ATOMIC_HELPER(and_fetch)
 180 GEN_ATOMIC_HELPER(or_fetch)
 181 GEN_ATOMIC_HELPER(xor_fetch)
 182
 183 #undef GEN_ATOMIC_HELPER
 184
 185 /* Note that for addition, we need to use a separate cmpxchg loop instead
 186    of bswaps for the reverse-host-endian helpers.  */
 187 ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr,
 188                          ABI_TYPE val EXTRA_ARGS)
 189 {
 190     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 191     DATA_TYPE ldo, ldn, ret, sto;
 192
 193     ldo = atomic_read__nocheck(haddr);
 194     while (1) {
 195         ret = BSWAP(ldo);
 196         sto = BSWAP(ret + val);
 197         ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
 198         if (ldn == ldo) {
 199             ATOMIC_MMU_CLEANUP;
 200             return ret;
 201         }
 202         ldo = ldn;
 203     }
 204 }
 205
 206 ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr,
 207                          ABI_TYPE val EXTRA_ARGS)
 208 {
 209     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 210     DATA_TYPE ldo, ldn, ret, sto;
 211
 212     ldo = atomic_read__nocheck(haddr);
 213     while (1) {
 214         ret = BSWAP(ldo) + val;
 215         sto = BSWAP(ret);
 216         ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
 217         if (ldn == ldo) {
 218             ATOMIC_MMU_CLEANUP;
 219             return ret;
 220         }
 221         ldo = ldn;
 222     }
 223 }
 224 #endif /* DATA_SIZE >= 16 */
 225
 226 #undef END
 227 #endif /* DATA_SIZE > 1 */
 228
 229 #undef BSWAP
 230 #undef ABI_TYPE
 231 #undef DATA_TYPE
 232 #undef SUFFIX
 233 #undef DATA_SIZE