arch/x86/include/asm/cmpxchg_32.h

   1 #ifndef _ASM_X86_CMPXCHG_32_H
   2 #define _ASM_X86_CMPXCHG_32_H
   3
   4 #include <asm/alternative.h> /* Provides LOCK_PREFIX */
   5
   6 /*
   7  * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
   8  *       you need to test for the feature in boot_cpu_data.
   9  */
  10
  11 extern void __xchg_wrong_size(void);
  12 extern void __cmpxchg_wrong_size(void);
  13
  14 /*
  15  * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
  16  * Since this is generally used to protect other memory information, we
  17  * use "asm volatile" and "memory" clobbers to prevent gcc from moving
  18  * information around.
  19  */
  20 #define __xchg(x, ptr, size)                                            \
  21 ({                                                                      \
  22         __typeof(*(ptr)) __x = (x);                                     \
  23         switch (size) {                                                 \
  24         case 1:                                                         \
  25         {                                                               \
  26                 volatile u8 *__ptr = (volatile u8 *)(ptr);              \
  27                 asm volatile("xchgb %0,%1"                              \
  28                              : "=q" (__x), "+m" (*__ptr)                \
  29                              : "0" (__x)                                \
  30                              : "memory");                               \
  31                 break;                                                  \
  32         }                                                               \
  33         case 2:                                                         \
  34         {                                                               \
  35                 volatile u16 *__ptr = (volatile u16 *)(ptr);            \
  36                 asm volatile("xchgw %0,%1"                              \
  37                              : "=r" (__x), "+m" (*__ptr)                \
  38                              : "0" (__x)                                \
  39                              : "memory");                               \
  40                 break;                                                  \
  41         }                                                               \
  42         case 4:                                                         \
  43         {                                                               \
  44                 volatile u32 *__ptr = (volatile u32 *)(ptr);            \
  45                 asm volatile("xchgl %0,%1"                              \
  46                              : "=r" (__x), "+m" (*__ptr)                \
  47                              : "0" (__x)                                \
  48                              : "memory");                               \
  49                 break;                                                  \
  50         }                                                               \
  51         default:                                                        \
  52                 __xchg_wrong_size();                                    \
  53         }                                                               \
  54         __x;                                                            \
  55 })
  56
  57 #define xchg(ptr, v)                                                    \
  58         __xchg((v), (ptr), sizeof(*ptr))
  59
  60 /*
  61  * CMPXCHG8B only writes to the target if we had the previous
  62  * value in registers, otherwise it acts as a read and gives us the
  63  * "new previous" value.  That is why there is a loop.  Preloading
  64  * EDX:EAX is a performance optimization: in the common case it means
  65  * we need only one locked operation.
  66  *
  67  * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
  68  * least an FPU save and/or %cr0.ts manipulation.
  69  *
  70  * cmpxchg8b must be used with the lock prefix here to allow the
  71  * instruction to be executed atomically.  We need to have the reader
  72  * side to see the coherent 64bit value.
  73  */
  74 static inline void set_64bit(volatile u64 *ptr, u64 value)
  75 {
  76         u32 low  = value;
  77         u32 high = value >> 32;
  78         u64 prev = *ptr;
  79
  80         asm volatile("\n1:\t"
  81                      LOCK_PREFIX "cmpxchg8b %0\n\t"
  82                      "jnz 1b"
  83                      : "=m" (*ptr), "+A" (prev)
  84                      : "b" (low), "c" (high)
  85                      : "memory");
  86 }
  87
  88 /*
  89  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
  90  * store NEW in MEM.  Return the initial value in MEM.  Success is
  91  * indicated by comparing RETURN with OLD.
  92  */
  93 #define __raw_cmpxchg(ptr, old, new, size, lock)                        \
  94 ({                                                                      \
  95         __typeof__(*(ptr)) __ret;                                       \
  96         __typeof__(*(ptr)) __old = (old);                               \
  97         __typeof__(*(ptr)) __new = (new);                               \
  98         switch (size) {                                                 \
  99         case 1:                                                         \
 100         {                                                               \
 101                 volatile u8 *__ptr = (volatile u8 *)(ptr);              \
 102                 asm volatile(lock "cmpxchgb %2,%1"                      \
 103                              : "=a" (__ret), "+m" (*__ptr)              \
 104                              : "q" (__new), "0" (__old)                 \
 105                              : "memory");                               \
 106                 break;                                                  \
 107         }                                                               \
 108         case 2:                                                         \
 109         {                                                               \
 110                 volatile u16 *__ptr = (volatile u16 *)(ptr);            \
 111                 asm volatile(lock "cmpxchgw %2,%1"                      \
 112                              : "=a" (__ret), "+m" (*__ptr)              \
 113                              : "r" (__new), "0" (__old)                 \
 114                              : "memory");                               \
 115                 break;                                                  \
 116         }                                                               \
 117         case 4:                                                         \
 118         {                                                               \
 119                 volatile u32 *__ptr = (volatile u32 *)(ptr);            \
 120                 asm volatile(lock "cmpxchgl %2,%1"                      \
 121                              : "=a" (__ret), "+m" (*__ptr)              \
 122                              : "r" (__new), "0" (__old)                 \
 123                              : "memory");                               \
 124                 break;                                                  \
 125         }                                                               \
 126         default:                                                        \
 127                 __cmpxchg_wrong_size();                                 \
 128         }                                                               \
 129         __ret;                                                          \
 130 })
 131
 132 #define __cmpxchg(ptr, old, new, size)                                  \
 133         __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
 134
 135 #define __sync_cmpxchg(ptr, old, new, size)                             \
 136         __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
 137
 138 #define __cmpxchg_local(ptr, old, new, size)                            \
 139         __raw_cmpxchg((ptr), (old), (new), (size), "")
 140
 141 #ifdef CONFIG_X86_CMPXCHG
 142 #define __HAVE_ARCH_CMPXCHG 1
 143
 144 #define cmpxchg(ptr, old, new)                                          \
 145         __cmpxchg((ptr), (old), (new), sizeof(*ptr))
 146
 147 #define sync_cmpxchg(ptr, old, new)                                     \
 148         __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
 149
 150 #define cmpxchg_local(ptr, old, new)                                    \
 151         __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
 152 #endif
 153
 154 #ifdef CONFIG_X86_CMPXCHG64
 155 #define cmpxchg64(ptr, o, n)                                            \
 156         ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
 157                                          (unsigned long long)(n)))
 158 #define cmpxchg64_local(ptr, o, n)                                      \
 159         ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
 160                                                (unsigned long long)(n)))
 161 #endif
 162
 163 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
 164 {
 165         u64 prev;
 166         asm volatile(LOCK_PREFIX "cmpxchg8b %1"
 167                      : "=A" (prev),
 168                        "+m" (*ptr)
 169                      : "b" ((u32)new),
 170                        "c" ((u32)(new >> 32)),
 171                        "0" (old)
 172                      : "memory");
 173         return prev;
 174 }
 175
 176 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
 177 {
 178         u64 prev;
 179         asm volatile("cmpxchg8b %1"
 180                      : "=A" (prev),
 181                        "+m" (*ptr)
 182                      : "b" ((u32)new),
 183                        "c" ((u32)(new >> 32)),
 184                        "0" (old)
 185                      : "memory");
 186         return prev;
 187 }
 188
 189 #ifndef CONFIG_X86_CMPXCHG
 190 /*
 191  * Building a kernel capable running on 80386. It may be necessary to
 192  * simulate the cmpxchg on the 80386 CPU. For that purpose we define
 193  * a function for each of the sizes we support.
 194  */
 195
 196 extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
 197 extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
 198 extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
 199
 200 static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
 201                                         unsigned long new, int size)
 202 {
 203         switch (size) {
 204         case 1:
 205                 return cmpxchg_386_u8(ptr, old, new);
 206         case 2:
 207                 return cmpxchg_386_u16(ptr, old, new);
 208         case 4:
 209                 return cmpxchg_386_u32(ptr, old, new);
 210         }
 211         return old;
 212 }
 213
 214 #define cmpxchg(ptr, o, n)                                              \
 215 ({                                                                      \
 216         __typeof__(*(ptr)) __ret;                                       \
 217         if (likely(boot_cpu_data.x86 > 3))                              \
 218                 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr),            \
 219                                 (unsigned long)(o), (unsigned long)(n), \
 220                                 sizeof(*(ptr)));                        \
 221         else                                                            \
 222                 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
 223                                 (unsigned long)(o), (unsigned long)(n), \
 224                                 sizeof(*(ptr)));                        \
 225         __ret;                                                          \
 226 })
 227 #define cmpxchg_local(ptr, o, n)                                        \
 228 ({                                                                      \
 229         __typeof__(*(ptr)) __ret;                                       \
 230         if (likely(boot_cpu_data.x86 > 3))                              \
 231                 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr),      \
 232                                 (unsigned long)(o), (unsigned long)(n), \
 233                                 sizeof(*(ptr)));                        \
 234         else                                                            \
 235                 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
 236                                 (unsigned long)(o), (unsigned long)(n), \
 237                                 sizeof(*(ptr)));                        \
 238         __ret;                                                          \
 239 })
 240 #endif
 241
 242 #ifndef CONFIG_X86_CMPXCHG64
 243 /*
 244  * Building a kernel capable running on 80386 and 80486. It may be necessary
 245  * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
 246  */
 247
 248 #define cmpxchg64(ptr, o, n)                                    \
 249 ({                                                              \
 250         __typeof__(*(ptr)) __ret;                               \
 251         __typeof__(*(ptr)) __old = (o);                         \
 252         __typeof__(*(ptr)) __new = (n);                         \
 253         alternative_io(LOCK_PREFIX_HERE                         \
 254                         "call cmpxchg8b_emu",                   \
 255                         "lock; cmpxchg8b (%%esi)" ,             \
 256                        X86_FEATURE_CX8,                         \
 257                        "=A" (__ret),                            \
 258                        "S" ((ptr)), "0" (__old),                \
 259                        "b" ((unsigned int)__new),               \
 260                        "c" ((unsigned int)(__new>>32))          \
 261                        : "memory");                             \
 262         __ret; })
 263
 264
 265 #define cmpxchg64_local(ptr, o, n)                              \
 266 ({                                                              \
 267         __typeof__(*(ptr)) __ret;                               \
 268         __typeof__(*(ptr)) __old = (o);                         \
 269         __typeof__(*(ptr)) __new = (n);                         \
 270         alternative_io("call cmpxchg8b_emu",                    \
 271                        "cmpxchg8b (%%esi)" ,                    \
 272                        X86_FEATURE_CX8,                         \
 273                        "=A" (__ret),                            \
 274                        "S" ((ptr)), "0" (__old),                \
 275                        "b" ((unsigned int)__new),               \
 276                        "c" ((unsigned int)(__new>>32))          \
 277                        : "memory");                             \
 278         __ret; })
 279
 280 #endif
 281
 282 #define cmpxchg8b(ptr, o1, o2, n1, n2)                          \
 283 ({                                                              \
 284         char __ret;                                             \
 285         __typeof__(o2) __dummy;                                 \
 286         __typeof__(*(ptr)) __old1 = (o1);                       \
 287         __typeof__(o2) __old2 = (o2);                           \
 288         __typeof__(*(ptr)) __new1 = (n1);                       \
 289         __typeof__(o2) __new2 = (n2);                           \
 290         asm volatile(LOCK_PREFIX "cmpxchg8b %2; setz %1"        \
 291                        : "=d"(__dummy), "=a" (__ret), "+m" (*ptr)\
 292                        : "a" (__old1), "d"(__old2),             \
 293                          "b" (__new1), "c" (__new2)             \
 294                        : "memory");                             \
 295         __ret; })
 296
 297
 298 #define cmpxchg8b_local(ptr, o1, o2, n1, n2)                    \
 299 ({                                                              \
 300         char __ret;                                             \
 301         __typeof__(o2) __dummy;                                 \
 302         __typeof__(*(ptr)) __old1 = (o1);                       \
 303         __typeof__(o2) __old2 = (o2);                           \
 304         __typeof__(*(ptr)) __new1 = (n1);                       \
 305         __typeof__(o2) __new2 = (n2);                           \
 306         asm volatile("cmpxchg8b %2; setz %1"                    \
 307                        : "=d"(__dummy), "=a"(__ret), "+m" (*ptr)\
 308                        : "a" (__old), "d"(__old2),              \
 309                          "b" (__new1), "c" (__new2),            \
 310                        : "memory");                             \
 311         __ret; })
 312
 313
 314 #define cmpxchg_double(ptr, o1, o2, n1, n2)                             \
 315 ({                                                                      \
 316         BUILD_BUG_ON(sizeof(*(ptr)) != 4);                              \
 317         VM_BUG_ON((unsigned long)(ptr) % 8);                            \
 318         cmpxchg8b((ptr), (o1), (o2), (n1), (n2));                       \
 319 })
 320
 321 #define cmpxchg_double_local(ptr, o1, o2, n1, n2)                       \
 322 ({                                                                      \
 323        BUILD_BUG_ON(sizeof(*(ptr)) != 4);                               \
 324        VM_BUG_ON((unsigned long)(ptr) % 8);                             \
 325        cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2));                 \
 326 })
 327
 328 #define system_has_cmpxchg_double() cpu_has_cx8
 329
 330 #endif /* _ASM_X86_CMPXCHG_32_H */