]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1965aae3 PA |
2 | #ifndef _ASM_X86_CMPXCHG_32_H |
3 | #define _ASM_X86_CMPXCHG_32_H | |
a436ed9c | 4 | |
2d9ce177 AK |
5 | /* |
6 | * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you | |
7 | * you need to test for the feature in boot_cpu_data. | |
8 | */ | |
9 | ||
a436ed9c | 10 | /* |
69309a05 PA |
11 | * CMPXCHG8B only writes to the target if we had the previous |
12 | * value in registers, otherwise it acts as a read and gives us the | |
13 | * "new previous" value. That is why there is a loop. Preloading | |
14 | * EDX:EAX is a performance optimization: in the common case it means | |
15 | * we need only one locked operation. | |
a436ed9c | 16 | * |
69309a05 PA |
17 | * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very |
18 | * least an FPU save and/or %cr0.ts manipulation. | |
19 | * | |
20 | * cmpxchg8b must be used with the lock prefix here to allow the | |
21 | * instruction to be executed atomically. We need to have the reader | |
22 | * side to see the coherent 64bit value. | |
a436ed9c | 23 | */ |
69309a05 | 24 | static inline void set_64bit(volatile u64 *ptr, u64 value) |
a436ed9c | 25 | { |
69309a05 PA |
26 | u32 low = value; |
27 | u32 high = value >> 32; | |
28 | u64 prev = *ptr; | |
29 | ||
8121019c | 30 | asm volatile("\n1:\t" |
69309a05 | 31 | LOCK_PREFIX "cmpxchg8b %0\n\t" |
8121019c | 32 | "jnz 1b" |
69309a05 PA |
33 | : "=m" (*ptr), "+A" (prev) |
34 | : "b" (low), "c" (high) | |
35 | : "memory"); | |
a436ed9c | 36 | } |
a436ed9c | 37 | |
2c0b8a75 | 38 | #ifdef CONFIG_X86_CMPXCHG64 |
8121019c JP |
39 | #define cmpxchg64(ptr, o, n) \ |
40 | ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ | |
41 | (unsigned long long)(n))) | |
42 | #define cmpxchg64_local(ptr, o, n) \ | |
43 | ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ | |
44 | (unsigned long long)(n))) | |
a436ed9c JD |
45 | #endif |
46 | ||
4532b305 | 47 | static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) |
2c0b8a75 | 48 | { |
4532b305 | 49 | u64 prev; |
113fc5a6 PA |
50 | asm volatile(LOCK_PREFIX "cmpxchg8b %1" |
51 | : "=A" (prev), | |
4532b305 PA |
52 | "+m" (*ptr) |
53 | : "b" ((u32)new), | |
54 | "c" ((u32)(new >> 32)), | |
113fc5a6 | 55 | "0" (old) |
8121019c | 56 | : "memory"); |
2c0b8a75 MD |
57 | return prev; |
58 | } | |
59 | ||
4532b305 | 60 | static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) |
2c0b8a75 | 61 | { |
4532b305 | 62 | u64 prev; |
113fc5a6 PA |
63 | asm volatile("cmpxchg8b %1" |
64 | : "=A" (prev), | |
4532b305 PA |
65 | "+m" (*ptr) |
66 | : "b" ((u32)new), | |
67 | "c" ((u32)(new >> 32)), | |
113fc5a6 | 68 | "0" (old) |
8121019c | 69 | : "memory"); |
2c0b8a75 MD |
70 | return prev; |
71 | } | |
72 | ||
2c0b8a75 MD |
73 | #ifndef CONFIG_X86_CMPXCHG64 |
74 | /* | |
75 | * Building a kernel capable running on 80386 and 80486. It may be necessary | |
76 | * to simulate the cmpxchg8b on the 80386 and 80486 CPU. | |
77 | */ | |
a436ed9c | 78 | |
79e1dd05 AV |
79 | #define cmpxchg64(ptr, o, n) \ |
80 | ({ \ | |
81 | __typeof__(*(ptr)) __ret; \ | |
82 | __typeof__(*(ptr)) __old = (o); \ | |
83 | __typeof__(*(ptr)) __new = (n); \ | |
9c76b384 LB |
84 | alternative_io(LOCK_PREFIX_HERE \ |
85 | "call cmpxchg8b_emu", \ | |
79e1dd05 AV |
86 | "lock; cmpxchg8b (%%esi)" , \ |
87 | X86_FEATURE_CX8, \ | |
88 | "=A" (__ret), \ | |
89 | "S" ((ptr)), "0" (__old), \ | |
90 | "b" ((unsigned int)__new), \ | |
91 | "c" ((unsigned int)(__new>>32)) \ | |
92 | : "memory"); \ | |
93 | __ret; }) | |
94 | ||
95 | ||
a378d933 PA |
96 | #define cmpxchg64_local(ptr, o, n) \ |
97 | ({ \ | |
98 | __typeof__(*(ptr)) __ret; \ | |
99 | __typeof__(*(ptr)) __old = (o); \ | |
100 | __typeof__(*(ptr)) __new = (n); \ | |
101 | alternative_io("call cmpxchg8b_emu", \ | |
102 | "cmpxchg8b (%%esi)" , \ | |
103 | X86_FEATURE_CX8, \ | |
104 | "=A" (__ret), \ | |
105 | "S" ((ptr)), "0" (__old), \ | |
106 | "b" ((unsigned int)__new), \ | |
107 | "c" ((unsigned int)(__new>>32)) \ | |
108 | : "memory"); \ | |
109 | __ret; }) | |
2c0b8a75 MD |
110 | |
111 | #endif | |
a436ed9c | 112 | |
362f924b | 113 | #define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8) |
3824abd1 | 114 | |
1965aae3 | 115 | #endif /* _ASM_X86_CMPXCHG_32_H */ |