]>
Commit | Line | Data |
---|---|---|
1965aae3 PA |
1 | #ifndef _ASM_X86_CMPXCHG_32_H |
2 | #define _ASM_X86_CMPXCHG_32_H | |
a436ed9c | 3 | |
2d9ce177 AK |
4 | /* |
5 | * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you | |
6 | * you need to test for the feature in boot_cpu_data. | |
7 | */ | |
8 | ||
a436ed9c | 9 | /* |
69309a05 PA |
10 | * CMPXCHG8B only writes to the target if we had the previous |
11 | * value in registers, otherwise it acts as a read and gives us the | |
12 | * "new previous" value. That is why there is a loop. Preloading | |
13 | * EDX:EAX is a performance optimization: in the common case it means | |
14 | * we need only one locked operation. | |
a436ed9c | 15 | * |
69309a05 PA |
16 | * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very |
17 | * least an FPU save and/or %cr0.ts manipulation. | |
18 | * | |
19 | * cmpxchg8b must be used with the lock prefix here to allow the | |
20 | * instruction to be executed atomically. We need to have the reader | |
21 | * side to see the coherent 64bit value. | |
a436ed9c | 22 | */ |
69309a05 | 23 | static inline void set_64bit(volatile u64 *ptr, u64 value) |
a436ed9c | 24 | { |
69309a05 PA |
25 | u32 low = value; |
26 | u32 high = value >> 32; | |
27 | u64 prev = *ptr; | |
28 | ||
8121019c | 29 | asm volatile("\n1:\t" |
69309a05 | 30 | LOCK_PREFIX "cmpxchg8b %0\n\t" |
8121019c | 31 | "jnz 1b" |
69309a05 PA |
32 | : "=m" (*ptr), "+A" (prev) |
33 | : "b" (low), "c" (high) | |
34 | : "memory"); | |
a436ed9c | 35 | } |
a436ed9c | 36 | |
2c0b8a75 | 37 | #ifdef CONFIG_X86_CMPXCHG64 |
8121019c JP |
38 | #define cmpxchg64(ptr, o, n) \ |
39 | ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ | |
40 | (unsigned long long)(n))) | |
41 | #define cmpxchg64_local(ptr, o, n) \ | |
42 | ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ | |
43 | (unsigned long long)(n))) | |
a436ed9c JD |
44 | #endif |
45 | ||
4532b305 | 46 | static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) |
2c0b8a75 | 47 | { |
4532b305 | 48 | u64 prev; |
113fc5a6 PA |
49 | asm volatile(LOCK_PREFIX "cmpxchg8b %1" |
50 | : "=A" (prev), | |
4532b305 PA |
51 | "+m" (*ptr) |
52 | : "b" ((u32)new), | |
53 | "c" ((u32)(new >> 32)), | |
113fc5a6 | 54 | "0" (old) |
8121019c | 55 | : "memory"); |
2c0b8a75 MD |
56 | return prev; |
57 | } | |
58 | ||
4532b305 | 59 | static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) |
2c0b8a75 | 60 | { |
4532b305 | 61 | u64 prev; |
113fc5a6 PA |
62 | asm volatile("cmpxchg8b %1" |
63 | : "=A" (prev), | |
4532b305 PA |
64 | "+m" (*ptr) |
65 | : "b" ((u32)new), | |
66 | "c" ((u32)(new >> 32)), | |
113fc5a6 | 67 | "0" (old) |
8121019c | 68 | : "memory"); |
2c0b8a75 MD |
69 | return prev; |
70 | } | |
71 | ||
2c0b8a75 MD |
72 | #ifndef CONFIG_X86_CMPXCHG64 |
73 | /* | |
74 | * Building a kernel capable running on 80386 and 80486. It may be necessary | |
75 | * to simulate the cmpxchg8b on the 80386 and 80486 CPU. | |
76 | */ | |
a436ed9c | 77 | |
79e1dd05 AV |
78 | #define cmpxchg64(ptr, o, n) \ |
79 | ({ \ | |
80 | __typeof__(*(ptr)) __ret; \ | |
81 | __typeof__(*(ptr)) __old = (o); \ | |
82 | __typeof__(*(ptr)) __new = (n); \ | |
9c76b384 LB |
83 | alternative_io(LOCK_PREFIX_HERE \ |
84 | "call cmpxchg8b_emu", \ | |
79e1dd05 AV |
85 | "lock; cmpxchg8b (%%esi)" , \ |
86 | X86_FEATURE_CX8, \ | |
87 | "=A" (__ret), \ | |
88 | "S" ((ptr)), "0" (__old), \ | |
89 | "b" ((unsigned int)__new), \ | |
90 | "c" ((unsigned int)(__new>>32)) \ | |
91 | : "memory"); \ | |
92 | __ret; }) | |
93 | ||
94 | ||
a378d933 PA |
95 | #define cmpxchg64_local(ptr, o, n) \ |
96 | ({ \ | |
97 | __typeof__(*(ptr)) __ret; \ | |
98 | __typeof__(*(ptr)) __old = (o); \ | |
99 | __typeof__(*(ptr)) __new = (n); \ | |
100 | alternative_io("call cmpxchg8b_emu", \ | |
101 | "cmpxchg8b (%%esi)" , \ | |
102 | X86_FEATURE_CX8, \ | |
103 | "=A" (__ret), \ | |
104 | "S" ((ptr)), "0" (__old), \ | |
105 | "b" ((unsigned int)__new), \ | |
106 | "c" ((unsigned int)(__new>>32)) \ | |
107 | : "memory"); \ | |
108 | __ret; }) | |
2c0b8a75 MD |
109 | |
110 | #endif | |
a436ed9c | 111 | |
362f924b | 112 | #define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8) |
3824abd1 | 113 | |
1965aae3 | 114 | #endif /* _ASM_X86_CMPXCHG_32_H */ |