]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
9f97da78 DH |
2 | #ifndef __ASM_ARM_CMPXCHG_H |
3 | #define __ASM_ARM_CMPXCHG_H | |
4 | ||
5 | #include <linux/irqflags.h> | |
c32ffce0 | 6 | #include <linux/prefetch.h> |
9f97da78 DH |
7 | #include <asm/barrier.h> |
8 | ||
9 | #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) | |
10 | /* | |
11 | * On the StrongARM, "swp" is terminally broken since it bypasses the | |
12 | * cache totally. This means that the cache becomes inconsistent, and, | |
13 | * since we use normal loads/stores as well, this is really bad. | |
14 | * Typically, this causes oopsen in filp_close, but could have other, | |
15 | * more disastrous effects. There are two work-arounds: | |
16 | * 1. Disable interrupts and emulate the atomic swap | |
17 | * 2. Clean the cache, perform atomic swap, flush the cache | |
18 | * | |
19 | * We choose (1) since its the "easiest" to achieve here and is not | |
20 | * dependent on the processor type. | |
21 | * | |
22 | * NOTE that this solution won't work on an SMP system, so explcitly | |
23 | * forbid it here. | |
24 | */ | |
25 | #define swp_is_buggy | |
26 | #endif | |
27 | ||
28 | static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) | |
29 | { | |
30 | extern void __bad_xchg(volatile void *, int); | |
31 | unsigned long ret; | |
32 | #ifdef swp_is_buggy | |
33 | unsigned long flags; | |
34 | #endif | |
35 | #if __LINUX_ARM_ARCH__ >= 6 | |
36 | unsigned int tmp; | |
37 | #endif | |
38 | ||
c32ffce0 | 39 | prefetchw((const void *)ptr); |
9f97da78 DH |
40 | |
41 | switch (size) { | |
42 | #if __LINUX_ARM_ARCH__ >= 6 | |
e8973a88 | 43 | #ifndef CONFIG_CPU_V6 /* MIN ARCH >= V6K */ |
9f97da78 DH |
44 | case 1: |
45 | asm volatile("@ __xchg1\n" | |
46 | "1: ldrexb %0, [%3]\n" | |
47 | " strexb %1, %2, [%3]\n" | |
48 | " teq %1, #0\n" | |
49 | " bne 1b" | |
50 | : "=&r" (ret), "=&r" (tmp) | |
51 | : "r" (x), "r" (ptr) | |
52 | : "memory", "cc"); | |
53 | break; | |
e8973a88 SG |
54 | case 2: |
55 | asm volatile("@ __xchg2\n" | |
56 | "1: ldrexh %0, [%3]\n" | |
57 | " strexh %1, %2, [%3]\n" | |
58 | " teq %1, #0\n" | |
59 | " bne 1b" | |
60 | : "=&r" (ret), "=&r" (tmp) | |
61 | : "r" (x), "r" (ptr) | |
62 | : "memory", "cc"); | |
63 | break; | |
64 | #endif | |
9f97da78 DH |
65 | case 4: |
66 | asm volatile("@ __xchg4\n" | |
67 | "1: ldrex %0, [%3]\n" | |
68 | " strex %1, %2, [%3]\n" | |
69 | " teq %1, #0\n" | |
70 | " bne 1b" | |
71 | : "=&r" (ret), "=&r" (tmp) | |
72 | : "r" (x), "r" (ptr) | |
73 | : "memory", "cc"); | |
74 | break; | |
75 | #elif defined(swp_is_buggy) | |
76 | #ifdef CONFIG_SMP | |
77 | #error SMP is not supported on this platform | |
78 | #endif | |
79 | case 1: | |
80 | raw_local_irq_save(flags); | |
81 | ret = *(volatile unsigned char *)ptr; | |
82 | *(volatile unsigned char *)ptr = x; | |
83 | raw_local_irq_restore(flags); | |
84 | break; | |
85 | ||
86 | case 4: | |
87 | raw_local_irq_save(flags); | |
88 | ret = *(volatile unsigned long *)ptr; | |
89 | *(volatile unsigned long *)ptr = x; | |
90 | raw_local_irq_restore(flags); | |
91 | break; | |
92 | #else | |
93 | case 1: | |
94 | asm volatile("@ __xchg1\n" | |
95 | " swpb %0, %1, [%2]" | |
96 | : "=&r" (ret) | |
97 | : "r" (x), "r" (ptr) | |
98 | : "memory", "cc"); | |
99 | break; | |
100 | case 4: | |
101 | asm volatile("@ __xchg4\n" | |
102 | " swp %0, %1, [%2]" | |
103 | : "=&r" (ret) | |
104 | : "r" (x), "r" (ptr) | |
105 | : "memory", "cc"); | |
106 | break; | |
107 | #endif | |
108 | default: | |
31cd08c3 | 109 | /* Cause a link-time error, the xchg() size is not supported */ |
9f97da78 DH |
110 | __bad_xchg(ptr, size), ret = 0; |
111 | break; | |
112 | } | |
9f97da78 DH |
113 | |
114 | return ret; | |
115 | } | |
116 | ||
0ca326de | 117 | #define xchg_relaxed(ptr, x) ({ \ |
e001bbae RK |
118 | (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \ |
119 | sizeof(*(ptr))); \ | |
120 | }) | |
9f97da78 DH |
121 | |
122 | #include <asm-generic/cmpxchg-local.h> | |
123 | ||
124 | #if __LINUX_ARM_ARCH__ < 6 | |
125 | /* min ARCH < ARMv6 */ | |
126 | ||
127 | #ifdef CONFIG_SMP | |
128 | #error "SMP is not supported on this platform" | |
129 | #endif | |
130 | ||
0ca326de WD |
131 | #define xchg xchg_relaxed |
132 | ||
9f97da78 DH |
133 | /* |
134 | * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make | |
135 | * them available. | |
136 | */ | |
e001bbae RK |
137 | #define cmpxchg_local(ptr, o, n) ({ \ |
138 | (__typeof(*ptr))__cmpxchg_local_generic((ptr), \ | |
139 | (unsigned long)(o), \ | |
140 | (unsigned long)(n), \ | |
141 | sizeof(*(ptr))); \ | |
142 | }) | |
143 | ||
9f97da78 DH |
144 | #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) |
145 | ||
9f97da78 | 146 | #include <asm-generic/cmpxchg.h> |
9f97da78 DH |
147 | |
148 | #else /* min ARCH >= ARMv6 */ | |
149 | ||
150 | extern void __bad_cmpxchg(volatile void *ptr, int size); | |
151 | ||
152 | /* | |
153 | * cmpxchg only support 32-bits operands on ARMv6. | |
154 | */ | |
155 | ||
156 | static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, | |
157 | unsigned long new, int size) | |
158 | { | |
159 | unsigned long oldval, res; | |
160 | ||
c32ffce0 WD |
161 | prefetchw((const void *)ptr); |
162 | ||
9f97da78 DH |
163 | switch (size) { |
164 | #ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ | |
165 | case 1: | |
166 | do { | |
167 | asm volatile("@ __cmpxchg1\n" | |
168 | " ldrexb %1, [%2]\n" | |
169 | " mov %0, #0\n" | |
170 | " teq %1, %3\n" | |
171 | " strexbeq %0, %4, [%2]\n" | |
172 | : "=&r" (res), "=&r" (oldval) | |
173 | : "r" (ptr), "Ir" (old), "r" (new) | |
174 | : "memory", "cc"); | |
175 | } while (res); | |
176 | break; | |
177 | case 2: | |
178 | do { | |
179 | asm volatile("@ __cmpxchg1\n" | |
180 | " ldrexh %1, [%2]\n" | |
181 | " mov %0, #0\n" | |
182 | " teq %1, %3\n" | |
183 | " strexheq %0, %4, [%2]\n" | |
184 | : "=&r" (res), "=&r" (oldval) | |
185 | : "r" (ptr), "Ir" (old), "r" (new) | |
186 | : "memory", "cc"); | |
187 | } while (res); | |
188 | break; | |
189 | #endif | |
190 | case 4: | |
191 | do { | |
192 | asm volatile("@ __cmpxchg4\n" | |
193 | " ldrex %1, [%2]\n" | |
194 | " mov %0, #0\n" | |
195 | " teq %1, %3\n" | |
196 | " strexeq %0, %4, [%2]\n" | |
197 | : "=&r" (res), "=&r" (oldval) | |
198 | : "r" (ptr), "Ir" (old), "r" (new) | |
199 | : "memory", "cc"); | |
200 | } while (res); | |
201 | break; | |
202 | default: | |
203 | __bad_cmpxchg(ptr, size); | |
204 | oldval = 0; | |
205 | } | |
206 | ||
207 | return oldval; | |
208 | } | |
209 | ||
0ca326de WD |
210 | #define cmpxchg_relaxed(ptr,o,n) ({ \ |
211 | (__typeof__(*(ptr)))__cmpxchg((ptr), \ | |
212 | (unsigned long)(o), \ | |
213 | (unsigned long)(n), \ | |
214 | sizeof(*(ptr))); \ | |
e001bbae | 215 | }) |
9f97da78 DH |
216 | |
217 | static inline unsigned long __cmpxchg_local(volatile void *ptr, | |
218 | unsigned long old, | |
219 | unsigned long new, int size) | |
220 | { | |
221 | unsigned long ret; | |
222 | ||
223 | switch (size) { | |
224 | #ifdef CONFIG_CPU_V6 /* min ARCH == ARMv6 */ | |
225 | case 1: | |
226 | case 2: | |
227 | ret = __cmpxchg_local_generic(ptr, old, new, size); | |
228 | break; | |
229 | #endif | |
230 | default: | |
231 | ret = __cmpxchg(ptr, old, new, size); | |
232 | } | |
233 | ||
234 | return ret; | |
235 | } | |
236 | ||
e001bbae RK |
237 | #define cmpxchg_local(ptr, o, n) ({ \ |
238 | (__typeof(*ptr))__cmpxchg_local((ptr), \ | |
239 | (unsigned long)(o), \ | |
240 | (unsigned long)(n), \ | |
241 | sizeof(*(ptr))); \ | |
242 | }) | |
243 | ||
2523c67b WD |
244 | static inline unsigned long long __cmpxchg64(unsigned long long *ptr, |
245 | unsigned long long old, | |
246 | unsigned long long new) | |
247 | { | |
248 | unsigned long long oldval; | |
249 | unsigned long res; | |
250 | ||
c32ffce0 WD |
251 | prefetchw(ptr); |
252 | ||
2523c67b WD |
253 | __asm__ __volatile__( |
254 | "1: ldrexd %1, %H1, [%3]\n" | |
255 | " teq %1, %4\n" | |
256 | " teqeq %H1, %H4\n" | |
257 | " bne 2f\n" | |
258 | " strexd %0, %5, %H5, [%3]\n" | |
259 | " teq %0, #0\n" | |
260 | " bne 1b\n" | |
261 | "2:" | |
262 | : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) | |
263 | : "r" (ptr), "r" (old), "r" (new) | |
264 | : "cc"); | |
265 | ||
266 | return oldval; | |
267 | } | |
268 | ||
e001bbae RK |
269 | #define cmpxchg64_relaxed(ptr, o, n) ({ \ |
270 | (__typeof__(*(ptr)))__cmpxchg64((ptr), \ | |
271 | (unsigned long long)(o), \ | |
272 | (unsigned long long)(n)); \ | |
273 | }) | |
274 | ||
275 | #define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n)) | |
276 | ||
9f97da78 DH |
277 | #endif /* __LINUX_ARM_ARCH__ >= 6 */ |
278 | ||
279 | #endif /* __ASM_ARM_CMPXCHG_H */ |