]>
Commit | Line | Data |
---|---|---|
9f97da78 DH |
1 | #ifndef __ASM_ARM_CMPXCHG_H |
2 | #define __ASM_ARM_CMPXCHG_H | |
3 | ||
4 | #include <linux/irqflags.h> | |
c32ffce0 | 5 | #include <linux/prefetch.h> |
9f97da78 DH |
6 | #include <asm/barrier.h> |
7 | ||
8 | #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) | |
9 | /* | |
10 | * On the StrongARM, "swp" is terminally broken since it bypasses the | |
11 | * cache totally. This means that the cache becomes inconsistent, and, | |
12 | * since we use normal loads/stores as well, this is really bad. | |
13 | * Typically, this causes oopsen in filp_close, but could have other, | |
14 | * more disastrous effects. There are two work-arounds: | |
15 | * 1. Disable interrupts and emulate the atomic swap | |
16 | * 2. Clean the cache, perform atomic swap, flush the cache | |
17 | * | |
18 | * We choose (1) since its the "easiest" to achieve here and is not | |
19 | * dependent on the processor type. | |
20 | * | |
21 | * NOTE that this solution won't work on an SMP system, so explcitly | |
22 | * forbid it here. | |
23 | */ | |
24 | #define swp_is_buggy | |
25 | #endif | |
26 | ||
27 | static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) | |
28 | { | |
29 | extern void __bad_xchg(volatile void *, int); | |
30 | unsigned long ret; | |
31 | #ifdef swp_is_buggy | |
32 | unsigned long flags; | |
33 | #endif | |
34 | #if __LINUX_ARM_ARCH__ >= 6 | |
35 | unsigned int tmp; | |
36 | #endif | |
37 | ||
c32ffce0 | 38 | prefetchw((const void *)ptr); |
9f97da78 DH |
39 | |
40 | switch (size) { | |
41 | #if __LINUX_ARM_ARCH__ >= 6 | |
e8973a88 | 42 | #ifndef CONFIG_CPU_V6 /* MIN ARCH >= V6K */ |
9f97da78 DH |
43 | case 1: |
44 | asm volatile("@ __xchg1\n" | |
45 | "1: ldrexb %0, [%3]\n" | |
46 | " strexb %1, %2, [%3]\n" | |
47 | " teq %1, #0\n" | |
48 | " bne 1b" | |
49 | : "=&r" (ret), "=&r" (tmp) | |
50 | : "r" (x), "r" (ptr) | |
51 | : "memory", "cc"); | |
52 | break; | |
e8973a88 SG |
53 | case 2: |
54 | asm volatile("@ __xchg2\n" | |
55 | "1: ldrexh %0, [%3]\n" | |
56 | " strexh %1, %2, [%3]\n" | |
57 | " teq %1, #0\n" | |
58 | " bne 1b" | |
59 | : "=&r" (ret), "=&r" (tmp) | |
60 | : "r" (x), "r" (ptr) | |
61 | : "memory", "cc"); | |
62 | break; | |
63 | #endif | |
9f97da78 DH |
64 | case 4: |
65 | asm volatile("@ __xchg4\n" | |
66 | "1: ldrex %0, [%3]\n" | |
67 | " strex %1, %2, [%3]\n" | |
68 | " teq %1, #0\n" | |
69 | " bne 1b" | |
70 | : "=&r" (ret), "=&r" (tmp) | |
71 | : "r" (x), "r" (ptr) | |
72 | : "memory", "cc"); | |
73 | break; | |
74 | #elif defined(swp_is_buggy) | |
75 | #ifdef CONFIG_SMP | |
76 | #error SMP is not supported on this platform | |
77 | #endif | |
78 | case 1: | |
79 | raw_local_irq_save(flags); | |
80 | ret = *(volatile unsigned char *)ptr; | |
81 | *(volatile unsigned char *)ptr = x; | |
82 | raw_local_irq_restore(flags); | |
83 | break; | |
84 | ||
85 | case 4: | |
86 | raw_local_irq_save(flags); | |
87 | ret = *(volatile unsigned long *)ptr; | |
88 | *(volatile unsigned long *)ptr = x; | |
89 | raw_local_irq_restore(flags); | |
90 | break; | |
91 | #else | |
92 | case 1: | |
93 | asm volatile("@ __xchg1\n" | |
94 | " swpb %0, %1, [%2]" | |
95 | : "=&r" (ret) | |
96 | : "r" (x), "r" (ptr) | |
97 | : "memory", "cc"); | |
98 | break; | |
99 | case 4: | |
100 | asm volatile("@ __xchg4\n" | |
101 | " swp %0, %1, [%2]" | |
102 | : "=&r" (ret) | |
103 | : "r" (x), "r" (ptr) | |
104 | : "memory", "cc"); | |
105 | break; | |
106 | #endif | |
107 | default: | |
31cd08c3 | 108 | /* Cause a link-time error, the xchg() size is not supported */ |
9f97da78 DH |
109 | __bad_xchg(ptr, size), ret = 0; |
110 | break; | |
111 | } | |
9f97da78 DH |
112 | |
113 | return ret; | |
114 | } | |
115 | ||
0ca326de | 116 | #define xchg_relaxed(ptr, x) ({ \ |
e001bbae RK |
117 | (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \ |
118 | sizeof(*(ptr))); \ | |
119 | }) | |
9f97da78 DH |
120 | |
121 | #include <asm-generic/cmpxchg-local.h> | |
122 | ||
123 | #if __LINUX_ARM_ARCH__ < 6 | |
124 | /* min ARCH < ARMv6 */ | |
125 | ||
126 | #ifdef CONFIG_SMP | |
127 | #error "SMP is not supported on this platform" | |
128 | #endif | |
129 | ||
0ca326de WD |
130 | #define xchg xchg_relaxed |
131 | ||
9f97da78 DH |
132 | /* |
133 | * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make | |
134 | * them available. | |
135 | */ | |
e001bbae RK |
136 | #define cmpxchg_local(ptr, o, n) ({ \ |
137 | (__typeof(*ptr))__cmpxchg_local_generic((ptr), \ | |
138 | (unsigned long)(o), \ | |
139 | (unsigned long)(n), \ | |
140 | sizeof(*(ptr))); \ | |
141 | }) | |
142 | ||
9f97da78 DH |
143 | #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) |
144 | ||
9f97da78 | 145 | #include <asm-generic/cmpxchg.h> |
9f97da78 DH |
146 | |
147 | #else /* min ARCH >= ARMv6 */ | |
148 | ||
149 | extern void __bad_cmpxchg(volatile void *ptr, int size); | |
150 | ||
151 | /* | |
152 | * cmpxchg only support 32-bits operands on ARMv6. | |
153 | */ | |
154 | ||
155 | static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, | |
156 | unsigned long new, int size) | |
157 | { | |
158 | unsigned long oldval, res; | |
159 | ||
c32ffce0 WD |
160 | prefetchw((const void *)ptr); |
161 | ||
9f97da78 DH |
162 | switch (size) { |
163 | #ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ | |
164 | case 1: | |
165 | do { | |
166 | asm volatile("@ __cmpxchg1\n" | |
167 | " ldrexb %1, [%2]\n" | |
168 | " mov %0, #0\n" | |
169 | " teq %1, %3\n" | |
170 | " strexbeq %0, %4, [%2]\n" | |
171 | : "=&r" (res), "=&r" (oldval) | |
172 | : "r" (ptr), "Ir" (old), "r" (new) | |
173 | : "memory", "cc"); | |
174 | } while (res); | |
175 | break; | |
176 | case 2: | |
177 | do { | |
178 | asm volatile("@ __cmpxchg1\n" | |
179 | " ldrexh %1, [%2]\n" | |
180 | " mov %0, #0\n" | |
181 | " teq %1, %3\n" | |
182 | " strexheq %0, %4, [%2]\n" | |
183 | : "=&r" (res), "=&r" (oldval) | |
184 | : "r" (ptr), "Ir" (old), "r" (new) | |
185 | : "memory", "cc"); | |
186 | } while (res); | |
187 | break; | |
188 | #endif | |
189 | case 4: | |
190 | do { | |
191 | asm volatile("@ __cmpxchg4\n" | |
192 | " ldrex %1, [%2]\n" | |
193 | " mov %0, #0\n" | |
194 | " teq %1, %3\n" | |
195 | " strexeq %0, %4, [%2]\n" | |
196 | : "=&r" (res), "=&r" (oldval) | |
197 | : "r" (ptr), "Ir" (old), "r" (new) | |
198 | : "memory", "cc"); | |
199 | } while (res); | |
200 | break; | |
201 | default: | |
202 | __bad_cmpxchg(ptr, size); | |
203 | oldval = 0; | |
204 | } | |
205 | ||
206 | return oldval; | |
207 | } | |
208 | ||
0ca326de WD |
209 | #define cmpxchg_relaxed(ptr,o,n) ({ \ |
210 | (__typeof__(*(ptr)))__cmpxchg((ptr), \ | |
211 | (unsigned long)(o), \ | |
212 | (unsigned long)(n), \ | |
213 | sizeof(*(ptr))); \ | |
e001bbae | 214 | }) |
9f97da78 DH |
215 | |
216 | static inline unsigned long __cmpxchg_local(volatile void *ptr, | |
217 | unsigned long old, | |
218 | unsigned long new, int size) | |
219 | { | |
220 | unsigned long ret; | |
221 | ||
222 | switch (size) { | |
223 | #ifdef CONFIG_CPU_V6 /* min ARCH == ARMv6 */ | |
224 | case 1: | |
225 | case 2: | |
226 | ret = __cmpxchg_local_generic(ptr, old, new, size); | |
227 | break; | |
228 | #endif | |
229 | default: | |
230 | ret = __cmpxchg(ptr, old, new, size); | |
231 | } | |
232 | ||
233 | return ret; | |
234 | } | |
235 | ||
e001bbae RK |
236 | #define cmpxchg_local(ptr, o, n) ({ \ |
237 | (__typeof(*ptr))__cmpxchg_local((ptr), \ | |
238 | (unsigned long)(o), \ | |
239 | (unsigned long)(n), \ | |
240 | sizeof(*(ptr))); \ | |
241 | }) | |
242 | ||
2523c67b WD |
243 | static inline unsigned long long __cmpxchg64(unsigned long long *ptr, |
244 | unsigned long long old, | |
245 | unsigned long long new) | |
246 | { | |
247 | unsigned long long oldval; | |
248 | unsigned long res; | |
249 | ||
c32ffce0 WD |
250 | prefetchw(ptr); |
251 | ||
2523c67b WD |
252 | __asm__ __volatile__( |
253 | "1: ldrexd %1, %H1, [%3]\n" | |
254 | " teq %1, %4\n" | |
255 | " teqeq %H1, %H4\n" | |
256 | " bne 2f\n" | |
257 | " strexd %0, %5, %H5, [%3]\n" | |
258 | " teq %0, #0\n" | |
259 | " bne 1b\n" | |
260 | "2:" | |
261 | : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) | |
262 | : "r" (ptr), "r" (old), "r" (new) | |
263 | : "cc"); | |
264 | ||
265 | return oldval; | |
266 | } | |
267 | ||
e001bbae RK |
268 | #define cmpxchg64_relaxed(ptr, o, n) ({ \ |
269 | (__typeof__(*(ptr)))__cmpxchg64((ptr), \ | |
270 | (unsigned long long)(o), \ | |
271 | (unsigned long long)(n)); \ | |
272 | }) | |
273 | ||
274 | #define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n)) | |
275 | ||
9f97da78 DH |
276 | #endif /* __LINUX_ARM_ARCH__ >= 6 */ |
277 | ||
278 | #endif /* __ASM_ARM_CMPXCHG_H */ |