]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __ARCH_S390_ATOMIC__ |
2 | #define __ARCH_S390_ATOMIC__ | |
3 | ||
4 | /* | |
12751058 HC |
5 | * Copyright 1999,2009 IBM Corp. |
6 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, | |
7 | * Denis Joseph Barrow, | |
8 | * Arnd Bergmann <arndb@de.ibm.com>, | |
1da177e4 | 9 | * |
12751058 HC |
10 | * Atomic operations that C can't guarantee us. |
11 | * Useful for resource counting etc. | |
25985edc | 12 | * s390 uses 'Compare And Swap' for atomicity in SMP environment. |
1da177e4 LT |
13 | * |
14 | */ | |
15 | ||
12751058 HC |
16 | #include <linux/compiler.h> |
17 | #include <linux/types.h> | |
2ddb3ec4 | 18 | #include <asm/system.h> |
1da177e4 | 19 | |
1da177e4 LT |
20 | #define ATOMIC_INIT(i) { (i) } |
21 | ||
1da177e4 | 22 | #define __CS_LOOP(ptr, op_val, op_string) ({ \ |
39475179 | 23 | int old_val, new_val; \ |
94c12cc7 MS |
24 | asm volatile( \ |
25 | " l %0,%2\n" \ | |
26 | "0: lr %1,%0\n" \ | |
27 | op_string " %1,%3\n" \ | |
28 | " cs %0,%1,%2\n" \ | |
29 | " jl 0b" \ | |
30 | : "=&d" (old_val), "=&d" (new_val), \ | |
31 | "=Q" (((atomic_t *)(ptr))->counter) \ | |
32 | : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \ | |
33 | : "cc", "memory"); \ | |
1da177e4 LT |
34 | new_val; \ |
35 | }) | |
94c12cc7 | 36 | |
c51b9621 HC |
37 | static inline int atomic_read(const atomic_t *v) |
38 | { | |
7657e41a HC |
39 | int c; |
40 | ||
41 | asm volatile( | |
42 | " l %0,%1\n" | |
43 | : "=d" (c) : "Q" (v->counter)); | |
44 | return c; | |
c51b9621 HC |
45 | } |
46 | ||
47 | static inline void atomic_set(atomic_t *v, int i) | |
48 | { | |
7657e41a HC |
49 | asm volatile( |
50 | " st %1,%0\n" | |
51 | : "=Q" (v->counter) : "d" (i)); | |
c51b9621 | 52 | } |
1da177e4 | 53 | |
bfe3349b | 54 | static inline int atomic_add_return(int i, atomic_t *v) |
1da177e4 LT |
55 | { |
56 | return __CS_LOOP(v, i, "ar"); | |
57 | } | |
973bd993 MS |
58 | #define atomic_add(_i, _v) atomic_add_return(_i, _v) |
59 | #define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0) | |
60 | #define atomic_inc(_v) atomic_add_return(1, _v) | |
61 | #define atomic_inc_return(_v) atomic_add_return(1, _v) | |
62 | #define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0) | |
63 | ||
bfe3349b | 64 | static inline int atomic_sub_return(int i, atomic_t *v) |
1da177e4 LT |
65 | { |
66 | return __CS_LOOP(v, i, "sr"); | |
67 | } | |
973bd993 MS |
68 | #define atomic_sub(_i, _v) atomic_sub_return(_i, _v) |
69 | #define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0) | |
70 | #define atomic_dec(_v) atomic_sub_return(1, _v) | |
71 | #define atomic_dec_return(_v) atomic_sub_return(1, _v) | |
72 | #define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) | |
1da177e4 | 73 | |
bfe3349b | 74 | static inline void atomic_clear_mask(unsigned long mask, atomic_t *v) |
1da177e4 | 75 | { |
bfe3349b | 76 | __CS_LOOP(v, ~mask, "nr"); |
1da177e4 | 77 | } |
973bd993 | 78 | |
bfe3349b | 79 | static inline void atomic_set_mask(unsigned long mask, atomic_t *v) |
1da177e4 | 80 | { |
bfe3349b | 81 | __CS_LOOP(v, mask, "or"); |
1da177e4 | 82 | } |
973bd993 | 83 | |
ffbf670f IM |
84 | #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) |
85 | ||
bfe3349b | 86 | static inline int atomic_cmpxchg(atomic_t *v, int old, int new) |
973bd993 | 87 | { |
94c12cc7 MS |
88 | asm volatile( |
89 | " cs %0,%2,%1" | |
90 | : "+d" (old), "=Q" (v->counter) | |
91 | : "d" (new), "Q" (v->counter) | |
92 | : "cc", "memory"); | |
973bd993 MS |
93 | return old; |
94 | } | |
95 | ||
bfe3349b | 96 | static inline int atomic_add_unless(atomic_t *v, int a, int u) |
973bd993 MS |
97 | { |
98 | int c, old; | |
973bd993 | 99 | c = atomic_read(v); |
0b2fcfdb NP |
100 | for (;;) { |
101 | if (unlikely(c == u)) | |
102 | break; | |
103 | old = atomic_cmpxchg(v, c, c + a); | |
104 | if (likely(old == c)) | |
105 | break; | |
973bd993 | 106 | c = old; |
0b2fcfdb | 107 | } |
973bd993 MS |
108 | return c != u; |
109 | } | |
110 | ||
973bd993 | 111 | |
1da177e4 LT |
112 | #undef __CS_LOOP |
113 | ||
1da177e4 LT |
114 | #define ATOMIC64_INIT(i) { (i) } |
115 | ||
12751058 HC |
116 | #ifdef CONFIG_64BIT |
117 | ||
1da177e4 | 118 | #define __CSG_LOOP(ptr, op_val, op_string) ({ \ |
39475179 | 119 | long long old_val, new_val; \ |
94c12cc7 MS |
120 | asm volatile( \ |
121 | " lg %0,%2\n" \ | |
122 | "0: lgr %1,%0\n" \ | |
123 | op_string " %1,%3\n" \ | |
124 | " csg %0,%1,%2\n" \ | |
125 | " jl 0b" \ | |
126 | : "=&d" (old_val), "=&d" (new_val), \ | |
127 | "=Q" (((atomic_t *)(ptr))->counter) \ | |
128 | : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \ | |
bfe3349b | 129 | : "cc", "memory"); \ |
1da177e4 LT |
130 | new_val; \ |
131 | }) | |
94c12cc7 | 132 | |
c51b9621 HC |
133 | static inline long long atomic64_read(const atomic64_t *v) |
134 | { | |
7657e41a HC |
135 | long long c; |
136 | ||
137 | asm volatile( | |
138 | " lg %0,%1\n" | |
139 | : "=d" (c) : "Q" (v->counter)); | |
140 | return c; | |
c51b9621 HC |
141 | } |
142 | ||
143 | static inline void atomic64_set(atomic64_t *v, long long i) | |
144 | { | |
7657e41a HC |
145 | asm volatile( |
146 | " stg %1,%0\n" | |
147 | : "=Q" (v->counter) : "d" (i)); | |
c51b9621 | 148 | } |
1da177e4 | 149 | |
bfe3349b | 150 | static inline long long atomic64_add_return(long long i, atomic64_t *v) |
1da177e4 LT |
151 | { |
152 | return __CSG_LOOP(v, i, "agr"); | |
153 | } | |
973bd993 | 154 | |
bfe3349b | 155 | static inline long long atomic64_sub_return(long long i, atomic64_t *v) |
1da177e4 | 156 | { |
973bd993 | 157 | return __CSG_LOOP(v, i, "sgr"); |
1da177e4 | 158 | } |
973bd993 | 159 | |
bfe3349b | 160 | static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v) |
1da177e4 | 161 | { |
bfe3349b | 162 | __CSG_LOOP(v, ~mask, "ngr"); |
1da177e4 | 163 | } |
973bd993 | 164 | |
bfe3349b | 165 | static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v) |
1da177e4 | 166 | { |
bfe3349b | 167 | __CSG_LOOP(v, mask, "ogr"); |
1da177e4 LT |
168 | } |
169 | ||
3a5f10e3 MD |
170 | #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) |
171 | ||
bfe3349b | 172 | static inline long long atomic64_cmpxchg(atomic64_t *v, |
973bd993 MS |
173 | long long old, long long new) |
174 | { | |
94c12cc7 MS |
175 | asm volatile( |
176 | " csg %0,%2,%1" | |
177 | : "+d" (old), "=Q" (v->counter) | |
178 | : "d" (new), "Q" (v->counter) | |
179 | : "cc", "memory"); | |
973bd993 MS |
180 | return old; |
181 | } | |
1da177e4 | 182 | |
12751058 HC |
183 | #undef __CSG_LOOP |
184 | ||
185 | #else /* CONFIG_64BIT */ | |
186 | ||
187 | typedef struct { | |
188 | long long counter; | |
189 | } atomic64_t; | |
190 | ||
191 | static inline long long atomic64_read(const atomic64_t *v) | |
192 | { | |
193 | register_pair rp; | |
194 | ||
195 | asm volatile( | |
987bcdac MS |
196 | " lm %0,%N0,%1" |
197 | : "=&d" (rp) : "Q" (v->counter) ); | |
12751058 HC |
198 | return rp.pair; |
199 | } | |
200 | ||
201 | static inline void atomic64_set(atomic64_t *v, long long i) | |
202 | { | |
203 | register_pair rp = {.pair = i}; | |
204 | ||
205 | asm volatile( | |
987bcdac MS |
206 | " stm %1,%N1,%0" |
207 | : "=Q" (v->counter) : "d" (rp) ); | |
12751058 HC |
208 | } |
209 | ||
210 | static inline long long atomic64_xchg(atomic64_t *v, long long new) | |
211 | { | |
212 | register_pair rp_new = {.pair = new}; | |
213 | register_pair rp_old; | |
214 | ||
215 | asm volatile( | |
987bcdac MS |
216 | " lm %0,%N0,%1\n" |
217 | "0: cds %0,%2,%1\n" | |
12751058 | 218 | " jl 0b\n" |
987bcdac MS |
219 | : "=&d" (rp_old), "=Q" (v->counter) |
220 | : "d" (rp_new), "Q" (v->counter) | |
12751058 HC |
221 | : "cc"); |
222 | return rp_old.pair; | |
223 | } | |
224 | ||
225 | static inline long long atomic64_cmpxchg(atomic64_t *v, | |
226 | long long old, long long new) | |
227 | { | |
228 | register_pair rp_old = {.pair = old}; | |
229 | register_pair rp_new = {.pair = new}; | |
230 | ||
231 | asm volatile( | |
987bcdac MS |
232 | " cds %0,%2,%1" |
233 | : "+&d" (rp_old), "=Q" (v->counter) | |
234 | : "d" (rp_new), "Q" (v->counter) | |
12751058 HC |
235 | : "cc"); |
236 | return rp_old.pair; | |
237 | } | |
238 | ||
239 | ||
240 | static inline long long atomic64_add_return(long long i, atomic64_t *v) | |
241 | { | |
242 | long long old, new; | |
243 | ||
244 | do { | |
245 | old = atomic64_read(v); | |
246 | new = old + i; | |
247 | } while (atomic64_cmpxchg(v, old, new) != old); | |
248 | return new; | |
249 | } | |
250 | ||
251 | static inline long long atomic64_sub_return(long long i, atomic64_t *v) | |
252 | { | |
253 | long long old, new; | |
254 | ||
255 | do { | |
256 | old = atomic64_read(v); | |
257 | new = old - i; | |
258 | } while (atomic64_cmpxchg(v, old, new) != old); | |
259 | return new; | |
260 | } | |
261 | ||
262 | static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v) | |
263 | { | |
264 | long long old, new; | |
265 | ||
266 | do { | |
267 | old = atomic64_read(v); | |
268 | new = old | mask; | |
269 | } while (atomic64_cmpxchg(v, old, new) != old); | |
270 | } | |
271 | ||
272 | static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v) | |
273 | { | |
274 | long long old, new; | |
275 | ||
276 | do { | |
277 | old = atomic64_read(v); | |
278 | new = old & mask; | |
279 | } while (atomic64_cmpxchg(v, old, new) != old); | |
280 | } | |
281 | ||
282 | #endif /* CONFIG_64BIT */ | |
283 | ||
bfe3349b | 284 | static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) |
1da177e4 | 285 | { |
973bd993 | 286 | long long c, old; |
2ddb3ec4 | 287 | |
973bd993 | 288 | c = atomic64_read(v); |
0b2fcfdb NP |
289 | for (;;) { |
290 | if (unlikely(c == u)) | |
291 | break; | |
292 | old = atomic64_cmpxchg(v, c, c + a); | |
293 | if (likely(old == c)) | |
294 | break; | |
973bd993 | 295 | c = old; |
0b2fcfdb | 296 | } |
973bd993 | 297 | return c != u; |
1da177e4 LT |
298 | } |
299 | ||
2ddb3ec4 HC |
300 | static inline long long atomic64_dec_if_positive(atomic64_t *v) |
301 | { | |
302 | long long c, old, dec; | |
303 | ||
304 | c = atomic64_read(v); | |
305 | for (;;) { | |
306 | dec = c - 1; | |
307 | if (unlikely(dec < 0)) | |
308 | break; | |
309 | old = atomic64_cmpxchg((v), c, dec); | |
310 | if (likely(old == c)) | |
311 | break; | |
312 | c = old; | |
313 | } | |
314 | return dec; | |
315 | } | |
316 | ||
12751058 HC |
317 | #define atomic64_add(_i, _v) atomic64_add_return(_i, _v) |
318 | #define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0) | |
319 | #define atomic64_inc(_v) atomic64_add_return(1, _v) | |
320 | #define atomic64_inc_return(_v) atomic64_add_return(1, _v) | |
321 | #define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) | |
322 | #define atomic64_sub(_i, _v) atomic64_sub_return(_i, _v) | |
323 | #define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) | |
324 | #define atomic64_dec(_v) atomic64_sub_return(1, _v) | |
325 | #define atomic64_dec_return(_v) atomic64_sub_return(1, _v) | |
326 | #define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0) | |
327 | #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) | |
8426e1f6 | 328 | |
1da177e4 LT |
329 | #define smp_mb__before_atomic_dec() smp_mb() |
330 | #define smp_mb__after_atomic_dec() smp_mb() | |
331 | #define smp_mb__before_atomic_inc() smp_mb() | |
332 | #define smp_mb__after_atomic_inc() smp_mb() | |
333 | ||
72099ed2 | 334 | #include <asm-generic/atomic-long.h> |
bfe3349b | 335 | |
1da177e4 | 336 | #endif /* __ARCH_S390_ATOMIC__ */ |