]>
Commit | Line | Data |
---|---|---|
1a3b1d89 BG |
1 | #ifndef _ASM_X86_ATOMIC64_32_H |
2 | #define _ASM_X86_ATOMIC64_32_H | |
3 | ||
4 | #include <linux/compiler.h> | |
5 | #include <linux/types.h> | |
6 | #include <asm/processor.h> | |
7 | //#include <asm/cmpxchg.h> | |
8 | ||
9 | /* An 64bit atomic type */ | |
10 | ||
11 | typedef struct { | |
12 | u64 __aligned(8) counter; | |
13 | } atomic64_t; | |
14 | ||
15 | #define ATOMIC64_INIT(val) { (val) } | |
16 | ||
819165fb JB |
17 | #define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...) |
18 | #ifndef ATOMIC64_EXPORT | |
19 | #define ATOMIC64_DECL_ONE __ATOMIC64_DECL | |
20 | #else | |
21 | #define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \ | |
22 | ATOMIC64_EXPORT(atomic64_##sym) | |
23 | #endif | |
24 | ||
a7e926ab | 25 | #ifdef CONFIG_X86_CMPXCHG64 |
819165fb JB |
26 | #define __alternative_atomic64(f, g, out, in...) \ |
27 | asm volatile("call %P[func]" \ | |
28 | : out : [func] "i" (atomic64_##g##_cx8), ## in) | |
29 | ||
30 | #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8) | |
a7e926ab | 31 | #else |
819165fb JB |
32 | #define __alternative_atomic64(f, g, out, in...) \ |
33 | alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \ | |
34 | X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in) | |
35 | ||
36 | #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \ | |
37 | ATOMIC64_DECL_ONE(sym##_386) | |
38 | ||
39 | ATOMIC64_DECL_ONE(add_386); | |
40 | ATOMIC64_DECL_ONE(sub_386); | |
41 | ATOMIC64_DECL_ONE(inc_386); | |
42 | ATOMIC64_DECL_ONE(dec_386); | |
a7e926ab LB |
43 | #endif |
44 | ||
819165fb JB |
45 | #define alternative_atomic64(f, out, in...) \ |
46 | __alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in) | |
47 | ||
48 | ATOMIC64_DECL(read); | |
49 | ATOMIC64_DECL(set); | |
50 | ATOMIC64_DECL(xchg); | |
51 | ATOMIC64_DECL(add_return); | |
52 | ATOMIC64_DECL(sub_return); | |
53 | ATOMIC64_DECL(inc_return); | |
54 | ATOMIC64_DECL(dec_return); | |
55 | ATOMIC64_DECL(dec_if_positive); | |
56 | ATOMIC64_DECL(inc_not_zero); | |
57 | ATOMIC64_DECL(add_unless); | |
58 | ||
59 | #undef ATOMIC64_DECL | |
60 | #undef ATOMIC64_DECL_ONE | |
61 | #undef __ATOMIC64_DECL | |
62 | #undef ATOMIC64_EXPORT | |
a7e926ab LB |
63 | |
64 | /** | |
65 | * atomic64_cmpxchg - cmpxchg atomic64 variable | |
66 | * @p: pointer to type atomic64_t | |
67 | * @o: expected value | |
68 | * @n: new value | |
69 | * | |
70 | * Atomically sets @v to @n if it was equal to @o and returns | |
71 | * the old value. | |
72 | */ | |
73 | ||
74 | static inline long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) | |
75 | { | |
76 | return cmpxchg64(&v->counter, o, n); | |
77 | } | |
1a3b1d89 BG |
78 | |
79 | /** | |
80 | * atomic64_xchg - xchg atomic64 variable | |
a7e926ab LB |
81 | * @v: pointer to type atomic64_t |
82 | * @n: value to assign | |
1a3b1d89 | 83 | * |
a7e926ab | 84 | * Atomically xchgs the value of @v to @n and returns |
1a3b1d89 BG |
85 | * the old value. |
86 | */ | |
a7e926ab LB |
87 | static inline long long atomic64_xchg(atomic64_t *v, long long n) |
88 | { | |
89 | long long o; | |
90 | unsigned high = (unsigned)(n >> 32); | |
91 | unsigned low = (unsigned)n; | |
819165fb JB |
92 | alternative_atomic64(xchg, "=&A" (o), |
93 | "S" (v), "b" (low), "c" (high) | |
94 | : "memory"); | |
a7e926ab LB |
95 | return o; |
96 | } | |
1a3b1d89 BG |
97 | |
98 | /** | |
99 | * atomic64_set - set atomic64 variable | |
a7e926ab LB |
100 | * @v: pointer to type atomic64_t |
101 | * @n: value to assign | |
1a3b1d89 | 102 | * |
a7e926ab | 103 | * Atomically sets the value of @v to @n. |
1a3b1d89 | 104 | */ |
a7e926ab LB |
105 | static inline void atomic64_set(atomic64_t *v, long long i) |
106 | { | |
107 | unsigned high = (unsigned)(i >> 32); | |
108 | unsigned low = (unsigned)i; | |
819165fb JB |
109 | alternative_atomic64(set, /* no output */, |
110 | "S" (v), "b" (low), "c" (high) | |
111 | : "eax", "edx", "memory"); | |
a7e926ab | 112 | } |
1a3b1d89 BG |
113 | |
114 | /** | |
115 | * atomic64_read - read atomic64 variable | |
a7e926ab | 116 | * @v: pointer to type atomic64_t |
1a3b1d89 | 117 | * |
a7e926ab | 118 | * Atomically reads the value of @v and returns it. |
1a3b1d89 | 119 | */ |
8030c36d | 120 | static inline long long atomic64_read(const atomic64_t *v) |
1a3b1d89 | 121 | { |
a7e926ab | 122 | long long r; |
819165fb | 123 | alternative_atomic64(read, "=&A" (r), "c" (v) : "memory"); |
a7e926ab LB |
124 | return r; |
125 | } | |
1a3b1d89 BG |
126 | |
127 | /** | |
128 | * atomic64_add_return - add and return | |
a7e926ab LB |
129 | * @i: integer value to add |
130 | * @v: pointer to type atomic64_t | |
1a3b1d89 | 131 | * |
a7e926ab | 132 | * Atomically adds @i to @v and returns @i + *@v |
1a3b1d89 | 133 | */ |
a7e926ab LB |
134 | static inline long long atomic64_add_return(long long i, atomic64_t *v) |
135 | { | |
819165fb JB |
136 | alternative_atomic64(add_return, |
137 | ASM_OUTPUT2("+A" (i), "+c" (v)), | |
138 | ASM_NO_INPUT_CLOBBER("memory")); | |
a7e926ab LB |
139 | return i; |
140 | } | |
1a3b1d89 BG |
141 | |
142 | /* | |
143 | * Other variants with different arithmetic operators: | |
144 | */ | |
a7e926ab LB |
145 | static inline long long atomic64_sub_return(long long i, atomic64_t *v) |
146 | { | |
819165fb JB |
147 | alternative_atomic64(sub_return, |
148 | ASM_OUTPUT2("+A" (i), "+c" (v)), | |
149 | ASM_NO_INPUT_CLOBBER("memory")); | |
a7e926ab LB |
150 | return i; |
151 | } | |
152 | ||
153 | static inline long long atomic64_inc_return(atomic64_t *v) | |
154 | { | |
155 | long long a; | |
819165fb JB |
156 | alternative_atomic64(inc_return, "=&A" (a), |
157 | "S" (v) : "memory", "ecx"); | |
a7e926ab LB |
158 | return a; |
159 | } | |
160 | ||
161 | static inline long long atomic64_dec_return(atomic64_t *v) | |
162 | { | |
163 | long long a; | |
819165fb JB |
164 | alternative_atomic64(dec_return, "=&A" (a), |
165 | "S" (v) : "memory", "ecx"); | |
a7e926ab LB |
166 | return a; |
167 | } | |
1a3b1d89 BG |
168 | |
169 | /** | |
170 | * atomic64_add - add integer to atomic64 variable | |
a7e926ab LB |
171 | * @i: integer value to add |
172 | * @v: pointer to type atomic64_t | |
1a3b1d89 | 173 | * |
a7e926ab | 174 | * Atomically adds @i to @v. |
1a3b1d89 | 175 | */ |
a7e926ab LB |
176 | static inline long long atomic64_add(long long i, atomic64_t *v) |
177 | { | |
819165fb JB |
178 | __alternative_atomic64(add, add_return, |
179 | ASM_OUTPUT2("+A" (i), "+c" (v)), | |
180 | ASM_NO_INPUT_CLOBBER("memory")); | |
a7e926ab LB |
181 | return i; |
182 | } | |
1a3b1d89 BG |
183 | |
184 | /** | |
185 | * atomic64_sub - subtract the atomic64 variable | |
a7e926ab LB |
186 | * @i: integer value to subtract |
187 | * @v: pointer to type atomic64_t | |
1a3b1d89 | 188 | * |
a7e926ab | 189 | * Atomically subtracts @i from @v. |
1a3b1d89 | 190 | */ |
a7e926ab LB |
191 | static inline long long atomic64_sub(long long i, atomic64_t *v) |
192 | { | |
819165fb JB |
193 | __alternative_atomic64(sub, sub_return, |
194 | ASM_OUTPUT2("+A" (i), "+c" (v)), | |
195 | ASM_NO_INPUT_CLOBBER("memory")); | |
a7e926ab LB |
196 | return i; |
197 | } | |
1a3b1d89 BG |
198 | |
199 | /** | |
200 | * atomic64_sub_and_test - subtract value from variable and test result | |
a7e926ab LB |
201 | * @i: integer value to subtract |
202 | * @v: pointer to type atomic64_t | |
203 | * | |
204 | * Atomically subtracts @i from @v and returns | |
1a3b1d89 BG |
205 | * true if the result is zero, or false for all |
206 | * other cases. | |
207 | */ | |
a7e926ab LB |
208 | static inline int atomic64_sub_and_test(long long i, atomic64_t *v) |
209 | { | |
210 | return atomic64_sub_return(i, v) == 0; | |
211 | } | |
1a3b1d89 BG |
212 | |
213 | /** | |
214 | * atomic64_inc - increment atomic64 variable | |
a7e926ab | 215 | * @v: pointer to type atomic64_t |
1a3b1d89 | 216 | * |
a7e926ab | 217 | * Atomically increments @v by 1. |
1a3b1d89 | 218 | */ |
a7e926ab LB |
219 | static inline void atomic64_inc(atomic64_t *v) |
220 | { | |
819165fb JB |
221 | __alternative_atomic64(inc, inc_return, /* no output */, |
222 | "S" (v) : "memory", "eax", "ecx", "edx"); | |
a7e926ab | 223 | } |
1a3b1d89 BG |
224 | |
225 | /** | |
226 | * atomic64_dec - decrement atomic64 variable | |
227 | * @ptr: pointer to type atomic64_t | |
228 | * | |
229 | * Atomically decrements @ptr by 1. | |
230 | */ | |
a7e926ab LB |
231 | static inline void atomic64_dec(atomic64_t *v) |
232 | { | |
819165fb JB |
233 | __alternative_atomic64(dec, dec_return, /* no output */, |
234 | "S" (v) : "memory", "eax", "ecx", "edx"); | |
a7e926ab | 235 | } |
1a3b1d89 BG |
236 | |
237 | /** | |
238 | * atomic64_dec_and_test - decrement and test | |
a7e926ab | 239 | * @v: pointer to type atomic64_t |
1a3b1d89 | 240 | * |
a7e926ab | 241 | * Atomically decrements @v by 1 and |
1a3b1d89 BG |
242 | * returns true if the result is 0, or false for all other |
243 | * cases. | |
244 | */ | |
a7e926ab LB |
245 | static inline int atomic64_dec_and_test(atomic64_t *v) |
246 | { | |
247 | return atomic64_dec_return(v) == 0; | |
248 | } | |
1a3b1d89 BG |
249 | |
250 | /** | |
251 | * atomic64_inc_and_test - increment and test | |
a7e926ab | 252 | * @v: pointer to type atomic64_t |
1a3b1d89 | 253 | * |
a7e926ab | 254 | * Atomically increments @v by 1 |
1a3b1d89 BG |
255 | * and returns true if the result is zero, or false for all |
256 | * other cases. | |
257 | */ | |
a7e926ab LB |
258 | static inline int atomic64_inc_and_test(atomic64_t *v) |
259 | { | |
260 | return atomic64_inc_return(v) == 0; | |
261 | } | |
1a3b1d89 BG |
262 | |
263 | /** | |
264 | * atomic64_add_negative - add and test if negative | |
a7e926ab LB |
265 | * @i: integer value to add |
266 | * @v: pointer to type atomic64_t | |
1a3b1d89 | 267 | * |
a7e926ab | 268 | * Atomically adds @i to @v and returns true |
1a3b1d89 BG |
269 | * if the result is negative, or false when |
270 | * result is greater than or equal to zero. | |
271 | */ | |
a7e926ab LB |
272 | static inline int atomic64_add_negative(long long i, atomic64_t *v) |
273 | { | |
274 | return atomic64_add_return(i, v) < 0; | |
275 | } | |
276 | ||
277 | /** | |
278 | * atomic64_add_unless - add unless the number is a given value | |
279 | * @v: pointer of type atomic64_t | |
280 | * @a: the amount to add to v... | |
281 | * @u: ...unless v is equal to u. | |
282 | * | |
283 | * Atomically adds @a to @v, so long as it was not @u. | |
819165fb | 284 | * Returns non-zero if the add was done, zero otherwise. |
a7e926ab LB |
285 | */ |
286 | static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) | |
287 | { | |
288 | unsigned low = (unsigned)u; | |
289 | unsigned high = (unsigned)(u >> 32); | |
819165fb | 290 | alternative_atomic64(add_unless, |
cb8095bb JB |
291 | ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)), |
292 | "S" (v) : "memory"); | |
a7e926ab LB |
293 | return (int)a; |
294 | } | |
295 | ||
296 | ||
297 | static inline int atomic64_inc_not_zero(atomic64_t *v) | |
298 | { | |
299 | int r; | |
819165fb JB |
300 | alternative_atomic64(inc_not_zero, "=&a" (r), |
301 | "S" (v) : "ecx", "edx", "memory"); | |
a7e926ab LB |
302 | return r; |
303 | } | |
304 | ||
305 | static inline long long atomic64_dec_if_positive(atomic64_t *v) | |
306 | { | |
307 | long long r; | |
819165fb JB |
308 | alternative_atomic64(dec_if_positive, "=&A" (r), |
309 | "S" (v) : "ecx", "memory"); | |
a7e926ab LB |
310 | return r; |
311 | } | |
312 | ||
819165fb JB |
313 | #undef alternative_atomic64 |
314 | #undef __alternative_atomic64 | |
1a3b1d89 BG |
315 | |
316 | #endif /* _ASM_X86_ATOMIC64_32_H */ |