]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - arch/arm64/include/asm/atomic_lse.h
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'for-linus-20170825' of git://git.infradead.org/linux-mtd
[mirror_ubuntu-artful-kernel.git] / arch / arm64 / include / asm / atomic_lse.h
1 /*
2 * Based on arch/arm/include/asm/atomic.h
3 *
4 * Copyright (C) 1996 Russell King.
5 * Copyright (C) 2002 Deep Blue Solutions Ltd.
6 * Copyright (C) 2012 ARM Ltd.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #ifndef __ASM_ATOMIC_LSE_H
22 #define __ASM_ATOMIC_LSE_H
23
24 #ifndef __ARM64_IN_ATOMIC_IMPL
25 #error "please don't include this file directly"
26 #endif
27
28 #define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
29 #define ATOMIC_OP(op, asm_op) \
30 static inline void atomic_##op(int i, atomic_t *v) \
31 { \
32 register int w0 asm ("w0") = i; \
33 register atomic_t *x1 asm ("x1") = v; \
34 \
35 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op), \
36 " " #asm_op " %w[i], %[v]\n") \
37 : [i] "+r" (w0), [v] "+Q" (v->counter) \
38 : "r" (x1) \
39 : __LL_SC_CLOBBERS); \
40 }
41
42 ATOMIC_OP(andnot, stclr)
43 ATOMIC_OP(or, stset)
44 ATOMIC_OP(xor, steor)
45 ATOMIC_OP(add, stadd)
46
47 #undef ATOMIC_OP
48
49 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \
50 static inline int atomic_fetch_##op##name(int i, atomic_t *v) \
51 { \
52 register int w0 asm ("w0") = i; \
53 register atomic_t *x1 asm ("x1") = v; \
54 \
55 asm volatile(ARM64_LSE_ATOMIC_INSN( \
56 /* LL/SC */ \
57 __LL_SC_ATOMIC(fetch_##op##name), \
58 /* LSE atomics */ \
59 " " #asm_op #mb " %w[i], %w[i], %[v]") \
60 : [i] "+r" (w0), [v] "+Q" (v->counter) \
61 : "r" (x1) \
62 : __LL_SC_CLOBBERS, ##cl); \
63 \
64 return w0; \
65 }
66
67 #define ATOMIC_FETCH_OPS(op, asm_op) \
68 ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \
69 ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \
70 ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \
71 ATOMIC_FETCH_OP( , al, op, asm_op, "memory")
72
73 ATOMIC_FETCH_OPS(andnot, ldclr)
74 ATOMIC_FETCH_OPS(or, ldset)
75 ATOMIC_FETCH_OPS(xor, ldeor)
76 ATOMIC_FETCH_OPS(add, ldadd)
77
78 #undef ATOMIC_FETCH_OP
79 #undef ATOMIC_FETCH_OPS
80
81 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
82 static inline int atomic_add_return##name(int i, atomic_t *v) \
83 { \
84 register int w0 asm ("w0") = i; \
85 register atomic_t *x1 asm ("x1") = v; \
86 \
87 asm volatile(ARM64_LSE_ATOMIC_INSN( \
88 /* LL/SC */ \
89 __LL_SC_ATOMIC(add_return##name) \
90 __nops(1), \
91 /* LSE atomics */ \
92 " ldadd" #mb " %w[i], w30, %[v]\n" \
93 " add %w[i], %w[i], w30") \
94 : [i] "+r" (w0), [v] "+Q" (v->counter) \
95 : "r" (x1) \
96 : __LL_SC_CLOBBERS, ##cl); \
97 \
98 return w0; \
99 }
100
101 ATOMIC_OP_ADD_RETURN(_relaxed, )
102 ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
103 ATOMIC_OP_ADD_RETURN(_release, l, "memory")
104 ATOMIC_OP_ADD_RETURN( , al, "memory")
105
106 #undef ATOMIC_OP_ADD_RETURN
107
108 static inline void atomic_and(int i, atomic_t *v)
109 {
110 register int w0 asm ("w0") = i;
111 register atomic_t *x1 asm ("x1") = v;
112
113 asm volatile(ARM64_LSE_ATOMIC_INSN(
114 /* LL/SC */
115 __LL_SC_ATOMIC(and)
116 __nops(1),
117 /* LSE atomics */
118 " mvn %w[i], %w[i]\n"
119 " stclr %w[i], %[v]")
120 : [i] "+r" (w0), [v] "+Q" (v->counter)
121 : "r" (x1)
122 : __LL_SC_CLOBBERS);
123 }
124
125 #define ATOMIC_FETCH_OP_AND(name, mb, cl...) \
126 static inline int atomic_fetch_and##name(int i, atomic_t *v) \
127 { \
128 register int w0 asm ("w0") = i; \
129 register atomic_t *x1 asm ("x1") = v; \
130 \
131 asm volatile(ARM64_LSE_ATOMIC_INSN( \
132 /* LL/SC */ \
133 __LL_SC_ATOMIC(fetch_and##name) \
134 __nops(1), \
135 /* LSE atomics */ \
136 " mvn %w[i], %w[i]\n" \
137 " ldclr" #mb " %w[i], %w[i], %[v]") \
138 : [i] "+r" (w0), [v] "+Q" (v->counter) \
139 : "r" (x1) \
140 : __LL_SC_CLOBBERS, ##cl); \
141 \
142 return w0; \
143 }
144
145 ATOMIC_FETCH_OP_AND(_relaxed, )
146 ATOMIC_FETCH_OP_AND(_acquire, a, "memory")
147 ATOMIC_FETCH_OP_AND(_release, l, "memory")
148 ATOMIC_FETCH_OP_AND( , al, "memory")
149
150 #undef ATOMIC_FETCH_OP_AND
151
152 static inline void atomic_sub(int i, atomic_t *v)
153 {
154 register int w0 asm ("w0") = i;
155 register atomic_t *x1 asm ("x1") = v;
156
157 asm volatile(ARM64_LSE_ATOMIC_INSN(
158 /* LL/SC */
159 __LL_SC_ATOMIC(sub)
160 __nops(1),
161 /* LSE atomics */
162 " neg %w[i], %w[i]\n"
163 " stadd %w[i], %[v]")
164 : [i] "+r" (w0), [v] "+Q" (v->counter)
165 : "r" (x1)
166 : __LL_SC_CLOBBERS);
167 }
168
169 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
170 static inline int atomic_sub_return##name(int i, atomic_t *v) \
171 { \
172 register int w0 asm ("w0") = i; \
173 register atomic_t *x1 asm ("x1") = v; \
174 \
175 asm volatile(ARM64_LSE_ATOMIC_INSN( \
176 /* LL/SC */ \
177 __LL_SC_ATOMIC(sub_return##name) \
178 __nops(2), \
179 /* LSE atomics */ \
180 " neg %w[i], %w[i]\n" \
181 " ldadd" #mb " %w[i], w30, %[v]\n" \
182 " add %w[i], %w[i], w30") \
183 : [i] "+r" (w0), [v] "+Q" (v->counter) \
184 : "r" (x1) \
185 : __LL_SC_CLOBBERS , ##cl); \
186 \
187 return w0; \
188 }
189
190 ATOMIC_OP_SUB_RETURN(_relaxed, )
191 ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
192 ATOMIC_OP_SUB_RETURN(_release, l, "memory")
193 ATOMIC_OP_SUB_RETURN( , al, "memory")
194
195 #undef ATOMIC_OP_SUB_RETURN
196
197 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \
198 static inline int atomic_fetch_sub##name(int i, atomic_t *v) \
199 { \
200 register int w0 asm ("w0") = i; \
201 register atomic_t *x1 asm ("x1") = v; \
202 \
203 asm volatile(ARM64_LSE_ATOMIC_INSN( \
204 /* LL/SC */ \
205 __LL_SC_ATOMIC(fetch_sub##name) \
206 __nops(1), \
207 /* LSE atomics */ \
208 " neg %w[i], %w[i]\n" \
209 " ldadd" #mb " %w[i], %w[i], %[v]") \
210 : [i] "+r" (w0), [v] "+Q" (v->counter) \
211 : "r" (x1) \
212 : __LL_SC_CLOBBERS, ##cl); \
213 \
214 return w0; \
215 }
216
217 ATOMIC_FETCH_OP_SUB(_relaxed, )
218 ATOMIC_FETCH_OP_SUB(_acquire, a, "memory")
219 ATOMIC_FETCH_OP_SUB(_release, l, "memory")
220 ATOMIC_FETCH_OP_SUB( , al, "memory")
221
222 #undef ATOMIC_FETCH_OP_SUB
223 #undef __LL_SC_ATOMIC
224
225 #define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
226 #define ATOMIC64_OP(op, asm_op) \
227 static inline void atomic64_##op(long i, atomic64_t *v) \
228 { \
229 register long x0 asm ("x0") = i; \
230 register atomic64_t *x1 asm ("x1") = v; \
231 \
232 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op), \
233 " " #asm_op " %[i], %[v]\n") \
234 : [i] "+r" (x0), [v] "+Q" (v->counter) \
235 : "r" (x1) \
236 : __LL_SC_CLOBBERS); \
237 }
238
239 ATOMIC64_OP(andnot, stclr)
240 ATOMIC64_OP(or, stset)
241 ATOMIC64_OP(xor, steor)
242 ATOMIC64_OP(add, stadd)
243
244 #undef ATOMIC64_OP
245
246 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \
247 static inline long atomic64_fetch_##op##name(long i, atomic64_t *v) \
248 { \
249 register long x0 asm ("x0") = i; \
250 register atomic64_t *x1 asm ("x1") = v; \
251 \
252 asm volatile(ARM64_LSE_ATOMIC_INSN( \
253 /* LL/SC */ \
254 __LL_SC_ATOMIC64(fetch_##op##name), \
255 /* LSE atomics */ \
256 " " #asm_op #mb " %[i], %[i], %[v]") \
257 : [i] "+r" (x0), [v] "+Q" (v->counter) \
258 : "r" (x1) \
259 : __LL_SC_CLOBBERS, ##cl); \
260 \
261 return x0; \
262 }
263
264 #define ATOMIC64_FETCH_OPS(op, asm_op) \
265 ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \
266 ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \
267 ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \
268 ATOMIC64_FETCH_OP( , al, op, asm_op, "memory")
269
270 ATOMIC64_FETCH_OPS(andnot, ldclr)
271 ATOMIC64_FETCH_OPS(or, ldset)
272 ATOMIC64_FETCH_OPS(xor, ldeor)
273 ATOMIC64_FETCH_OPS(add, ldadd)
274
275 #undef ATOMIC64_FETCH_OP
276 #undef ATOMIC64_FETCH_OPS
277
278 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
279 static inline long atomic64_add_return##name(long i, atomic64_t *v) \
280 { \
281 register long x0 asm ("x0") = i; \
282 register atomic64_t *x1 asm ("x1") = v; \
283 \
284 asm volatile(ARM64_LSE_ATOMIC_INSN( \
285 /* LL/SC */ \
286 __LL_SC_ATOMIC64(add_return##name) \
287 __nops(1), \
288 /* LSE atomics */ \
289 " ldadd" #mb " %[i], x30, %[v]\n" \
290 " add %[i], %[i], x30") \
291 : [i] "+r" (x0), [v] "+Q" (v->counter) \
292 : "r" (x1) \
293 : __LL_SC_CLOBBERS, ##cl); \
294 \
295 return x0; \
296 }
297
298 ATOMIC64_OP_ADD_RETURN(_relaxed, )
299 ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
300 ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
301 ATOMIC64_OP_ADD_RETURN( , al, "memory")
302
303 #undef ATOMIC64_OP_ADD_RETURN
304
305 static inline void atomic64_and(long i, atomic64_t *v)
306 {
307 register long x0 asm ("x0") = i;
308 register atomic64_t *x1 asm ("x1") = v;
309
310 asm volatile(ARM64_LSE_ATOMIC_INSN(
311 /* LL/SC */
312 __LL_SC_ATOMIC64(and)
313 __nops(1),
314 /* LSE atomics */
315 " mvn %[i], %[i]\n"
316 " stclr %[i], %[v]")
317 : [i] "+r" (x0), [v] "+Q" (v->counter)
318 : "r" (x1)
319 : __LL_SC_CLOBBERS);
320 }
321
322 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \
323 static inline long atomic64_fetch_and##name(long i, atomic64_t *v) \
324 { \
325 register long x0 asm ("x0") = i; \
326 register atomic64_t *x1 asm ("x1") = v; \
327 \
328 asm volatile(ARM64_LSE_ATOMIC_INSN( \
329 /* LL/SC */ \
330 __LL_SC_ATOMIC64(fetch_and##name) \
331 __nops(1), \
332 /* LSE atomics */ \
333 " mvn %[i], %[i]\n" \
334 " ldclr" #mb " %[i], %[i], %[v]") \
335 : [i] "+r" (x0), [v] "+Q" (v->counter) \
336 : "r" (x1) \
337 : __LL_SC_CLOBBERS, ##cl); \
338 \
339 return x0; \
340 }
341
342 ATOMIC64_FETCH_OP_AND(_relaxed, )
343 ATOMIC64_FETCH_OP_AND(_acquire, a, "memory")
344 ATOMIC64_FETCH_OP_AND(_release, l, "memory")
345 ATOMIC64_FETCH_OP_AND( , al, "memory")
346
347 #undef ATOMIC64_FETCH_OP_AND
348
349 static inline void atomic64_sub(long i, atomic64_t *v)
350 {
351 register long x0 asm ("x0") = i;
352 register atomic64_t *x1 asm ("x1") = v;
353
354 asm volatile(ARM64_LSE_ATOMIC_INSN(
355 /* LL/SC */
356 __LL_SC_ATOMIC64(sub)
357 __nops(1),
358 /* LSE atomics */
359 " neg %[i], %[i]\n"
360 " stadd %[i], %[v]")
361 : [i] "+r" (x0), [v] "+Q" (v->counter)
362 : "r" (x1)
363 : __LL_SC_CLOBBERS);
364 }
365
366 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
367 static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
368 { \
369 register long x0 asm ("x0") = i; \
370 register atomic64_t *x1 asm ("x1") = v; \
371 \
372 asm volatile(ARM64_LSE_ATOMIC_INSN( \
373 /* LL/SC */ \
374 __LL_SC_ATOMIC64(sub_return##name) \
375 __nops(2), \
376 /* LSE atomics */ \
377 " neg %[i], %[i]\n" \
378 " ldadd" #mb " %[i], x30, %[v]\n" \
379 " add %[i], %[i], x30") \
380 : [i] "+r" (x0), [v] "+Q" (v->counter) \
381 : "r" (x1) \
382 : __LL_SC_CLOBBERS, ##cl); \
383 \
384 return x0; \
385 }
386
387 ATOMIC64_OP_SUB_RETURN(_relaxed, )
388 ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
389 ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
390 ATOMIC64_OP_SUB_RETURN( , al, "memory")
391
392 #undef ATOMIC64_OP_SUB_RETURN
393
394 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \
395 static inline long atomic64_fetch_sub##name(long i, atomic64_t *v) \
396 { \
397 register long x0 asm ("x0") = i; \
398 register atomic64_t *x1 asm ("x1") = v; \
399 \
400 asm volatile(ARM64_LSE_ATOMIC_INSN( \
401 /* LL/SC */ \
402 __LL_SC_ATOMIC64(fetch_sub##name) \
403 __nops(1), \
404 /* LSE atomics */ \
405 " neg %[i], %[i]\n" \
406 " ldadd" #mb " %[i], %[i], %[v]") \
407 : [i] "+r" (x0), [v] "+Q" (v->counter) \
408 : "r" (x1) \
409 : __LL_SC_CLOBBERS, ##cl); \
410 \
411 return x0; \
412 }
413
414 ATOMIC64_FETCH_OP_SUB(_relaxed, )
415 ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory")
416 ATOMIC64_FETCH_OP_SUB(_release, l, "memory")
417 ATOMIC64_FETCH_OP_SUB( , al, "memory")
418
419 #undef ATOMIC64_FETCH_OP_SUB
420
421 static inline long atomic64_dec_if_positive(atomic64_t *v)
422 {
423 register long x0 asm ("x0") = (long)v;
424
425 asm volatile(ARM64_LSE_ATOMIC_INSN(
426 /* LL/SC */
427 __LL_SC_ATOMIC64(dec_if_positive)
428 __nops(6),
429 /* LSE atomics */
430 "1: ldr x30, %[v]\n"
431 " subs %[ret], x30, #1\n"
432 " b.lt 2f\n"
433 " casal x30, %[ret], %[v]\n"
434 " sub x30, x30, #1\n"
435 " sub x30, x30, %[ret]\n"
436 " cbnz x30, 1b\n"
437 "2:")
438 : [ret] "+r" (x0), [v] "+Q" (v->counter)
439 :
440 : __LL_SC_CLOBBERS, "cc", "memory");
441
442 return x0;
443 }
444
445 #undef __LL_SC_ATOMIC64
446
447 #define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
448
449 #define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
450 static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
451 unsigned long old, \
452 unsigned long new) \
453 { \
454 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
455 register unsigned long x1 asm ("x1") = old; \
456 register unsigned long x2 asm ("x2") = new; \
457 \
458 asm volatile(ARM64_LSE_ATOMIC_INSN( \
459 /* LL/SC */ \
460 __LL_SC_CMPXCHG(name) \
461 __nops(2), \
462 /* LSE atomics */ \
463 " mov " #w "30, %" #w "[old]\n" \
464 " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
465 " mov %" #w "[ret], " #w "30") \
466 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
467 : [old] "r" (x1), [new] "r" (x2) \
468 : __LL_SC_CLOBBERS, ##cl); \
469 \
470 return x0; \
471 }
472
473 __CMPXCHG_CASE(w, b, 1, )
474 __CMPXCHG_CASE(w, h, 2, )
475 __CMPXCHG_CASE(w, , 4, )
476 __CMPXCHG_CASE(x, , 8, )
477 __CMPXCHG_CASE(w, b, acq_1, a, "memory")
478 __CMPXCHG_CASE(w, h, acq_2, a, "memory")
479 __CMPXCHG_CASE(w, , acq_4, a, "memory")
480 __CMPXCHG_CASE(x, , acq_8, a, "memory")
481 __CMPXCHG_CASE(w, b, rel_1, l, "memory")
482 __CMPXCHG_CASE(w, h, rel_2, l, "memory")
483 __CMPXCHG_CASE(w, , rel_4, l, "memory")
484 __CMPXCHG_CASE(x, , rel_8, l, "memory")
485 __CMPXCHG_CASE(w, b, mb_1, al, "memory")
486 __CMPXCHG_CASE(w, h, mb_2, al, "memory")
487 __CMPXCHG_CASE(w, , mb_4, al, "memory")
488 __CMPXCHG_CASE(x, , mb_8, al, "memory")
489
490 #undef __LL_SC_CMPXCHG
491 #undef __CMPXCHG_CASE
492
493 #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
494
495 #define __CMPXCHG_DBL(name, mb, cl...) \
496 static inline long __cmpxchg_double##name(unsigned long old1, \
497 unsigned long old2, \
498 unsigned long new1, \
499 unsigned long new2, \
500 volatile void *ptr) \
501 { \
502 unsigned long oldval1 = old1; \
503 unsigned long oldval2 = old2; \
504 register unsigned long x0 asm ("x0") = old1; \
505 register unsigned long x1 asm ("x1") = old2; \
506 register unsigned long x2 asm ("x2") = new1; \
507 register unsigned long x3 asm ("x3") = new2; \
508 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
509 \
510 asm volatile(ARM64_LSE_ATOMIC_INSN( \
511 /* LL/SC */ \
512 __LL_SC_CMPXCHG_DBL(name) \
513 __nops(3), \
514 /* LSE atomics */ \
515 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
516 " eor %[old1], %[old1], %[oldval1]\n" \
517 " eor %[old2], %[old2], %[oldval2]\n" \
518 " orr %[old1], %[old1], %[old2]") \
519 : [old1] "+r" (x0), [old2] "+r" (x1), \
520 [v] "+Q" (*(unsigned long *)ptr) \
521 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
522 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
523 : __LL_SC_CLOBBERS, ##cl); \
524 \
525 return x0; \
526 }
527
528 __CMPXCHG_DBL( , )
529 __CMPXCHG_DBL(_mb, al, "memory")
530
531 #undef __LL_SC_CMPXCHG_DBL
532 #undef __CMPXCHG_DBL
533
534 #endif /* __ASM_ATOMIC_LSE_H */
Ubuntu Artful kernel mirror