]>
Commit | Line | Data |
---|---|---|
24f287e4 | 1 | /* atomic.S: These things are too big to do inline. |
1da177e4 | 2 | * |
193d2aad | 3 | * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net) |
1da177e4 LT |
4 | */ |
5 | ||
8695c37d | 6 | #include <linux/linkage.h> |
1da177e4 | 7 | #include <asm/asi.h> |
24f287e4 | 8 | #include <asm/backoff.h> |
1da177e4 | 9 | |
1da177e4 LT |
10 | .text |
11 | ||
3a1adb23 | 12 | /* Three versions of the atomic routines, one that |
1da177e4 | 13 | * does not return a value and does not perform |
3a1adb23 PZ |
14 | * memory barriers, and a two which return |
15 | * a value, the new and old value resp. and does the | |
16 | * barriers. | |
1da177e4 | 17 | */ |
1da177e4 | 18 | |
4f3316c2 PZ |
19 | #define ATOMIC_OP(op) \ |
20 | ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ | |
21 | BACKOFF_SETUP(%o2); \ | |
22 | 1: lduw [%o1], %g1; \ | |
23 | op %g1, %o0, %g7; \ | |
24 | cas [%o1], %g1, %g7; \ | |
25 | cmp %g1, %g7; \ | |
26 | bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ | |
27 | nop; \ | |
28 | retl; \ | |
29 | nop; \ | |
30 | 2: BACKOFF_SPIN(%o2, %o3, 1b); \ | |
31 | ENDPROC(atomic_##op); \ | |
1da177e4 | 32 | |
4f3316c2 PZ |
33 | #define ATOMIC_OP_RETURN(op) \ |
34 | ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ | |
35 | BACKOFF_SETUP(%o2); \ | |
36 | 1: lduw [%o1], %g1; \ | |
37 | op %g1, %o0, %g7; \ | |
38 | cas [%o1], %g1, %g7; \ | |
39 | cmp %g1, %g7; \ | |
40 | bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ | |
caa17d49 | 41 | op %g1, %o0, %g1; \ |
4f3316c2 PZ |
42 | retl; \ |
43 | sra %g1, 0, %o0; \ | |
44 | 2: BACKOFF_SPIN(%o2, %o3, 1b); \ | |
45 | ENDPROC(atomic_##op##_return); | |
1da177e4 | 46 | |
3a1adb23 PZ |
47 | #define ATOMIC_FETCH_OP(op) \ |
48 | ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ | |
49 | BACKOFF_SETUP(%o2); \ | |
50 | 1: lduw [%o1], %g1; \ | |
51 | op %g1, %o0, %g7; \ | |
52 | cas [%o1], %g1, %g7; \ | |
53 | cmp %g1, %g7; \ | |
54 | bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ | |
55 | nop; \ | |
56 | retl; \ | |
57 | sra %g1, 0, %o0; \ | |
58 | 2: BACKOFF_SPIN(%o2, %o3, 1b); \ | |
59 | ENDPROC(atomic_fetch_##op); | |
60 | ||
61 | #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) | |
1da177e4 | 62 | |
4f3316c2 PZ |
63 | ATOMIC_OPS(add) |
64 | ATOMIC_OPS(sub) | |
1da177e4 | 65 | |
4f3316c2 | 66 | #undef ATOMIC_OPS |
3a1adb23 PZ |
67 | #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) |
68 | ||
69 | ATOMIC_OPS(and) | |
70 | ATOMIC_OPS(or) | |
71 | ATOMIC_OPS(xor) | |
72 | ||
73 | #undef ATOMIC_OPS | |
74 | #undef ATOMIC_FETCH_OP | |
4f3316c2 PZ |
75 | #undef ATOMIC_OP_RETURN |
76 | #undef ATOMIC_OP | |
1da177e4 | 77 | |
4f3316c2 PZ |
78 | #define ATOMIC64_OP(op) \ |
79 | ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ | |
80 | BACKOFF_SETUP(%o2); \ | |
81 | 1: ldx [%o1], %g1; \ | |
82 | op %g1, %o0, %g7; \ | |
83 | casx [%o1], %g1, %g7; \ | |
84 | cmp %g1, %g7; \ | |
85 | bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ | |
86 | nop; \ | |
87 | retl; \ | |
88 | nop; \ | |
89 | 2: BACKOFF_SPIN(%o2, %o3, 1b); \ | |
90 | ENDPROC(atomic64_##op); \ | |
1da177e4 | 91 | |
4f3316c2 PZ |
92 | #define ATOMIC64_OP_RETURN(op) \ |
93 | ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ | |
94 | BACKOFF_SETUP(%o2); \ | |
95 | 1: ldx [%o1], %g1; \ | |
96 | op %g1, %o0, %g7; \ | |
97 | casx [%o1], %g1, %g7; \ | |
98 | cmp %g1, %g7; \ | |
99 | bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ | |
100 | nop; \ | |
101 | retl; \ | |
caa17d49 | 102 | op %g1, %o0, %o0; \ |
4f3316c2 PZ |
103 | 2: BACKOFF_SPIN(%o2, %o3, 1b); \ |
104 | ENDPROC(atomic64_##op##_return); | |
105 | ||
3a1adb23 PZ |
106 | #define ATOMIC64_FETCH_OP(op) \ |
107 | ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ | |
108 | BACKOFF_SETUP(%o2); \ | |
109 | 1: ldx [%o1], %g1; \ | |
110 | op %g1, %o0, %g7; \ | |
111 | casx [%o1], %g1, %g7; \ | |
112 | cmp %g1, %g7; \ | |
113 | bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ | |
114 | nop; \ | |
115 | retl; \ | |
116 | mov %g1, %o0; \ | |
117 | 2: BACKOFF_SPIN(%o2, %o3, 1b); \ | |
118 | ENDPROC(atomic64_fetch_##op); | |
119 | ||
120 | #define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op) | |
4f3316c2 PZ |
121 | |
122 | ATOMIC64_OPS(add) | |
123 | ATOMIC64_OPS(sub) | |
124 | ||
125 | #undef ATOMIC64_OPS | |
3a1adb23 PZ |
126 | #define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_FETCH_OP(op) |
127 | ||
128 | ATOMIC64_OPS(and) | |
129 | ATOMIC64_OPS(or) | |
130 | ATOMIC64_OPS(xor) | |
131 | ||
132 | #undef ATOMIC64_OPS | |
133 | #undef ATOMIC64_FETCH_OP | |
4f3316c2 PZ |
134 | #undef ATOMIC64_OP_RETURN |
135 | #undef ATOMIC64_OP | |
193d2aad DM |
136 | |
137 | ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */ | |
138 | BACKOFF_SETUP(%o2) | |
139 | 1: ldx [%o0], %g1 | |
140 | brlez,pn %g1, 3f | |
141 | sub %g1, 1, %g7 | |
142 | casx [%o0], %g1, %g7 | |
143 | cmp %g1, %g7 | |
144 | bne,pn %xcc, BACKOFF_LABEL(2f, 1b) | |
145 | nop | |
146 | 3: retl | |
147 | sub %g1, 1, %o0 | |
148 | 2: BACKOFF_SPIN(%o2, %o3, 1b) | |
149 | ENDPROC(atomic64_dec_if_positive) |