]>
Commit | Line | Data |
---|---|---|
c482cb11 RH |
1 | /* |
2 | * Atomic helper templates | |
3 | * Included from tcg-runtime.c and cputlb.c. | |
4 | * | |
5 | * Copyright (c) 2016 Red Hat, Inc | |
6 | * | |
7 | * This library is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU Lesser General Public | |
9 | * License as published by the Free Software Foundation; either | |
fb0343d5 | 10 | * version 2.1 of the License, or (at your option) any later version. |
c482cb11 RH |
11 | * |
12 | * This library is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 | * Lesser General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU Lesser General Public | |
18 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. | |
19 | */ | |
20 | ||
e6d86bed | 21 | #include "qemu/plugin.h" |
d071f4cd | 22 | |
7ebee43e RH |
23 | #if DATA_SIZE == 16 |
24 | # define SUFFIX o | |
25 | # define DATA_TYPE Int128 | |
26 | # define BSWAP bswap128 | |
d071f4cd | 27 | # define SHIFT 4 |
7ebee43e | 28 | #elif DATA_SIZE == 8 |
c482cb11 | 29 | # define SUFFIX q |
9ef0c6d6 RH |
30 | # define DATA_TYPE aligned_uint64_t |
31 | # define SDATA_TYPE aligned_int64_t | |
c482cb11 | 32 | # define BSWAP bswap64 |
d071f4cd | 33 | # define SHIFT 3 |
c482cb11 RH |
34 | #elif DATA_SIZE == 4 |
35 | # define SUFFIX l | |
36 | # define DATA_TYPE uint32_t | |
5507c2bf | 37 | # define SDATA_TYPE int32_t |
c482cb11 | 38 | # define BSWAP bswap32 |
d071f4cd | 39 | # define SHIFT 2 |
c482cb11 RH |
40 | #elif DATA_SIZE == 2 |
41 | # define SUFFIX w | |
42 | # define DATA_TYPE uint16_t | |
5507c2bf | 43 | # define SDATA_TYPE int16_t |
c482cb11 | 44 | # define BSWAP bswap16 |
d071f4cd | 45 | # define SHIFT 1 |
c482cb11 RH |
46 | #elif DATA_SIZE == 1 |
47 | # define SUFFIX b | |
48 | # define DATA_TYPE uint8_t | |
5507c2bf | 49 | # define SDATA_TYPE int8_t |
c482cb11 | 50 | # define BSWAP |
d071f4cd | 51 | # define SHIFT 0 |
c482cb11 RH |
52 | #else |
53 | # error unsupported data size | |
54 | #endif | |
55 | ||
56 | #if DATA_SIZE >= 4 | |
57 | # define ABI_TYPE DATA_TYPE | |
58 | #else | |
59 | # define ABI_TYPE uint32_t | |
60 | #endif | |
61 | ||
62 | /* Define host-endian atomic operations. Note that END is used within | |
63 | the ATOMIC_NAME macro, and redefined below. */ | |
64 | #if DATA_SIZE == 1 | |
65 | # define END | |
66 | #elif defined(HOST_WORDS_BIGENDIAN) | |
67 | # define END _be | |
68 | #else | |
69 | # define END _le | |
70 | #endif | |
71 | ||
72 | ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, | |
48688faf | 73 | ABI_TYPE cmpv, ABI_TYPE newv, |
9002ffcb | 74 | MemOpIdx oi, uintptr_t retaddr) |
c482cb11 | 75 | { |
a754f7f3 RH |
76 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, |
77 | PAGE_READ | PAGE_WRITE, retaddr); | |
d071f4cd EC |
78 | DATA_TYPE ret; |
79 | ||
e6cd4bb5 RH |
80 | #if DATA_SIZE == 16 |
81 | ret = atomic16_cmpxchg(haddr, cmpv, newv); | |
82 | #else | |
d73415a3 | 83 | ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv); |
e6cd4bb5 | 84 | #endif |
ec603b55 | 85 | ATOMIC_MMU_CLEANUP; |
c3e83e37 | 86 | atomic_trace_rmw_post(env, addr, oi); |
ec603b55 | 87 | return ret; |
c482cb11 RH |
88 | } |
89 | ||
7ebee43e | 90 | #if DATA_SIZE >= 16 |
e6cd4bb5 | 91 | #if HAVE_ATOMIC128 |
48688faf | 92 | ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, |
9002ffcb | 93 | MemOpIdx oi, uintptr_t retaddr) |
7ebee43e | 94 | { |
a754f7f3 RH |
95 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, |
96 | PAGE_READ, retaddr); | |
97 | DATA_TYPE val; | |
d071f4cd | 98 | |
e6cd4bb5 | 99 | val = atomic16_read(haddr); |
ec603b55 | 100 | ATOMIC_MMU_CLEANUP; |
c3e83e37 | 101 | atomic_trace_ld_post(env, addr, oi); |
7ebee43e RH |
102 | return val; |
103 | } | |
104 | ||
48688faf | 105 | void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, |
9002ffcb | 106 | MemOpIdx oi, uintptr_t retaddr) |
7ebee43e | 107 | { |
a754f7f3 RH |
108 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, |
109 | PAGE_WRITE, retaddr); | |
d071f4cd | 110 | |
e6cd4bb5 | 111 | atomic16_set(haddr, val); |
ec603b55 | 112 | ATOMIC_MMU_CLEANUP; |
c3e83e37 | 113 | atomic_trace_st_post(env, addr, oi); |
7ebee43e | 114 | } |
e6cd4bb5 | 115 | #endif |
7ebee43e | 116 | #else |
48688faf | 117 | ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, |
9002ffcb | 118 | MemOpIdx oi, uintptr_t retaddr) |
c482cb11 | 119 | { |
a754f7f3 RH |
120 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, |
121 | PAGE_READ | PAGE_WRITE, retaddr); | |
d071f4cd EC |
122 | DATA_TYPE ret; |
123 | ||
d73415a3 | 124 | ret = qatomic_xchg__nocheck(haddr, val); |
ec603b55 | 125 | ATOMIC_MMU_CLEANUP; |
c3e83e37 | 126 | atomic_trace_rmw_post(env, addr, oi); |
ec603b55 | 127 | return ret; |
c482cb11 RH |
128 | } |
129 | ||
130 | #define GEN_ATOMIC_HELPER(X) \ | |
131 | ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ | |
9002ffcb | 132 | ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ |
c482cb11 | 133 | { \ |
a754f7f3 RH |
134 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ |
135 | PAGE_READ | PAGE_WRITE, retaddr); \ | |
d071f4cd | 136 | DATA_TYPE ret; \ |
d73415a3 | 137 | ret = qatomic_##X(haddr, val); \ |
ec603b55 | 138 | ATOMIC_MMU_CLEANUP; \ |
c3e83e37 | 139 | atomic_trace_rmw_post(env, addr, oi); \ |
ec603b55 RH |
140 | return ret; \ |
141 | } | |
c482cb11 RH |
142 | |
143 | GEN_ATOMIC_HELPER(fetch_add) | |
144 | GEN_ATOMIC_HELPER(fetch_and) | |
145 | GEN_ATOMIC_HELPER(fetch_or) | |
146 | GEN_ATOMIC_HELPER(fetch_xor) | |
147 | GEN_ATOMIC_HELPER(add_fetch) | |
148 | GEN_ATOMIC_HELPER(and_fetch) | |
149 | GEN_ATOMIC_HELPER(or_fetch) | |
150 | GEN_ATOMIC_HELPER(xor_fetch) | |
151 | ||
152 | #undef GEN_ATOMIC_HELPER | |
5507c2bf | 153 | |
a754f7f3 RH |
154 | /* |
155 | * These helpers are, as a whole, full barriers. Within the helper, | |
5507c2bf RH |
156 | * the leading barrier is explicit and the trailing barrier is within |
157 | * cmpxchg primitive. | |
d071f4cd EC |
158 | * |
159 | * Trace this load + RMW loop as a single RMW op. This way, regardless | |
160 | * of CF_PARALLEL's value, we'll trace just a read and a write. | |
5507c2bf RH |
161 | */ |
162 | #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ | |
163 | ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ | |
9002ffcb | 164 | ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ |
5507c2bf | 165 | { \ |
a754f7f3 RH |
166 | XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ |
167 | PAGE_READ | PAGE_WRITE, retaddr); \ | |
5507c2bf RH |
168 | XDATA_TYPE cmp, old, new, val = xval; \ |
169 | smp_mb(); \ | |
d73415a3 | 170 | cmp = qatomic_read__nocheck(haddr); \ |
5507c2bf RH |
171 | do { \ |
172 | old = cmp; new = FN(old, val); \ | |
d73415a3 | 173 | cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \ |
5507c2bf RH |
174 | } while (cmp != old); \ |
175 | ATOMIC_MMU_CLEANUP; \ | |
c3e83e37 | 176 | atomic_trace_rmw_post(env, addr, oi); \ |
5507c2bf RH |
177 | return RET; \ |
178 | } | |
179 | ||
180 | GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) | |
181 | GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) | |
182 | GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) | |
183 | GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) | |
184 | ||
185 | GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) | |
186 | GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) | |
187 | GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) | |
188 | GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) | |
189 | ||
190 | #undef GEN_ATOMIC_HELPER_FN | |
7ebee43e RH |
191 | #endif /* DATA SIZE >= 16 */ |
192 | ||
c482cb11 RH |
193 | #undef END |
194 | ||
195 | #if DATA_SIZE > 1 | |
196 | ||
197 | /* Define reverse-host-endian atomic operations. Note that END is used | |
198 | within the ATOMIC_NAME macro. */ | |
199 | #ifdef HOST_WORDS_BIGENDIAN | |
200 | # define END _le | |
201 | #else | |
202 | # define END _be | |
203 | #endif | |
204 | ||
205 | ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, | |
48688faf | 206 | ABI_TYPE cmpv, ABI_TYPE newv, |
9002ffcb | 207 | MemOpIdx oi, uintptr_t retaddr) |
c482cb11 | 208 | { |
a754f7f3 RH |
209 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, |
210 | PAGE_READ | PAGE_WRITE, retaddr); | |
d071f4cd EC |
211 | DATA_TYPE ret; |
212 | ||
e6cd4bb5 RH |
213 | #if DATA_SIZE == 16 |
214 | ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); | |
215 | #else | |
d73415a3 | 216 | ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); |
e6cd4bb5 | 217 | #endif |
ec603b55 | 218 | ATOMIC_MMU_CLEANUP; |
c3e83e37 | 219 | atomic_trace_rmw_post(env, addr, oi); |
ec603b55 | 220 | return BSWAP(ret); |
c482cb11 RH |
221 | } |
222 | ||
7ebee43e | 223 | #if DATA_SIZE >= 16 |
e6cd4bb5 | 224 | #if HAVE_ATOMIC128 |
48688faf | 225 | ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, |
9002ffcb | 226 | MemOpIdx oi, uintptr_t retaddr) |
7ebee43e | 227 | { |
a754f7f3 RH |
228 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, |
229 | PAGE_READ, retaddr); | |
230 | DATA_TYPE val; | |
d071f4cd | 231 | |
e6cd4bb5 | 232 | val = atomic16_read(haddr); |
ec603b55 | 233 | ATOMIC_MMU_CLEANUP; |
c3e83e37 | 234 | atomic_trace_ld_post(env, addr, oi); |
7ebee43e RH |
235 | return BSWAP(val); |
236 | } | |
237 | ||
48688faf | 238 | void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, |
9002ffcb | 239 | MemOpIdx oi, uintptr_t retaddr) |
7ebee43e | 240 | { |
a754f7f3 RH |
241 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, |
242 | PAGE_WRITE, retaddr); | |
d071f4cd | 243 | |
7ebee43e | 244 | val = BSWAP(val); |
e6cd4bb5 | 245 | atomic16_set(haddr, val); |
ec603b55 | 246 | ATOMIC_MMU_CLEANUP; |
c3e83e37 | 247 | atomic_trace_st_post(env, addr, oi); |
7ebee43e | 248 | } |
e6cd4bb5 | 249 | #endif |
7ebee43e | 250 | #else |
48688faf | 251 | ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, |
9002ffcb | 252 | MemOpIdx oi, uintptr_t retaddr) |
c482cb11 | 253 | { |
a754f7f3 RH |
254 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, |
255 | PAGE_READ | PAGE_WRITE, retaddr); | |
d071f4cd EC |
256 | ABI_TYPE ret; |
257 | ||
d73415a3 | 258 | ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); |
ec603b55 | 259 | ATOMIC_MMU_CLEANUP; |
c3e83e37 | 260 | atomic_trace_rmw_post(env, addr, oi); |
ec603b55 | 261 | return BSWAP(ret); |
c482cb11 RH |
262 | } |
263 | ||
264 | #define GEN_ATOMIC_HELPER(X) \ | |
265 | ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ | |
9002ffcb | 266 | ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ |
c482cb11 | 267 | { \ |
a754f7f3 RH |
268 | DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ |
269 | PAGE_READ | PAGE_WRITE, retaddr); \ | |
d071f4cd | 270 | DATA_TYPE ret; \ |
d73415a3 | 271 | ret = qatomic_##X(haddr, BSWAP(val)); \ |
ec603b55 | 272 | ATOMIC_MMU_CLEANUP; \ |
c3e83e37 | 273 | atomic_trace_rmw_post(env, addr, oi); \ |
ec603b55 | 274 | return BSWAP(ret); \ |
c482cb11 RH |
275 | } |
276 | ||
277 | GEN_ATOMIC_HELPER(fetch_and) | |
278 | GEN_ATOMIC_HELPER(fetch_or) | |
279 | GEN_ATOMIC_HELPER(fetch_xor) | |
280 | GEN_ATOMIC_HELPER(and_fetch) | |
281 | GEN_ATOMIC_HELPER(or_fetch) | |
282 | GEN_ATOMIC_HELPER(xor_fetch) | |
283 | ||
284 | #undef GEN_ATOMIC_HELPER | |
285 | ||
5507c2bf RH |
286 | /* These helpers are, as a whole, full barriers. Within the helper, |
287 | * the leading barrier is explicit and the trailing barrier is within | |
288 | * cmpxchg primitive. | |
d071f4cd EC |
289 | * |
290 | * Trace this load + RMW loop as a single RMW op. This way, regardless | |
291 | * of CF_PARALLEL's value, we'll trace just a read and a write. | |
5507c2bf RH |
292 | */ |
293 | #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ | |
294 | ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ | |
9002ffcb | 295 | ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ |
5507c2bf | 296 | { \ |
a754f7f3 RH |
297 | XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ |
298 | PAGE_READ | PAGE_WRITE, retaddr); \ | |
5507c2bf RH |
299 | XDATA_TYPE ldo, ldn, old, new, val = xval; \ |
300 | smp_mb(); \ | |
d73415a3 | 301 | ldn = qatomic_read__nocheck(haddr); \ |
5507c2bf RH |
302 | do { \ |
303 | ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ | |
d73415a3 | 304 | ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ |
5507c2bf RH |
305 | } while (ldo != ldn); \ |
306 | ATOMIC_MMU_CLEANUP; \ | |
c3e83e37 | 307 | atomic_trace_rmw_post(env, addr, oi); \ |
5507c2bf RH |
308 | return RET; \ |
309 | } | |
310 | ||
311 | GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) | |
312 | GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) | |
313 | GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) | |
314 | GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) | |
315 | ||
316 | GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) | |
317 | GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) | |
318 | GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) | |
319 | GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) | |
320 | ||
58edf9ee RH |
321 | /* Note that for addition, we need to use a separate cmpxchg loop instead |
322 | of bswaps for the reverse-host-endian helpers. */ | |
323 | #define ADD(X, Y) (X + Y) | |
324 | GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) | |
325 | GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) | |
326 | #undef ADD | |
327 | ||
5507c2bf | 328 | #undef GEN_ATOMIC_HELPER_FN |
7ebee43e | 329 | #endif /* DATA_SIZE >= 16 */ |
c482cb11 RH |
330 | |
331 | #undef END | |
332 | #endif /* DATA_SIZE > 1 */ | |
333 | ||
334 | #undef BSWAP | |
335 | #undef ABI_TYPE | |
336 | #undef DATA_TYPE | |
5507c2bf | 337 | #undef SDATA_TYPE |
c482cb11 RH |
338 | #undef SUFFIX |
339 | #undef DATA_SIZE | |
d071f4cd | 340 | #undef SHIFT |