2 * Atomic helper templates
3 * Included from tcg-runtime.c and cputlb.c.
5 * Copyright (c) 2016 Red Hat, Inc
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include "trace/mem.h"
25 # define DATA_TYPE Int128
26 # define BSWAP bswap128
30 # define DATA_TYPE uint64_t
31 # define SDATA_TYPE int64_t
32 # define BSWAP bswap64
36 # define DATA_TYPE uint32_t
37 # define SDATA_TYPE int32_t
38 # define BSWAP bswap32
42 # define DATA_TYPE uint16_t
43 # define SDATA_TYPE int16_t
44 # define BSWAP bswap16
48 # define DATA_TYPE uint8_t
49 # define SDATA_TYPE int8_t
53 # error unsupported data size
57 # define ABI_TYPE DATA_TYPE
59 # define ABI_TYPE uint32_t
62 #define ATOMIC_TRACE_RMW do { \
63 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
65 trace_guest_mem_before_exec(env_cpu(env), addr, info); \
66 trace_guest_mem_before_exec(env_cpu(env), addr, \
67 info | TRACE_MEM_ST); \
70 #define ATOMIC_TRACE_LD do { \
71 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
73 trace_guest_mem_before_exec(env_cpu(env), addr, info); \
76 # define ATOMIC_TRACE_ST do { \
77 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
79 trace_guest_mem_before_exec(env_cpu(env), addr, info); \
82 /* Define host-endian atomic operations. Note that END is used within
83 the ATOMIC_NAME macro, and redefined below. */
86 # define MEND _be /* either le or be would be fine */
87 #elif defined(HOST_WORDS_BIGENDIAN)
95 ABI_TYPE
ATOMIC_NAME(cmpxchg
)(CPUArchState
*env
, target_ulong addr
,
96 ABI_TYPE cmpv
, ABI_TYPE newv EXTRA_ARGS
)
99 DATA_TYPE
*haddr
= ATOMIC_MMU_LOOKUP
;
104 ret
= atomic16_cmpxchg(haddr
, cmpv
, newv
);
106 ret
= atomic_cmpxchg__nocheck(haddr
, cmpv
, newv
);
114 ABI_TYPE
ATOMIC_NAME(ld
)(CPUArchState
*env
, target_ulong addr EXTRA_ARGS
)
117 DATA_TYPE val
, *haddr
= ATOMIC_MMU_LOOKUP
;
120 val
= atomic16_read(haddr
);
125 void ATOMIC_NAME(st
)(CPUArchState
*env
, target_ulong addr
,
126 ABI_TYPE val EXTRA_ARGS
)
129 DATA_TYPE
*haddr
= ATOMIC_MMU_LOOKUP
;
132 atomic16_set(haddr
, val
);
137 ABI_TYPE
ATOMIC_NAME(xchg
)(CPUArchState
*env
, target_ulong addr
,
138 ABI_TYPE val EXTRA_ARGS
)
141 DATA_TYPE
*haddr
= ATOMIC_MMU_LOOKUP
;
145 ret
= atomic_xchg__nocheck(haddr
, val
);
150 #define GEN_ATOMIC_HELPER(X) \
151 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
152 ABI_TYPE val EXTRA_ARGS) \
155 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
159 ret = atomic_##X(haddr, val); \
160 ATOMIC_MMU_CLEANUP; \
164 GEN_ATOMIC_HELPER(fetch_add
)
165 GEN_ATOMIC_HELPER(fetch_and
)
166 GEN_ATOMIC_HELPER(fetch_or
)
167 GEN_ATOMIC_HELPER(fetch_xor
)
168 GEN_ATOMIC_HELPER(add_fetch
)
169 GEN_ATOMIC_HELPER(and_fetch
)
170 GEN_ATOMIC_HELPER(or_fetch
)
171 GEN_ATOMIC_HELPER(xor_fetch
)
173 #undef GEN_ATOMIC_HELPER
175 /* These helpers are, as a whole, full barriers. Within the helper,
176 * the leading barrier is explicit and the trailing barrier is within
179 * Trace this load + RMW loop as a single RMW op. This way, regardless
180 * of CF_PARALLEL's value, we'll trace just a read and a write.
182 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
183 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
184 ABI_TYPE xval EXTRA_ARGS) \
187 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
188 XDATA_TYPE cmp, old, new, val = xval; \
192 cmp = atomic_read__nocheck(haddr); \
194 old = cmp; new = FN(old, val); \
195 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
196 } while (cmp != old); \
197 ATOMIC_MMU_CLEANUP; \
201 GEN_ATOMIC_HELPER_FN(fetch_smin
, MIN
, SDATA_TYPE
, old
)
202 GEN_ATOMIC_HELPER_FN(fetch_umin
, MIN
, DATA_TYPE
, old
)
203 GEN_ATOMIC_HELPER_FN(fetch_smax
, MAX
, SDATA_TYPE
, old
)
204 GEN_ATOMIC_HELPER_FN(fetch_umax
, MAX
, DATA_TYPE
, old
)
206 GEN_ATOMIC_HELPER_FN(smin_fetch
, MIN
, SDATA_TYPE
, new)
207 GEN_ATOMIC_HELPER_FN(umin_fetch
, MIN
, DATA_TYPE
, new)
208 GEN_ATOMIC_HELPER_FN(smax_fetch
, MAX
, SDATA_TYPE
, new)
209 GEN_ATOMIC_HELPER_FN(umax_fetch
, MAX
, DATA_TYPE
, new)
211 #undef GEN_ATOMIC_HELPER_FN
212 #endif /* DATA SIZE >= 16 */
219 /* Define reverse-host-endian atomic operations. Note that END is used
220 within the ATOMIC_NAME macro. */
221 #ifdef HOST_WORDS_BIGENDIAN
229 ABI_TYPE
ATOMIC_NAME(cmpxchg
)(CPUArchState
*env
, target_ulong addr
,
230 ABI_TYPE cmpv
, ABI_TYPE newv EXTRA_ARGS
)
233 DATA_TYPE
*haddr
= ATOMIC_MMU_LOOKUP
;
238 ret
= atomic16_cmpxchg(haddr
, BSWAP(cmpv
), BSWAP(newv
));
240 ret
= atomic_cmpxchg__nocheck(haddr
, BSWAP(cmpv
), BSWAP(newv
));
248 ABI_TYPE
ATOMIC_NAME(ld
)(CPUArchState
*env
, target_ulong addr EXTRA_ARGS
)
251 DATA_TYPE val
, *haddr
= ATOMIC_MMU_LOOKUP
;
254 val
= atomic16_read(haddr
);
259 void ATOMIC_NAME(st
)(CPUArchState
*env
, target_ulong addr
,
260 ABI_TYPE val EXTRA_ARGS
)
263 DATA_TYPE
*haddr
= ATOMIC_MMU_LOOKUP
;
267 atomic16_set(haddr
, val
);
272 ABI_TYPE
ATOMIC_NAME(xchg
)(CPUArchState
*env
, target_ulong addr
,
273 ABI_TYPE val EXTRA_ARGS
)
276 DATA_TYPE
*haddr
= ATOMIC_MMU_LOOKUP
;
280 ret
= atomic_xchg__nocheck(haddr
, BSWAP(val
));
285 #define GEN_ATOMIC_HELPER(X) \
286 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
287 ABI_TYPE val EXTRA_ARGS) \
290 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
294 ret = atomic_##X(haddr, BSWAP(val)); \
295 ATOMIC_MMU_CLEANUP; \
299 GEN_ATOMIC_HELPER(fetch_and
)
300 GEN_ATOMIC_HELPER(fetch_or
)
301 GEN_ATOMIC_HELPER(fetch_xor
)
302 GEN_ATOMIC_HELPER(and_fetch
)
303 GEN_ATOMIC_HELPER(or_fetch
)
304 GEN_ATOMIC_HELPER(xor_fetch
)
306 #undef GEN_ATOMIC_HELPER
308 /* These helpers are, as a whole, full barriers. Within the helper,
309 * the leading barrier is explicit and the trailing barrier is within
312 * Trace this load + RMW loop as a single RMW op. This way, regardless
313 * of CF_PARALLEL's value, we'll trace just a read and a write.
315 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
316 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
317 ABI_TYPE xval EXTRA_ARGS) \
320 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
321 XDATA_TYPE ldo, ldn, old, new, val = xval; \
325 ldn = atomic_read__nocheck(haddr); \
327 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
328 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
329 } while (ldo != ldn); \
330 ATOMIC_MMU_CLEANUP; \
334 GEN_ATOMIC_HELPER_FN(fetch_smin
, MIN
, SDATA_TYPE
, old
)
335 GEN_ATOMIC_HELPER_FN(fetch_umin
, MIN
, DATA_TYPE
, old
)
336 GEN_ATOMIC_HELPER_FN(fetch_smax
, MAX
, SDATA_TYPE
, old
)
337 GEN_ATOMIC_HELPER_FN(fetch_umax
, MAX
, DATA_TYPE
, old
)
339 GEN_ATOMIC_HELPER_FN(smin_fetch
, MIN
, SDATA_TYPE
, new)
340 GEN_ATOMIC_HELPER_FN(umin_fetch
, MIN
, DATA_TYPE
, new)
341 GEN_ATOMIC_HELPER_FN(smax_fetch
, MAX
, SDATA_TYPE
, new)
342 GEN_ATOMIC_HELPER_FN(umax_fetch
, MAX
, DATA_TYPE
, new)
344 /* Note that for addition, we need to use a separate cmpxchg loop instead
345 of bswaps for the reverse-host-endian helpers. */
346 #define ADD(X, Y) (X + Y)
347 GEN_ATOMIC_HELPER_FN(fetch_add
, ADD
, DATA_TYPE
, old
)
348 GEN_ATOMIC_HELPER_FN(add_fetch
, ADD
, DATA_TYPE
, new)
351 #undef GEN_ATOMIC_HELPER_FN
352 #endif /* DATA_SIZE >= 16 */
356 #endif /* DATA_SIZE > 1 */
358 #undef ATOMIC_TRACE_ST
359 #undef ATOMIC_TRACE_LD
360 #undef ATOMIC_TRACE_RMW