2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 #include "qemu/osdep.h"
26 #include "exec/exec-all.h"
28 #include "tcg/tcg-op.h"
29 #include "tcg/tcg-mo.h"
30 #include "trace-tcg.h"
31 #include "trace/mem.h"
32 #include "exec/plugin-gen.h"
34 /* Reduce the number of ifdefs below. This assumes that all uses of
35 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
36 the compiler can eliminate. */
37 #if TCG_TARGET_REG_BITS == 64
38 extern TCGv_i32
TCGV_LOW_link_error(TCGv_i64
);
39 extern TCGv_i32
TCGV_HIGH_link_error(TCGv_i64
);
40 #define TCGV_LOW TCGV_LOW_link_error
41 #define TCGV_HIGH TCGV_HIGH_link_error
44 void tcg_gen_op1(TCGOpcode opc
, TCGArg a1
)
46 TCGOp
*op
= tcg_emit_op(opc
);
50 void tcg_gen_op2(TCGOpcode opc
, TCGArg a1
, TCGArg a2
)
52 TCGOp
*op
= tcg_emit_op(opc
);
57 void tcg_gen_op3(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
)
59 TCGOp
*op
= tcg_emit_op(opc
);
65 void tcg_gen_op4(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
, TCGArg a4
)
67 TCGOp
*op
= tcg_emit_op(opc
);
74 void tcg_gen_op5(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
,
77 TCGOp
*op
= tcg_emit_op(opc
);
85 void tcg_gen_op6(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
,
86 TCGArg a4
, TCGArg a5
, TCGArg a6
)
88 TCGOp
*op
= tcg_emit_op(opc
);
97 void tcg_gen_mb(TCGBar mb_type
)
99 if (tcg_ctx
->tb_cflags
& CF_PARALLEL
) {
100 tcg_gen_op1(INDEX_op_mb
, mb_type
);
106 void tcg_gen_movi_i32(TCGv_i32 ret
, int32_t arg
)
108 tcg_gen_mov_i32(ret
, tcg_constant_i32(arg
));
111 void tcg_gen_addi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
113 /* some cases can be optimized here */
115 tcg_gen_mov_i32(ret
, arg1
);
117 tcg_gen_add_i32(ret
, arg1
, tcg_constant_i32(arg2
));
121 void tcg_gen_subfi_i32(TCGv_i32 ret
, int32_t arg1
, TCGv_i32 arg2
)
123 if (arg1
== 0 && TCG_TARGET_HAS_neg_i32
) {
124 /* Don't recurse with tcg_gen_neg_i32. */
125 tcg_gen_op2_i32(INDEX_op_neg_i32
, ret
, arg2
);
127 tcg_gen_sub_i32(ret
, tcg_constant_i32(arg1
), arg2
);
131 void tcg_gen_subi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
133 /* some cases can be optimized here */
135 tcg_gen_mov_i32(ret
, arg1
);
137 tcg_gen_sub_i32(ret
, arg1
, tcg_constant_i32(arg2
));
141 void tcg_gen_andi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
143 /* Some cases can be optimized here. */
146 tcg_gen_movi_i32(ret
, 0);
149 tcg_gen_mov_i32(ret
, arg1
);
152 /* Don't recurse with tcg_gen_ext8u_i32. */
153 if (TCG_TARGET_HAS_ext8u_i32
) {
154 tcg_gen_op2_i32(INDEX_op_ext8u_i32
, ret
, arg1
);
159 if (TCG_TARGET_HAS_ext16u_i32
) {
160 tcg_gen_op2_i32(INDEX_op_ext16u_i32
, ret
, arg1
);
166 tcg_gen_and_i32(ret
, arg1
, tcg_constant_i32(arg2
));
169 void tcg_gen_ori_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
171 /* Some cases can be optimized here. */
173 tcg_gen_movi_i32(ret
, -1);
174 } else if (arg2
== 0) {
175 tcg_gen_mov_i32(ret
, arg1
);
177 tcg_gen_or_i32(ret
, arg1
, tcg_constant_i32(arg2
));
181 void tcg_gen_xori_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
183 /* Some cases can be optimized here. */
185 tcg_gen_mov_i32(ret
, arg1
);
186 } else if (arg2
== -1 && TCG_TARGET_HAS_not_i32
) {
187 /* Don't recurse with tcg_gen_not_i32. */
188 tcg_gen_op2_i32(INDEX_op_not_i32
, ret
, arg1
);
190 tcg_gen_xor_i32(ret
, arg1
, tcg_constant_i32(arg2
));
194 void tcg_gen_shli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
196 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
198 tcg_gen_mov_i32(ret
, arg1
);
200 tcg_gen_shl_i32(ret
, arg1
, tcg_constant_i32(arg2
));
204 void tcg_gen_shri_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
206 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
208 tcg_gen_mov_i32(ret
, arg1
);
210 tcg_gen_shr_i32(ret
, arg1
, tcg_constant_i32(arg2
));
214 void tcg_gen_sari_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
216 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
218 tcg_gen_mov_i32(ret
, arg1
);
220 tcg_gen_sar_i32(ret
, arg1
, tcg_constant_i32(arg2
));
224 void tcg_gen_brcond_i32(TCGCond cond
, TCGv_i32 arg1
, TCGv_i32 arg2
, TCGLabel
*l
)
226 if (cond
== TCG_COND_ALWAYS
) {
228 } else if (cond
!= TCG_COND_NEVER
) {
230 tcg_gen_op4ii_i32(INDEX_op_brcond_i32
, arg1
, arg2
, cond
, label_arg(l
));
234 void tcg_gen_brcondi_i32(TCGCond cond
, TCGv_i32 arg1
, int32_t arg2
, TCGLabel
*l
)
236 if (cond
== TCG_COND_ALWAYS
) {
238 } else if (cond
!= TCG_COND_NEVER
) {
239 tcg_gen_brcond_i32(cond
, arg1
, tcg_constant_i32(arg2
), l
);
243 void tcg_gen_setcond_i32(TCGCond cond
, TCGv_i32 ret
,
244 TCGv_i32 arg1
, TCGv_i32 arg2
)
246 if (cond
== TCG_COND_ALWAYS
) {
247 tcg_gen_movi_i32(ret
, 1);
248 } else if (cond
== TCG_COND_NEVER
) {
249 tcg_gen_movi_i32(ret
, 0);
251 tcg_gen_op4i_i32(INDEX_op_setcond_i32
, ret
, arg1
, arg2
, cond
);
255 void tcg_gen_setcondi_i32(TCGCond cond
, TCGv_i32 ret
,
256 TCGv_i32 arg1
, int32_t arg2
)
258 tcg_gen_setcond_i32(cond
, ret
, arg1
, tcg_constant_i32(arg2
));
261 void tcg_gen_muli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
264 tcg_gen_movi_i32(ret
, 0);
265 } else if (is_power_of_2(arg2
)) {
266 tcg_gen_shli_i32(ret
, arg1
, ctz32(arg2
));
268 tcg_gen_mul_i32(ret
, arg1
, tcg_constant_i32(arg2
));
272 void tcg_gen_div_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
274 if (TCG_TARGET_HAS_div_i32
) {
275 tcg_gen_op3_i32(INDEX_op_div_i32
, ret
, arg1
, arg2
);
276 } else if (TCG_TARGET_HAS_div2_i32
) {
277 TCGv_i32 t0
= tcg_temp_new_i32();
278 tcg_gen_sari_i32(t0
, arg1
, 31);
279 tcg_gen_op5_i32(INDEX_op_div2_i32
, ret
, t0
, arg1
, t0
, arg2
);
280 tcg_temp_free_i32(t0
);
282 gen_helper_div_i32(ret
, arg1
, arg2
);
286 void tcg_gen_rem_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
288 if (TCG_TARGET_HAS_rem_i32
) {
289 tcg_gen_op3_i32(INDEX_op_rem_i32
, ret
, arg1
, arg2
);
290 } else if (TCG_TARGET_HAS_div_i32
) {
291 TCGv_i32 t0
= tcg_temp_new_i32();
292 tcg_gen_op3_i32(INDEX_op_div_i32
, t0
, arg1
, arg2
);
293 tcg_gen_mul_i32(t0
, t0
, arg2
);
294 tcg_gen_sub_i32(ret
, arg1
, t0
);
295 tcg_temp_free_i32(t0
);
296 } else if (TCG_TARGET_HAS_div2_i32
) {
297 TCGv_i32 t0
= tcg_temp_new_i32();
298 tcg_gen_sari_i32(t0
, arg1
, 31);
299 tcg_gen_op5_i32(INDEX_op_div2_i32
, t0
, ret
, arg1
, t0
, arg2
);
300 tcg_temp_free_i32(t0
);
302 gen_helper_rem_i32(ret
, arg1
, arg2
);
306 void tcg_gen_divu_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
308 if (TCG_TARGET_HAS_div_i32
) {
309 tcg_gen_op3_i32(INDEX_op_divu_i32
, ret
, arg1
, arg2
);
310 } else if (TCG_TARGET_HAS_div2_i32
) {
311 TCGv_i32 t0
= tcg_temp_new_i32();
312 tcg_gen_movi_i32(t0
, 0);
313 tcg_gen_op5_i32(INDEX_op_divu2_i32
, ret
, t0
, arg1
, t0
, arg2
);
314 tcg_temp_free_i32(t0
);
316 gen_helper_divu_i32(ret
, arg1
, arg2
);
320 void tcg_gen_remu_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
322 if (TCG_TARGET_HAS_rem_i32
) {
323 tcg_gen_op3_i32(INDEX_op_remu_i32
, ret
, arg1
, arg2
);
324 } else if (TCG_TARGET_HAS_div_i32
) {
325 TCGv_i32 t0
= tcg_temp_new_i32();
326 tcg_gen_op3_i32(INDEX_op_divu_i32
, t0
, arg1
, arg2
);
327 tcg_gen_mul_i32(t0
, t0
, arg2
);
328 tcg_gen_sub_i32(ret
, arg1
, t0
);
329 tcg_temp_free_i32(t0
);
330 } else if (TCG_TARGET_HAS_div2_i32
) {
331 TCGv_i32 t0
= tcg_temp_new_i32();
332 tcg_gen_movi_i32(t0
, 0);
333 tcg_gen_op5_i32(INDEX_op_divu2_i32
, t0
, ret
, arg1
, t0
, arg2
);
334 tcg_temp_free_i32(t0
);
336 gen_helper_remu_i32(ret
, arg1
, arg2
);
340 void tcg_gen_andc_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
342 if (TCG_TARGET_HAS_andc_i32
) {
343 tcg_gen_op3_i32(INDEX_op_andc_i32
, ret
, arg1
, arg2
);
345 TCGv_i32 t0
= tcg_temp_new_i32();
346 tcg_gen_not_i32(t0
, arg2
);
347 tcg_gen_and_i32(ret
, arg1
, t0
);
348 tcg_temp_free_i32(t0
);
352 void tcg_gen_eqv_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
354 if (TCG_TARGET_HAS_eqv_i32
) {
355 tcg_gen_op3_i32(INDEX_op_eqv_i32
, ret
, arg1
, arg2
);
357 tcg_gen_xor_i32(ret
, arg1
, arg2
);
358 tcg_gen_not_i32(ret
, ret
);
362 void tcg_gen_nand_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
364 if (TCG_TARGET_HAS_nand_i32
) {
365 tcg_gen_op3_i32(INDEX_op_nand_i32
, ret
, arg1
, arg2
);
367 tcg_gen_and_i32(ret
, arg1
, arg2
);
368 tcg_gen_not_i32(ret
, ret
);
372 void tcg_gen_nor_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
374 if (TCG_TARGET_HAS_nor_i32
) {
375 tcg_gen_op3_i32(INDEX_op_nor_i32
, ret
, arg1
, arg2
);
377 tcg_gen_or_i32(ret
, arg1
, arg2
);
378 tcg_gen_not_i32(ret
, ret
);
382 void tcg_gen_orc_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
384 if (TCG_TARGET_HAS_orc_i32
) {
385 tcg_gen_op3_i32(INDEX_op_orc_i32
, ret
, arg1
, arg2
);
387 TCGv_i32 t0
= tcg_temp_new_i32();
388 tcg_gen_not_i32(t0
, arg2
);
389 tcg_gen_or_i32(ret
, arg1
, t0
);
390 tcg_temp_free_i32(t0
);
394 void tcg_gen_clz_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
396 if (TCG_TARGET_HAS_clz_i32
) {
397 tcg_gen_op3_i32(INDEX_op_clz_i32
, ret
, arg1
, arg2
);
398 } else if (TCG_TARGET_HAS_clz_i64
) {
399 TCGv_i64 t1
= tcg_temp_new_i64();
400 TCGv_i64 t2
= tcg_temp_new_i64();
401 tcg_gen_extu_i32_i64(t1
, arg1
);
402 tcg_gen_extu_i32_i64(t2
, arg2
);
403 tcg_gen_addi_i64(t2
, t2
, 32);
404 tcg_gen_clz_i64(t1
, t1
, t2
);
405 tcg_gen_extrl_i64_i32(ret
, t1
);
406 tcg_temp_free_i64(t1
);
407 tcg_temp_free_i64(t2
);
408 tcg_gen_subi_i32(ret
, ret
, 32);
410 gen_helper_clz_i32(ret
, arg1
, arg2
);
414 void tcg_gen_clzi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, uint32_t arg2
)
416 tcg_gen_clz_i32(ret
, arg1
, tcg_constant_i32(arg2
));
419 void tcg_gen_ctz_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
421 if (TCG_TARGET_HAS_ctz_i32
) {
422 tcg_gen_op3_i32(INDEX_op_ctz_i32
, ret
, arg1
, arg2
);
423 } else if (TCG_TARGET_HAS_ctz_i64
) {
424 TCGv_i64 t1
= tcg_temp_new_i64();
425 TCGv_i64 t2
= tcg_temp_new_i64();
426 tcg_gen_extu_i32_i64(t1
, arg1
);
427 tcg_gen_extu_i32_i64(t2
, arg2
);
428 tcg_gen_ctz_i64(t1
, t1
, t2
);
429 tcg_gen_extrl_i64_i32(ret
, t1
);
430 tcg_temp_free_i64(t1
);
431 tcg_temp_free_i64(t2
);
432 } else if (TCG_TARGET_HAS_ctpop_i32
433 || TCG_TARGET_HAS_ctpop_i64
434 || TCG_TARGET_HAS_clz_i32
435 || TCG_TARGET_HAS_clz_i64
) {
436 TCGv_i32 z
, t
= tcg_temp_new_i32();
438 if (TCG_TARGET_HAS_ctpop_i32
|| TCG_TARGET_HAS_ctpop_i64
) {
439 tcg_gen_subi_i32(t
, arg1
, 1);
440 tcg_gen_andc_i32(t
, t
, arg1
);
441 tcg_gen_ctpop_i32(t
, t
);
443 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
444 tcg_gen_neg_i32(t
, arg1
);
445 tcg_gen_and_i32(t
, t
, arg1
);
446 tcg_gen_clzi_i32(t
, t
, 32);
447 tcg_gen_xori_i32(t
, t
, 31);
449 z
= tcg_constant_i32(0);
450 tcg_gen_movcond_i32(TCG_COND_EQ
, ret
, arg1
, z
, arg2
, t
);
451 tcg_temp_free_i32(t
);
453 gen_helper_ctz_i32(ret
, arg1
, arg2
);
457 void tcg_gen_ctzi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, uint32_t arg2
)
459 if (!TCG_TARGET_HAS_ctz_i32
&& TCG_TARGET_HAS_ctpop_i32
&& arg2
== 32) {
460 /* This equivalence has the advantage of not requiring a fixup. */
461 TCGv_i32 t
= tcg_temp_new_i32();
462 tcg_gen_subi_i32(t
, arg1
, 1);
463 tcg_gen_andc_i32(t
, t
, arg1
);
464 tcg_gen_ctpop_i32(ret
, t
);
465 tcg_temp_free_i32(t
);
467 tcg_gen_ctz_i32(ret
, arg1
, tcg_constant_i32(arg2
));
471 void tcg_gen_clrsb_i32(TCGv_i32 ret
, TCGv_i32 arg
)
473 if (TCG_TARGET_HAS_clz_i32
) {
474 TCGv_i32 t
= tcg_temp_new_i32();
475 tcg_gen_sari_i32(t
, arg
, 31);
476 tcg_gen_xor_i32(t
, t
, arg
);
477 tcg_gen_clzi_i32(t
, t
, 32);
478 tcg_gen_subi_i32(ret
, t
, 1);
479 tcg_temp_free_i32(t
);
481 gen_helper_clrsb_i32(ret
, arg
);
485 void tcg_gen_ctpop_i32(TCGv_i32 ret
, TCGv_i32 arg1
)
487 if (TCG_TARGET_HAS_ctpop_i32
) {
488 tcg_gen_op2_i32(INDEX_op_ctpop_i32
, ret
, arg1
);
489 } else if (TCG_TARGET_HAS_ctpop_i64
) {
490 TCGv_i64 t
= tcg_temp_new_i64();
491 tcg_gen_extu_i32_i64(t
, arg1
);
492 tcg_gen_ctpop_i64(t
, t
);
493 tcg_gen_extrl_i64_i32(ret
, t
);
494 tcg_temp_free_i64(t
);
496 gen_helper_ctpop_i32(ret
, arg1
);
500 void tcg_gen_rotl_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
502 if (TCG_TARGET_HAS_rot_i32
) {
503 tcg_gen_op3_i32(INDEX_op_rotl_i32
, ret
, arg1
, arg2
);
507 t0
= tcg_temp_new_i32();
508 t1
= tcg_temp_new_i32();
509 tcg_gen_shl_i32(t0
, arg1
, arg2
);
510 tcg_gen_subfi_i32(t1
, 32, arg2
);
511 tcg_gen_shr_i32(t1
, arg1
, t1
);
512 tcg_gen_or_i32(ret
, t0
, t1
);
513 tcg_temp_free_i32(t0
);
514 tcg_temp_free_i32(t1
);
518 void tcg_gen_rotli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
520 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
521 /* some cases can be optimized here */
523 tcg_gen_mov_i32(ret
, arg1
);
524 } else if (TCG_TARGET_HAS_rot_i32
) {
525 tcg_gen_rotl_i32(ret
, arg1
, tcg_constant_i32(arg2
));
528 t0
= tcg_temp_new_i32();
529 t1
= tcg_temp_new_i32();
530 tcg_gen_shli_i32(t0
, arg1
, arg2
);
531 tcg_gen_shri_i32(t1
, arg1
, 32 - arg2
);
532 tcg_gen_or_i32(ret
, t0
, t1
);
533 tcg_temp_free_i32(t0
);
534 tcg_temp_free_i32(t1
);
538 void tcg_gen_rotr_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
540 if (TCG_TARGET_HAS_rot_i32
) {
541 tcg_gen_op3_i32(INDEX_op_rotr_i32
, ret
, arg1
, arg2
);
545 t0
= tcg_temp_new_i32();
546 t1
= tcg_temp_new_i32();
547 tcg_gen_shr_i32(t0
, arg1
, arg2
);
548 tcg_gen_subfi_i32(t1
, 32, arg2
);
549 tcg_gen_shl_i32(t1
, arg1
, t1
);
550 tcg_gen_or_i32(ret
, t0
, t1
);
551 tcg_temp_free_i32(t0
);
552 tcg_temp_free_i32(t1
);
556 void tcg_gen_rotri_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
558 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
559 /* some cases can be optimized here */
561 tcg_gen_mov_i32(ret
, arg1
);
563 tcg_gen_rotli_i32(ret
, arg1
, 32 - arg2
);
567 void tcg_gen_deposit_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
,
568 unsigned int ofs
, unsigned int len
)
573 tcg_debug_assert(ofs
< 32);
574 tcg_debug_assert(len
> 0);
575 tcg_debug_assert(len
<= 32);
576 tcg_debug_assert(ofs
+ len
<= 32);
579 tcg_gen_mov_i32(ret
, arg2
);
582 if (TCG_TARGET_HAS_deposit_i32
&& TCG_TARGET_deposit_i32_valid(ofs
, len
)) {
583 tcg_gen_op5ii_i32(INDEX_op_deposit_i32
, ret
, arg1
, arg2
, ofs
, len
);
587 t1
= tcg_temp_new_i32();
589 if (TCG_TARGET_HAS_extract2_i32
) {
590 if (ofs
+ len
== 32) {
591 tcg_gen_shli_i32(t1
, arg1
, len
);
592 tcg_gen_extract2_i32(ret
, t1
, arg2
, len
);
596 tcg_gen_extract2_i32(ret
, arg1
, arg2
, len
);
597 tcg_gen_rotli_i32(ret
, ret
, len
);
602 mask
= (1u << len
) - 1;
603 if (ofs
+ len
< 32) {
604 tcg_gen_andi_i32(t1
, arg2
, mask
);
605 tcg_gen_shli_i32(t1
, t1
, ofs
);
607 tcg_gen_shli_i32(t1
, arg2
, ofs
);
609 tcg_gen_andi_i32(ret
, arg1
, ~(mask
<< ofs
));
610 tcg_gen_or_i32(ret
, ret
, t1
);
612 tcg_temp_free_i32(t1
);
615 void tcg_gen_deposit_z_i32(TCGv_i32 ret
, TCGv_i32 arg
,
616 unsigned int ofs
, unsigned int len
)
618 tcg_debug_assert(ofs
< 32);
619 tcg_debug_assert(len
> 0);
620 tcg_debug_assert(len
<= 32);
621 tcg_debug_assert(ofs
+ len
<= 32);
623 if (ofs
+ len
== 32) {
624 tcg_gen_shli_i32(ret
, arg
, ofs
);
625 } else if (ofs
== 0) {
626 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
627 } else if (TCG_TARGET_HAS_deposit_i32
628 && TCG_TARGET_deposit_i32_valid(ofs
, len
)) {
629 TCGv_i32 zero
= tcg_constant_i32(0);
630 tcg_gen_op5ii_i32(INDEX_op_deposit_i32
, ret
, zero
, arg
, ofs
, len
);
632 /* To help two-operand hosts we prefer to zero-extend first,
633 which allows ARG to stay live. */
636 if (TCG_TARGET_HAS_ext16u_i32
) {
637 tcg_gen_ext16u_i32(ret
, arg
);
638 tcg_gen_shli_i32(ret
, ret
, ofs
);
643 if (TCG_TARGET_HAS_ext8u_i32
) {
644 tcg_gen_ext8u_i32(ret
, arg
);
645 tcg_gen_shli_i32(ret
, ret
, ofs
);
650 /* Otherwise prefer zero-extension over AND for code size. */
653 if (TCG_TARGET_HAS_ext16u_i32
) {
654 tcg_gen_shli_i32(ret
, arg
, ofs
);
655 tcg_gen_ext16u_i32(ret
, ret
);
660 if (TCG_TARGET_HAS_ext8u_i32
) {
661 tcg_gen_shli_i32(ret
, arg
, ofs
);
662 tcg_gen_ext8u_i32(ret
, ret
);
667 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
668 tcg_gen_shli_i32(ret
, ret
, ofs
);
672 void tcg_gen_extract_i32(TCGv_i32 ret
, TCGv_i32 arg
,
673 unsigned int ofs
, unsigned int len
)
675 tcg_debug_assert(ofs
< 32);
676 tcg_debug_assert(len
> 0);
677 tcg_debug_assert(len
<= 32);
678 tcg_debug_assert(ofs
+ len
<= 32);
680 /* Canonicalize certain special cases, even if extract is supported. */
681 if (ofs
+ len
== 32) {
682 tcg_gen_shri_i32(ret
, arg
, 32 - len
);
686 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
690 if (TCG_TARGET_HAS_extract_i32
691 && TCG_TARGET_extract_i32_valid(ofs
, len
)) {
692 tcg_gen_op4ii_i32(INDEX_op_extract_i32
, ret
, arg
, ofs
, len
);
696 /* Assume that zero-extension, if available, is cheaper than a shift. */
699 if (TCG_TARGET_HAS_ext16u_i32
) {
700 tcg_gen_ext16u_i32(ret
, arg
);
701 tcg_gen_shri_i32(ret
, ret
, ofs
);
706 if (TCG_TARGET_HAS_ext8u_i32
) {
707 tcg_gen_ext8u_i32(ret
, arg
);
708 tcg_gen_shri_i32(ret
, ret
, ofs
);
714 /* ??? Ideally we'd know what values are available for immediate AND.
715 Assume that 8 bits are available, plus the special case of 16,
716 so that we get ext8u, ext16u. */
718 case 1 ... 8: case 16:
719 tcg_gen_shri_i32(ret
, arg
, ofs
);
720 tcg_gen_andi_i32(ret
, ret
, (1u << len
) - 1);
723 tcg_gen_shli_i32(ret
, arg
, 32 - len
- ofs
);
724 tcg_gen_shri_i32(ret
, ret
, 32 - len
);
729 void tcg_gen_sextract_i32(TCGv_i32 ret
, TCGv_i32 arg
,
730 unsigned int ofs
, unsigned int len
)
732 tcg_debug_assert(ofs
< 32);
733 tcg_debug_assert(len
> 0);
734 tcg_debug_assert(len
<= 32);
735 tcg_debug_assert(ofs
+ len
<= 32);
737 /* Canonicalize certain special cases, even if extract is supported. */
738 if (ofs
+ len
== 32) {
739 tcg_gen_sari_i32(ret
, arg
, 32 - len
);
745 tcg_gen_ext16s_i32(ret
, arg
);
748 tcg_gen_ext8s_i32(ret
, arg
);
753 if (TCG_TARGET_HAS_sextract_i32
754 && TCG_TARGET_extract_i32_valid(ofs
, len
)) {
755 tcg_gen_op4ii_i32(INDEX_op_sextract_i32
, ret
, arg
, ofs
, len
);
759 /* Assume that sign-extension, if available, is cheaper than a shift. */
762 if (TCG_TARGET_HAS_ext16s_i32
) {
763 tcg_gen_ext16s_i32(ret
, arg
);
764 tcg_gen_sari_i32(ret
, ret
, ofs
);
769 if (TCG_TARGET_HAS_ext8s_i32
) {
770 tcg_gen_ext8s_i32(ret
, arg
);
771 tcg_gen_sari_i32(ret
, ret
, ofs
);
778 if (TCG_TARGET_HAS_ext16s_i32
) {
779 tcg_gen_shri_i32(ret
, arg
, ofs
);
780 tcg_gen_ext16s_i32(ret
, ret
);
785 if (TCG_TARGET_HAS_ext8s_i32
) {
786 tcg_gen_shri_i32(ret
, arg
, ofs
);
787 tcg_gen_ext8s_i32(ret
, ret
);
793 tcg_gen_shli_i32(ret
, arg
, 32 - len
- ofs
);
794 tcg_gen_sari_i32(ret
, ret
, 32 - len
);
798 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
799 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
801 void tcg_gen_extract2_i32(TCGv_i32 ret
, TCGv_i32 al
, TCGv_i32 ah
,
804 tcg_debug_assert(ofs
<= 32);
806 tcg_gen_mov_i32(ret
, al
);
807 } else if (ofs
== 32) {
808 tcg_gen_mov_i32(ret
, ah
);
809 } else if (al
== ah
) {
810 tcg_gen_rotri_i32(ret
, al
, ofs
);
811 } else if (TCG_TARGET_HAS_extract2_i32
) {
812 tcg_gen_op4i_i32(INDEX_op_extract2_i32
, ret
, al
, ah
, ofs
);
814 TCGv_i32 t0
= tcg_temp_new_i32();
815 tcg_gen_shri_i32(t0
, al
, ofs
);
816 tcg_gen_deposit_i32(ret
, t0
, ah
, 32 - ofs
, ofs
);
817 tcg_temp_free_i32(t0
);
821 void tcg_gen_movcond_i32(TCGCond cond
, TCGv_i32 ret
, TCGv_i32 c1
,
822 TCGv_i32 c2
, TCGv_i32 v1
, TCGv_i32 v2
)
824 if (cond
== TCG_COND_ALWAYS
) {
825 tcg_gen_mov_i32(ret
, v1
);
826 } else if (cond
== TCG_COND_NEVER
) {
827 tcg_gen_mov_i32(ret
, v2
);
828 } else if (TCG_TARGET_HAS_movcond_i32
) {
829 tcg_gen_op6i_i32(INDEX_op_movcond_i32
, ret
, c1
, c2
, v1
, v2
, cond
);
831 TCGv_i32 t0
= tcg_temp_new_i32();
832 TCGv_i32 t1
= tcg_temp_new_i32();
833 tcg_gen_setcond_i32(cond
, t0
, c1
, c2
);
834 tcg_gen_neg_i32(t0
, t0
);
835 tcg_gen_and_i32(t1
, v1
, t0
);
836 tcg_gen_andc_i32(ret
, v2
, t0
);
837 tcg_gen_or_i32(ret
, ret
, t1
);
838 tcg_temp_free_i32(t0
);
839 tcg_temp_free_i32(t1
);
843 void tcg_gen_add2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 al
,
844 TCGv_i32 ah
, TCGv_i32 bl
, TCGv_i32 bh
)
846 if (TCG_TARGET_HAS_add2_i32
) {
847 tcg_gen_op6_i32(INDEX_op_add2_i32
, rl
, rh
, al
, ah
, bl
, bh
);
849 TCGv_i64 t0
= tcg_temp_new_i64();
850 TCGv_i64 t1
= tcg_temp_new_i64();
851 tcg_gen_concat_i32_i64(t0
, al
, ah
);
852 tcg_gen_concat_i32_i64(t1
, bl
, bh
);
853 tcg_gen_add_i64(t0
, t0
, t1
);
854 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
855 tcg_temp_free_i64(t0
);
856 tcg_temp_free_i64(t1
);
860 void tcg_gen_sub2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 al
,
861 TCGv_i32 ah
, TCGv_i32 bl
, TCGv_i32 bh
)
863 if (TCG_TARGET_HAS_sub2_i32
) {
864 tcg_gen_op6_i32(INDEX_op_sub2_i32
, rl
, rh
, al
, ah
, bl
, bh
);
866 TCGv_i64 t0
= tcg_temp_new_i64();
867 TCGv_i64 t1
= tcg_temp_new_i64();
868 tcg_gen_concat_i32_i64(t0
, al
, ah
);
869 tcg_gen_concat_i32_i64(t1
, bl
, bh
);
870 tcg_gen_sub_i64(t0
, t0
, t1
);
871 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
872 tcg_temp_free_i64(t0
);
873 tcg_temp_free_i64(t1
);
877 void tcg_gen_mulu2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
879 if (TCG_TARGET_HAS_mulu2_i32
) {
880 tcg_gen_op4_i32(INDEX_op_mulu2_i32
, rl
, rh
, arg1
, arg2
);
881 } else if (TCG_TARGET_HAS_muluh_i32
) {
882 TCGv_i32 t
= tcg_temp_new_i32();
883 tcg_gen_op3_i32(INDEX_op_mul_i32
, t
, arg1
, arg2
);
884 tcg_gen_op3_i32(INDEX_op_muluh_i32
, rh
, arg1
, arg2
);
885 tcg_gen_mov_i32(rl
, t
);
886 tcg_temp_free_i32(t
);
888 TCGv_i64 t0
= tcg_temp_new_i64();
889 TCGv_i64 t1
= tcg_temp_new_i64();
890 tcg_gen_extu_i32_i64(t0
, arg1
);
891 tcg_gen_extu_i32_i64(t1
, arg2
);
892 tcg_gen_mul_i64(t0
, t0
, t1
);
893 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
894 tcg_temp_free_i64(t0
);
895 tcg_temp_free_i64(t1
);
899 void tcg_gen_muls2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
901 if (TCG_TARGET_HAS_muls2_i32
) {
902 tcg_gen_op4_i32(INDEX_op_muls2_i32
, rl
, rh
, arg1
, arg2
);
903 } else if (TCG_TARGET_HAS_mulsh_i32
) {
904 TCGv_i32 t
= tcg_temp_new_i32();
905 tcg_gen_op3_i32(INDEX_op_mul_i32
, t
, arg1
, arg2
);
906 tcg_gen_op3_i32(INDEX_op_mulsh_i32
, rh
, arg1
, arg2
);
907 tcg_gen_mov_i32(rl
, t
);
908 tcg_temp_free_i32(t
);
909 } else if (TCG_TARGET_REG_BITS
== 32) {
910 TCGv_i32 t0
= tcg_temp_new_i32();
911 TCGv_i32 t1
= tcg_temp_new_i32();
912 TCGv_i32 t2
= tcg_temp_new_i32();
913 TCGv_i32 t3
= tcg_temp_new_i32();
914 tcg_gen_mulu2_i32(t0
, t1
, arg1
, arg2
);
915 /* Adjust for negative inputs. */
916 tcg_gen_sari_i32(t2
, arg1
, 31);
917 tcg_gen_sari_i32(t3
, arg2
, 31);
918 tcg_gen_and_i32(t2
, t2
, arg2
);
919 tcg_gen_and_i32(t3
, t3
, arg1
);
920 tcg_gen_sub_i32(rh
, t1
, t2
);
921 tcg_gen_sub_i32(rh
, rh
, t3
);
922 tcg_gen_mov_i32(rl
, t0
);
923 tcg_temp_free_i32(t0
);
924 tcg_temp_free_i32(t1
);
925 tcg_temp_free_i32(t2
);
926 tcg_temp_free_i32(t3
);
928 TCGv_i64 t0
= tcg_temp_new_i64();
929 TCGv_i64 t1
= tcg_temp_new_i64();
930 tcg_gen_ext_i32_i64(t0
, arg1
);
931 tcg_gen_ext_i32_i64(t1
, arg2
);
932 tcg_gen_mul_i64(t0
, t0
, t1
);
933 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
934 tcg_temp_free_i64(t0
);
935 tcg_temp_free_i64(t1
);
939 void tcg_gen_mulsu2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
941 if (TCG_TARGET_REG_BITS
== 32) {
942 TCGv_i32 t0
= tcg_temp_new_i32();
943 TCGv_i32 t1
= tcg_temp_new_i32();
944 TCGv_i32 t2
= tcg_temp_new_i32();
945 tcg_gen_mulu2_i32(t0
, t1
, arg1
, arg2
);
946 /* Adjust for negative input for the signed arg1. */
947 tcg_gen_sari_i32(t2
, arg1
, 31);
948 tcg_gen_and_i32(t2
, t2
, arg2
);
949 tcg_gen_sub_i32(rh
, t1
, t2
);
950 tcg_gen_mov_i32(rl
, t0
);
951 tcg_temp_free_i32(t0
);
952 tcg_temp_free_i32(t1
);
953 tcg_temp_free_i32(t2
);
955 TCGv_i64 t0
= tcg_temp_new_i64();
956 TCGv_i64 t1
= tcg_temp_new_i64();
957 tcg_gen_ext_i32_i64(t0
, arg1
);
958 tcg_gen_extu_i32_i64(t1
, arg2
);
959 tcg_gen_mul_i64(t0
, t0
, t1
);
960 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
961 tcg_temp_free_i64(t0
);
962 tcg_temp_free_i64(t1
);
966 void tcg_gen_ext8s_i32(TCGv_i32 ret
, TCGv_i32 arg
)
968 if (TCG_TARGET_HAS_ext8s_i32
) {
969 tcg_gen_op2_i32(INDEX_op_ext8s_i32
, ret
, arg
);
971 tcg_gen_shli_i32(ret
, arg
, 24);
972 tcg_gen_sari_i32(ret
, ret
, 24);
976 void tcg_gen_ext16s_i32(TCGv_i32 ret
, TCGv_i32 arg
)
978 if (TCG_TARGET_HAS_ext16s_i32
) {
979 tcg_gen_op2_i32(INDEX_op_ext16s_i32
, ret
, arg
);
981 tcg_gen_shli_i32(ret
, arg
, 16);
982 tcg_gen_sari_i32(ret
, ret
, 16);
986 void tcg_gen_ext8u_i32(TCGv_i32 ret
, TCGv_i32 arg
)
988 if (TCG_TARGET_HAS_ext8u_i32
) {
989 tcg_gen_op2_i32(INDEX_op_ext8u_i32
, ret
, arg
);
991 tcg_gen_andi_i32(ret
, arg
, 0xffu
);
995 void tcg_gen_ext16u_i32(TCGv_i32 ret
, TCGv_i32 arg
)
997 if (TCG_TARGET_HAS_ext16u_i32
) {
998 tcg_gen_op2_i32(INDEX_op_ext16u_i32
, ret
, arg
);
1000 tcg_gen_andi_i32(ret
, arg
, 0xffffu
);
1004 /* Note: we assume the two high bytes are set to zero */
1005 void tcg_gen_bswap16_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1007 if (TCG_TARGET_HAS_bswap16_i32
) {
1008 tcg_gen_op3i_i32(INDEX_op_bswap16_i32
, ret
, arg
,
1009 TCG_BSWAP_IZ
| TCG_BSWAP_OZ
);
1011 TCGv_i32 t0
= tcg_temp_new_i32();
1013 tcg_gen_ext8u_i32(t0
, arg
);
1014 tcg_gen_shli_i32(t0
, t0
, 8);
1015 tcg_gen_shri_i32(ret
, arg
, 8);
1016 tcg_gen_or_i32(ret
, ret
, t0
);
1017 tcg_temp_free_i32(t0
);
1021 void tcg_gen_bswap32_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1023 if (TCG_TARGET_HAS_bswap32_i32
) {
1024 tcg_gen_op3i_i32(INDEX_op_bswap32_i32
, ret
, arg
, 0);
1026 TCGv_i32 t0
= tcg_temp_new_i32();
1027 TCGv_i32 t1
= tcg_temp_new_i32();
1028 TCGv_i32 t2
= tcg_constant_i32(0x00ff00ff);
1031 tcg_gen_shri_i32(t0
, arg
, 8); /* t0 = .abc */
1032 tcg_gen_and_i32(t1
, arg
, t2
); /* t1 = .b.d */
1033 tcg_gen_and_i32(t0
, t0
, t2
); /* t0 = .a.c */
1034 tcg_gen_shli_i32(t1
, t1
, 8); /* t1 = b.d. */
1035 tcg_gen_or_i32(ret
, t0
, t1
); /* ret = badc */
1037 tcg_gen_shri_i32(t0
, ret
, 16); /* t0 = ..ba */
1038 tcg_gen_shli_i32(t1
, ret
, 16); /* t1 = dc.. */
1039 tcg_gen_or_i32(ret
, t0
, t1
); /* ret = dcba */
1041 tcg_temp_free_i32(t0
);
1042 tcg_temp_free_i32(t1
);
1046 void tcg_gen_smin_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1048 tcg_gen_movcond_i32(TCG_COND_LT
, ret
, a
, b
, a
, b
);
1051 void tcg_gen_umin_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1053 tcg_gen_movcond_i32(TCG_COND_LTU
, ret
, a
, b
, a
, b
);
1056 void tcg_gen_smax_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1058 tcg_gen_movcond_i32(TCG_COND_LT
, ret
, a
, b
, b
, a
);
1061 void tcg_gen_umax_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1063 tcg_gen_movcond_i32(TCG_COND_LTU
, ret
, a
, b
, b
, a
);
1066 void tcg_gen_abs_i32(TCGv_i32 ret
, TCGv_i32 a
)
1068 TCGv_i32 t
= tcg_temp_new_i32();
1070 tcg_gen_sari_i32(t
, a
, 31);
1071 tcg_gen_xor_i32(ret
, a
, t
);
1072 tcg_gen_sub_i32(ret
, ret
, t
);
1073 tcg_temp_free_i32(t
);
1078 #if TCG_TARGET_REG_BITS == 32
1079 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1081 void tcg_gen_discard_i64(TCGv_i64 arg
)
1083 tcg_gen_discard_i32(TCGV_LOW(arg
));
1084 tcg_gen_discard_i32(TCGV_HIGH(arg
));
1087 void tcg_gen_mov_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1089 TCGTemp
*ts
= tcgv_i64_temp(arg
);
1091 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1092 if (ts
->kind
== TEMP_CONST
) {
1093 tcg_gen_movi_i64(ret
, ts
->val
);
1095 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1096 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
));
1100 void tcg_gen_movi_i64(TCGv_i64 ret
, int64_t arg
)
1102 tcg_gen_movi_i32(TCGV_LOW(ret
), arg
);
1103 tcg_gen_movi_i32(TCGV_HIGH(ret
), arg
>> 32);
1106 void tcg_gen_ld8u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1108 tcg_gen_ld8u_i32(TCGV_LOW(ret
), arg2
, offset
);
1109 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1112 void tcg_gen_ld8s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1114 tcg_gen_ld8s_i32(TCGV_LOW(ret
), arg2
, offset
);
1115 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1118 void tcg_gen_ld16u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1120 tcg_gen_ld16u_i32(TCGV_LOW(ret
), arg2
, offset
);
1121 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1124 void tcg_gen_ld16s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1126 tcg_gen_ld16s_i32(TCGV_LOW(ret
), arg2
, offset
);
1127 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1130 void tcg_gen_ld32u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1132 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1133 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1136 void tcg_gen_ld32s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1138 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1139 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1142 void tcg_gen_ld_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1144 /* Since arg2 and ret have different types,
1145 they cannot be the same temporary */
1146 #ifdef HOST_WORDS_BIGENDIAN
1147 tcg_gen_ld_i32(TCGV_HIGH(ret
), arg2
, offset
);
1148 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
+ 4);
1150 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1151 tcg_gen_ld_i32(TCGV_HIGH(ret
), arg2
, offset
+ 4);
1155 void tcg_gen_st_i64(TCGv_i64 arg1
, TCGv_ptr arg2
, tcg_target_long offset
)
1157 #ifdef HOST_WORDS_BIGENDIAN
1158 tcg_gen_st_i32(TCGV_HIGH(arg1
), arg2
, offset
);
1159 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
+ 4);
1161 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
);
1162 tcg_gen_st_i32(TCGV_HIGH(arg1
), arg2
, offset
+ 4);
1166 void tcg_gen_and_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1168 tcg_gen_and_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1169 tcg_gen_and_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1172 void tcg_gen_or_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1174 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1175 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1178 void tcg_gen_xor_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1180 tcg_gen_xor_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1181 tcg_gen_xor_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1184 void tcg_gen_shl_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1186 gen_helper_shl_i64(ret
, arg1
, arg2
);
1189 void tcg_gen_shr_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1191 gen_helper_shr_i64(ret
, arg1
, arg2
);
1194 void tcg_gen_sar_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1196 gen_helper_sar_i64(ret
, arg1
, arg2
);
1199 void tcg_gen_mul_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1204 t0
= tcg_temp_new_i64();
1205 t1
= tcg_temp_new_i32();
1207 tcg_gen_mulu2_i32(TCGV_LOW(t0
), TCGV_HIGH(t0
),
1208 TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1210 tcg_gen_mul_i32(t1
, TCGV_LOW(arg1
), TCGV_HIGH(arg2
));
1211 tcg_gen_add_i32(TCGV_HIGH(t0
), TCGV_HIGH(t0
), t1
);
1212 tcg_gen_mul_i32(t1
, TCGV_HIGH(arg1
), TCGV_LOW(arg2
));
1213 tcg_gen_add_i32(TCGV_HIGH(t0
), TCGV_HIGH(t0
), t1
);
1215 tcg_gen_mov_i64(ret
, t0
);
1216 tcg_temp_free_i64(t0
);
1217 tcg_temp_free_i32(t1
);
1222 void tcg_gen_movi_i64(TCGv_i64 ret
, int64_t arg
)
1224 tcg_gen_mov_i64(ret
, tcg_constant_i64(arg
));
1227 #endif /* TCG_TARGET_REG_SIZE == 32 */
1229 void tcg_gen_addi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1231 /* some cases can be optimized here */
1233 tcg_gen_mov_i64(ret
, arg1
);
1234 } else if (TCG_TARGET_REG_BITS
== 64) {
1235 tcg_gen_add_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1237 tcg_gen_add2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1238 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1239 tcg_constant_i32(arg2
), tcg_constant_i32(arg2
>> 32));
1243 void tcg_gen_subfi_i64(TCGv_i64 ret
, int64_t arg1
, TCGv_i64 arg2
)
1245 if (arg1
== 0 && TCG_TARGET_HAS_neg_i64
) {
1246 /* Don't recurse with tcg_gen_neg_i64. */
1247 tcg_gen_op2_i64(INDEX_op_neg_i64
, ret
, arg2
);
1248 } else if (TCG_TARGET_REG_BITS
== 64) {
1249 tcg_gen_sub_i64(ret
, tcg_constant_i64(arg1
), arg2
);
1251 tcg_gen_sub2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1252 tcg_constant_i32(arg1
), tcg_constant_i32(arg1
>> 32),
1253 TCGV_LOW(arg2
), TCGV_HIGH(arg2
));
1257 void tcg_gen_subi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1259 /* some cases can be optimized here */
1261 tcg_gen_mov_i64(ret
, arg1
);
1262 } else if (TCG_TARGET_REG_BITS
== 64) {
1263 tcg_gen_sub_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1265 tcg_gen_sub2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1266 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1267 tcg_constant_i32(arg2
), tcg_constant_i32(arg2
>> 32));
1271 void tcg_gen_andi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1273 if (TCG_TARGET_REG_BITS
== 32) {
1274 tcg_gen_andi_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1275 tcg_gen_andi_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1279 /* Some cases can be optimized here. */
1282 tcg_gen_movi_i64(ret
, 0);
1285 tcg_gen_mov_i64(ret
, arg1
);
1288 /* Don't recurse with tcg_gen_ext8u_i64. */
1289 if (TCG_TARGET_HAS_ext8u_i64
) {
1290 tcg_gen_op2_i64(INDEX_op_ext8u_i64
, ret
, arg1
);
1295 if (TCG_TARGET_HAS_ext16u_i64
) {
1296 tcg_gen_op2_i64(INDEX_op_ext16u_i64
, ret
, arg1
);
1301 if (TCG_TARGET_HAS_ext32u_i64
) {
1302 tcg_gen_op2_i64(INDEX_op_ext32u_i64
, ret
, arg1
);
1308 tcg_gen_and_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1311 void tcg_gen_ori_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1313 if (TCG_TARGET_REG_BITS
== 32) {
1314 tcg_gen_ori_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1315 tcg_gen_ori_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1318 /* Some cases can be optimized here. */
1320 tcg_gen_movi_i64(ret
, -1);
1321 } else if (arg2
== 0) {
1322 tcg_gen_mov_i64(ret
, arg1
);
1324 tcg_gen_or_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1328 void tcg_gen_xori_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1330 if (TCG_TARGET_REG_BITS
== 32) {
1331 tcg_gen_xori_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1332 tcg_gen_xori_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1335 /* Some cases can be optimized here. */
1337 tcg_gen_mov_i64(ret
, arg1
);
1338 } else if (arg2
== -1 && TCG_TARGET_HAS_not_i64
) {
1339 /* Don't recurse with tcg_gen_not_i64. */
1340 tcg_gen_op2_i64(INDEX_op_not_i64
, ret
, arg1
);
1342 tcg_gen_xor_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1346 static inline void tcg_gen_shifti_i64(TCGv_i64 ret
, TCGv_i64 arg1
,
1347 unsigned c
, bool right
, bool arith
)
1349 tcg_debug_assert(c
< 64);
1351 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
1352 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
1353 } else if (c
>= 32) {
1357 tcg_gen_sari_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), c
);
1358 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), 31);
1360 tcg_gen_shri_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), c
);
1361 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1364 tcg_gen_shli_i32(TCGV_HIGH(ret
), TCGV_LOW(arg1
), c
);
1365 tcg_gen_movi_i32(TCGV_LOW(ret
), 0);
1368 if (TCG_TARGET_HAS_extract2_i32
) {
1369 tcg_gen_extract2_i32(TCGV_LOW(ret
),
1370 TCGV_LOW(arg1
), TCGV_HIGH(arg1
), c
);
1372 tcg_gen_shri_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), c
);
1373 tcg_gen_deposit_i32(TCGV_LOW(ret
), TCGV_LOW(ret
),
1374 TCGV_HIGH(arg1
), 32 - c
, c
);
1377 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), c
);
1379 tcg_gen_shri_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), c
);
1382 if (TCG_TARGET_HAS_extract2_i32
) {
1383 tcg_gen_extract2_i32(TCGV_HIGH(ret
),
1384 TCGV_LOW(arg1
), TCGV_HIGH(arg1
), 32 - c
);
1386 TCGv_i32 t0
= tcg_temp_new_i32();
1387 tcg_gen_shri_i32(t0
, TCGV_LOW(arg1
), 32 - c
);
1388 tcg_gen_deposit_i32(TCGV_HIGH(ret
), t0
,
1389 TCGV_HIGH(arg1
), c
, 32 - c
);
1390 tcg_temp_free_i32(t0
);
1392 tcg_gen_shli_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), c
);
1396 void tcg_gen_shli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1398 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1399 if (TCG_TARGET_REG_BITS
== 32) {
1400 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 0, 0);
1401 } else if (arg2
== 0) {
1402 tcg_gen_mov_i64(ret
, arg1
);
1404 tcg_gen_shl_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1408 void tcg_gen_shri_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1410 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1411 if (TCG_TARGET_REG_BITS
== 32) {
1412 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 1, 0);
1413 } else if (arg2
== 0) {
1414 tcg_gen_mov_i64(ret
, arg1
);
1416 tcg_gen_shr_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1420 void tcg_gen_sari_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1422 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1423 if (TCG_TARGET_REG_BITS
== 32) {
1424 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 1, 1);
1425 } else if (arg2
== 0) {
1426 tcg_gen_mov_i64(ret
, arg1
);
1428 tcg_gen_sar_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1432 void tcg_gen_brcond_i64(TCGCond cond
, TCGv_i64 arg1
, TCGv_i64 arg2
, TCGLabel
*l
)
1434 if (cond
== TCG_COND_ALWAYS
) {
1436 } else if (cond
!= TCG_COND_NEVER
) {
1438 if (TCG_TARGET_REG_BITS
== 32) {
1439 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32
, TCGV_LOW(arg1
),
1440 TCGV_HIGH(arg1
), TCGV_LOW(arg2
),
1441 TCGV_HIGH(arg2
), cond
, label_arg(l
));
1443 tcg_gen_op4ii_i64(INDEX_op_brcond_i64
, arg1
, arg2
, cond
,
1449 void tcg_gen_brcondi_i64(TCGCond cond
, TCGv_i64 arg1
, int64_t arg2
, TCGLabel
*l
)
1451 if (TCG_TARGET_REG_BITS
== 64) {
1452 tcg_gen_brcond_i64(cond
, arg1
, tcg_constant_i64(arg2
), l
);
1453 } else if (cond
== TCG_COND_ALWAYS
) {
1455 } else if (cond
!= TCG_COND_NEVER
) {
1457 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32
,
1458 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1459 tcg_constant_i32(arg2
),
1460 tcg_constant_i32(arg2
>> 32),
1461 cond
, label_arg(l
));
1465 void tcg_gen_setcond_i64(TCGCond cond
, TCGv_i64 ret
,
1466 TCGv_i64 arg1
, TCGv_i64 arg2
)
1468 if (cond
== TCG_COND_ALWAYS
) {
1469 tcg_gen_movi_i64(ret
, 1);
1470 } else if (cond
== TCG_COND_NEVER
) {
1471 tcg_gen_movi_i64(ret
, 0);
1473 if (TCG_TARGET_REG_BITS
== 32) {
1474 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, TCGV_LOW(ret
),
1475 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1476 TCGV_LOW(arg2
), TCGV_HIGH(arg2
), cond
);
1477 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1479 tcg_gen_op4i_i64(INDEX_op_setcond_i64
, ret
, arg1
, arg2
, cond
);
1484 void tcg_gen_setcondi_i64(TCGCond cond
, TCGv_i64 ret
,
1485 TCGv_i64 arg1
, int64_t arg2
)
1487 if (TCG_TARGET_REG_BITS
== 64) {
1488 tcg_gen_setcond_i64(cond
, ret
, arg1
, tcg_constant_i64(arg2
));
1489 } else if (cond
== TCG_COND_ALWAYS
) {
1490 tcg_gen_movi_i64(ret
, 1);
1491 } else if (cond
== TCG_COND_NEVER
) {
1492 tcg_gen_movi_i64(ret
, 0);
1494 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, TCGV_LOW(ret
),
1495 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1496 tcg_constant_i32(arg2
),
1497 tcg_constant_i32(arg2
>> 32), cond
);
1498 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1502 void tcg_gen_muli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1505 tcg_gen_movi_i64(ret
, 0);
1506 } else if (is_power_of_2(arg2
)) {
1507 tcg_gen_shli_i64(ret
, arg1
, ctz64(arg2
));
1509 TCGv_i64 t0
= tcg_const_i64(arg2
);
1510 tcg_gen_mul_i64(ret
, arg1
, t0
);
1511 tcg_temp_free_i64(t0
);
1515 void tcg_gen_div_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1517 if (TCG_TARGET_HAS_div_i64
) {
1518 tcg_gen_op3_i64(INDEX_op_div_i64
, ret
, arg1
, arg2
);
1519 } else if (TCG_TARGET_HAS_div2_i64
) {
1520 TCGv_i64 t0
= tcg_temp_new_i64();
1521 tcg_gen_sari_i64(t0
, arg1
, 63);
1522 tcg_gen_op5_i64(INDEX_op_div2_i64
, ret
, t0
, arg1
, t0
, arg2
);
1523 tcg_temp_free_i64(t0
);
1525 gen_helper_div_i64(ret
, arg1
, arg2
);
1529 void tcg_gen_rem_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1531 if (TCG_TARGET_HAS_rem_i64
) {
1532 tcg_gen_op3_i64(INDEX_op_rem_i64
, ret
, arg1
, arg2
);
1533 } else if (TCG_TARGET_HAS_div_i64
) {
1534 TCGv_i64 t0
= tcg_temp_new_i64();
1535 tcg_gen_op3_i64(INDEX_op_div_i64
, t0
, arg1
, arg2
);
1536 tcg_gen_mul_i64(t0
, t0
, arg2
);
1537 tcg_gen_sub_i64(ret
, arg1
, t0
);
1538 tcg_temp_free_i64(t0
);
1539 } else if (TCG_TARGET_HAS_div2_i64
) {
1540 TCGv_i64 t0
= tcg_temp_new_i64();
1541 tcg_gen_sari_i64(t0
, arg1
, 63);
1542 tcg_gen_op5_i64(INDEX_op_div2_i64
, t0
, ret
, arg1
, t0
, arg2
);
1543 tcg_temp_free_i64(t0
);
1545 gen_helper_rem_i64(ret
, arg1
, arg2
);
1549 void tcg_gen_divu_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1551 if (TCG_TARGET_HAS_div_i64
) {
1552 tcg_gen_op3_i64(INDEX_op_divu_i64
, ret
, arg1
, arg2
);
1553 } else if (TCG_TARGET_HAS_div2_i64
) {
1554 TCGv_i64 t0
= tcg_temp_new_i64();
1555 tcg_gen_movi_i64(t0
, 0);
1556 tcg_gen_op5_i64(INDEX_op_divu2_i64
, ret
, t0
, arg1
, t0
, arg2
);
1557 tcg_temp_free_i64(t0
);
1559 gen_helper_divu_i64(ret
, arg1
, arg2
);
1563 void tcg_gen_remu_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1565 if (TCG_TARGET_HAS_rem_i64
) {
1566 tcg_gen_op3_i64(INDEX_op_remu_i64
, ret
, arg1
, arg2
);
1567 } else if (TCG_TARGET_HAS_div_i64
) {
1568 TCGv_i64 t0
= tcg_temp_new_i64();
1569 tcg_gen_op3_i64(INDEX_op_divu_i64
, t0
, arg1
, arg2
);
1570 tcg_gen_mul_i64(t0
, t0
, arg2
);
1571 tcg_gen_sub_i64(ret
, arg1
, t0
);
1572 tcg_temp_free_i64(t0
);
1573 } else if (TCG_TARGET_HAS_div2_i64
) {
1574 TCGv_i64 t0
= tcg_temp_new_i64();
1575 tcg_gen_movi_i64(t0
, 0);
1576 tcg_gen_op5_i64(INDEX_op_divu2_i64
, t0
, ret
, arg1
, t0
, arg2
);
1577 tcg_temp_free_i64(t0
);
1579 gen_helper_remu_i64(ret
, arg1
, arg2
);
1583 void tcg_gen_ext8s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1585 if (TCG_TARGET_REG_BITS
== 32) {
1586 tcg_gen_ext8s_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1587 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1588 } else if (TCG_TARGET_HAS_ext8s_i64
) {
1589 tcg_gen_op2_i64(INDEX_op_ext8s_i64
, ret
, arg
);
1591 tcg_gen_shli_i64(ret
, arg
, 56);
1592 tcg_gen_sari_i64(ret
, ret
, 56);
1596 void tcg_gen_ext16s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1598 if (TCG_TARGET_REG_BITS
== 32) {
1599 tcg_gen_ext16s_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1600 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1601 } else if (TCG_TARGET_HAS_ext16s_i64
) {
1602 tcg_gen_op2_i64(INDEX_op_ext16s_i64
, ret
, arg
);
1604 tcg_gen_shli_i64(ret
, arg
, 48);
1605 tcg_gen_sari_i64(ret
, ret
, 48);
1609 void tcg_gen_ext32s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1611 if (TCG_TARGET_REG_BITS
== 32) {
1612 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1613 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1614 } else if (TCG_TARGET_HAS_ext32s_i64
) {
1615 tcg_gen_op2_i64(INDEX_op_ext32s_i64
, ret
, arg
);
1617 tcg_gen_shli_i64(ret
, arg
, 32);
1618 tcg_gen_sari_i64(ret
, ret
, 32);
1622 void tcg_gen_ext8u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1624 if (TCG_TARGET_REG_BITS
== 32) {
1625 tcg_gen_ext8u_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1626 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1627 } else if (TCG_TARGET_HAS_ext8u_i64
) {
1628 tcg_gen_op2_i64(INDEX_op_ext8u_i64
, ret
, arg
);
1630 tcg_gen_andi_i64(ret
, arg
, 0xffu
);
1634 void tcg_gen_ext16u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1636 if (TCG_TARGET_REG_BITS
== 32) {
1637 tcg_gen_ext16u_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1638 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1639 } else if (TCG_TARGET_HAS_ext16u_i64
) {
1640 tcg_gen_op2_i64(INDEX_op_ext16u_i64
, ret
, arg
);
1642 tcg_gen_andi_i64(ret
, arg
, 0xffffu
);
1646 void tcg_gen_ext32u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1648 if (TCG_TARGET_REG_BITS
== 32) {
1649 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1650 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1651 } else if (TCG_TARGET_HAS_ext32u_i64
) {
1652 tcg_gen_op2_i64(INDEX_op_ext32u_i64
, ret
, arg
);
1654 tcg_gen_andi_i64(ret
, arg
, 0xffffffffu
);
1658 /* Note: we assume the six high bytes are set to zero */
1659 void tcg_gen_bswap16_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1661 if (TCG_TARGET_REG_BITS
== 32) {
1662 tcg_gen_bswap16_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1663 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1664 } else if (TCG_TARGET_HAS_bswap16_i64
) {
1665 tcg_gen_op3i_i64(INDEX_op_bswap16_i64
, ret
, arg
,
1666 TCG_BSWAP_IZ
| TCG_BSWAP_OZ
);
1668 TCGv_i64 t0
= tcg_temp_new_i64();
1670 tcg_gen_ext8u_i64(t0
, arg
);
1671 tcg_gen_shli_i64(t0
, t0
, 8);
1672 tcg_gen_shri_i64(ret
, arg
, 8);
1673 tcg_gen_or_i64(ret
, ret
, t0
);
1674 tcg_temp_free_i64(t0
);
1678 /* Note: we assume the four high bytes are set to zero */
1679 void tcg_gen_bswap32_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1681 if (TCG_TARGET_REG_BITS
== 32) {
1682 tcg_gen_bswap32_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1683 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1684 } else if (TCG_TARGET_HAS_bswap32_i64
) {
1685 tcg_gen_op3i_i64(INDEX_op_bswap32_i64
, ret
, arg
,
1686 TCG_BSWAP_IZ
| TCG_BSWAP_OZ
);
1688 TCGv_i64 t0
= tcg_temp_new_i64();
1689 TCGv_i64 t1
= tcg_temp_new_i64();
1690 TCGv_i64 t2
= tcg_constant_i64(0x00ff00ff);
1692 /* arg = ....abcd */
1693 tcg_gen_shri_i64(t0
, arg
, 8); /* t0 = .....abc */
1694 tcg_gen_and_i64(t1
, arg
, t2
); /* t1 = .....b.d */
1695 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = .....a.c */
1696 tcg_gen_shli_i64(t1
, t1
, 8); /* t1 = ....b.d. */
1697 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = ....badc */
1699 tcg_gen_shli_i64(t1
, ret
, 48); /* t1 = dc...... */
1700 tcg_gen_shri_i64(t0
, ret
, 16); /* t0 = ......ba */
1701 tcg_gen_shri_i64(t1
, t1
, 32); /* t1 = ....dc.. */
1702 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = ....dcba */
1704 tcg_temp_free_i64(t0
);
1705 tcg_temp_free_i64(t1
);
1709 void tcg_gen_bswap64_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1711 if (TCG_TARGET_REG_BITS
== 32) {
1713 t0
= tcg_temp_new_i32();
1714 t1
= tcg_temp_new_i32();
1716 tcg_gen_bswap32_i32(t0
, TCGV_LOW(arg
));
1717 tcg_gen_bswap32_i32(t1
, TCGV_HIGH(arg
));
1718 tcg_gen_mov_i32(TCGV_LOW(ret
), t1
);
1719 tcg_gen_mov_i32(TCGV_HIGH(ret
), t0
);
1720 tcg_temp_free_i32(t0
);
1721 tcg_temp_free_i32(t1
);
1722 } else if (TCG_TARGET_HAS_bswap64_i64
) {
1723 tcg_gen_op3i_i64(INDEX_op_bswap64_i64
, ret
, arg
, 0);
1725 TCGv_i64 t0
= tcg_temp_new_i64();
1726 TCGv_i64 t1
= tcg_temp_new_i64();
1727 TCGv_i64 t2
= tcg_temp_new_i64();
1729 /* arg = abcdefgh */
1730 tcg_gen_movi_i64(t2
, 0x00ff00ff00ff00ffull
);
1731 tcg_gen_shri_i64(t0
, arg
, 8); /* t0 = .abcdefg */
1732 tcg_gen_and_i64(t1
, arg
, t2
); /* t1 = .b.d.f.h */
1733 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = .a.c.e.g */
1734 tcg_gen_shli_i64(t1
, t1
, 8); /* t1 = b.d.f.h. */
1735 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = badcfehg */
1737 tcg_gen_movi_i64(t2
, 0x0000ffff0000ffffull
);
1738 tcg_gen_shri_i64(t0
, ret
, 16); /* t0 = ..badcfe */
1739 tcg_gen_and_i64(t1
, ret
, t2
); /* t1 = ..dc..hg */
1740 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = ..ba..fe */
1741 tcg_gen_shli_i64(t1
, t1
, 16); /* t1 = dc..hg.. */
1742 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = dcbahgfe */
1744 tcg_gen_shri_i64(t0
, ret
, 32); /* t0 = ....dcba */
1745 tcg_gen_shli_i64(t1
, ret
, 32); /* t1 = hgfe.... */
1746 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = hgfedcba */
1748 tcg_temp_free_i64(t0
);
1749 tcg_temp_free_i64(t1
);
1750 tcg_temp_free_i64(t2
);
1754 void tcg_gen_not_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1756 if (TCG_TARGET_REG_BITS
== 32) {
1757 tcg_gen_not_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1758 tcg_gen_not_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
));
1759 } else if (TCG_TARGET_HAS_not_i64
) {
1760 tcg_gen_op2_i64(INDEX_op_not_i64
, ret
, arg
);
1762 tcg_gen_xori_i64(ret
, arg
, -1);
1766 void tcg_gen_andc_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1768 if (TCG_TARGET_REG_BITS
== 32) {
1769 tcg_gen_andc_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1770 tcg_gen_andc_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1771 } else if (TCG_TARGET_HAS_andc_i64
) {
1772 tcg_gen_op3_i64(INDEX_op_andc_i64
, ret
, arg1
, arg2
);
1774 TCGv_i64 t0
= tcg_temp_new_i64();
1775 tcg_gen_not_i64(t0
, arg2
);
1776 tcg_gen_and_i64(ret
, arg1
, t0
);
1777 tcg_temp_free_i64(t0
);
1781 void tcg_gen_eqv_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1783 if (TCG_TARGET_REG_BITS
== 32) {
1784 tcg_gen_eqv_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1785 tcg_gen_eqv_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1786 } else if (TCG_TARGET_HAS_eqv_i64
) {
1787 tcg_gen_op3_i64(INDEX_op_eqv_i64
, ret
, arg1
, arg2
);
1789 tcg_gen_xor_i64(ret
, arg1
, arg2
);
1790 tcg_gen_not_i64(ret
, ret
);
1794 void tcg_gen_nand_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1796 if (TCG_TARGET_REG_BITS
== 32) {
1797 tcg_gen_nand_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1798 tcg_gen_nand_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1799 } else if (TCG_TARGET_HAS_nand_i64
) {
1800 tcg_gen_op3_i64(INDEX_op_nand_i64
, ret
, arg1
, arg2
);
1802 tcg_gen_and_i64(ret
, arg1
, arg2
);
1803 tcg_gen_not_i64(ret
, ret
);
1807 void tcg_gen_nor_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1809 if (TCG_TARGET_REG_BITS
== 32) {
1810 tcg_gen_nor_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1811 tcg_gen_nor_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1812 } else if (TCG_TARGET_HAS_nor_i64
) {
1813 tcg_gen_op3_i64(INDEX_op_nor_i64
, ret
, arg1
, arg2
);
1815 tcg_gen_or_i64(ret
, arg1
, arg2
);
1816 tcg_gen_not_i64(ret
, ret
);
1820 void tcg_gen_orc_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1822 if (TCG_TARGET_REG_BITS
== 32) {
1823 tcg_gen_orc_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1824 tcg_gen_orc_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1825 } else if (TCG_TARGET_HAS_orc_i64
) {
1826 tcg_gen_op3_i64(INDEX_op_orc_i64
, ret
, arg1
, arg2
);
1828 TCGv_i64 t0
= tcg_temp_new_i64();
1829 tcg_gen_not_i64(t0
, arg2
);
1830 tcg_gen_or_i64(ret
, arg1
, t0
);
1831 tcg_temp_free_i64(t0
);
1835 void tcg_gen_clz_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1837 if (TCG_TARGET_HAS_clz_i64
) {
1838 tcg_gen_op3_i64(INDEX_op_clz_i64
, ret
, arg1
, arg2
);
1840 gen_helper_clz_i64(ret
, arg1
, arg2
);
1844 void tcg_gen_clzi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, uint64_t arg2
)
1846 if (TCG_TARGET_REG_BITS
== 32
1847 && TCG_TARGET_HAS_clz_i32
1848 && arg2
<= 0xffffffffu
) {
1849 TCGv_i32 t
= tcg_temp_new_i32();
1850 tcg_gen_clzi_i32(t
, TCGV_LOW(arg1
), arg2
- 32);
1851 tcg_gen_addi_i32(t
, t
, 32);
1852 tcg_gen_clz_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), t
);
1853 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1854 tcg_temp_free_i32(t
);
1856 TCGv_i64 t0
= tcg_const_i64(arg2
);
1857 tcg_gen_clz_i64(ret
, arg1
, t0
);
1858 tcg_temp_free_i64(t0
);
1862 void tcg_gen_ctz_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1864 if (TCG_TARGET_HAS_ctz_i64
) {
1865 tcg_gen_op3_i64(INDEX_op_ctz_i64
, ret
, arg1
, arg2
);
1866 } else if (TCG_TARGET_HAS_ctpop_i64
|| TCG_TARGET_HAS_clz_i64
) {
1867 TCGv_i64 z
, t
= tcg_temp_new_i64();
1869 if (TCG_TARGET_HAS_ctpop_i64
) {
1870 tcg_gen_subi_i64(t
, arg1
, 1);
1871 tcg_gen_andc_i64(t
, t
, arg1
);
1872 tcg_gen_ctpop_i64(t
, t
);
1874 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1875 tcg_gen_neg_i64(t
, arg1
);
1876 tcg_gen_and_i64(t
, t
, arg1
);
1877 tcg_gen_clzi_i64(t
, t
, 64);
1878 tcg_gen_xori_i64(t
, t
, 63);
1880 z
= tcg_constant_i64(0);
1881 tcg_gen_movcond_i64(TCG_COND_EQ
, ret
, arg1
, z
, arg2
, t
);
1882 tcg_temp_free_i64(t
);
1883 tcg_temp_free_i64(z
);
1885 gen_helper_ctz_i64(ret
, arg1
, arg2
);
1889 void tcg_gen_ctzi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, uint64_t arg2
)
1891 if (TCG_TARGET_REG_BITS
== 32
1892 && TCG_TARGET_HAS_ctz_i32
1893 && arg2
<= 0xffffffffu
) {
1894 TCGv_i32 t32
= tcg_temp_new_i32();
1895 tcg_gen_ctzi_i32(t32
, TCGV_HIGH(arg1
), arg2
- 32);
1896 tcg_gen_addi_i32(t32
, t32
, 32);
1897 tcg_gen_ctz_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), t32
);
1898 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1899 tcg_temp_free_i32(t32
);
1900 } else if (!TCG_TARGET_HAS_ctz_i64
1901 && TCG_TARGET_HAS_ctpop_i64
1903 /* This equivalence has the advantage of not requiring a fixup. */
1904 TCGv_i64 t
= tcg_temp_new_i64();
1905 tcg_gen_subi_i64(t
, arg1
, 1);
1906 tcg_gen_andc_i64(t
, t
, arg1
);
1907 tcg_gen_ctpop_i64(ret
, t
);
1908 tcg_temp_free_i64(t
);
1910 TCGv_i64 t0
= tcg_const_i64(arg2
);
1911 tcg_gen_ctz_i64(ret
, arg1
, t0
);
1912 tcg_temp_free_i64(t0
);
1916 void tcg_gen_clrsb_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1918 if (TCG_TARGET_HAS_clz_i64
|| TCG_TARGET_HAS_clz_i32
) {
1919 TCGv_i64 t
= tcg_temp_new_i64();
1920 tcg_gen_sari_i64(t
, arg
, 63);
1921 tcg_gen_xor_i64(t
, t
, arg
);
1922 tcg_gen_clzi_i64(t
, t
, 64);
1923 tcg_gen_subi_i64(ret
, t
, 1);
1924 tcg_temp_free_i64(t
);
1926 gen_helper_clrsb_i64(ret
, arg
);
1930 void tcg_gen_ctpop_i64(TCGv_i64 ret
, TCGv_i64 arg1
)
1932 if (TCG_TARGET_HAS_ctpop_i64
) {
1933 tcg_gen_op2_i64(INDEX_op_ctpop_i64
, ret
, arg1
);
1934 } else if (TCG_TARGET_REG_BITS
== 32 && TCG_TARGET_HAS_ctpop_i32
) {
1935 tcg_gen_ctpop_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
1936 tcg_gen_ctpop_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
1937 tcg_gen_add_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), TCGV_HIGH(ret
));
1938 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1940 gen_helper_ctpop_i64(ret
, arg1
);
1944 void tcg_gen_rotl_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1946 if (TCG_TARGET_HAS_rot_i64
) {
1947 tcg_gen_op3_i64(INDEX_op_rotl_i64
, ret
, arg1
, arg2
);
1950 t0
= tcg_temp_new_i64();
1951 t1
= tcg_temp_new_i64();
1952 tcg_gen_shl_i64(t0
, arg1
, arg2
);
1953 tcg_gen_subfi_i64(t1
, 64, arg2
);
1954 tcg_gen_shr_i64(t1
, arg1
, t1
);
1955 tcg_gen_or_i64(ret
, t0
, t1
);
1956 tcg_temp_free_i64(t0
);
1957 tcg_temp_free_i64(t1
);
1961 void tcg_gen_rotli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1963 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1964 /* some cases can be optimized here */
1966 tcg_gen_mov_i64(ret
, arg1
);
1967 } else if (TCG_TARGET_HAS_rot_i64
) {
1968 tcg_gen_rotl_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1971 t0
= tcg_temp_new_i64();
1972 t1
= tcg_temp_new_i64();
1973 tcg_gen_shli_i64(t0
, arg1
, arg2
);
1974 tcg_gen_shri_i64(t1
, arg1
, 64 - arg2
);
1975 tcg_gen_or_i64(ret
, t0
, t1
);
1976 tcg_temp_free_i64(t0
);
1977 tcg_temp_free_i64(t1
);
1981 void tcg_gen_rotr_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1983 if (TCG_TARGET_HAS_rot_i64
) {
1984 tcg_gen_op3_i64(INDEX_op_rotr_i64
, ret
, arg1
, arg2
);
1987 t0
= tcg_temp_new_i64();
1988 t1
= tcg_temp_new_i64();
1989 tcg_gen_shr_i64(t0
, arg1
, arg2
);
1990 tcg_gen_subfi_i64(t1
, 64, arg2
);
1991 tcg_gen_shl_i64(t1
, arg1
, t1
);
1992 tcg_gen_or_i64(ret
, t0
, t1
);
1993 tcg_temp_free_i64(t0
);
1994 tcg_temp_free_i64(t1
);
1998 void tcg_gen_rotri_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
2000 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
2001 /* some cases can be optimized here */
2003 tcg_gen_mov_i64(ret
, arg1
);
2005 tcg_gen_rotli_i64(ret
, arg1
, 64 - arg2
);
2009 void tcg_gen_deposit_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
,
2010 unsigned int ofs
, unsigned int len
)
2015 tcg_debug_assert(ofs
< 64);
2016 tcg_debug_assert(len
> 0);
2017 tcg_debug_assert(len
<= 64);
2018 tcg_debug_assert(ofs
+ len
<= 64);
2021 tcg_gen_mov_i64(ret
, arg2
);
2024 if (TCG_TARGET_HAS_deposit_i64
&& TCG_TARGET_deposit_i64_valid(ofs
, len
)) {
2025 tcg_gen_op5ii_i64(INDEX_op_deposit_i64
, ret
, arg1
, arg2
, ofs
, len
);
2029 if (TCG_TARGET_REG_BITS
== 32) {
2031 tcg_gen_deposit_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
),
2032 TCGV_LOW(arg2
), ofs
- 32, len
);
2033 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
2036 if (ofs
+ len
<= 32) {
2037 tcg_gen_deposit_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
),
2038 TCGV_LOW(arg2
), ofs
, len
);
2039 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
2044 t1
= tcg_temp_new_i64();
2046 if (TCG_TARGET_HAS_extract2_i64
) {
2047 if (ofs
+ len
== 64) {
2048 tcg_gen_shli_i64(t1
, arg1
, len
);
2049 tcg_gen_extract2_i64(ret
, t1
, arg2
, len
);
2053 tcg_gen_extract2_i64(ret
, arg1
, arg2
, len
);
2054 tcg_gen_rotli_i64(ret
, ret
, len
);
2059 mask
= (1ull << len
) - 1;
2060 if (ofs
+ len
< 64) {
2061 tcg_gen_andi_i64(t1
, arg2
, mask
);
2062 tcg_gen_shli_i64(t1
, t1
, ofs
);
2064 tcg_gen_shli_i64(t1
, arg2
, ofs
);
2066 tcg_gen_andi_i64(ret
, arg1
, ~(mask
<< ofs
));
2067 tcg_gen_or_i64(ret
, ret
, t1
);
2069 tcg_temp_free_i64(t1
);
2072 void tcg_gen_deposit_z_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2073 unsigned int ofs
, unsigned int len
)
2075 tcg_debug_assert(ofs
< 64);
2076 tcg_debug_assert(len
> 0);
2077 tcg_debug_assert(len
<= 64);
2078 tcg_debug_assert(ofs
+ len
<= 64);
2080 if (ofs
+ len
== 64) {
2081 tcg_gen_shli_i64(ret
, arg
, ofs
);
2082 } else if (ofs
== 0) {
2083 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2084 } else if (TCG_TARGET_HAS_deposit_i64
2085 && TCG_TARGET_deposit_i64_valid(ofs
, len
)) {
2086 TCGv_i64 zero
= tcg_constant_i64(0);
2087 tcg_gen_op5ii_i64(INDEX_op_deposit_i64
, ret
, zero
, arg
, ofs
, len
);
2089 if (TCG_TARGET_REG_BITS
== 32) {
2091 tcg_gen_deposit_z_i32(TCGV_HIGH(ret
), TCGV_LOW(arg
),
2093 tcg_gen_movi_i32(TCGV_LOW(ret
), 0);
2096 if (ofs
+ len
<= 32) {
2097 tcg_gen_deposit_z_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2098 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2102 /* To help two-operand hosts we prefer to zero-extend first,
2103 which allows ARG to stay live. */
2106 if (TCG_TARGET_HAS_ext32u_i64
) {
2107 tcg_gen_ext32u_i64(ret
, arg
);
2108 tcg_gen_shli_i64(ret
, ret
, ofs
);
2113 if (TCG_TARGET_HAS_ext16u_i64
) {
2114 tcg_gen_ext16u_i64(ret
, arg
);
2115 tcg_gen_shli_i64(ret
, ret
, ofs
);
2120 if (TCG_TARGET_HAS_ext8u_i64
) {
2121 tcg_gen_ext8u_i64(ret
, arg
);
2122 tcg_gen_shli_i64(ret
, ret
, ofs
);
2127 /* Otherwise prefer zero-extension over AND for code size. */
2128 switch (ofs
+ len
) {
2130 if (TCG_TARGET_HAS_ext32u_i64
) {
2131 tcg_gen_shli_i64(ret
, arg
, ofs
);
2132 tcg_gen_ext32u_i64(ret
, ret
);
2137 if (TCG_TARGET_HAS_ext16u_i64
) {
2138 tcg_gen_shli_i64(ret
, arg
, ofs
);
2139 tcg_gen_ext16u_i64(ret
, ret
);
2144 if (TCG_TARGET_HAS_ext8u_i64
) {
2145 tcg_gen_shli_i64(ret
, arg
, ofs
);
2146 tcg_gen_ext8u_i64(ret
, ret
);
2151 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2152 tcg_gen_shli_i64(ret
, ret
, ofs
);
2156 void tcg_gen_extract_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2157 unsigned int ofs
, unsigned int len
)
2159 tcg_debug_assert(ofs
< 64);
2160 tcg_debug_assert(len
> 0);
2161 tcg_debug_assert(len
<= 64);
2162 tcg_debug_assert(ofs
+ len
<= 64);
2164 /* Canonicalize certain special cases, even if extract is supported. */
2165 if (ofs
+ len
== 64) {
2166 tcg_gen_shri_i64(ret
, arg
, 64 - len
);
2170 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2174 if (TCG_TARGET_REG_BITS
== 32) {
2175 /* Look for a 32-bit extract within one of the two words. */
2177 tcg_gen_extract_i32(TCGV_LOW(ret
), TCGV_HIGH(arg
), ofs
- 32, len
);
2178 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2181 if (ofs
+ len
<= 32) {
2182 tcg_gen_extract_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2183 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2186 /* The field is split across two words. One double-word
2187 shift is better than two double-word shifts. */
2191 if (TCG_TARGET_HAS_extract_i64
2192 && TCG_TARGET_extract_i64_valid(ofs
, len
)) {
2193 tcg_gen_op4ii_i64(INDEX_op_extract_i64
, ret
, arg
, ofs
, len
);
2197 /* Assume that zero-extension, if available, is cheaper than a shift. */
2198 switch (ofs
+ len
) {
2200 if (TCG_TARGET_HAS_ext32u_i64
) {
2201 tcg_gen_ext32u_i64(ret
, arg
);
2202 tcg_gen_shri_i64(ret
, ret
, ofs
);
2207 if (TCG_TARGET_HAS_ext16u_i64
) {
2208 tcg_gen_ext16u_i64(ret
, arg
);
2209 tcg_gen_shri_i64(ret
, ret
, ofs
);
2214 if (TCG_TARGET_HAS_ext8u_i64
) {
2215 tcg_gen_ext8u_i64(ret
, arg
);
2216 tcg_gen_shri_i64(ret
, ret
, ofs
);
2222 /* ??? Ideally we'd know what values are available for immediate AND.
2223 Assume that 8 bits are available, plus the special cases of 16 and 32,
2224 so that we get ext8u, ext16u, and ext32u. */
2226 case 1 ... 8: case 16: case 32:
2228 tcg_gen_shri_i64(ret
, arg
, ofs
);
2229 tcg_gen_andi_i64(ret
, ret
, (1ull << len
) - 1);
2232 tcg_gen_shli_i64(ret
, arg
, 64 - len
- ofs
);
2233 tcg_gen_shri_i64(ret
, ret
, 64 - len
);
2238 void tcg_gen_sextract_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2239 unsigned int ofs
, unsigned int len
)
2241 tcg_debug_assert(ofs
< 64);
2242 tcg_debug_assert(len
> 0);
2243 tcg_debug_assert(len
<= 64);
2244 tcg_debug_assert(ofs
+ len
<= 64);
2246 /* Canonicalize certain special cases, even if sextract is supported. */
2247 if (ofs
+ len
== 64) {
2248 tcg_gen_sari_i64(ret
, arg
, 64 - len
);
2254 tcg_gen_ext32s_i64(ret
, arg
);
2257 tcg_gen_ext16s_i64(ret
, arg
);
2260 tcg_gen_ext8s_i64(ret
, arg
);
2265 if (TCG_TARGET_REG_BITS
== 32) {
2266 /* Look for a 32-bit extract within one of the two words. */
2268 tcg_gen_sextract_i32(TCGV_LOW(ret
), TCGV_HIGH(arg
), ofs
- 32, len
);
2269 } else if (ofs
+ len
<= 32) {
2270 tcg_gen_sextract_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2271 } else if (ofs
== 0) {
2272 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
2273 tcg_gen_sextract_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
), 0, len
- 32);
2275 } else if (len
> 32) {
2276 TCGv_i32 t
= tcg_temp_new_i32();
2277 /* Extract the bits for the high word normally. */
2278 tcg_gen_sextract_i32(t
, TCGV_HIGH(arg
), ofs
+ 32, len
- 32);
2279 /* Shift the field down for the low part. */
2280 tcg_gen_shri_i64(ret
, arg
, ofs
);
2281 /* Overwrite the shift into the high part. */
2282 tcg_gen_mov_i32(TCGV_HIGH(ret
), t
);
2283 tcg_temp_free_i32(t
);
2286 /* Shift the field down for the low part, such that the
2287 field sits at the MSB. */
2288 tcg_gen_shri_i64(ret
, arg
, ofs
+ len
- 32);
2289 /* Shift the field down from the MSB, sign extending. */
2290 tcg_gen_sari_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), 32 - len
);
2292 /* Sign-extend the field from 32 bits. */
2293 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
2297 if (TCG_TARGET_HAS_sextract_i64
2298 && TCG_TARGET_extract_i64_valid(ofs
, len
)) {
2299 tcg_gen_op4ii_i64(INDEX_op_sextract_i64
, ret
, arg
, ofs
, len
);
2303 /* Assume that sign-extension, if available, is cheaper than a shift. */
2304 switch (ofs
+ len
) {
2306 if (TCG_TARGET_HAS_ext32s_i64
) {
2307 tcg_gen_ext32s_i64(ret
, arg
);
2308 tcg_gen_sari_i64(ret
, ret
, ofs
);
2313 if (TCG_TARGET_HAS_ext16s_i64
) {
2314 tcg_gen_ext16s_i64(ret
, arg
);
2315 tcg_gen_sari_i64(ret
, ret
, ofs
);
2320 if (TCG_TARGET_HAS_ext8s_i64
) {
2321 tcg_gen_ext8s_i64(ret
, arg
);
2322 tcg_gen_sari_i64(ret
, ret
, ofs
);
2329 if (TCG_TARGET_HAS_ext32s_i64
) {
2330 tcg_gen_shri_i64(ret
, arg
, ofs
);
2331 tcg_gen_ext32s_i64(ret
, ret
);
2336 if (TCG_TARGET_HAS_ext16s_i64
) {
2337 tcg_gen_shri_i64(ret
, arg
, ofs
);
2338 tcg_gen_ext16s_i64(ret
, ret
);
2343 if (TCG_TARGET_HAS_ext8s_i64
) {
2344 tcg_gen_shri_i64(ret
, arg
, ofs
);
2345 tcg_gen_ext8s_i64(ret
, ret
);
2350 tcg_gen_shli_i64(ret
, arg
, 64 - len
- ofs
);
2351 tcg_gen_sari_i64(ret
, ret
, 64 - len
);
2355 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2356 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2358 void tcg_gen_extract2_i64(TCGv_i64 ret
, TCGv_i64 al
, TCGv_i64 ah
,
2361 tcg_debug_assert(ofs
<= 64);
2363 tcg_gen_mov_i64(ret
, al
);
2364 } else if (ofs
== 64) {
2365 tcg_gen_mov_i64(ret
, ah
);
2366 } else if (al
== ah
) {
2367 tcg_gen_rotri_i64(ret
, al
, ofs
);
2368 } else if (TCG_TARGET_HAS_extract2_i64
) {
2369 tcg_gen_op4i_i64(INDEX_op_extract2_i64
, ret
, al
, ah
, ofs
);
2371 TCGv_i64 t0
= tcg_temp_new_i64();
2372 tcg_gen_shri_i64(t0
, al
, ofs
);
2373 tcg_gen_deposit_i64(ret
, t0
, ah
, 64 - ofs
, ofs
);
2374 tcg_temp_free_i64(t0
);
2378 void tcg_gen_movcond_i64(TCGCond cond
, TCGv_i64 ret
, TCGv_i64 c1
,
2379 TCGv_i64 c2
, TCGv_i64 v1
, TCGv_i64 v2
)
2381 if (cond
== TCG_COND_ALWAYS
) {
2382 tcg_gen_mov_i64(ret
, v1
);
2383 } else if (cond
== TCG_COND_NEVER
) {
2384 tcg_gen_mov_i64(ret
, v2
);
2385 } else if (TCG_TARGET_REG_BITS
== 32) {
2386 TCGv_i32 t0
= tcg_temp_new_i32();
2387 TCGv_i32 t1
= tcg_temp_new_i32();
2388 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, t0
,
2389 TCGV_LOW(c1
), TCGV_HIGH(c1
),
2390 TCGV_LOW(c2
), TCGV_HIGH(c2
), cond
);
2392 if (TCG_TARGET_HAS_movcond_i32
) {
2393 tcg_gen_movi_i32(t1
, 0);
2394 tcg_gen_movcond_i32(TCG_COND_NE
, TCGV_LOW(ret
), t0
, t1
,
2395 TCGV_LOW(v1
), TCGV_LOW(v2
));
2396 tcg_gen_movcond_i32(TCG_COND_NE
, TCGV_HIGH(ret
), t0
, t1
,
2397 TCGV_HIGH(v1
), TCGV_HIGH(v2
));
2399 tcg_gen_neg_i32(t0
, t0
);
2401 tcg_gen_and_i32(t1
, TCGV_LOW(v1
), t0
);
2402 tcg_gen_andc_i32(TCGV_LOW(ret
), TCGV_LOW(v2
), t0
);
2403 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), t1
);
2405 tcg_gen_and_i32(t1
, TCGV_HIGH(v1
), t0
);
2406 tcg_gen_andc_i32(TCGV_HIGH(ret
), TCGV_HIGH(v2
), t0
);
2407 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(ret
), t1
);
2409 tcg_temp_free_i32(t0
);
2410 tcg_temp_free_i32(t1
);
2411 } else if (TCG_TARGET_HAS_movcond_i64
) {
2412 tcg_gen_op6i_i64(INDEX_op_movcond_i64
, ret
, c1
, c2
, v1
, v2
, cond
);
2414 TCGv_i64 t0
= tcg_temp_new_i64();
2415 TCGv_i64 t1
= tcg_temp_new_i64();
2416 tcg_gen_setcond_i64(cond
, t0
, c1
, c2
);
2417 tcg_gen_neg_i64(t0
, t0
);
2418 tcg_gen_and_i64(t1
, v1
, t0
);
2419 tcg_gen_andc_i64(ret
, v2
, t0
);
2420 tcg_gen_or_i64(ret
, ret
, t1
);
2421 tcg_temp_free_i64(t0
);
2422 tcg_temp_free_i64(t1
);
2426 void tcg_gen_add2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 al
,
2427 TCGv_i64 ah
, TCGv_i64 bl
, TCGv_i64 bh
)
2429 if (TCG_TARGET_HAS_add2_i64
) {
2430 tcg_gen_op6_i64(INDEX_op_add2_i64
, rl
, rh
, al
, ah
, bl
, bh
);
2432 TCGv_i64 t0
= tcg_temp_new_i64();
2433 TCGv_i64 t1
= tcg_temp_new_i64();
2434 tcg_gen_add_i64(t0
, al
, bl
);
2435 tcg_gen_setcond_i64(TCG_COND_LTU
, t1
, t0
, al
);
2436 tcg_gen_add_i64(rh
, ah
, bh
);
2437 tcg_gen_add_i64(rh
, rh
, t1
);
2438 tcg_gen_mov_i64(rl
, t0
);
2439 tcg_temp_free_i64(t0
);
2440 tcg_temp_free_i64(t1
);
2444 void tcg_gen_sub2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 al
,
2445 TCGv_i64 ah
, TCGv_i64 bl
, TCGv_i64 bh
)
2447 if (TCG_TARGET_HAS_sub2_i64
) {
2448 tcg_gen_op6_i64(INDEX_op_sub2_i64
, rl
, rh
, al
, ah
, bl
, bh
);
2450 TCGv_i64 t0
= tcg_temp_new_i64();
2451 TCGv_i64 t1
= tcg_temp_new_i64();
2452 tcg_gen_sub_i64(t0
, al
, bl
);
2453 tcg_gen_setcond_i64(TCG_COND_LTU
, t1
, al
, bl
);
2454 tcg_gen_sub_i64(rh
, ah
, bh
);
2455 tcg_gen_sub_i64(rh
, rh
, t1
);
2456 tcg_gen_mov_i64(rl
, t0
);
2457 tcg_temp_free_i64(t0
);
2458 tcg_temp_free_i64(t1
);
2462 void tcg_gen_mulu2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2464 if (TCG_TARGET_HAS_mulu2_i64
) {
2465 tcg_gen_op4_i64(INDEX_op_mulu2_i64
, rl
, rh
, arg1
, arg2
);
2466 } else if (TCG_TARGET_HAS_muluh_i64
) {
2467 TCGv_i64 t
= tcg_temp_new_i64();
2468 tcg_gen_op3_i64(INDEX_op_mul_i64
, t
, arg1
, arg2
);
2469 tcg_gen_op3_i64(INDEX_op_muluh_i64
, rh
, arg1
, arg2
);
2470 tcg_gen_mov_i64(rl
, t
);
2471 tcg_temp_free_i64(t
);
2473 TCGv_i64 t0
= tcg_temp_new_i64();
2474 tcg_gen_mul_i64(t0
, arg1
, arg2
);
2475 gen_helper_muluh_i64(rh
, arg1
, arg2
);
2476 tcg_gen_mov_i64(rl
, t0
);
2477 tcg_temp_free_i64(t0
);
2481 void tcg_gen_muls2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2483 if (TCG_TARGET_HAS_muls2_i64
) {
2484 tcg_gen_op4_i64(INDEX_op_muls2_i64
, rl
, rh
, arg1
, arg2
);
2485 } else if (TCG_TARGET_HAS_mulsh_i64
) {
2486 TCGv_i64 t
= tcg_temp_new_i64();
2487 tcg_gen_op3_i64(INDEX_op_mul_i64
, t
, arg1
, arg2
);
2488 tcg_gen_op3_i64(INDEX_op_mulsh_i64
, rh
, arg1
, arg2
);
2489 tcg_gen_mov_i64(rl
, t
);
2490 tcg_temp_free_i64(t
);
2491 } else if (TCG_TARGET_HAS_mulu2_i64
|| TCG_TARGET_HAS_muluh_i64
) {
2492 TCGv_i64 t0
= tcg_temp_new_i64();
2493 TCGv_i64 t1
= tcg_temp_new_i64();
2494 TCGv_i64 t2
= tcg_temp_new_i64();
2495 TCGv_i64 t3
= tcg_temp_new_i64();
2496 tcg_gen_mulu2_i64(t0
, t1
, arg1
, arg2
);
2497 /* Adjust for negative inputs. */
2498 tcg_gen_sari_i64(t2
, arg1
, 63);
2499 tcg_gen_sari_i64(t3
, arg2
, 63);
2500 tcg_gen_and_i64(t2
, t2
, arg2
);
2501 tcg_gen_and_i64(t3
, t3
, arg1
);
2502 tcg_gen_sub_i64(rh
, t1
, t2
);
2503 tcg_gen_sub_i64(rh
, rh
, t3
);
2504 tcg_gen_mov_i64(rl
, t0
);
2505 tcg_temp_free_i64(t0
);
2506 tcg_temp_free_i64(t1
);
2507 tcg_temp_free_i64(t2
);
2508 tcg_temp_free_i64(t3
);
2510 TCGv_i64 t0
= tcg_temp_new_i64();
2511 tcg_gen_mul_i64(t0
, arg1
, arg2
);
2512 gen_helper_mulsh_i64(rh
, arg1
, arg2
);
2513 tcg_gen_mov_i64(rl
, t0
);
2514 tcg_temp_free_i64(t0
);
2518 void tcg_gen_mulsu2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2520 TCGv_i64 t0
= tcg_temp_new_i64();
2521 TCGv_i64 t1
= tcg_temp_new_i64();
2522 TCGv_i64 t2
= tcg_temp_new_i64();
2523 tcg_gen_mulu2_i64(t0
, t1
, arg1
, arg2
);
2524 /* Adjust for negative input for the signed arg1. */
2525 tcg_gen_sari_i64(t2
, arg1
, 63);
2526 tcg_gen_and_i64(t2
, t2
, arg2
);
2527 tcg_gen_sub_i64(rh
, t1
, t2
);
2528 tcg_gen_mov_i64(rl
, t0
);
2529 tcg_temp_free_i64(t0
);
2530 tcg_temp_free_i64(t1
);
2531 tcg_temp_free_i64(t2
);
2534 void tcg_gen_smin_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2536 tcg_gen_movcond_i64(TCG_COND_LT
, ret
, a
, b
, a
, b
);
2539 void tcg_gen_umin_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2541 tcg_gen_movcond_i64(TCG_COND_LTU
, ret
, a
, b
, a
, b
);
2544 void tcg_gen_smax_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2546 tcg_gen_movcond_i64(TCG_COND_LT
, ret
, a
, b
, b
, a
);
2549 void tcg_gen_umax_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2551 tcg_gen_movcond_i64(TCG_COND_LTU
, ret
, a
, b
, b
, a
);
2554 void tcg_gen_abs_i64(TCGv_i64 ret
, TCGv_i64 a
)
2556 TCGv_i64 t
= tcg_temp_new_i64();
2558 tcg_gen_sari_i64(t
, a
, 63);
2559 tcg_gen_xor_i64(ret
, a
, t
);
2560 tcg_gen_sub_i64(ret
, ret
, t
);
2561 tcg_temp_free_i64(t
);
2564 /* Size changing operations. */
2566 void tcg_gen_extrl_i64_i32(TCGv_i32 ret
, TCGv_i64 arg
)
2568 if (TCG_TARGET_REG_BITS
== 32) {
2569 tcg_gen_mov_i32(ret
, TCGV_LOW(arg
));
2570 } else if (TCG_TARGET_HAS_extrl_i64_i32
) {
2571 tcg_gen_op2(INDEX_op_extrl_i64_i32
,
2572 tcgv_i32_arg(ret
), tcgv_i64_arg(arg
));
2574 tcg_gen_mov_i32(ret
, (TCGv_i32
)arg
);
2578 void tcg_gen_extrh_i64_i32(TCGv_i32 ret
, TCGv_i64 arg
)
2580 if (TCG_TARGET_REG_BITS
== 32) {
2581 tcg_gen_mov_i32(ret
, TCGV_HIGH(arg
));
2582 } else if (TCG_TARGET_HAS_extrh_i64_i32
) {
2583 tcg_gen_op2(INDEX_op_extrh_i64_i32
,
2584 tcgv_i32_arg(ret
), tcgv_i64_arg(arg
));
2586 TCGv_i64 t
= tcg_temp_new_i64();
2587 tcg_gen_shri_i64(t
, arg
, 32);
2588 tcg_gen_mov_i32(ret
, (TCGv_i32
)t
);
2589 tcg_temp_free_i64(t
);
2593 void tcg_gen_extu_i32_i64(TCGv_i64 ret
, TCGv_i32 arg
)
2595 if (TCG_TARGET_REG_BITS
== 32) {
2596 tcg_gen_mov_i32(TCGV_LOW(ret
), arg
);
2597 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2599 tcg_gen_op2(INDEX_op_extu_i32_i64
,
2600 tcgv_i64_arg(ret
), tcgv_i32_arg(arg
));
2604 void tcg_gen_ext_i32_i64(TCGv_i64 ret
, TCGv_i32 arg
)
2606 if (TCG_TARGET_REG_BITS
== 32) {
2607 tcg_gen_mov_i32(TCGV_LOW(ret
), arg
);
2608 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
2610 tcg_gen_op2(INDEX_op_ext_i32_i64
,
2611 tcgv_i64_arg(ret
), tcgv_i32_arg(arg
));
2615 void tcg_gen_concat_i32_i64(TCGv_i64 dest
, TCGv_i32 low
, TCGv_i32 high
)
2619 if (TCG_TARGET_REG_BITS
== 32) {
2620 tcg_gen_mov_i32(TCGV_LOW(dest
), low
);
2621 tcg_gen_mov_i32(TCGV_HIGH(dest
), high
);
2625 tmp
= tcg_temp_new_i64();
2626 /* These extensions are only needed for type correctness.
2627 We may be able to do better given target specific information. */
2628 tcg_gen_extu_i32_i64(tmp
, high
);
2629 tcg_gen_extu_i32_i64(dest
, low
);
2630 /* If deposit is available, use it. Otherwise use the extra
2631 knowledge that we have of the zero-extensions above. */
2632 if (TCG_TARGET_HAS_deposit_i64
&& TCG_TARGET_deposit_i64_valid(32, 32)) {
2633 tcg_gen_deposit_i64(dest
, dest
, tmp
, 32, 32);
2635 tcg_gen_shli_i64(tmp
, tmp
, 32);
2636 tcg_gen_or_i64(dest
, dest
, tmp
);
2638 tcg_temp_free_i64(tmp
);
2641 void tcg_gen_extr_i64_i32(TCGv_i32 lo
, TCGv_i32 hi
, TCGv_i64 arg
)
2643 if (TCG_TARGET_REG_BITS
== 32) {
2644 tcg_gen_mov_i32(lo
, TCGV_LOW(arg
));
2645 tcg_gen_mov_i32(hi
, TCGV_HIGH(arg
));
2647 tcg_gen_extrl_i64_i32(lo
, arg
);
2648 tcg_gen_extrh_i64_i32(hi
, arg
);
2652 void tcg_gen_extr32_i64(TCGv_i64 lo
, TCGv_i64 hi
, TCGv_i64 arg
)
2654 tcg_gen_ext32u_i64(lo
, arg
);
2655 tcg_gen_shri_i64(hi
, arg
, 32);
2658 /* QEMU specific operations. */
2660 void tcg_gen_exit_tb(const TranslationBlock
*tb
, unsigned idx
)
2663 * Let the jit code return the read-only version of the
2664 * TranslationBlock, so that we minimize the pc-relative
2665 * distance of the address of the exit_tb code to TB.
2666 * This will improve utilization of pc-relative address loads.
2668 * TODO: Move this to translator_loop, so that all const
2669 * TranslationBlock pointers refer to read-only memory.
2670 * This requires coordination with targets that do not use
2671 * the translator_loop.
2673 uintptr_t val
= (uintptr_t)tcg_splitwx_to_rx((void *)tb
) + idx
;
2676 tcg_debug_assert(idx
== 0);
2677 } else if (idx
<= TB_EXIT_IDXMAX
) {
2678 #ifdef CONFIG_DEBUG_TCG
2679 /* This is an exit following a goto_tb. Verify that we have
2680 seen this numbered exit before, via tcg_gen_goto_tb. */
2681 tcg_debug_assert(tcg_ctx
->goto_tb_issue_mask
& (1 << idx
));
2683 /* When not chaining, exit without indicating a link. */
2684 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN
)) {
2688 /* This is an exit via the exitreq label. */
2689 tcg_debug_assert(idx
== TB_EXIT_REQUESTED
);
2692 plugin_gen_disable_mem_helpers();
2693 tcg_gen_op1i(INDEX_op_exit_tb
, val
);
2696 void tcg_gen_goto_tb(unsigned idx
)
2698 /* We only support two chained exits. */
2699 tcg_debug_assert(idx
<= TB_EXIT_IDXMAX
);
2700 #ifdef CONFIG_DEBUG_TCG
2701 /* Verify that we havn't seen this numbered exit before. */
2702 tcg_debug_assert((tcg_ctx
->goto_tb_issue_mask
& (1 << idx
)) == 0);
2703 tcg_ctx
->goto_tb_issue_mask
|= 1 << idx
;
2705 plugin_gen_disable_mem_helpers();
2706 /* When not chaining, we simply fall through to the "fallback" exit. */
2707 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN
)) {
2708 tcg_gen_op1i(INDEX_op_goto_tb
, idx
);
2712 void tcg_gen_lookup_and_goto_ptr(void)
2714 if (TCG_TARGET_HAS_goto_ptr
&& !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN
)) {
2717 plugin_gen_disable_mem_helpers();
2718 ptr
= tcg_temp_new_ptr();
2719 gen_helper_lookup_tb_ptr(ptr
, cpu_env
);
2720 tcg_gen_op1i(INDEX_op_goto_ptr
, tcgv_ptr_arg(ptr
));
2721 tcg_temp_free_ptr(ptr
);
2723 tcg_gen_exit_tb(NULL
, 0);
2727 static inline MemOp
tcg_canonicalize_memop(MemOp op
, bool is64
, bool st
)
2729 /* Trigger the asserts within as early as possible. */
2730 (void)get_alignment_bits(op
);
2732 switch (op
& MO_SIZE
) {
2755 static void gen_ldst_i32(TCGOpcode opc
, TCGv_i32 val
, TCGv addr
,
2756 MemOp memop
, TCGArg idx
)
2758 TCGMemOpIdx oi
= make_memop_idx(memop
, idx
);
2759 #if TARGET_LONG_BITS == 32
2760 tcg_gen_op3i_i32(opc
, val
, addr
, oi
);
2762 if (TCG_TARGET_REG_BITS
== 32) {
2763 tcg_gen_op4i_i32(opc
, val
, TCGV_LOW(addr
), TCGV_HIGH(addr
), oi
);
2765 tcg_gen_op3(opc
, tcgv_i32_arg(val
), tcgv_i64_arg(addr
), oi
);
2770 static void gen_ldst_i64(TCGOpcode opc
, TCGv_i64 val
, TCGv addr
,
2771 MemOp memop
, TCGArg idx
)
2773 TCGMemOpIdx oi
= make_memop_idx(memop
, idx
);
2774 #if TARGET_LONG_BITS == 32
2775 if (TCG_TARGET_REG_BITS
== 32) {
2776 tcg_gen_op4i_i32(opc
, TCGV_LOW(val
), TCGV_HIGH(val
), addr
, oi
);
2778 tcg_gen_op3(opc
, tcgv_i64_arg(val
), tcgv_i32_arg(addr
), oi
);
2781 if (TCG_TARGET_REG_BITS
== 32) {
2782 tcg_gen_op5i_i32(opc
, TCGV_LOW(val
), TCGV_HIGH(val
),
2783 TCGV_LOW(addr
), TCGV_HIGH(addr
), oi
);
2785 tcg_gen_op3i_i64(opc
, val
, addr
, oi
);
2790 static void tcg_gen_req_mo(TCGBar type
)
2792 #ifdef TCG_GUEST_DEFAULT_MO
2793 type
&= TCG_GUEST_DEFAULT_MO
;
2795 type
&= ~TCG_TARGET_DEFAULT_MO
;
2797 tcg_gen_mb(type
| TCG_BAR_SC
);
2801 static inline TCGv
plugin_prep_mem_callbacks(TCGv vaddr
)
2803 #ifdef CONFIG_PLUGIN
2804 if (tcg_ctx
->plugin_insn
!= NULL
) {
2805 /* Save a copy of the vaddr for use after a load. */
2806 TCGv temp
= tcg_temp_new();
2807 tcg_gen_mov_tl(temp
, vaddr
);
2814 static inline void plugin_gen_mem_callbacks(TCGv vaddr
, uint16_t info
)
2816 #ifdef CONFIG_PLUGIN
2817 if (tcg_ctx
->plugin_insn
!= NULL
) {
2818 plugin_gen_empty_mem_callback(vaddr
, info
);
2819 tcg_temp_free(vaddr
);
2824 void tcg_gen_qemu_ld_i32(TCGv_i32 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2827 uint16_t info
= trace_mem_get_info(memop
, idx
, 0);
2829 tcg_gen_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2830 memop
= tcg_canonicalize_memop(memop
, 0, 0);
2831 trace_guest_mem_before_tcg(tcg_ctx
->cpu
, cpu_env
, addr
, info
);
2834 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2836 /* The bswap primitive requires zero-extended input. */
2837 if ((memop
& MO_SSIZE
) == MO_SW
) {
2842 addr
= plugin_prep_mem_callbacks(addr
);
2843 gen_ldst_i32(INDEX_op_qemu_ld_i32
, val
, addr
, memop
, idx
);
2844 plugin_gen_mem_callbacks(addr
, info
);
2846 if ((orig_memop
^ memop
) & MO_BSWAP
) {
2847 switch (orig_memop
& MO_SIZE
) {
2849 tcg_gen_bswap16_i32(val
, val
);
2850 if (orig_memop
& MO_SIGN
) {
2851 tcg_gen_ext16s_i32(val
, val
);
2855 tcg_gen_bswap32_i32(val
, val
);
2858 g_assert_not_reached();
2863 void tcg_gen_qemu_st_i32(TCGv_i32 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2865 TCGv_i32 swap
= NULL
;
2866 uint16_t info
= trace_mem_get_info(memop
, idx
, 1);
2868 tcg_gen_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2869 memop
= tcg_canonicalize_memop(memop
, 0, 1);
2870 trace_guest_mem_before_tcg(tcg_ctx
->cpu
, cpu_env
, addr
, info
);
2872 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2873 swap
= tcg_temp_new_i32();
2874 switch (memop
& MO_SIZE
) {
2876 tcg_gen_ext16u_i32(swap
, val
);
2877 tcg_gen_bswap16_i32(swap
, swap
);
2880 tcg_gen_bswap32_i32(swap
, val
);
2883 g_assert_not_reached();
2889 addr
= plugin_prep_mem_callbacks(addr
);
2890 if (TCG_TARGET_HAS_qemu_st8_i32
&& (memop
& MO_SIZE
) == MO_8
) {
2891 gen_ldst_i32(INDEX_op_qemu_st8_i32
, val
, addr
, memop
, idx
);
2893 gen_ldst_i32(INDEX_op_qemu_st_i32
, val
, addr
, memop
, idx
);
2895 plugin_gen_mem_callbacks(addr
, info
);
2898 tcg_temp_free_i32(swap
);
2902 void tcg_gen_qemu_ld_i64(TCGv_i64 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2907 if (TCG_TARGET_REG_BITS
== 32 && (memop
& MO_SIZE
) < MO_64
) {
2908 tcg_gen_qemu_ld_i32(TCGV_LOW(val
), addr
, idx
, memop
);
2909 if (memop
& MO_SIGN
) {
2910 tcg_gen_sari_i32(TCGV_HIGH(val
), TCGV_LOW(val
), 31);
2912 tcg_gen_movi_i32(TCGV_HIGH(val
), 0);
2917 tcg_gen_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2918 memop
= tcg_canonicalize_memop(memop
, 1, 0);
2919 info
= trace_mem_get_info(memop
, idx
, 0);
2920 trace_guest_mem_before_tcg(tcg_ctx
->cpu
, cpu_env
, addr
, info
);
2923 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2925 /* The bswap primitive requires zero-extended input. */
2926 if ((memop
& MO_SIGN
) && (memop
& MO_SIZE
) < MO_64
) {
2931 addr
= plugin_prep_mem_callbacks(addr
);
2932 gen_ldst_i64(INDEX_op_qemu_ld_i64
, val
, addr
, memop
, idx
);
2933 plugin_gen_mem_callbacks(addr
, info
);
2935 if ((orig_memop
^ memop
) & MO_BSWAP
) {
2936 switch (orig_memop
& MO_SIZE
) {
2938 tcg_gen_bswap16_i64(val
, val
);
2939 if (orig_memop
& MO_SIGN
) {
2940 tcg_gen_ext16s_i64(val
, val
);
2944 tcg_gen_bswap32_i64(val
, val
);
2945 if (orig_memop
& MO_SIGN
) {
2946 tcg_gen_ext32s_i64(val
, val
);
2950 tcg_gen_bswap64_i64(val
, val
);
2953 g_assert_not_reached();
2958 void tcg_gen_qemu_st_i64(TCGv_i64 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2960 TCGv_i64 swap
= NULL
;
2963 if (TCG_TARGET_REG_BITS
== 32 && (memop
& MO_SIZE
) < MO_64
) {
2964 tcg_gen_qemu_st_i32(TCGV_LOW(val
), addr
, idx
, memop
);
2968 tcg_gen_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2969 memop
= tcg_canonicalize_memop(memop
, 1, 1);
2970 info
= trace_mem_get_info(memop
, idx
, 1);
2971 trace_guest_mem_before_tcg(tcg_ctx
->cpu
, cpu_env
, addr
, info
);
2973 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2974 swap
= tcg_temp_new_i64();
2975 switch (memop
& MO_SIZE
) {
2977 tcg_gen_ext16u_i64(swap
, val
);
2978 tcg_gen_bswap16_i64(swap
, swap
);
2981 tcg_gen_ext32u_i64(swap
, val
);
2982 tcg_gen_bswap32_i64(swap
, swap
);
2985 tcg_gen_bswap64_i64(swap
, val
);
2988 g_assert_not_reached();
2994 addr
= plugin_prep_mem_callbacks(addr
);
2995 gen_ldst_i64(INDEX_op_qemu_st_i64
, val
, addr
, memop
, idx
);
2996 plugin_gen_mem_callbacks(addr
, info
);
2999 tcg_temp_free_i64(swap
);
3003 static void tcg_gen_ext_i32(TCGv_i32 ret
, TCGv_i32 val
, MemOp opc
)
3005 switch (opc
& MO_SSIZE
) {
3007 tcg_gen_ext8s_i32(ret
, val
);
3010 tcg_gen_ext8u_i32(ret
, val
);
3013 tcg_gen_ext16s_i32(ret
, val
);
3016 tcg_gen_ext16u_i32(ret
, val
);
3019 tcg_gen_mov_i32(ret
, val
);
3024 static void tcg_gen_ext_i64(TCGv_i64 ret
, TCGv_i64 val
, MemOp opc
)
3026 switch (opc
& MO_SSIZE
) {
3028 tcg_gen_ext8s_i64(ret
, val
);
3031 tcg_gen_ext8u_i64(ret
, val
);
3034 tcg_gen_ext16s_i64(ret
, val
);
3037 tcg_gen_ext16u_i64(ret
, val
);
3040 tcg_gen_ext32s_i64(ret
, val
);
3043 tcg_gen_ext32u_i64(ret
, val
);
3046 tcg_gen_mov_i64(ret
, val
);
3051 #ifdef CONFIG_SOFTMMU
3052 typedef void (*gen_atomic_cx_i32
)(TCGv_i32
, TCGv_env
, TCGv
,
3053 TCGv_i32
, TCGv_i32
, TCGv_i32
);
3054 typedef void (*gen_atomic_cx_i64
)(TCGv_i64
, TCGv_env
, TCGv
,
3055 TCGv_i64
, TCGv_i64
, TCGv_i32
);
3056 typedef void (*gen_atomic_op_i32
)(TCGv_i32
, TCGv_env
, TCGv
,
3057 TCGv_i32
, TCGv_i32
);
3058 typedef void (*gen_atomic_op_i64
)(TCGv_i64
, TCGv_env
, TCGv
,
3059 TCGv_i64
, TCGv_i32
);
3061 typedef void (*gen_atomic_cx_i32
)(TCGv_i32
, TCGv_env
, TCGv
, TCGv_i32
, TCGv_i32
);
3062 typedef void (*gen_atomic_cx_i64
)(TCGv_i64
, TCGv_env
, TCGv
, TCGv_i64
, TCGv_i64
);
3063 typedef void (*gen_atomic_op_i32
)(TCGv_i32
, TCGv_env
, TCGv
, TCGv_i32
);
3064 typedef void (*gen_atomic_op_i64
)(TCGv_i64
, TCGv_env
, TCGv
, TCGv_i64
);
3067 #ifdef CONFIG_ATOMIC64
3068 # define WITH_ATOMIC64(X) X,
3070 # define WITH_ATOMIC64(X)
3073 static void * const table_cmpxchg
[16] = {
3074 [MO_8
] = gen_helper_atomic_cmpxchgb
,
3075 [MO_16
| MO_LE
] = gen_helper_atomic_cmpxchgw_le
,
3076 [MO_16
| MO_BE
] = gen_helper_atomic_cmpxchgw_be
,
3077 [MO_32
| MO_LE
] = gen_helper_atomic_cmpxchgl_le
,
3078 [MO_32
| MO_BE
] = gen_helper_atomic_cmpxchgl_be
,
3079 WITH_ATOMIC64([MO_64
| MO_LE
] = gen_helper_atomic_cmpxchgq_le
)
3080 WITH_ATOMIC64([MO_64
| MO_BE
] = gen_helper_atomic_cmpxchgq_be
)
3083 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv
, TCGv addr
, TCGv_i32 cmpv
,
3084 TCGv_i32 newv
, TCGArg idx
, MemOp memop
)
3086 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3088 if (!(tcg_ctx
->tb_cflags
& CF_PARALLEL
)) {
3089 TCGv_i32 t1
= tcg_temp_new_i32();
3090 TCGv_i32 t2
= tcg_temp_new_i32();
3092 tcg_gen_ext_i32(t2
, cmpv
, memop
& MO_SIZE
);
3094 tcg_gen_qemu_ld_i32(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3095 tcg_gen_movcond_i32(TCG_COND_EQ
, t2
, t1
, t2
, newv
, t1
);
3096 tcg_gen_qemu_st_i32(t2
, addr
, idx
, memop
);
3097 tcg_temp_free_i32(t2
);
3099 if (memop
& MO_SIGN
) {
3100 tcg_gen_ext_i32(retv
, t1
, memop
);
3102 tcg_gen_mov_i32(retv
, t1
);
3104 tcg_temp_free_i32(t1
);
3106 gen_atomic_cx_i32 gen
;
3108 gen
= table_cmpxchg
[memop
& (MO_SIZE
| MO_BSWAP
)];
3109 tcg_debug_assert(gen
!= NULL
);
3111 #ifdef CONFIG_SOFTMMU
3113 TCGMemOpIdx oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3114 gen(retv
, cpu_env
, addr
, cmpv
, newv
, tcg_constant_i32(oi
));
3117 gen(retv
, cpu_env
, addr
, cmpv
, newv
);
3120 if (memop
& MO_SIGN
) {
3121 tcg_gen_ext_i32(retv
, retv
, memop
);
3126 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv
, TCGv addr
, TCGv_i64 cmpv
,
3127 TCGv_i64 newv
, TCGArg idx
, MemOp memop
)
3129 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3131 if (!(tcg_ctx
->tb_cflags
& CF_PARALLEL
)) {
3132 TCGv_i64 t1
= tcg_temp_new_i64();
3133 TCGv_i64 t2
= tcg_temp_new_i64();
3135 tcg_gen_ext_i64(t2
, cmpv
, memop
& MO_SIZE
);
3137 tcg_gen_qemu_ld_i64(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3138 tcg_gen_movcond_i64(TCG_COND_EQ
, t2
, t1
, t2
, newv
, t1
);
3139 tcg_gen_qemu_st_i64(t2
, addr
, idx
, memop
);
3140 tcg_temp_free_i64(t2
);
3142 if (memop
& MO_SIGN
) {
3143 tcg_gen_ext_i64(retv
, t1
, memop
);
3145 tcg_gen_mov_i64(retv
, t1
);
3147 tcg_temp_free_i64(t1
);
3148 } else if ((memop
& MO_SIZE
) == MO_64
) {
3149 #ifdef CONFIG_ATOMIC64
3150 gen_atomic_cx_i64 gen
;
3152 gen
= table_cmpxchg
[memop
& (MO_SIZE
| MO_BSWAP
)];
3153 tcg_debug_assert(gen
!= NULL
);
3155 #ifdef CONFIG_SOFTMMU
3157 TCGMemOpIdx oi
= make_memop_idx(memop
, idx
);
3158 gen(retv
, cpu_env
, addr
, cmpv
, newv
, tcg_constant_i32(oi
));
3161 gen(retv
, cpu_env
, addr
, cmpv
, newv
);
3164 gen_helper_exit_atomic(cpu_env
);
3165 /* Produce a result, so that we have a well-formed opcode stream
3166 with respect to uses of the result in the (dead) code following. */
3167 tcg_gen_movi_i64(retv
, 0);
3168 #endif /* CONFIG_ATOMIC64 */
3170 TCGv_i32 c32
= tcg_temp_new_i32();
3171 TCGv_i32 n32
= tcg_temp_new_i32();
3172 TCGv_i32 r32
= tcg_temp_new_i32();
3174 tcg_gen_extrl_i64_i32(c32
, cmpv
);
3175 tcg_gen_extrl_i64_i32(n32
, newv
);
3176 tcg_gen_atomic_cmpxchg_i32(r32
, addr
, c32
, n32
, idx
, memop
& ~MO_SIGN
);
3177 tcg_temp_free_i32(c32
);
3178 tcg_temp_free_i32(n32
);
3180 tcg_gen_extu_i32_i64(retv
, r32
);
3181 tcg_temp_free_i32(r32
);
3183 if (memop
& MO_SIGN
) {
3184 tcg_gen_ext_i64(retv
, retv
, memop
);
3189 static void do_nonatomic_op_i32(TCGv_i32 ret
, TCGv addr
, TCGv_i32 val
,
3190 TCGArg idx
, MemOp memop
, bool new_val
,
3191 void (*gen
)(TCGv_i32
, TCGv_i32
, TCGv_i32
))
3193 TCGv_i32 t1
= tcg_temp_new_i32();
3194 TCGv_i32 t2
= tcg_temp_new_i32();
3196 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3198 tcg_gen_qemu_ld_i32(t1
, addr
, idx
, memop
);
3199 tcg_gen_ext_i32(t2
, val
, memop
);
3201 tcg_gen_qemu_st_i32(t2
, addr
, idx
, memop
);
3203 tcg_gen_ext_i32(ret
, (new_val
? t2
: t1
), memop
);
3204 tcg_temp_free_i32(t1
);
3205 tcg_temp_free_i32(t2
);
3208 static void do_atomic_op_i32(TCGv_i32 ret
, TCGv addr
, TCGv_i32 val
,
3209 TCGArg idx
, MemOp memop
, void * const table
[])
3211 gen_atomic_op_i32 gen
;
3213 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3215 gen
= table
[memop
& (MO_SIZE
| MO_BSWAP
)];
3216 tcg_debug_assert(gen
!= NULL
);
3218 #ifdef CONFIG_SOFTMMU
3220 TCGMemOpIdx oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3221 gen(ret
, cpu_env
, addr
, val
, tcg_constant_i32(oi
));
3224 gen(ret
, cpu_env
, addr
, val
);
3227 if (memop
& MO_SIGN
) {
3228 tcg_gen_ext_i32(ret
, ret
, memop
);
3232 static void do_nonatomic_op_i64(TCGv_i64 ret
, TCGv addr
, TCGv_i64 val
,
3233 TCGArg idx
, MemOp memop
, bool new_val
,
3234 void (*gen
)(TCGv_i64
, TCGv_i64
, TCGv_i64
))
3236 TCGv_i64 t1
= tcg_temp_new_i64();
3237 TCGv_i64 t2
= tcg_temp_new_i64();
3239 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3241 tcg_gen_qemu_ld_i64(t1
, addr
, idx
, memop
);
3242 tcg_gen_ext_i64(t2
, val
, memop
);
3244 tcg_gen_qemu_st_i64(t2
, addr
, idx
, memop
);
3246 tcg_gen_ext_i64(ret
, (new_val
? t2
: t1
), memop
);
3247 tcg_temp_free_i64(t1
);
3248 tcg_temp_free_i64(t2
);
3251 static void do_atomic_op_i64(TCGv_i64 ret
, TCGv addr
, TCGv_i64 val
,
3252 TCGArg idx
, MemOp memop
, void * const table
[])
3254 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3256 if ((memop
& MO_SIZE
) == MO_64
) {
3257 #ifdef CONFIG_ATOMIC64
3258 gen_atomic_op_i64 gen
;
3260 gen
= table
[memop
& (MO_SIZE
| MO_BSWAP
)];
3261 tcg_debug_assert(gen
!= NULL
);
3263 #ifdef CONFIG_SOFTMMU
3265 TCGMemOpIdx oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3266 gen(ret
, cpu_env
, addr
, val
, tcg_constant_i32(oi
));
3269 gen(ret
, cpu_env
, addr
, val
);
3272 gen_helper_exit_atomic(cpu_env
);
3273 /* Produce a result, so that we have a well-formed opcode stream
3274 with respect to uses of the result in the (dead) code following. */
3275 tcg_gen_movi_i64(ret
, 0);
3276 #endif /* CONFIG_ATOMIC64 */
3278 TCGv_i32 v32
= tcg_temp_new_i32();
3279 TCGv_i32 r32
= tcg_temp_new_i32();
3281 tcg_gen_extrl_i64_i32(v32
, val
);
3282 do_atomic_op_i32(r32
, addr
, v32
, idx
, memop
& ~MO_SIGN
, table
);
3283 tcg_temp_free_i32(v32
);
3285 tcg_gen_extu_i32_i64(ret
, r32
);
3286 tcg_temp_free_i32(r32
);
3288 if (memop
& MO_SIGN
) {
3289 tcg_gen_ext_i64(ret
, ret
, memop
);
3294 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3295 static void * const table_##NAME[16] = { \
3296 [MO_8] = gen_helper_atomic_##NAME##b, \
3297 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3298 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3299 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3300 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
3301 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3302 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
3304 void tcg_gen_atomic_##NAME##_i32 \
3305 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \
3307 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3308 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3310 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3311 tcg_gen_##OP##_i32); \
3314 void tcg_gen_atomic_##NAME##_i64 \
3315 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \
3317 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3318 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3320 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3321 tcg_gen_##OP##_i64); \
3325 GEN_ATOMIC_HELPER(fetch_add
, add
, 0)
3326 GEN_ATOMIC_HELPER(fetch_and
, and, 0)
3327 GEN_ATOMIC_HELPER(fetch_or
, or, 0)
3328 GEN_ATOMIC_HELPER(fetch_xor
, xor, 0)
3329 GEN_ATOMIC_HELPER(fetch_smin
, smin
, 0)
3330 GEN_ATOMIC_HELPER(fetch_umin
, umin
, 0)
3331 GEN_ATOMIC_HELPER(fetch_smax
, smax
, 0)
3332 GEN_ATOMIC_HELPER(fetch_umax
, umax
, 0)
3334 GEN_ATOMIC_HELPER(add_fetch
, add
, 1)
3335 GEN_ATOMIC_HELPER(and_fetch
, and, 1)
3336 GEN_ATOMIC_HELPER(or_fetch
, or, 1)
3337 GEN_ATOMIC_HELPER(xor_fetch
, xor, 1)
3338 GEN_ATOMIC_HELPER(smin_fetch
, smin
, 1)
3339 GEN_ATOMIC_HELPER(umin_fetch
, umin
, 1)
3340 GEN_ATOMIC_HELPER(smax_fetch
, smax
, 1)
3341 GEN_ATOMIC_HELPER(umax_fetch
, umax
, 1)
3343 static void tcg_gen_mov2_i32(TCGv_i32 r
, TCGv_i32 a
, TCGv_i32 b
)
3345 tcg_gen_mov_i32(r
, b
);
3348 static void tcg_gen_mov2_i64(TCGv_i64 r
, TCGv_i64 a
, TCGv_i64 b
)
3350 tcg_gen_mov_i64(r
, b
);
3353 GEN_ATOMIC_HELPER(xchg
, mov2
, 0)
3355 #undef GEN_ATOMIC_HELPER