2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 #include "qemu/osdep.h"
26 #include "qemu-common.h"
28 #include "exec/exec-all.h"
32 #include "trace-tcg.h"
33 #include "trace/mem.h"
35 /* Reduce the number of ifdefs below. This assumes that all uses of
36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
37 the compiler can eliminate. */
38 #if TCG_TARGET_REG_BITS == 64
39 extern TCGv_i32
TCGV_LOW_link_error(TCGv_i64
);
40 extern TCGv_i32
TCGV_HIGH_link_error(TCGv_i64
);
41 #define TCGV_LOW TCGV_LOW_link_error
42 #define TCGV_HIGH TCGV_HIGH_link_error
45 void tcg_gen_op1(TCGOpcode opc
, TCGArg a1
)
47 TCGOp
*op
= tcg_emit_op(opc
);
51 void tcg_gen_op2(TCGOpcode opc
, TCGArg a1
, TCGArg a2
)
53 TCGOp
*op
= tcg_emit_op(opc
);
58 void tcg_gen_op3(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
)
60 TCGOp
*op
= tcg_emit_op(opc
);
66 void tcg_gen_op4(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
, TCGArg a4
)
68 TCGOp
*op
= tcg_emit_op(opc
);
75 void tcg_gen_op5(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
,
78 TCGOp
*op
= tcg_emit_op(opc
);
86 void tcg_gen_op6(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
,
87 TCGArg a4
, TCGArg a5
, TCGArg a6
)
89 TCGOp
*op
= tcg_emit_op(opc
);
98 void tcg_gen_mb(TCGBar mb_type
)
100 if (tcg_ctx
->tb_cflags
& CF_PARALLEL
) {
101 tcg_gen_op1(INDEX_op_mb
, mb_type
);
107 void tcg_gen_addi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
109 /* some cases can be optimized here */
111 tcg_gen_mov_i32(ret
, arg1
);
113 TCGv_i32 t0
= tcg_const_i32(arg2
);
114 tcg_gen_add_i32(ret
, arg1
, t0
);
115 tcg_temp_free_i32(t0
);
119 void tcg_gen_subfi_i32(TCGv_i32 ret
, int32_t arg1
, TCGv_i32 arg2
)
121 if (arg1
== 0 && TCG_TARGET_HAS_neg_i32
) {
122 /* Don't recurse with tcg_gen_neg_i32. */
123 tcg_gen_op2_i32(INDEX_op_neg_i32
, ret
, arg2
);
125 TCGv_i32 t0
= tcg_const_i32(arg1
);
126 tcg_gen_sub_i32(ret
, t0
, arg2
);
127 tcg_temp_free_i32(t0
);
131 void tcg_gen_subi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
133 /* some cases can be optimized here */
135 tcg_gen_mov_i32(ret
, arg1
);
137 TCGv_i32 t0
= tcg_const_i32(arg2
);
138 tcg_gen_sub_i32(ret
, arg1
, t0
);
139 tcg_temp_free_i32(t0
);
143 void tcg_gen_andi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
146 /* Some cases can be optimized here. */
149 tcg_gen_movi_i32(ret
, 0);
152 tcg_gen_mov_i32(ret
, arg1
);
155 /* Don't recurse with tcg_gen_ext8u_i32. */
156 if (TCG_TARGET_HAS_ext8u_i32
) {
157 tcg_gen_op2_i32(INDEX_op_ext8u_i32
, ret
, arg1
);
162 if (TCG_TARGET_HAS_ext16u_i32
) {
163 tcg_gen_op2_i32(INDEX_op_ext16u_i32
, ret
, arg1
);
168 t0
= tcg_const_i32(arg2
);
169 tcg_gen_and_i32(ret
, arg1
, t0
);
170 tcg_temp_free_i32(t0
);
173 void tcg_gen_ori_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
175 /* Some cases can be optimized here. */
177 tcg_gen_movi_i32(ret
, -1);
178 } else if (arg2
== 0) {
179 tcg_gen_mov_i32(ret
, arg1
);
181 TCGv_i32 t0
= tcg_const_i32(arg2
);
182 tcg_gen_or_i32(ret
, arg1
, t0
);
183 tcg_temp_free_i32(t0
);
187 void tcg_gen_xori_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
189 /* Some cases can be optimized here. */
191 tcg_gen_mov_i32(ret
, arg1
);
192 } else if (arg2
== -1 && TCG_TARGET_HAS_not_i32
) {
193 /* Don't recurse with tcg_gen_not_i32. */
194 tcg_gen_op2_i32(INDEX_op_not_i32
, ret
, arg1
);
196 TCGv_i32 t0
= tcg_const_i32(arg2
);
197 tcg_gen_xor_i32(ret
, arg1
, t0
);
198 tcg_temp_free_i32(t0
);
202 void tcg_gen_shli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
204 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
206 tcg_gen_mov_i32(ret
, arg1
);
208 TCGv_i32 t0
= tcg_const_i32(arg2
);
209 tcg_gen_shl_i32(ret
, arg1
, t0
);
210 tcg_temp_free_i32(t0
);
214 void tcg_gen_shri_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
216 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
218 tcg_gen_mov_i32(ret
, arg1
);
220 TCGv_i32 t0
= tcg_const_i32(arg2
);
221 tcg_gen_shr_i32(ret
, arg1
, t0
);
222 tcg_temp_free_i32(t0
);
226 void tcg_gen_sari_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
228 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
230 tcg_gen_mov_i32(ret
, arg1
);
232 TCGv_i32 t0
= tcg_const_i32(arg2
);
233 tcg_gen_sar_i32(ret
, arg1
, t0
);
234 tcg_temp_free_i32(t0
);
238 void tcg_gen_brcond_i32(TCGCond cond
, TCGv_i32 arg1
, TCGv_i32 arg2
, TCGLabel
*l
)
240 if (cond
== TCG_COND_ALWAYS
) {
242 } else if (cond
!= TCG_COND_NEVER
) {
244 tcg_gen_op4ii_i32(INDEX_op_brcond_i32
, arg1
, arg2
, cond
, label_arg(l
));
248 void tcg_gen_brcondi_i32(TCGCond cond
, TCGv_i32 arg1
, int32_t arg2
, TCGLabel
*l
)
250 if (cond
== TCG_COND_ALWAYS
) {
252 } else if (cond
!= TCG_COND_NEVER
) {
253 TCGv_i32 t0
= tcg_const_i32(arg2
);
254 tcg_gen_brcond_i32(cond
, arg1
, t0
, l
);
255 tcg_temp_free_i32(t0
);
259 void tcg_gen_setcond_i32(TCGCond cond
, TCGv_i32 ret
,
260 TCGv_i32 arg1
, TCGv_i32 arg2
)
262 if (cond
== TCG_COND_ALWAYS
) {
263 tcg_gen_movi_i32(ret
, 1);
264 } else if (cond
== TCG_COND_NEVER
) {
265 tcg_gen_movi_i32(ret
, 0);
267 tcg_gen_op4i_i32(INDEX_op_setcond_i32
, ret
, arg1
, arg2
, cond
);
271 void tcg_gen_setcondi_i32(TCGCond cond
, TCGv_i32 ret
,
272 TCGv_i32 arg1
, int32_t arg2
)
274 TCGv_i32 t0
= tcg_const_i32(arg2
);
275 tcg_gen_setcond_i32(cond
, ret
, arg1
, t0
);
276 tcg_temp_free_i32(t0
);
279 void tcg_gen_muli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
282 tcg_gen_movi_i32(ret
, 0);
283 } else if (is_power_of_2(arg2
)) {
284 tcg_gen_shli_i32(ret
, arg1
, ctz32(arg2
));
286 TCGv_i32 t0
= tcg_const_i32(arg2
);
287 tcg_gen_mul_i32(ret
, arg1
, t0
);
288 tcg_temp_free_i32(t0
);
292 void tcg_gen_div_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
294 if (TCG_TARGET_HAS_div_i32
) {
295 tcg_gen_op3_i32(INDEX_op_div_i32
, ret
, arg1
, arg2
);
296 } else if (TCG_TARGET_HAS_div2_i32
) {
297 TCGv_i32 t0
= tcg_temp_new_i32();
298 tcg_gen_sari_i32(t0
, arg1
, 31);
299 tcg_gen_op5_i32(INDEX_op_div2_i32
, ret
, t0
, arg1
, t0
, arg2
);
300 tcg_temp_free_i32(t0
);
302 gen_helper_div_i32(ret
, arg1
, arg2
);
306 void tcg_gen_rem_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
308 if (TCG_TARGET_HAS_rem_i32
) {
309 tcg_gen_op3_i32(INDEX_op_rem_i32
, ret
, arg1
, arg2
);
310 } else if (TCG_TARGET_HAS_div_i32
) {
311 TCGv_i32 t0
= tcg_temp_new_i32();
312 tcg_gen_op3_i32(INDEX_op_div_i32
, t0
, arg1
, arg2
);
313 tcg_gen_mul_i32(t0
, t0
, arg2
);
314 tcg_gen_sub_i32(ret
, arg1
, t0
);
315 tcg_temp_free_i32(t0
);
316 } else if (TCG_TARGET_HAS_div2_i32
) {
317 TCGv_i32 t0
= tcg_temp_new_i32();
318 tcg_gen_sari_i32(t0
, arg1
, 31);
319 tcg_gen_op5_i32(INDEX_op_div2_i32
, t0
, ret
, arg1
, t0
, arg2
);
320 tcg_temp_free_i32(t0
);
322 gen_helper_rem_i32(ret
, arg1
, arg2
);
326 void tcg_gen_divu_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
328 if (TCG_TARGET_HAS_div_i32
) {
329 tcg_gen_op3_i32(INDEX_op_divu_i32
, ret
, arg1
, arg2
);
330 } else if (TCG_TARGET_HAS_div2_i32
) {
331 TCGv_i32 t0
= tcg_temp_new_i32();
332 tcg_gen_movi_i32(t0
, 0);
333 tcg_gen_op5_i32(INDEX_op_divu2_i32
, ret
, t0
, arg1
, t0
, arg2
);
334 tcg_temp_free_i32(t0
);
336 gen_helper_divu_i32(ret
, arg1
, arg2
);
340 void tcg_gen_remu_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
342 if (TCG_TARGET_HAS_rem_i32
) {
343 tcg_gen_op3_i32(INDEX_op_remu_i32
, ret
, arg1
, arg2
);
344 } else if (TCG_TARGET_HAS_div_i32
) {
345 TCGv_i32 t0
= tcg_temp_new_i32();
346 tcg_gen_op3_i32(INDEX_op_divu_i32
, t0
, arg1
, arg2
);
347 tcg_gen_mul_i32(t0
, t0
, arg2
);
348 tcg_gen_sub_i32(ret
, arg1
, t0
);
349 tcg_temp_free_i32(t0
);
350 } else if (TCG_TARGET_HAS_div2_i32
) {
351 TCGv_i32 t0
= tcg_temp_new_i32();
352 tcg_gen_movi_i32(t0
, 0);
353 tcg_gen_op5_i32(INDEX_op_divu2_i32
, t0
, ret
, arg1
, t0
, arg2
);
354 tcg_temp_free_i32(t0
);
356 gen_helper_remu_i32(ret
, arg1
, arg2
);
360 void tcg_gen_andc_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
362 if (TCG_TARGET_HAS_andc_i32
) {
363 tcg_gen_op3_i32(INDEX_op_andc_i32
, ret
, arg1
, arg2
);
365 TCGv_i32 t0
= tcg_temp_new_i32();
366 tcg_gen_not_i32(t0
, arg2
);
367 tcg_gen_and_i32(ret
, arg1
, t0
);
368 tcg_temp_free_i32(t0
);
372 void tcg_gen_eqv_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
374 if (TCG_TARGET_HAS_eqv_i32
) {
375 tcg_gen_op3_i32(INDEX_op_eqv_i32
, ret
, arg1
, arg2
);
377 tcg_gen_xor_i32(ret
, arg1
, arg2
);
378 tcg_gen_not_i32(ret
, ret
);
382 void tcg_gen_nand_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
384 if (TCG_TARGET_HAS_nand_i32
) {
385 tcg_gen_op3_i32(INDEX_op_nand_i32
, ret
, arg1
, arg2
);
387 tcg_gen_and_i32(ret
, arg1
, arg2
);
388 tcg_gen_not_i32(ret
, ret
);
392 void tcg_gen_nor_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
394 if (TCG_TARGET_HAS_nor_i32
) {
395 tcg_gen_op3_i32(INDEX_op_nor_i32
, ret
, arg1
, arg2
);
397 tcg_gen_or_i32(ret
, arg1
, arg2
);
398 tcg_gen_not_i32(ret
, ret
);
402 void tcg_gen_orc_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
404 if (TCG_TARGET_HAS_orc_i32
) {
405 tcg_gen_op3_i32(INDEX_op_orc_i32
, ret
, arg1
, arg2
);
407 TCGv_i32 t0
= tcg_temp_new_i32();
408 tcg_gen_not_i32(t0
, arg2
);
409 tcg_gen_or_i32(ret
, arg1
, t0
);
410 tcg_temp_free_i32(t0
);
414 void tcg_gen_clz_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
416 if (TCG_TARGET_HAS_clz_i32
) {
417 tcg_gen_op3_i32(INDEX_op_clz_i32
, ret
, arg1
, arg2
);
418 } else if (TCG_TARGET_HAS_clz_i64
) {
419 TCGv_i64 t1
= tcg_temp_new_i64();
420 TCGv_i64 t2
= tcg_temp_new_i64();
421 tcg_gen_extu_i32_i64(t1
, arg1
);
422 tcg_gen_extu_i32_i64(t2
, arg2
);
423 tcg_gen_addi_i64(t2
, t2
, 32);
424 tcg_gen_clz_i64(t1
, t1
, t2
);
425 tcg_gen_extrl_i64_i32(ret
, t1
);
426 tcg_temp_free_i64(t1
);
427 tcg_temp_free_i64(t2
);
428 tcg_gen_subi_i32(ret
, ret
, 32);
430 gen_helper_clz_i32(ret
, arg1
, arg2
);
434 void tcg_gen_clzi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, uint32_t arg2
)
436 TCGv_i32 t
= tcg_const_i32(arg2
);
437 tcg_gen_clz_i32(ret
, arg1
, t
);
438 tcg_temp_free_i32(t
);
441 void tcg_gen_ctz_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
443 if (TCG_TARGET_HAS_ctz_i32
) {
444 tcg_gen_op3_i32(INDEX_op_ctz_i32
, ret
, arg1
, arg2
);
445 } else if (TCG_TARGET_HAS_ctz_i64
) {
446 TCGv_i64 t1
= tcg_temp_new_i64();
447 TCGv_i64 t2
= tcg_temp_new_i64();
448 tcg_gen_extu_i32_i64(t1
, arg1
);
449 tcg_gen_extu_i32_i64(t2
, arg2
);
450 tcg_gen_ctz_i64(t1
, t1
, t2
);
451 tcg_gen_extrl_i64_i32(ret
, t1
);
452 tcg_temp_free_i64(t1
);
453 tcg_temp_free_i64(t2
);
454 } else if (TCG_TARGET_HAS_ctpop_i32
455 || TCG_TARGET_HAS_ctpop_i64
456 || TCG_TARGET_HAS_clz_i32
457 || TCG_TARGET_HAS_clz_i64
) {
458 TCGv_i32 z
, t
= tcg_temp_new_i32();
460 if (TCG_TARGET_HAS_ctpop_i32
|| TCG_TARGET_HAS_ctpop_i64
) {
461 tcg_gen_subi_i32(t
, arg1
, 1);
462 tcg_gen_andc_i32(t
, t
, arg1
);
463 tcg_gen_ctpop_i32(t
, t
);
465 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
466 tcg_gen_neg_i32(t
, arg1
);
467 tcg_gen_and_i32(t
, t
, arg1
);
468 tcg_gen_clzi_i32(t
, t
, 32);
469 tcg_gen_xori_i32(t
, t
, 31);
471 z
= tcg_const_i32(0);
472 tcg_gen_movcond_i32(TCG_COND_EQ
, ret
, arg1
, z
, arg2
, t
);
473 tcg_temp_free_i32(t
);
474 tcg_temp_free_i32(z
);
476 gen_helper_ctz_i32(ret
, arg1
, arg2
);
480 void tcg_gen_ctzi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, uint32_t arg2
)
482 if (!TCG_TARGET_HAS_ctz_i32
&& TCG_TARGET_HAS_ctpop_i32
&& arg2
== 32) {
483 /* This equivalence has the advantage of not requiring a fixup. */
484 TCGv_i32 t
= tcg_temp_new_i32();
485 tcg_gen_subi_i32(t
, arg1
, 1);
486 tcg_gen_andc_i32(t
, t
, arg1
);
487 tcg_gen_ctpop_i32(ret
, t
);
488 tcg_temp_free_i32(t
);
490 TCGv_i32 t
= tcg_const_i32(arg2
);
491 tcg_gen_ctz_i32(ret
, arg1
, t
);
492 tcg_temp_free_i32(t
);
496 void tcg_gen_clrsb_i32(TCGv_i32 ret
, TCGv_i32 arg
)
498 if (TCG_TARGET_HAS_clz_i32
) {
499 TCGv_i32 t
= tcg_temp_new_i32();
500 tcg_gen_sari_i32(t
, arg
, 31);
501 tcg_gen_xor_i32(t
, t
, arg
);
502 tcg_gen_clzi_i32(t
, t
, 32);
503 tcg_gen_subi_i32(ret
, t
, 1);
504 tcg_temp_free_i32(t
);
506 gen_helper_clrsb_i32(ret
, arg
);
510 void tcg_gen_ctpop_i32(TCGv_i32 ret
, TCGv_i32 arg1
)
512 if (TCG_TARGET_HAS_ctpop_i32
) {
513 tcg_gen_op2_i32(INDEX_op_ctpop_i32
, ret
, arg1
);
514 } else if (TCG_TARGET_HAS_ctpop_i64
) {
515 TCGv_i64 t
= tcg_temp_new_i64();
516 tcg_gen_extu_i32_i64(t
, arg1
);
517 tcg_gen_ctpop_i64(t
, t
);
518 tcg_gen_extrl_i64_i32(ret
, t
);
519 tcg_temp_free_i64(t
);
521 gen_helper_ctpop_i32(ret
, arg1
);
525 void tcg_gen_rotl_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
527 if (TCG_TARGET_HAS_rot_i32
) {
528 tcg_gen_op3_i32(INDEX_op_rotl_i32
, ret
, arg1
, arg2
);
532 t0
= tcg_temp_new_i32();
533 t1
= tcg_temp_new_i32();
534 tcg_gen_shl_i32(t0
, arg1
, arg2
);
535 tcg_gen_subfi_i32(t1
, 32, arg2
);
536 tcg_gen_shr_i32(t1
, arg1
, t1
);
537 tcg_gen_or_i32(ret
, t0
, t1
);
538 tcg_temp_free_i32(t0
);
539 tcg_temp_free_i32(t1
);
543 void tcg_gen_rotli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, unsigned arg2
)
545 tcg_debug_assert(arg2
< 32);
546 /* some cases can be optimized here */
548 tcg_gen_mov_i32(ret
, arg1
);
549 } else if (TCG_TARGET_HAS_rot_i32
) {
550 TCGv_i32 t0
= tcg_const_i32(arg2
);
551 tcg_gen_rotl_i32(ret
, arg1
, t0
);
552 tcg_temp_free_i32(t0
);
555 t0
= tcg_temp_new_i32();
556 t1
= tcg_temp_new_i32();
557 tcg_gen_shli_i32(t0
, arg1
, arg2
);
558 tcg_gen_shri_i32(t1
, arg1
, 32 - arg2
);
559 tcg_gen_or_i32(ret
, t0
, t1
);
560 tcg_temp_free_i32(t0
);
561 tcg_temp_free_i32(t1
);
565 void tcg_gen_rotr_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
567 if (TCG_TARGET_HAS_rot_i32
) {
568 tcg_gen_op3_i32(INDEX_op_rotr_i32
, ret
, arg1
, arg2
);
572 t0
= tcg_temp_new_i32();
573 t1
= tcg_temp_new_i32();
574 tcg_gen_shr_i32(t0
, arg1
, arg2
);
575 tcg_gen_subfi_i32(t1
, 32, arg2
);
576 tcg_gen_shl_i32(t1
, arg1
, t1
);
577 tcg_gen_or_i32(ret
, t0
, t1
);
578 tcg_temp_free_i32(t0
);
579 tcg_temp_free_i32(t1
);
583 void tcg_gen_rotri_i32(TCGv_i32 ret
, TCGv_i32 arg1
, unsigned arg2
)
585 tcg_debug_assert(arg2
< 32);
586 /* some cases can be optimized here */
588 tcg_gen_mov_i32(ret
, arg1
);
590 tcg_gen_rotli_i32(ret
, arg1
, 32 - arg2
);
594 void tcg_gen_deposit_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
,
595 unsigned int ofs
, unsigned int len
)
600 tcg_debug_assert(ofs
< 32);
601 tcg_debug_assert(len
> 0);
602 tcg_debug_assert(len
<= 32);
603 tcg_debug_assert(ofs
+ len
<= 32);
606 tcg_gen_mov_i32(ret
, arg2
);
609 if (TCG_TARGET_HAS_deposit_i32
&& TCG_TARGET_deposit_i32_valid(ofs
, len
)) {
610 tcg_gen_op5ii_i32(INDEX_op_deposit_i32
, ret
, arg1
, arg2
, ofs
, len
);
614 mask
= (1u << len
) - 1;
615 t1
= tcg_temp_new_i32();
617 if (ofs
+ len
< 32) {
618 tcg_gen_andi_i32(t1
, arg2
, mask
);
619 tcg_gen_shli_i32(t1
, t1
, ofs
);
621 tcg_gen_shli_i32(t1
, arg2
, ofs
);
623 tcg_gen_andi_i32(ret
, arg1
, ~(mask
<< ofs
));
624 tcg_gen_or_i32(ret
, ret
, t1
);
626 tcg_temp_free_i32(t1
);
629 void tcg_gen_deposit_z_i32(TCGv_i32 ret
, TCGv_i32 arg
,
630 unsigned int ofs
, unsigned int len
)
632 tcg_debug_assert(ofs
< 32);
633 tcg_debug_assert(len
> 0);
634 tcg_debug_assert(len
<= 32);
635 tcg_debug_assert(ofs
+ len
<= 32);
637 if (ofs
+ len
== 32) {
638 tcg_gen_shli_i32(ret
, arg
, ofs
);
639 } else if (ofs
== 0) {
640 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
641 } else if (TCG_TARGET_HAS_deposit_i32
642 && TCG_TARGET_deposit_i32_valid(ofs
, len
)) {
643 TCGv_i32 zero
= tcg_const_i32(0);
644 tcg_gen_op5ii_i32(INDEX_op_deposit_i32
, ret
, zero
, arg
, ofs
, len
);
645 tcg_temp_free_i32(zero
);
647 /* To help two-operand hosts we prefer to zero-extend first,
648 which allows ARG to stay live. */
651 if (TCG_TARGET_HAS_ext16u_i32
) {
652 tcg_gen_ext16u_i32(ret
, arg
);
653 tcg_gen_shli_i32(ret
, ret
, ofs
);
658 if (TCG_TARGET_HAS_ext8u_i32
) {
659 tcg_gen_ext8u_i32(ret
, arg
);
660 tcg_gen_shli_i32(ret
, ret
, ofs
);
665 /* Otherwise prefer zero-extension over AND for code size. */
668 if (TCG_TARGET_HAS_ext16u_i32
) {
669 tcg_gen_shli_i32(ret
, arg
, ofs
);
670 tcg_gen_ext16u_i32(ret
, ret
);
675 if (TCG_TARGET_HAS_ext8u_i32
) {
676 tcg_gen_shli_i32(ret
, arg
, ofs
);
677 tcg_gen_ext8u_i32(ret
, ret
);
682 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
683 tcg_gen_shli_i32(ret
, ret
, ofs
);
687 void tcg_gen_extract_i32(TCGv_i32 ret
, TCGv_i32 arg
,
688 unsigned int ofs
, unsigned int len
)
690 tcg_debug_assert(ofs
< 32);
691 tcg_debug_assert(len
> 0);
692 tcg_debug_assert(len
<= 32);
693 tcg_debug_assert(ofs
+ len
<= 32);
695 /* Canonicalize certain special cases, even if extract is supported. */
696 if (ofs
+ len
== 32) {
697 tcg_gen_shri_i32(ret
, arg
, 32 - len
);
701 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
705 if (TCG_TARGET_HAS_extract_i32
706 && TCG_TARGET_extract_i32_valid(ofs
, len
)) {
707 tcg_gen_op4ii_i32(INDEX_op_extract_i32
, ret
, arg
, ofs
, len
);
711 /* Assume that zero-extension, if available, is cheaper than a shift. */
714 if (TCG_TARGET_HAS_ext16u_i32
) {
715 tcg_gen_ext16u_i32(ret
, arg
);
716 tcg_gen_shri_i32(ret
, ret
, ofs
);
721 if (TCG_TARGET_HAS_ext8u_i32
) {
722 tcg_gen_ext8u_i32(ret
, arg
);
723 tcg_gen_shri_i32(ret
, ret
, ofs
);
729 /* ??? Ideally we'd know what values are available for immediate AND.
730 Assume that 8 bits are available, plus the special case of 16,
731 so that we get ext8u, ext16u. */
733 case 1 ... 8: case 16:
734 tcg_gen_shri_i32(ret
, arg
, ofs
);
735 tcg_gen_andi_i32(ret
, ret
, (1u << len
) - 1);
738 tcg_gen_shli_i32(ret
, arg
, 32 - len
- ofs
);
739 tcg_gen_shri_i32(ret
, ret
, 32 - len
);
744 void tcg_gen_sextract_i32(TCGv_i32 ret
, TCGv_i32 arg
,
745 unsigned int ofs
, unsigned int len
)
747 tcg_debug_assert(ofs
< 32);
748 tcg_debug_assert(len
> 0);
749 tcg_debug_assert(len
<= 32);
750 tcg_debug_assert(ofs
+ len
<= 32);
752 /* Canonicalize certain special cases, even if extract is supported. */
753 if (ofs
+ len
== 32) {
754 tcg_gen_sari_i32(ret
, arg
, 32 - len
);
760 tcg_gen_ext16s_i32(ret
, arg
);
763 tcg_gen_ext8s_i32(ret
, arg
);
768 if (TCG_TARGET_HAS_sextract_i32
769 && TCG_TARGET_extract_i32_valid(ofs
, len
)) {
770 tcg_gen_op4ii_i32(INDEX_op_sextract_i32
, ret
, arg
, ofs
, len
);
774 /* Assume that sign-extension, if available, is cheaper than a shift. */
777 if (TCG_TARGET_HAS_ext16s_i32
) {
778 tcg_gen_ext16s_i32(ret
, arg
);
779 tcg_gen_sari_i32(ret
, ret
, ofs
);
784 if (TCG_TARGET_HAS_ext8s_i32
) {
785 tcg_gen_ext8s_i32(ret
, arg
);
786 tcg_gen_sari_i32(ret
, ret
, ofs
);
793 if (TCG_TARGET_HAS_ext16s_i32
) {
794 tcg_gen_shri_i32(ret
, arg
, ofs
);
795 tcg_gen_ext16s_i32(ret
, ret
);
800 if (TCG_TARGET_HAS_ext8s_i32
) {
801 tcg_gen_shri_i32(ret
, arg
, ofs
);
802 tcg_gen_ext8s_i32(ret
, ret
);
808 tcg_gen_shli_i32(ret
, arg
, 32 - len
- ofs
);
809 tcg_gen_sari_i32(ret
, ret
, 32 - len
);
813 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
814 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
816 void tcg_gen_extract2_i32(TCGv_i32 ret
, TCGv_i32 al
, TCGv_i32 ah
,
819 tcg_debug_assert(ofs
<= 32);
821 tcg_gen_mov_i32(ret
, al
);
822 } else if (ofs
== 32) {
823 tcg_gen_mov_i32(ret
, ah
);
824 } else if (al
== ah
) {
825 tcg_gen_rotri_i32(ret
, al
, ofs
);
827 TCGv_i32 t0
= tcg_temp_new_i32();
828 tcg_gen_shri_i32(t0
, al
, ofs
);
829 tcg_gen_deposit_i32(ret
, t0
, ah
, 32 - ofs
, ofs
);
830 tcg_temp_free_i32(t0
);
834 void tcg_gen_movcond_i32(TCGCond cond
, TCGv_i32 ret
, TCGv_i32 c1
,
835 TCGv_i32 c2
, TCGv_i32 v1
, TCGv_i32 v2
)
837 if (cond
== TCG_COND_ALWAYS
) {
838 tcg_gen_mov_i32(ret
, v1
);
839 } else if (cond
== TCG_COND_NEVER
) {
840 tcg_gen_mov_i32(ret
, v2
);
841 } else if (TCG_TARGET_HAS_movcond_i32
) {
842 tcg_gen_op6i_i32(INDEX_op_movcond_i32
, ret
, c1
, c2
, v1
, v2
, cond
);
844 TCGv_i32 t0
= tcg_temp_new_i32();
845 TCGv_i32 t1
= tcg_temp_new_i32();
846 tcg_gen_setcond_i32(cond
, t0
, c1
, c2
);
847 tcg_gen_neg_i32(t0
, t0
);
848 tcg_gen_and_i32(t1
, v1
, t0
);
849 tcg_gen_andc_i32(ret
, v2
, t0
);
850 tcg_gen_or_i32(ret
, ret
, t1
);
851 tcg_temp_free_i32(t0
);
852 tcg_temp_free_i32(t1
);
856 void tcg_gen_add2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 al
,
857 TCGv_i32 ah
, TCGv_i32 bl
, TCGv_i32 bh
)
859 if (TCG_TARGET_HAS_add2_i32
) {
860 tcg_gen_op6_i32(INDEX_op_add2_i32
, rl
, rh
, al
, ah
, bl
, bh
);
862 TCGv_i64 t0
= tcg_temp_new_i64();
863 TCGv_i64 t1
= tcg_temp_new_i64();
864 tcg_gen_concat_i32_i64(t0
, al
, ah
);
865 tcg_gen_concat_i32_i64(t1
, bl
, bh
);
866 tcg_gen_add_i64(t0
, t0
, t1
);
867 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
868 tcg_temp_free_i64(t0
);
869 tcg_temp_free_i64(t1
);
873 void tcg_gen_sub2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 al
,
874 TCGv_i32 ah
, TCGv_i32 bl
, TCGv_i32 bh
)
876 if (TCG_TARGET_HAS_sub2_i32
) {
877 tcg_gen_op6_i32(INDEX_op_sub2_i32
, rl
, rh
, al
, ah
, bl
, bh
);
879 TCGv_i64 t0
= tcg_temp_new_i64();
880 TCGv_i64 t1
= tcg_temp_new_i64();
881 tcg_gen_concat_i32_i64(t0
, al
, ah
);
882 tcg_gen_concat_i32_i64(t1
, bl
, bh
);
883 tcg_gen_sub_i64(t0
, t0
, t1
);
884 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
885 tcg_temp_free_i64(t0
);
886 tcg_temp_free_i64(t1
);
890 void tcg_gen_mulu2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
892 if (TCG_TARGET_HAS_mulu2_i32
) {
893 tcg_gen_op4_i32(INDEX_op_mulu2_i32
, rl
, rh
, arg1
, arg2
);
894 } else if (TCG_TARGET_HAS_muluh_i32
) {
895 TCGv_i32 t
= tcg_temp_new_i32();
896 tcg_gen_op3_i32(INDEX_op_mul_i32
, t
, arg1
, arg2
);
897 tcg_gen_op3_i32(INDEX_op_muluh_i32
, rh
, arg1
, arg2
);
898 tcg_gen_mov_i32(rl
, t
);
899 tcg_temp_free_i32(t
);
901 TCGv_i64 t0
= tcg_temp_new_i64();
902 TCGv_i64 t1
= tcg_temp_new_i64();
903 tcg_gen_extu_i32_i64(t0
, arg1
);
904 tcg_gen_extu_i32_i64(t1
, arg2
);
905 tcg_gen_mul_i64(t0
, t0
, t1
);
906 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
907 tcg_temp_free_i64(t0
);
908 tcg_temp_free_i64(t1
);
912 void tcg_gen_muls2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
914 if (TCG_TARGET_HAS_muls2_i32
) {
915 tcg_gen_op4_i32(INDEX_op_muls2_i32
, rl
, rh
, arg1
, arg2
);
916 } else if (TCG_TARGET_HAS_mulsh_i32
) {
917 TCGv_i32 t
= tcg_temp_new_i32();
918 tcg_gen_op3_i32(INDEX_op_mul_i32
, t
, arg1
, arg2
);
919 tcg_gen_op3_i32(INDEX_op_mulsh_i32
, rh
, arg1
, arg2
);
920 tcg_gen_mov_i32(rl
, t
);
921 tcg_temp_free_i32(t
);
922 } else if (TCG_TARGET_REG_BITS
== 32) {
923 TCGv_i32 t0
= tcg_temp_new_i32();
924 TCGv_i32 t1
= tcg_temp_new_i32();
925 TCGv_i32 t2
= tcg_temp_new_i32();
926 TCGv_i32 t3
= tcg_temp_new_i32();
927 tcg_gen_mulu2_i32(t0
, t1
, arg1
, arg2
);
928 /* Adjust for negative inputs. */
929 tcg_gen_sari_i32(t2
, arg1
, 31);
930 tcg_gen_sari_i32(t3
, arg2
, 31);
931 tcg_gen_and_i32(t2
, t2
, arg2
);
932 tcg_gen_and_i32(t3
, t3
, arg1
);
933 tcg_gen_sub_i32(rh
, t1
, t2
);
934 tcg_gen_sub_i32(rh
, rh
, t3
);
935 tcg_gen_mov_i32(rl
, t0
);
936 tcg_temp_free_i32(t0
);
937 tcg_temp_free_i32(t1
);
938 tcg_temp_free_i32(t2
);
939 tcg_temp_free_i32(t3
);
941 TCGv_i64 t0
= tcg_temp_new_i64();
942 TCGv_i64 t1
= tcg_temp_new_i64();
943 tcg_gen_ext_i32_i64(t0
, arg1
);
944 tcg_gen_ext_i32_i64(t1
, arg2
);
945 tcg_gen_mul_i64(t0
, t0
, t1
);
946 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
947 tcg_temp_free_i64(t0
);
948 tcg_temp_free_i64(t1
);
952 void tcg_gen_mulsu2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
954 if (TCG_TARGET_REG_BITS
== 32) {
955 TCGv_i32 t0
= tcg_temp_new_i32();
956 TCGv_i32 t1
= tcg_temp_new_i32();
957 TCGv_i32 t2
= tcg_temp_new_i32();
958 tcg_gen_mulu2_i32(t0
, t1
, arg1
, arg2
);
959 /* Adjust for negative input for the signed arg1. */
960 tcg_gen_sari_i32(t2
, arg1
, 31);
961 tcg_gen_and_i32(t2
, t2
, arg2
);
962 tcg_gen_sub_i32(rh
, t1
, t2
);
963 tcg_gen_mov_i32(rl
, t0
);
964 tcg_temp_free_i32(t0
);
965 tcg_temp_free_i32(t1
);
966 tcg_temp_free_i32(t2
);
968 TCGv_i64 t0
= tcg_temp_new_i64();
969 TCGv_i64 t1
= tcg_temp_new_i64();
970 tcg_gen_ext_i32_i64(t0
, arg1
);
971 tcg_gen_extu_i32_i64(t1
, arg2
);
972 tcg_gen_mul_i64(t0
, t0
, t1
);
973 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
974 tcg_temp_free_i64(t0
);
975 tcg_temp_free_i64(t1
);
979 void tcg_gen_ext8s_i32(TCGv_i32 ret
, TCGv_i32 arg
)
981 if (TCG_TARGET_HAS_ext8s_i32
) {
982 tcg_gen_op2_i32(INDEX_op_ext8s_i32
, ret
, arg
);
984 tcg_gen_shli_i32(ret
, arg
, 24);
985 tcg_gen_sari_i32(ret
, ret
, 24);
989 void tcg_gen_ext16s_i32(TCGv_i32 ret
, TCGv_i32 arg
)
991 if (TCG_TARGET_HAS_ext16s_i32
) {
992 tcg_gen_op2_i32(INDEX_op_ext16s_i32
, ret
, arg
);
994 tcg_gen_shli_i32(ret
, arg
, 16);
995 tcg_gen_sari_i32(ret
, ret
, 16);
999 void tcg_gen_ext8u_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1001 if (TCG_TARGET_HAS_ext8u_i32
) {
1002 tcg_gen_op2_i32(INDEX_op_ext8u_i32
, ret
, arg
);
1004 tcg_gen_andi_i32(ret
, arg
, 0xffu
);
1008 void tcg_gen_ext16u_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1010 if (TCG_TARGET_HAS_ext16u_i32
) {
1011 tcg_gen_op2_i32(INDEX_op_ext16u_i32
, ret
, arg
);
1013 tcg_gen_andi_i32(ret
, arg
, 0xffffu
);
1017 /* Note: we assume the two high bytes are set to zero */
1018 void tcg_gen_bswap16_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1020 if (TCG_TARGET_HAS_bswap16_i32
) {
1021 tcg_gen_op2_i32(INDEX_op_bswap16_i32
, ret
, arg
);
1023 TCGv_i32 t0
= tcg_temp_new_i32();
1025 tcg_gen_ext8u_i32(t0
, arg
);
1026 tcg_gen_shli_i32(t0
, t0
, 8);
1027 tcg_gen_shri_i32(ret
, arg
, 8);
1028 tcg_gen_or_i32(ret
, ret
, t0
);
1029 tcg_temp_free_i32(t0
);
1033 void tcg_gen_bswap32_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1035 if (TCG_TARGET_HAS_bswap32_i32
) {
1036 tcg_gen_op2_i32(INDEX_op_bswap32_i32
, ret
, arg
);
1038 TCGv_i32 t0
= tcg_temp_new_i32();
1039 TCGv_i32 t1
= tcg_temp_new_i32();
1040 TCGv_i32 t2
= tcg_const_i32(0x00ff00ff);
1043 tcg_gen_shri_i32(t0
, arg
, 8); /* t0 = .abc */
1044 tcg_gen_and_i32(t1
, arg
, t2
); /* t1 = .b.d */
1045 tcg_gen_and_i32(t0
, t0
, t2
); /* t0 = .a.c */
1046 tcg_gen_shli_i32(t1
, t1
, 8); /* t1 = b.d. */
1047 tcg_gen_or_i32(ret
, t0
, t1
); /* ret = badc */
1049 tcg_gen_shri_i32(t0
, ret
, 16); /* t0 = ..ba */
1050 tcg_gen_shli_i32(t1
, ret
, 16); /* t1 = dc.. */
1051 tcg_gen_or_i32(ret
, t0
, t1
); /* ret = dcba */
1053 tcg_temp_free_i32(t0
);
1054 tcg_temp_free_i32(t1
);
1055 tcg_temp_free_i32(t2
);
1059 void tcg_gen_smin_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1061 tcg_gen_movcond_i32(TCG_COND_LT
, ret
, a
, b
, a
, b
);
1064 void tcg_gen_umin_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1066 tcg_gen_movcond_i32(TCG_COND_LTU
, ret
, a
, b
, a
, b
);
1069 void tcg_gen_smax_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1071 tcg_gen_movcond_i32(TCG_COND_LT
, ret
, a
, b
, b
, a
);
1074 void tcg_gen_umax_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1076 tcg_gen_movcond_i32(TCG_COND_LTU
, ret
, a
, b
, b
, a
);
1081 #if TCG_TARGET_REG_BITS == 32
1082 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1084 void tcg_gen_discard_i64(TCGv_i64 arg
)
1086 tcg_gen_discard_i32(TCGV_LOW(arg
));
1087 tcg_gen_discard_i32(TCGV_HIGH(arg
));
1090 void tcg_gen_mov_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1092 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1093 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
));
1096 void tcg_gen_movi_i64(TCGv_i64 ret
, int64_t arg
)
1098 tcg_gen_movi_i32(TCGV_LOW(ret
), arg
);
1099 tcg_gen_movi_i32(TCGV_HIGH(ret
), arg
>> 32);
1102 void tcg_gen_ld8u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1104 tcg_gen_ld8u_i32(TCGV_LOW(ret
), arg2
, offset
);
1105 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1108 void tcg_gen_ld8s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1110 tcg_gen_ld8s_i32(TCGV_LOW(ret
), arg2
, offset
);
1111 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1114 void tcg_gen_ld16u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1116 tcg_gen_ld16u_i32(TCGV_LOW(ret
), arg2
, offset
);
1117 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1120 void tcg_gen_ld16s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1122 tcg_gen_ld16s_i32(TCGV_LOW(ret
), arg2
, offset
);
1123 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1126 void tcg_gen_ld32u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1128 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1129 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1132 void tcg_gen_ld32s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1134 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1135 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1138 void tcg_gen_ld_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1140 /* Since arg2 and ret have different types,
1141 they cannot be the same temporary */
1142 #ifdef HOST_WORDS_BIGENDIAN
1143 tcg_gen_ld_i32(TCGV_HIGH(ret
), arg2
, offset
);
1144 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
+ 4);
1146 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1147 tcg_gen_ld_i32(TCGV_HIGH(ret
), arg2
, offset
+ 4);
1151 void tcg_gen_st_i64(TCGv_i64 arg1
, TCGv_ptr arg2
, tcg_target_long offset
)
1153 #ifdef HOST_WORDS_BIGENDIAN
1154 tcg_gen_st_i32(TCGV_HIGH(arg1
), arg2
, offset
);
1155 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
+ 4);
1157 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
);
1158 tcg_gen_st_i32(TCGV_HIGH(arg1
), arg2
, offset
+ 4);
1162 void tcg_gen_and_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1164 tcg_gen_and_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1165 tcg_gen_and_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1168 void tcg_gen_or_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1170 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1171 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1174 void tcg_gen_xor_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1176 tcg_gen_xor_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1177 tcg_gen_xor_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1180 void tcg_gen_shl_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1182 gen_helper_shl_i64(ret
, arg1
, arg2
);
1185 void tcg_gen_shr_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1187 gen_helper_shr_i64(ret
, arg1
, arg2
);
1190 void tcg_gen_sar_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1192 gen_helper_sar_i64(ret
, arg1
, arg2
);
1195 void tcg_gen_mul_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1200 t0
= tcg_temp_new_i64();
1201 t1
= tcg_temp_new_i32();
1203 tcg_gen_mulu2_i32(TCGV_LOW(t0
), TCGV_HIGH(t0
),
1204 TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1206 tcg_gen_mul_i32(t1
, TCGV_LOW(arg1
), TCGV_HIGH(arg2
));
1207 tcg_gen_add_i32(TCGV_HIGH(t0
), TCGV_HIGH(t0
), t1
);
1208 tcg_gen_mul_i32(t1
, TCGV_HIGH(arg1
), TCGV_LOW(arg2
));
1209 tcg_gen_add_i32(TCGV_HIGH(t0
), TCGV_HIGH(t0
), t1
);
1211 tcg_gen_mov_i64(ret
, t0
);
1212 tcg_temp_free_i64(t0
);
1213 tcg_temp_free_i32(t1
);
1215 #endif /* TCG_TARGET_REG_SIZE == 32 */
1217 void tcg_gen_addi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1219 /* some cases can be optimized here */
1221 tcg_gen_mov_i64(ret
, arg1
);
1223 TCGv_i64 t0
= tcg_const_i64(arg2
);
1224 tcg_gen_add_i64(ret
, arg1
, t0
);
1225 tcg_temp_free_i64(t0
);
1229 void tcg_gen_subfi_i64(TCGv_i64 ret
, int64_t arg1
, TCGv_i64 arg2
)
1231 if (arg1
== 0 && TCG_TARGET_HAS_neg_i64
) {
1232 /* Don't recurse with tcg_gen_neg_i64. */
1233 tcg_gen_op2_i64(INDEX_op_neg_i64
, ret
, arg2
);
1235 TCGv_i64 t0
= tcg_const_i64(arg1
);
1236 tcg_gen_sub_i64(ret
, t0
, arg2
);
1237 tcg_temp_free_i64(t0
);
1241 void tcg_gen_subi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1243 /* some cases can be optimized here */
1245 tcg_gen_mov_i64(ret
, arg1
);
1247 TCGv_i64 t0
= tcg_const_i64(arg2
);
1248 tcg_gen_sub_i64(ret
, arg1
, t0
);
1249 tcg_temp_free_i64(t0
);
1253 void tcg_gen_andi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1257 if (TCG_TARGET_REG_BITS
== 32) {
1258 tcg_gen_andi_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1259 tcg_gen_andi_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1263 /* Some cases can be optimized here. */
1266 tcg_gen_movi_i64(ret
, 0);
1269 tcg_gen_mov_i64(ret
, arg1
);
1272 /* Don't recurse with tcg_gen_ext8u_i64. */
1273 if (TCG_TARGET_HAS_ext8u_i64
) {
1274 tcg_gen_op2_i64(INDEX_op_ext8u_i64
, ret
, arg1
);
1279 if (TCG_TARGET_HAS_ext16u_i64
) {
1280 tcg_gen_op2_i64(INDEX_op_ext16u_i64
, ret
, arg1
);
1285 if (TCG_TARGET_HAS_ext32u_i64
) {
1286 tcg_gen_op2_i64(INDEX_op_ext32u_i64
, ret
, arg1
);
1291 t0
= tcg_const_i64(arg2
);
1292 tcg_gen_and_i64(ret
, arg1
, t0
);
1293 tcg_temp_free_i64(t0
);
1296 void tcg_gen_ori_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1298 if (TCG_TARGET_REG_BITS
== 32) {
1299 tcg_gen_ori_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1300 tcg_gen_ori_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1303 /* Some cases can be optimized here. */
1305 tcg_gen_movi_i64(ret
, -1);
1306 } else if (arg2
== 0) {
1307 tcg_gen_mov_i64(ret
, arg1
);
1309 TCGv_i64 t0
= tcg_const_i64(arg2
);
1310 tcg_gen_or_i64(ret
, arg1
, t0
);
1311 tcg_temp_free_i64(t0
);
1315 void tcg_gen_xori_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1317 if (TCG_TARGET_REG_BITS
== 32) {
1318 tcg_gen_xori_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1319 tcg_gen_xori_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1322 /* Some cases can be optimized here. */
1324 tcg_gen_mov_i64(ret
, arg1
);
1325 } else if (arg2
== -1 && TCG_TARGET_HAS_not_i64
) {
1326 /* Don't recurse with tcg_gen_not_i64. */
1327 tcg_gen_op2_i64(INDEX_op_not_i64
, ret
, arg1
);
1329 TCGv_i64 t0
= tcg_const_i64(arg2
);
1330 tcg_gen_xor_i64(ret
, arg1
, t0
);
1331 tcg_temp_free_i64(t0
);
1335 static inline void tcg_gen_shifti_i64(TCGv_i64 ret
, TCGv_i64 arg1
,
1336 unsigned c
, bool right
, bool arith
)
1338 tcg_debug_assert(c
< 64);
1340 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
1341 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
1342 } else if (c
>= 32) {
1346 tcg_gen_sari_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), c
);
1347 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), 31);
1349 tcg_gen_shri_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), c
);
1350 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1353 tcg_gen_shli_i32(TCGV_HIGH(ret
), TCGV_LOW(arg1
), c
);
1354 tcg_gen_movi_i32(TCGV_LOW(ret
), 0);
1359 t0
= tcg_temp_new_i32();
1360 t1
= tcg_temp_new_i32();
1362 tcg_gen_shli_i32(t0
, TCGV_HIGH(arg1
), 32 - c
);
1364 tcg_gen_sari_i32(t1
, TCGV_HIGH(arg1
), c
);
1366 tcg_gen_shri_i32(t1
, TCGV_HIGH(arg1
), c
);
1368 tcg_gen_shri_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), c
);
1369 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), t0
);
1370 tcg_gen_mov_i32(TCGV_HIGH(ret
), t1
);
1372 tcg_gen_shri_i32(t0
, TCGV_LOW(arg1
), 32 - c
);
1373 /* Note: ret can be the same as arg1, so we use t1 */
1374 tcg_gen_shli_i32(t1
, TCGV_LOW(arg1
), c
);
1375 tcg_gen_shli_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), c
);
1376 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(ret
), t0
);
1377 tcg_gen_mov_i32(TCGV_LOW(ret
), t1
);
1379 tcg_temp_free_i32(t0
);
1380 tcg_temp_free_i32(t1
);
1384 void tcg_gen_shli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1386 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1387 if (TCG_TARGET_REG_BITS
== 32) {
1388 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 0, 0);
1389 } else if (arg2
== 0) {
1390 tcg_gen_mov_i64(ret
, arg1
);
1392 TCGv_i64 t0
= tcg_const_i64(arg2
);
1393 tcg_gen_shl_i64(ret
, arg1
, t0
);
1394 tcg_temp_free_i64(t0
);
1398 void tcg_gen_shri_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1400 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1401 if (TCG_TARGET_REG_BITS
== 32) {
1402 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 1, 0);
1403 } else if (arg2
== 0) {
1404 tcg_gen_mov_i64(ret
, arg1
);
1406 TCGv_i64 t0
= tcg_const_i64(arg2
);
1407 tcg_gen_shr_i64(ret
, arg1
, t0
);
1408 tcg_temp_free_i64(t0
);
1412 void tcg_gen_sari_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1414 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1415 if (TCG_TARGET_REG_BITS
== 32) {
1416 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 1, 1);
1417 } else if (arg2
== 0) {
1418 tcg_gen_mov_i64(ret
, arg1
);
1420 TCGv_i64 t0
= tcg_const_i64(arg2
);
1421 tcg_gen_sar_i64(ret
, arg1
, t0
);
1422 tcg_temp_free_i64(t0
);
1426 void tcg_gen_brcond_i64(TCGCond cond
, TCGv_i64 arg1
, TCGv_i64 arg2
, TCGLabel
*l
)
1428 if (cond
== TCG_COND_ALWAYS
) {
1430 } else if (cond
!= TCG_COND_NEVER
) {
1432 if (TCG_TARGET_REG_BITS
== 32) {
1433 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32
, TCGV_LOW(arg1
),
1434 TCGV_HIGH(arg1
), TCGV_LOW(arg2
),
1435 TCGV_HIGH(arg2
), cond
, label_arg(l
));
1437 tcg_gen_op4ii_i64(INDEX_op_brcond_i64
, arg1
, arg2
, cond
,
1443 void tcg_gen_brcondi_i64(TCGCond cond
, TCGv_i64 arg1
, int64_t arg2
, TCGLabel
*l
)
1445 if (cond
== TCG_COND_ALWAYS
) {
1447 } else if (cond
!= TCG_COND_NEVER
) {
1448 TCGv_i64 t0
= tcg_const_i64(arg2
);
1449 tcg_gen_brcond_i64(cond
, arg1
, t0
, l
);
1450 tcg_temp_free_i64(t0
);
1454 void tcg_gen_setcond_i64(TCGCond cond
, TCGv_i64 ret
,
1455 TCGv_i64 arg1
, TCGv_i64 arg2
)
1457 if (cond
== TCG_COND_ALWAYS
) {
1458 tcg_gen_movi_i64(ret
, 1);
1459 } else if (cond
== TCG_COND_NEVER
) {
1460 tcg_gen_movi_i64(ret
, 0);
1462 if (TCG_TARGET_REG_BITS
== 32) {
1463 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, TCGV_LOW(ret
),
1464 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1465 TCGV_LOW(arg2
), TCGV_HIGH(arg2
), cond
);
1466 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1468 tcg_gen_op4i_i64(INDEX_op_setcond_i64
, ret
, arg1
, arg2
, cond
);
1473 void tcg_gen_setcondi_i64(TCGCond cond
, TCGv_i64 ret
,
1474 TCGv_i64 arg1
, int64_t arg2
)
1476 TCGv_i64 t0
= tcg_const_i64(arg2
);
1477 tcg_gen_setcond_i64(cond
, ret
, arg1
, t0
);
1478 tcg_temp_free_i64(t0
);
1481 void tcg_gen_muli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1484 tcg_gen_movi_i64(ret
, 0);
1485 } else if (is_power_of_2(arg2
)) {
1486 tcg_gen_shli_i64(ret
, arg1
, ctz64(arg2
));
1488 TCGv_i64 t0
= tcg_const_i64(arg2
);
1489 tcg_gen_mul_i64(ret
, arg1
, t0
);
1490 tcg_temp_free_i64(t0
);
1494 void tcg_gen_div_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1496 if (TCG_TARGET_HAS_div_i64
) {
1497 tcg_gen_op3_i64(INDEX_op_div_i64
, ret
, arg1
, arg2
);
1498 } else if (TCG_TARGET_HAS_div2_i64
) {
1499 TCGv_i64 t0
= tcg_temp_new_i64();
1500 tcg_gen_sari_i64(t0
, arg1
, 63);
1501 tcg_gen_op5_i64(INDEX_op_div2_i64
, ret
, t0
, arg1
, t0
, arg2
);
1502 tcg_temp_free_i64(t0
);
1504 gen_helper_div_i64(ret
, arg1
, arg2
);
1508 void tcg_gen_rem_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1510 if (TCG_TARGET_HAS_rem_i64
) {
1511 tcg_gen_op3_i64(INDEX_op_rem_i64
, ret
, arg1
, arg2
);
1512 } else if (TCG_TARGET_HAS_div_i64
) {
1513 TCGv_i64 t0
= tcg_temp_new_i64();
1514 tcg_gen_op3_i64(INDEX_op_div_i64
, t0
, arg1
, arg2
);
1515 tcg_gen_mul_i64(t0
, t0
, arg2
);
1516 tcg_gen_sub_i64(ret
, arg1
, t0
);
1517 tcg_temp_free_i64(t0
);
1518 } else if (TCG_TARGET_HAS_div2_i64
) {
1519 TCGv_i64 t0
= tcg_temp_new_i64();
1520 tcg_gen_sari_i64(t0
, arg1
, 63);
1521 tcg_gen_op5_i64(INDEX_op_div2_i64
, t0
, ret
, arg1
, t0
, arg2
);
1522 tcg_temp_free_i64(t0
);
1524 gen_helper_rem_i64(ret
, arg1
, arg2
);
1528 void tcg_gen_divu_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1530 if (TCG_TARGET_HAS_div_i64
) {
1531 tcg_gen_op3_i64(INDEX_op_divu_i64
, ret
, arg1
, arg2
);
1532 } else if (TCG_TARGET_HAS_div2_i64
) {
1533 TCGv_i64 t0
= tcg_temp_new_i64();
1534 tcg_gen_movi_i64(t0
, 0);
1535 tcg_gen_op5_i64(INDEX_op_divu2_i64
, ret
, t0
, arg1
, t0
, arg2
);
1536 tcg_temp_free_i64(t0
);
1538 gen_helper_divu_i64(ret
, arg1
, arg2
);
1542 void tcg_gen_remu_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1544 if (TCG_TARGET_HAS_rem_i64
) {
1545 tcg_gen_op3_i64(INDEX_op_remu_i64
, ret
, arg1
, arg2
);
1546 } else if (TCG_TARGET_HAS_div_i64
) {
1547 TCGv_i64 t0
= tcg_temp_new_i64();
1548 tcg_gen_op3_i64(INDEX_op_divu_i64
, t0
, arg1
, arg2
);
1549 tcg_gen_mul_i64(t0
, t0
, arg2
);
1550 tcg_gen_sub_i64(ret
, arg1
, t0
);
1551 tcg_temp_free_i64(t0
);
1552 } else if (TCG_TARGET_HAS_div2_i64
) {
1553 TCGv_i64 t0
= tcg_temp_new_i64();
1554 tcg_gen_movi_i64(t0
, 0);
1555 tcg_gen_op5_i64(INDEX_op_divu2_i64
, t0
, ret
, arg1
, t0
, arg2
);
1556 tcg_temp_free_i64(t0
);
1558 gen_helper_remu_i64(ret
, arg1
, arg2
);
1562 void tcg_gen_ext8s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1564 if (TCG_TARGET_REG_BITS
== 32) {
1565 tcg_gen_ext8s_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1566 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1567 } else if (TCG_TARGET_HAS_ext8s_i64
) {
1568 tcg_gen_op2_i64(INDEX_op_ext8s_i64
, ret
, arg
);
1570 tcg_gen_shli_i64(ret
, arg
, 56);
1571 tcg_gen_sari_i64(ret
, ret
, 56);
1575 void tcg_gen_ext16s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1577 if (TCG_TARGET_REG_BITS
== 32) {
1578 tcg_gen_ext16s_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1579 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1580 } else if (TCG_TARGET_HAS_ext16s_i64
) {
1581 tcg_gen_op2_i64(INDEX_op_ext16s_i64
, ret
, arg
);
1583 tcg_gen_shli_i64(ret
, arg
, 48);
1584 tcg_gen_sari_i64(ret
, ret
, 48);
1588 void tcg_gen_ext32s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1590 if (TCG_TARGET_REG_BITS
== 32) {
1591 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1592 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1593 } else if (TCG_TARGET_HAS_ext32s_i64
) {
1594 tcg_gen_op2_i64(INDEX_op_ext32s_i64
, ret
, arg
);
1596 tcg_gen_shli_i64(ret
, arg
, 32);
1597 tcg_gen_sari_i64(ret
, ret
, 32);
1601 void tcg_gen_ext8u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1603 if (TCG_TARGET_REG_BITS
== 32) {
1604 tcg_gen_ext8u_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1605 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1606 } else if (TCG_TARGET_HAS_ext8u_i64
) {
1607 tcg_gen_op2_i64(INDEX_op_ext8u_i64
, ret
, arg
);
1609 tcg_gen_andi_i64(ret
, arg
, 0xffu
);
1613 void tcg_gen_ext16u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1615 if (TCG_TARGET_REG_BITS
== 32) {
1616 tcg_gen_ext16u_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1617 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1618 } else if (TCG_TARGET_HAS_ext16u_i64
) {
1619 tcg_gen_op2_i64(INDEX_op_ext16u_i64
, ret
, arg
);
1621 tcg_gen_andi_i64(ret
, arg
, 0xffffu
);
1625 void tcg_gen_ext32u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1627 if (TCG_TARGET_REG_BITS
== 32) {
1628 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1629 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1630 } else if (TCG_TARGET_HAS_ext32u_i64
) {
1631 tcg_gen_op2_i64(INDEX_op_ext32u_i64
, ret
, arg
);
1633 tcg_gen_andi_i64(ret
, arg
, 0xffffffffu
);
1637 /* Note: we assume the six high bytes are set to zero */
1638 void tcg_gen_bswap16_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1640 if (TCG_TARGET_REG_BITS
== 32) {
1641 tcg_gen_bswap16_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1642 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1643 } else if (TCG_TARGET_HAS_bswap16_i64
) {
1644 tcg_gen_op2_i64(INDEX_op_bswap16_i64
, ret
, arg
);
1646 TCGv_i64 t0
= tcg_temp_new_i64();
1648 tcg_gen_ext8u_i64(t0
, arg
);
1649 tcg_gen_shli_i64(t0
, t0
, 8);
1650 tcg_gen_shri_i64(ret
, arg
, 8);
1651 tcg_gen_or_i64(ret
, ret
, t0
);
1652 tcg_temp_free_i64(t0
);
1656 /* Note: we assume the four high bytes are set to zero */
1657 void tcg_gen_bswap32_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1659 if (TCG_TARGET_REG_BITS
== 32) {
1660 tcg_gen_bswap32_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1661 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1662 } else if (TCG_TARGET_HAS_bswap32_i64
) {
1663 tcg_gen_op2_i64(INDEX_op_bswap32_i64
, ret
, arg
);
1665 TCGv_i64 t0
= tcg_temp_new_i64();
1666 TCGv_i64 t1
= tcg_temp_new_i64();
1667 TCGv_i64 t2
= tcg_const_i64(0x00ff00ff);
1669 /* arg = ....abcd */
1670 tcg_gen_shri_i64(t0
, arg
, 8); /* t0 = .....abc */
1671 tcg_gen_and_i64(t1
, arg
, t2
); /* t1 = .....b.d */
1672 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = .....a.c */
1673 tcg_gen_shli_i64(t1
, t1
, 8); /* t1 = ....b.d. */
1674 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = ....badc */
1676 tcg_gen_shli_i64(t1
, ret
, 48); /* t1 = dc...... */
1677 tcg_gen_shri_i64(t0
, ret
, 16); /* t0 = ......ba */
1678 tcg_gen_shri_i64(t1
, t1
, 32); /* t1 = ....dc.. */
1679 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = ....dcba */
1681 tcg_temp_free_i64(t0
);
1682 tcg_temp_free_i64(t1
);
1683 tcg_temp_free_i64(t2
);
1687 void tcg_gen_bswap64_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1689 if (TCG_TARGET_REG_BITS
== 32) {
1691 t0
= tcg_temp_new_i32();
1692 t1
= tcg_temp_new_i32();
1694 tcg_gen_bswap32_i32(t0
, TCGV_LOW(arg
));
1695 tcg_gen_bswap32_i32(t1
, TCGV_HIGH(arg
));
1696 tcg_gen_mov_i32(TCGV_LOW(ret
), t1
);
1697 tcg_gen_mov_i32(TCGV_HIGH(ret
), t0
);
1698 tcg_temp_free_i32(t0
);
1699 tcg_temp_free_i32(t1
);
1700 } else if (TCG_TARGET_HAS_bswap64_i64
) {
1701 tcg_gen_op2_i64(INDEX_op_bswap64_i64
, ret
, arg
);
1703 TCGv_i64 t0
= tcg_temp_new_i64();
1704 TCGv_i64 t1
= tcg_temp_new_i64();
1705 TCGv_i64 t2
= tcg_temp_new_i64();
1707 /* arg = abcdefgh */
1708 tcg_gen_movi_i64(t2
, 0x00ff00ff00ff00ffull
);
1709 tcg_gen_shri_i64(t0
, arg
, 8); /* t0 = .abcdefg */
1710 tcg_gen_and_i64(t1
, arg
, t2
); /* t1 = .b.d.f.h */
1711 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = .a.c.e.g */
1712 tcg_gen_shli_i64(t1
, t1
, 8); /* t1 = b.d.f.h. */
1713 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = badcfehg */
1715 tcg_gen_movi_i64(t2
, 0x0000ffff0000ffffull
);
1716 tcg_gen_shri_i64(t0
, ret
, 16); /* t0 = ..badcfe */
1717 tcg_gen_and_i64(t1
, ret
, t2
); /* t1 = ..dc..hg */
1718 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = ..ba..fe */
1719 tcg_gen_shli_i64(t1
, t1
, 16); /* t1 = dc..hg.. */
1720 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = dcbahgfe */
1722 tcg_gen_shri_i64(t0
, ret
, 32); /* t0 = ....dcba */
1723 tcg_gen_shli_i64(t1
, ret
, 32); /* t1 = hgfe.... */
1724 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = hgfedcba */
1726 tcg_temp_free_i64(t0
);
1727 tcg_temp_free_i64(t1
);
1728 tcg_temp_free_i64(t2
);
1732 void tcg_gen_not_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1734 if (TCG_TARGET_REG_BITS
== 32) {
1735 tcg_gen_not_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1736 tcg_gen_not_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
));
1737 } else if (TCG_TARGET_HAS_not_i64
) {
1738 tcg_gen_op2_i64(INDEX_op_not_i64
, ret
, arg
);
1740 tcg_gen_xori_i64(ret
, arg
, -1);
1744 void tcg_gen_andc_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1746 if (TCG_TARGET_REG_BITS
== 32) {
1747 tcg_gen_andc_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1748 tcg_gen_andc_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1749 } else if (TCG_TARGET_HAS_andc_i64
) {
1750 tcg_gen_op3_i64(INDEX_op_andc_i64
, ret
, arg1
, arg2
);
1752 TCGv_i64 t0
= tcg_temp_new_i64();
1753 tcg_gen_not_i64(t0
, arg2
);
1754 tcg_gen_and_i64(ret
, arg1
, t0
);
1755 tcg_temp_free_i64(t0
);
1759 void tcg_gen_eqv_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1761 if (TCG_TARGET_REG_BITS
== 32) {
1762 tcg_gen_eqv_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1763 tcg_gen_eqv_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1764 } else if (TCG_TARGET_HAS_eqv_i64
) {
1765 tcg_gen_op3_i64(INDEX_op_eqv_i64
, ret
, arg1
, arg2
);
1767 tcg_gen_xor_i64(ret
, arg1
, arg2
);
1768 tcg_gen_not_i64(ret
, ret
);
1772 void tcg_gen_nand_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1774 if (TCG_TARGET_REG_BITS
== 32) {
1775 tcg_gen_nand_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1776 tcg_gen_nand_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1777 } else if (TCG_TARGET_HAS_nand_i64
) {
1778 tcg_gen_op3_i64(INDEX_op_nand_i64
, ret
, arg1
, arg2
);
1780 tcg_gen_and_i64(ret
, arg1
, arg2
);
1781 tcg_gen_not_i64(ret
, ret
);
1785 void tcg_gen_nor_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1787 if (TCG_TARGET_REG_BITS
== 32) {
1788 tcg_gen_nor_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1789 tcg_gen_nor_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1790 } else if (TCG_TARGET_HAS_nor_i64
) {
1791 tcg_gen_op3_i64(INDEX_op_nor_i64
, ret
, arg1
, arg2
);
1793 tcg_gen_or_i64(ret
, arg1
, arg2
);
1794 tcg_gen_not_i64(ret
, ret
);
1798 void tcg_gen_orc_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1800 if (TCG_TARGET_REG_BITS
== 32) {
1801 tcg_gen_orc_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1802 tcg_gen_orc_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1803 } else if (TCG_TARGET_HAS_orc_i64
) {
1804 tcg_gen_op3_i64(INDEX_op_orc_i64
, ret
, arg1
, arg2
);
1806 TCGv_i64 t0
= tcg_temp_new_i64();
1807 tcg_gen_not_i64(t0
, arg2
);
1808 tcg_gen_or_i64(ret
, arg1
, t0
);
1809 tcg_temp_free_i64(t0
);
1813 void tcg_gen_clz_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1815 if (TCG_TARGET_HAS_clz_i64
) {
1816 tcg_gen_op3_i64(INDEX_op_clz_i64
, ret
, arg1
, arg2
);
1818 gen_helper_clz_i64(ret
, arg1
, arg2
);
1822 void tcg_gen_clzi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, uint64_t arg2
)
1824 if (TCG_TARGET_REG_BITS
== 32
1825 && TCG_TARGET_HAS_clz_i32
1826 && arg2
<= 0xffffffffu
) {
1827 TCGv_i32 t
= tcg_const_i32((uint32_t)arg2
- 32);
1828 tcg_gen_clz_i32(t
, TCGV_LOW(arg1
), t
);
1829 tcg_gen_addi_i32(t
, t
, 32);
1830 tcg_gen_clz_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), t
);
1831 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1832 tcg_temp_free_i32(t
);
1834 TCGv_i64 t
= tcg_const_i64(arg2
);
1835 tcg_gen_clz_i64(ret
, arg1
, t
);
1836 tcg_temp_free_i64(t
);
1840 void tcg_gen_ctz_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1842 if (TCG_TARGET_HAS_ctz_i64
) {
1843 tcg_gen_op3_i64(INDEX_op_ctz_i64
, ret
, arg1
, arg2
);
1844 } else if (TCG_TARGET_HAS_ctpop_i64
|| TCG_TARGET_HAS_clz_i64
) {
1845 TCGv_i64 z
, t
= tcg_temp_new_i64();
1847 if (TCG_TARGET_HAS_ctpop_i64
) {
1848 tcg_gen_subi_i64(t
, arg1
, 1);
1849 tcg_gen_andc_i64(t
, t
, arg1
);
1850 tcg_gen_ctpop_i64(t
, t
);
1852 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1853 tcg_gen_neg_i64(t
, arg1
);
1854 tcg_gen_and_i64(t
, t
, arg1
);
1855 tcg_gen_clzi_i64(t
, t
, 64);
1856 tcg_gen_xori_i64(t
, t
, 63);
1858 z
= tcg_const_i64(0);
1859 tcg_gen_movcond_i64(TCG_COND_EQ
, ret
, arg1
, z
, arg2
, t
);
1860 tcg_temp_free_i64(t
);
1861 tcg_temp_free_i64(z
);
1863 gen_helper_ctz_i64(ret
, arg1
, arg2
);
1867 void tcg_gen_ctzi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, uint64_t arg2
)
1869 if (TCG_TARGET_REG_BITS
== 32
1870 && TCG_TARGET_HAS_ctz_i32
1871 && arg2
<= 0xffffffffu
) {
1872 TCGv_i32 t32
= tcg_const_i32((uint32_t)arg2
- 32);
1873 tcg_gen_ctz_i32(t32
, TCGV_HIGH(arg1
), t32
);
1874 tcg_gen_addi_i32(t32
, t32
, 32);
1875 tcg_gen_ctz_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), t32
);
1876 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1877 tcg_temp_free_i32(t32
);
1878 } else if (!TCG_TARGET_HAS_ctz_i64
1879 && TCG_TARGET_HAS_ctpop_i64
1881 /* This equivalence has the advantage of not requiring a fixup. */
1882 TCGv_i64 t
= tcg_temp_new_i64();
1883 tcg_gen_subi_i64(t
, arg1
, 1);
1884 tcg_gen_andc_i64(t
, t
, arg1
);
1885 tcg_gen_ctpop_i64(ret
, t
);
1886 tcg_temp_free_i64(t
);
1888 TCGv_i64 t64
= tcg_const_i64(arg2
);
1889 tcg_gen_ctz_i64(ret
, arg1
, t64
);
1890 tcg_temp_free_i64(t64
);
1894 void tcg_gen_clrsb_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1896 if (TCG_TARGET_HAS_clz_i64
|| TCG_TARGET_HAS_clz_i32
) {
1897 TCGv_i64 t
= tcg_temp_new_i64();
1898 tcg_gen_sari_i64(t
, arg
, 63);
1899 tcg_gen_xor_i64(t
, t
, arg
);
1900 tcg_gen_clzi_i64(t
, t
, 64);
1901 tcg_gen_subi_i64(ret
, t
, 1);
1902 tcg_temp_free_i64(t
);
1904 gen_helper_clrsb_i64(ret
, arg
);
1908 void tcg_gen_ctpop_i64(TCGv_i64 ret
, TCGv_i64 arg1
)
1910 if (TCG_TARGET_HAS_ctpop_i64
) {
1911 tcg_gen_op2_i64(INDEX_op_ctpop_i64
, ret
, arg1
);
1912 } else if (TCG_TARGET_REG_BITS
== 32 && TCG_TARGET_HAS_ctpop_i32
) {
1913 tcg_gen_ctpop_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
1914 tcg_gen_ctpop_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
1915 tcg_gen_add_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), TCGV_HIGH(ret
));
1916 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1918 gen_helper_ctpop_i64(ret
, arg1
);
1922 void tcg_gen_rotl_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1924 if (TCG_TARGET_HAS_rot_i64
) {
1925 tcg_gen_op3_i64(INDEX_op_rotl_i64
, ret
, arg1
, arg2
);
1928 t0
= tcg_temp_new_i64();
1929 t1
= tcg_temp_new_i64();
1930 tcg_gen_shl_i64(t0
, arg1
, arg2
);
1931 tcg_gen_subfi_i64(t1
, 64, arg2
);
1932 tcg_gen_shr_i64(t1
, arg1
, t1
);
1933 tcg_gen_or_i64(ret
, t0
, t1
);
1934 tcg_temp_free_i64(t0
);
1935 tcg_temp_free_i64(t1
);
1939 void tcg_gen_rotli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, unsigned arg2
)
1941 tcg_debug_assert(arg2
< 64);
1942 /* some cases can be optimized here */
1944 tcg_gen_mov_i64(ret
, arg1
);
1945 } else if (TCG_TARGET_HAS_rot_i64
) {
1946 TCGv_i64 t0
= tcg_const_i64(arg2
);
1947 tcg_gen_rotl_i64(ret
, arg1
, t0
);
1948 tcg_temp_free_i64(t0
);
1951 t0
= tcg_temp_new_i64();
1952 t1
= tcg_temp_new_i64();
1953 tcg_gen_shli_i64(t0
, arg1
, arg2
);
1954 tcg_gen_shri_i64(t1
, arg1
, 64 - arg2
);
1955 tcg_gen_or_i64(ret
, t0
, t1
);
1956 tcg_temp_free_i64(t0
);
1957 tcg_temp_free_i64(t1
);
1961 void tcg_gen_rotr_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1963 if (TCG_TARGET_HAS_rot_i64
) {
1964 tcg_gen_op3_i64(INDEX_op_rotr_i64
, ret
, arg1
, arg2
);
1967 t0
= tcg_temp_new_i64();
1968 t1
= tcg_temp_new_i64();
1969 tcg_gen_shr_i64(t0
, arg1
, arg2
);
1970 tcg_gen_subfi_i64(t1
, 64, arg2
);
1971 tcg_gen_shl_i64(t1
, arg1
, t1
);
1972 tcg_gen_or_i64(ret
, t0
, t1
);
1973 tcg_temp_free_i64(t0
);
1974 tcg_temp_free_i64(t1
);
1978 void tcg_gen_rotri_i64(TCGv_i64 ret
, TCGv_i64 arg1
, unsigned arg2
)
1980 tcg_debug_assert(arg2
< 64);
1981 /* some cases can be optimized here */
1983 tcg_gen_mov_i64(ret
, arg1
);
1985 tcg_gen_rotli_i64(ret
, arg1
, 64 - arg2
);
1989 void tcg_gen_deposit_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
,
1990 unsigned int ofs
, unsigned int len
)
1995 tcg_debug_assert(ofs
< 64);
1996 tcg_debug_assert(len
> 0);
1997 tcg_debug_assert(len
<= 64);
1998 tcg_debug_assert(ofs
+ len
<= 64);
2001 tcg_gen_mov_i64(ret
, arg2
);
2004 if (TCG_TARGET_HAS_deposit_i64
&& TCG_TARGET_deposit_i64_valid(ofs
, len
)) {
2005 tcg_gen_op5ii_i64(INDEX_op_deposit_i64
, ret
, arg1
, arg2
, ofs
, len
);
2009 if (TCG_TARGET_REG_BITS
== 32) {
2011 tcg_gen_deposit_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
),
2012 TCGV_LOW(arg2
), ofs
- 32, len
);
2013 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
2016 if (ofs
+ len
<= 32) {
2017 tcg_gen_deposit_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
),
2018 TCGV_LOW(arg2
), ofs
, len
);
2019 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
2024 mask
= (1ull << len
) - 1;
2025 t1
= tcg_temp_new_i64();
2027 if (ofs
+ len
< 64) {
2028 tcg_gen_andi_i64(t1
, arg2
, mask
);
2029 tcg_gen_shli_i64(t1
, t1
, ofs
);
2031 tcg_gen_shli_i64(t1
, arg2
, ofs
);
2033 tcg_gen_andi_i64(ret
, arg1
, ~(mask
<< ofs
));
2034 tcg_gen_or_i64(ret
, ret
, t1
);
2036 tcg_temp_free_i64(t1
);
2039 void tcg_gen_deposit_z_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2040 unsigned int ofs
, unsigned int len
)
2042 tcg_debug_assert(ofs
< 64);
2043 tcg_debug_assert(len
> 0);
2044 tcg_debug_assert(len
<= 64);
2045 tcg_debug_assert(ofs
+ len
<= 64);
2047 if (ofs
+ len
== 64) {
2048 tcg_gen_shli_i64(ret
, arg
, ofs
);
2049 } else if (ofs
== 0) {
2050 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2051 } else if (TCG_TARGET_HAS_deposit_i64
2052 && TCG_TARGET_deposit_i64_valid(ofs
, len
)) {
2053 TCGv_i64 zero
= tcg_const_i64(0);
2054 tcg_gen_op5ii_i64(INDEX_op_deposit_i64
, ret
, zero
, arg
, ofs
, len
);
2055 tcg_temp_free_i64(zero
);
2057 if (TCG_TARGET_REG_BITS
== 32) {
2059 tcg_gen_deposit_z_i32(TCGV_HIGH(ret
), TCGV_LOW(arg
),
2061 tcg_gen_movi_i32(TCGV_LOW(ret
), 0);
2064 if (ofs
+ len
<= 32) {
2065 tcg_gen_deposit_z_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2066 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2070 /* To help two-operand hosts we prefer to zero-extend first,
2071 which allows ARG to stay live. */
2074 if (TCG_TARGET_HAS_ext32u_i64
) {
2075 tcg_gen_ext32u_i64(ret
, arg
);
2076 tcg_gen_shli_i64(ret
, ret
, ofs
);
2081 if (TCG_TARGET_HAS_ext16u_i64
) {
2082 tcg_gen_ext16u_i64(ret
, arg
);
2083 tcg_gen_shli_i64(ret
, ret
, ofs
);
2088 if (TCG_TARGET_HAS_ext8u_i64
) {
2089 tcg_gen_ext8u_i64(ret
, arg
);
2090 tcg_gen_shli_i64(ret
, ret
, ofs
);
2095 /* Otherwise prefer zero-extension over AND for code size. */
2096 switch (ofs
+ len
) {
2098 if (TCG_TARGET_HAS_ext32u_i64
) {
2099 tcg_gen_shli_i64(ret
, arg
, ofs
);
2100 tcg_gen_ext32u_i64(ret
, ret
);
2105 if (TCG_TARGET_HAS_ext16u_i64
) {
2106 tcg_gen_shli_i64(ret
, arg
, ofs
);
2107 tcg_gen_ext16u_i64(ret
, ret
);
2112 if (TCG_TARGET_HAS_ext8u_i64
) {
2113 tcg_gen_shli_i64(ret
, arg
, ofs
);
2114 tcg_gen_ext8u_i64(ret
, ret
);
2119 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2120 tcg_gen_shli_i64(ret
, ret
, ofs
);
2124 void tcg_gen_extract_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2125 unsigned int ofs
, unsigned int len
)
2127 tcg_debug_assert(ofs
< 64);
2128 tcg_debug_assert(len
> 0);
2129 tcg_debug_assert(len
<= 64);
2130 tcg_debug_assert(ofs
+ len
<= 64);
2132 /* Canonicalize certain special cases, even if extract is supported. */
2133 if (ofs
+ len
== 64) {
2134 tcg_gen_shri_i64(ret
, arg
, 64 - len
);
2138 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2142 if (TCG_TARGET_REG_BITS
== 32) {
2143 /* Look for a 32-bit extract within one of the two words. */
2145 tcg_gen_extract_i32(TCGV_LOW(ret
), TCGV_HIGH(arg
), ofs
- 32, len
);
2146 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2149 if (ofs
+ len
<= 32) {
2150 tcg_gen_extract_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2151 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2154 /* The field is split across two words. One double-word
2155 shift is better than two double-word shifts. */
2159 if (TCG_TARGET_HAS_extract_i64
2160 && TCG_TARGET_extract_i64_valid(ofs
, len
)) {
2161 tcg_gen_op4ii_i64(INDEX_op_extract_i64
, ret
, arg
, ofs
, len
);
2165 /* Assume that zero-extension, if available, is cheaper than a shift. */
2166 switch (ofs
+ len
) {
2168 if (TCG_TARGET_HAS_ext32u_i64
) {
2169 tcg_gen_ext32u_i64(ret
, arg
);
2170 tcg_gen_shri_i64(ret
, ret
, ofs
);
2175 if (TCG_TARGET_HAS_ext16u_i64
) {
2176 tcg_gen_ext16u_i64(ret
, arg
);
2177 tcg_gen_shri_i64(ret
, ret
, ofs
);
2182 if (TCG_TARGET_HAS_ext8u_i64
) {
2183 tcg_gen_ext8u_i64(ret
, arg
);
2184 tcg_gen_shri_i64(ret
, ret
, ofs
);
2190 /* ??? Ideally we'd know what values are available for immediate AND.
2191 Assume that 8 bits are available, plus the special cases of 16 and 32,
2192 so that we get ext8u, ext16u, and ext32u. */
2194 case 1 ... 8: case 16: case 32:
2196 tcg_gen_shri_i64(ret
, arg
, ofs
);
2197 tcg_gen_andi_i64(ret
, ret
, (1ull << len
) - 1);
2200 tcg_gen_shli_i64(ret
, arg
, 64 - len
- ofs
);
2201 tcg_gen_shri_i64(ret
, ret
, 64 - len
);
2206 void tcg_gen_sextract_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2207 unsigned int ofs
, unsigned int len
)
2209 tcg_debug_assert(ofs
< 64);
2210 tcg_debug_assert(len
> 0);
2211 tcg_debug_assert(len
<= 64);
2212 tcg_debug_assert(ofs
+ len
<= 64);
2214 /* Canonicalize certain special cases, even if sextract is supported. */
2215 if (ofs
+ len
== 64) {
2216 tcg_gen_sari_i64(ret
, arg
, 64 - len
);
2222 tcg_gen_ext32s_i64(ret
, arg
);
2225 tcg_gen_ext16s_i64(ret
, arg
);
2228 tcg_gen_ext8s_i64(ret
, arg
);
2233 if (TCG_TARGET_REG_BITS
== 32) {
2234 /* Look for a 32-bit extract within one of the two words. */
2236 tcg_gen_sextract_i32(TCGV_LOW(ret
), TCGV_HIGH(arg
), ofs
- 32, len
);
2237 } else if (ofs
+ len
<= 32) {
2238 tcg_gen_sextract_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2239 } else if (ofs
== 0) {
2240 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
2241 tcg_gen_sextract_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
), 0, len
- 32);
2243 } else if (len
> 32) {
2244 TCGv_i32 t
= tcg_temp_new_i32();
2245 /* Extract the bits for the high word normally. */
2246 tcg_gen_sextract_i32(t
, TCGV_HIGH(arg
), ofs
+ 32, len
- 32);
2247 /* Shift the field down for the low part. */
2248 tcg_gen_shri_i64(ret
, arg
, ofs
);
2249 /* Overwrite the shift into the high part. */
2250 tcg_gen_mov_i32(TCGV_HIGH(ret
), t
);
2251 tcg_temp_free_i32(t
);
2254 /* Shift the field down for the low part, such that the
2255 field sits at the MSB. */
2256 tcg_gen_shri_i64(ret
, arg
, ofs
+ len
- 32);
2257 /* Shift the field down from the MSB, sign extending. */
2258 tcg_gen_sari_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), 32 - len
);
2260 /* Sign-extend the field from 32 bits. */
2261 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
2265 if (TCG_TARGET_HAS_sextract_i64
2266 && TCG_TARGET_extract_i64_valid(ofs
, len
)) {
2267 tcg_gen_op4ii_i64(INDEX_op_sextract_i64
, ret
, arg
, ofs
, len
);
2271 /* Assume that sign-extension, if available, is cheaper than a shift. */
2272 switch (ofs
+ len
) {
2274 if (TCG_TARGET_HAS_ext32s_i64
) {
2275 tcg_gen_ext32s_i64(ret
, arg
);
2276 tcg_gen_sari_i64(ret
, ret
, ofs
);
2281 if (TCG_TARGET_HAS_ext16s_i64
) {
2282 tcg_gen_ext16s_i64(ret
, arg
);
2283 tcg_gen_sari_i64(ret
, ret
, ofs
);
2288 if (TCG_TARGET_HAS_ext8s_i64
) {
2289 tcg_gen_ext8s_i64(ret
, arg
);
2290 tcg_gen_sari_i64(ret
, ret
, ofs
);
2297 if (TCG_TARGET_HAS_ext32s_i64
) {
2298 tcg_gen_shri_i64(ret
, arg
, ofs
);
2299 tcg_gen_ext32s_i64(ret
, ret
);
2304 if (TCG_TARGET_HAS_ext16s_i64
) {
2305 tcg_gen_shri_i64(ret
, arg
, ofs
);
2306 tcg_gen_ext16s_i64(ret
, ret
);
2311 if (TCG_TARGET_HAS_ext8s_i64
) {
2312 tcg_gen_shri_i64(ret
, arg
, ofs
);
2313 tcg_gen_ext8s_i64(ret
, ret
);
2318 tcg_gen_shli_i64(ret
, arg
, 64 - len
- ofs
);
2319 tcg_gen_sari_i64(ret
, ret
, 64 - len
);
2323 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2324 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2326 void tcg_gen_extract2_i64(TCGv_i64 ret
, TCGv_i64 al
, TCGv_i64 ah
,
2329 tcg_debug_assert(ofs
<= 64);
2331 tcg_gen_mov_i64(ret
, al
);
2332 } else if (ofs
== 64) {
2333 tcg_gen_mov_i64(ret
, ah
);
2334 } else if (al
== ah
) {
2335 tcg_gen_rotri_i64(ret
, al
, ofs
);
2337 TCGv_i64 t0
= tcg_temp_new_i64();
2338 tcg_gen_shri_i64(t0
, al
, ofs
);
2339 tcg_gen_deposit_i64(ret
, t0
, ah
, 64 - ofs
, ofs
);
2340 tcg_temp_free_i64(t0
);
2344 void tcg_gen_movcond_i64(TCGCond cond
, TCGv_i64 ret
, TCGv_i64 c1
,
2345 TCGv_i64 c2
, TCGv_i64 v1
, TCGv_i64 v2
)
2347 if (cond
== TCG_COND_ALWAYS
) {
2348 tcg_gen_mov_i64(ret
, v1
);
2349 } else if (cond
== TCG_COND_NEVER
) {
2350 tcg_gen_mov_i64(ret
, v2
);
2351 } else if (TCG_TARGET_REG_BITS
== 32) {
2352 TCGv_i32 t0
= tcg_temp_new_i32();
2353 TCGv_i32 t1
= tcg_temp_new_i32();
2354 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, t0
,
2355 TCGV_LOW(c1
), TCGV_HIGH(c1
),
2356 TCGV_LOW(c2
), TCGV_HIGH(c2
), cond
);
2358 if (TCG_TARGET_HAS_movcond_i32
) {
2359 tcg_gen_movi_i32(t1
, 0);
2360 tcg_gen_movcond_i32(TCG_COND_NE
, TCGV_LOW(ret
), t0
, t1
,
2361 TCGV_LOW(v1
), TCGV_LOW(v2
));
2362 tcg_gen_movcond_i32(TCG_COND_NE
, TCGV_HIGH(ret
), t0
, t1
,
2363 TCGV_HIGH(v1
), TCGV_HIGH(v2
));
2365 tcg_gen_neg_i32(t0
, t0
);
2367 tcg_gen_and_i32(t1
, TCGV_LOW(v1
), t0
);
2368 tcg_gen_andc_i32(TCGV_LOW(ret
), TCGV_LOW(v2
), t0
);
2369 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), t1
);
2371 tcg_gen_and_i32(t1
, TCGV_HIGH(v1
), t0
);
2372 tcg_gen_andc_i32(TCGV_HIGH(ret
), TCGV_HIGH(v2
), t0
);
2373 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(ret
), t1
);
2375 tcg_temp_free_i32(t0
);
2376 tcg_temp_free_i32(t1
);
2377 } else if (TCG_TARGET_HAS_movcond_i64
) {
2378 tcg_gen_op6i_i64(INDEX_op_movcond_i64
, ret
, c1
, c2
, v1
, v2
, cond
);
2380 TCGv_i64 t0
= tcg_temp_new_i64();
2381 TCGv_i64 t1
= tcg_temp_new_i64();
2382 tcg_gen_setcond_i64(cond
, t0
, c1
, c2
);
2383 tcg_gen_neg_i64(t0
, t0
);
2384 tcg_gen_and_i64(t1
, v1
, t0
);
2385 tcg_gen_andc_i64(ret
, v2
, t0
);
2386 tcg_gen_or_i64(ret
, ret
, t1
);
2387 tcg_temp_free_i64(t0
);
2388 tcg_temp_free_i64(t1
);
2392 void tcg_gen_add2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 al
,
2393 TCGv_i64 ah
, TCGv_i64 bl
, TCGv_i64 bh
)
2395 if (TCG_TARGET_HAS_add2_i64
) {
2396 tcg_gen_op6_i64(INDEX_op_add2_i64
, rl
, rh
, al
, ah
, bl
, bh
);
2398 TCGv_i64 t0
= tcg_temp_new_i64();
2399 TCGv_i64 t1
= tcg_temp_new_i64();
2400 tcg_gen_add_i64(t0
, al
, bl
);
2401 tcg_gen_setcond_i64(TCG_COND_LTU
, t1
, t0
, al
);
2402 tcg_gen_add_i64(rh
, ah
, bh
);
2403 tcg_gen_add_i64(rh
, rh
, t1
);
2404 tcg_gen_mov_i64(rl
, t0
);
2405 tcg_temp_free_i64(t0
);
2406 tcg_temp_free_i64(t1
);
2410 void tcg_gen_sub2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 al
,
2411 TCGv_i64 ah
, TCGv_i64 bl
, TCGv_i64 bh
)
2413 if (TCG_TARGET_HAS_sub2_i64
) {
2414 tcg_gen_op6_i64(INDEX_op_sub2_i64
, rl
, rh
, al
, ah
, bl
, bh
);
2416 TCGv_i64 t0
= tcg_temp_new_i64();
2417 TCGv_i64 t1
= tcg_temp_new_i64();
2418 tcg_gen_sub_i64(t0
, al
, bl
);
2419 tcg_gen_setcond_i64(TCG_COND_LTU
, t1
, al
, bl
);
2420 tcg_gen_sub_i64(rh
, ah
, bh
);
2421 tcg_gen_sub_i64(rh
, rh
, t1
);
2422 tcg_gen_mov_i64(rl
, t0
);
2423 tcg_temp_free_i64(t0
);
2424 tcg_temp_free_i64(t1
);
2428 void tcg_gen_mulu2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2430 if (TCG_TARGET_HAS_mulu2_i64
) {
2431 tcg_gen_op4_i64(INDEX_op_mulu2_i64
, rl
, rh
, arg1
, arg2
);
2432 } else if (TCG_TARGET_HAS_muluh_i64
) {
2433 TCGv_i64 t
= tcg_temp_new_i64();
2434 tcg_gen_op3_i64(INDEX_op_mul_i64
, t
, arg1
, arg2
);
2435 tcg_gen_op3_i64(INDEX_op_muluh_i64
, rh
, arg1
, arg2
);
2436 tcg_gen_mov_i64(rl
, t
);
2437 tcg_temp_free_i64(t
);
2439 TCGv_i64 t0
= tcg_temp_new_i64();
2440 tcg_gen_mul_i64(t0
, arg1
, arg2
);
2441 gen_helper_muluh_i64(rh
, arg1
, arg2
);
2442 tcg_gen_mov_i64(rl
, t0
);
2443 tcg_temp_free_i64(t0
);
2447 void tcg_gen_muls2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2449 if (TCG_TARGET_HAS_muls2_i64
) {
2450 tcg_gen_op4_i64(INDEX_op_muls2_i64
, rl
, rh
, arg1
, arg2
);
2451 } else if (TCG_TARGET_HAS_mulsh_i64
) {
2452 TCGv_i64 t
= tcg_temp_new_i64();
2453 tcg_gen_op3_i64(INDEX_op_mul_i64
, t
, arg1
, arg2
);
2454 tcg_gen_op3_i64(INDEX_op_mulsh_i64
, rh
, arg1
, arg2
);
2455 tcg_gen_mov_i64(rl
, t
);
2456 tcg_temp_free_i64(t
);
2457 } else if (TCG_TARGET_HAS_mulu2_i64
|| TCG_TARGET_HAS_muluh_i64
) {
2458 TCGv_i64 t0
= tcg_temp_new_i64();
2459 TCGv_i64 t1
= tcg_temp_new_i64();
2460 TCGv_i64 t2
= tcg_temp_new_i64();
2461 TCGv_i64 t3
= tcg_temp_new_i64();
2462 tcg_gen_mulu2_i64(t0
, t1
, arg1
, arg2
);
2463 /* Adjust for negative inputs. */
2464 tcg_gen_sari_i64(t2
, arg1
, 63);
2465 tcg_gen_sari_i64(t3
, arg2
, 63);
2466 tcg_gen_and_i64(t2
, t2
, arg2
);
2467 tcg_gen_and_i64(t3
, t3
, arg1
);
2468 tcg_gen_sub_i64(rh
, t1
, t2
);
2469 tcg_gen_sub_i64(rh
, rh
, t3
);
2470 tcg_gen_mov_i64(rl
, t0
);
2471 tcg_temp_free_i64(t0
);
2472 tcg_temp_free_i64(t1
);
2473 tcg_temp_free_i64(t2
);
2474 tcg_temp_free_i64(t3
);
2476 TCGv_i64 t0
= tcg_temp_new_i64();
2477 tcg_gen_mul_i64(t0
, arg1
, arg2
);
2478 gen_helper_mulsh_i64(rh
, arg1
, arg2
);
2479 tcg_gen_mov_i64(rl
, t0
);
2480 tcg_temp_free_i64(t0
);
2484 void tcg_gen_mulsu2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2486 TCGv_i64 t0
= tcg_temp_new_i64();
2487 TCGv_i64 t1
= tcg_temp_new_i64();
2488 TCGv_i64 t2
= tcg_temp_new_i64();
2489 tcg_gen_mulu2_i64(t0
, t1
, arg1
, arg2
);
2490 /* Adjust for negative input for the signed arg1. */
2491 tcg_gen_sari_i64(t2
, arg1
, 63);
2492 tcg_gen_and_i64(t2
, t2
, arg2
);
2493 tcg_gen_sub_i64(rh
, t1
, t2
);
2494 tcg_gen_mov_i64(rl
, t0
);
2495 tcg_temp_free_i64(t0
);
2496 tcg_temp_free_i64(t1
);
2497 tcg_temp_free_i64(t2
);
2500 void tcg_gen_smin_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2502 tcg_gen_movcond_i64(TCG_COND_LT
, ret
, a
, b
, a
, b
);
2505 void tcg_gen_umin_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2507 tcg_gen_movcond_i64(TCG_COND_LTU
, ret
, a
, b
, a
, b
);
2510 void tcg_gen_smax_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2512 tcg_gen_movcond_i64(TCG_COND_LT
, ret
, a
, b
, b
, a
);
2515 void tcg_gen_umax_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2517 tcg_gen_movcond_i64(TCG_COND_LTU
, ret
, a
, b
, b
, a
);
2520 /* Size changing operations. */
2522 void tcg_gen_extrl_i64_i32(TCGv_i32 ret
, TCGv_i64 arg
)
2524 if (TCG_TARGET_REG_BITS
== 32) {
2525 tcg_gen_mov_i32(ret
, TCGV_LOW(arg
));
2526 } else if (TCG_TARGET_HAS_extrl_i64_i32
) {
2527 tcg_gen_op2(INDEX_op_extrl_i64_i32
,
2528 tcgv_i32_arg(ret
), tcgv_i64_arg(arg
));
2530 tcg_gen_mov_i32(ret
, (TCGv_i32
)arg
);
2534 void tcg_gen_extrh_i64_i32(TCGv_i32 ret
, TCGv_i64 arg
)
2536 if (TCG_TARGET_REG_BITS
== 32) {
2537 tcg_gen_mov_i32(ret
, TCGV_HIGH(arg
));
2538 } else if (TCG_TARGET_HAS_extrh_i64_i32
) {
2539 tcg_gen_op2(INDEX_op_extrh_i64_i32
,
2540 tcgv_i32_arg(ret
), tcgv_i64_arg(arg
));
2542 TCGv_i64 t
= tcg_temp_new_i64();
2543 tcg_gen_shri_i64(t
, arg
, 32);
2544 tcg_gen_mov_i32(ret
, (TCGv_i32
)t
);
2545 tcg_temp_free_i64(t
);
2549 void tcg_gen_extu_i32_i64(TCGv_i64 ret
, TCGv_i32 arg
)
2551 if (TCG_TARGET_REG_BITS
== 32) {
2552 tcg_gen_mov_i32(TCGV_LOW(ret
), arg
);
2553 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2555 tcg_gen_op2(INDEX_op_extu_i32_i64
,
2556 tcgv_i64_arg(ret
), tcgv_i32_arg(arg
));
2560 void tcg_gen_ext_i32_i64(TCGv_i64 ret
, TCGv_i32 arg
)
2562 if (TCG_TARGET_REG_BITS
== 32) {
2563 tcg_gen_mov_i32(TCGV_LOW(ret
), arg
);
2564 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
2566 tcg_gen_op2(INDEX_op_ext_i32_i64
,
2567 tcgv_i64_arg(ret
), tcgv_i32_arg(arg
));
2571 void tcg_gen_concat_i32_i64(TCGv_i64 dest
, TCGv_i32 low
, TCGv_i32 high
)
2575 if (TCG_TARGET_REG_BITS
== 32) {
2576 tcg_gen_mov_i32(TCGV_LOW(dest
), low
);
2577 tcg_gen_mov_i32(TCGV_HIGH(dest
), high
);
2581 tmp
= tcg_temp_new_i64();
2582 /* These extensions are only needed for type correctness.
2583 We may be able to do better given target specific information. */
2584 tcg_gen_extu_i32_i64(tmp
, high
);
2585 tcg_gen_extu_i32_i64(dest
, low
);
2586 /* If deposit is available, use it. Otherwise use the extra
2587 knowledge that we have of the zero-extensions above. */
2588 if (TCG_TARGET_HAS_deposit_i64
&& TCG_TARGET_deposit_i64_valid(32, 32)) {
2589 tcg_gen_deposit_i64(dest
, dest
, tmp
, 32, 32);
2591 tcg_gen_shli_i64(tmp
, tmp
, 32);
2592 tcg_gen_or_i64(dest
, dest
, tmp
);
2594 tcg_temp_free_i64(tmp
);
2597 void tcg_gen_extr_i64_i32(TCGv_i32 lo
, TCGv_i32 hi
, TCGv_i64 arg
)
2599 if (TCG_TARGET_REG_BITS
== 32) {
2600 tcg_gen_mov_i32(lo
, TCGV_LOW(arg
));
2601 tcg_gen_mov_i32(hi
, TCGV_HIGH(arg
));
2603 tcg_gen_extrl_i64_i32(lo
, arg
);
2604 tcg_gen_extrh_i64_i32(hi
, arg
);
2608 void tcg_gen_extr32_i64(TCGv_i64 lo
, TCGv_i64 hi
, TCGv_i64 arg
)
2610 tcg_gen_ext32u_i64(lo
, arg
);
2611 tcg_gen_shri_i64(hi
, arg
, 32);
2614 /* QEMU specific operations. */
2616 void tcg_gen_exit_tb(TranslationBlock
*tb
, unsigned idx
)
2618 uintptr_t val
= (uintptr_t)tb
+ idx
;
2621 tcg_debug_assert(idx
== 0);
2622 } else if (idx
<= TB_EXIT_IDXMAX
) {
2623 #ifdef CONFIG_DEBUG_TCG
2624 /* This is an exit following a goto_tb. Verify that we have
2625 seen this numbered exit before, via tcg_gen_goto_tb. */
2626 tcg_debug_assert(tcg_ctx
->goto_tb_issue_mask
& (1 << idx
));
2628 /* When not chaining, exit without indicating a link. */
2629 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN
)) {
2633 /* This is an exit via the exitreq label. */
2634 tcg_debug_assert(idx
== TB_EXIT_REQUESTED
);
2637 tcg_gen_op1i(INDEX_op_exit_tb
, val
);
2640 void tcg_gen_goto_tb(unsigned idx
)
2642 /* We only support two chained exits. */
2643 tcg_debug_assert(idx
<= TB_EXIT_IDXMAX
);
2644 #ifdef CONFIG_DEBUG_TCG
2645 /* Verify that we havn't seen this numbered exit before. */
2646 tcg_debug_assert((tcg_ctx
->goto_tb_issue_mask
& (1 << idx
)) == 0);
2647 tcg_ctx
->goto_tb_issue_mask
|= 1 << idx
;
2649 /* When not chaining, we simply fall through to the "fallback" exit. */
2650 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN
)) {
2651 tcg_gen_op1i(INDEX_op_goto_tb
, idx
);
2655 void tcg_gen_lookup_and_goto_ptr(void)
2657 if (TCG_TARGET_HAS_goto_ptr
&& !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN
)) {
2658 TCGv_ptr ptr
= tcg_temp_new_ptr();
2659 gen_helper_lookup_tb_ptr(ptr
, cpu_env
);
2660 tcg_gen_op1i(INDEX_op_goto_ptr
, tcgv_ptr_arg(ptr
));
2661 tcg_temp_free_ptr(ptr
);
2663 tcg_gen_exit_tb(NULL
, 0);
2667 static inline TCGMemOp
tcg_canonicalize_memop(TCGMemOp op
, bool is64
, bool st
)
2669 /* Trigger the asserts within as early as possible. */
2670 (void)get_alignment_bits(op
);
2672 switch (op
& MO_SIZE
) {
2695 static void gen_ldst_i32(TCGOpcode opc
, TCGv_i32 val
, TCGv addr
,
2696 TCGMemOp memop
, TCGArg idx
)
2698 TCGMemOpIdx oi
= make_memop_idx(memop
, idx
);
2699 #if TARGET_LONG_BITS == 32
2700 tcg_gen_op3i_i32(opc
, val
, addr
, oi
);
2702 if (TCG_TARGET_REG_BITS
== 32) {
2703 tcg_gen_op4i_i32(opc
, val
, TCGV_LOW(addr
), TCGV_HIGH(addr
), oi
);
2705 tcg_gen_op3(opc
, tcgv_i32_arg(val
), tcgv_i64_arg(addr
), oi
);
2710 static void gen_ldst_i64(TCGOpcode opc
, TCGv_i64 val
, TCGv addr
,
2711 TCGMemOp memop
, TCGArg idx
)
2713 TCGMemOpIdx oi
= make_memop_idx(memop
, idx
);
2714 #if TARGET_LONG_BITS == 32
2715 if (TCG_TARGET_REG_BITS
== 32) {
2716 tcg_gen_op4i_i32(opc
, TCGV_LOW(val
), TCGV_HIGH(val
), addr
, oi
);
2718 tcg_gen_op3(opc
, tcgv_i64_arg(val
), tcgv_i32_arg(addr
), oi
);
2721 if (TCG_TARGET_REG_BITS
== 32) {
2722 tcg_gen_op5i_i32(opc
, TCGV_LOW(val
), TCGV_HIGH(val
),
2723 TCGV_LOW(addr
), TCGV_HIGH(addr
), oi
);
2725 tcg_gen_op3i_i64(opc
, val
, addr
, oi
);
2730 static void tcg_gen_req_mo(TCGBar type
)
2732 #ifdef TCG_GUEST_DEFAULT_MO
2733 type
&= TCG_GUEST_DEFAULT_MO
;
2735 type
&= ~TCG_TARGET_DEFAULT_MO
;
2737 tcg_gen_mb(type
| TCG_BAR_SC
);
2741 void tcg_gen_qemu_ld_i32(TCGv_i32 val
, TCGv addr
, TCGArg idx
, TCGMemOp memop
)
2743 TCGMemOp orig_memop
;
2745 tcg_gen_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2746 memop
= tcg_canonicalize_memop(memop
, 0, 0);
2747 trace_guest_mem_before_tcg(tcg_ctx
->cpu
, cpu_env
,
2748 addr
, trace_mem_get_info(memop
, 0));
2751 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2753 /* The bswap primitive requires zero-extended input. */
2754 if ((memop
& MO_SSIZE
) == MO_SW
) {
2759 gen_ldst_i32(INDEX_op_qemu_ld_i32
, val
, addr
, memop
, idx
);
2761 if ((orig_memop
^ memop
) & MO_BSWAP
) {
2762 switch (orig_memop
& MO_SIZE
) {
2764 tcg_gen_bswap16_i32(val
, val
);
2765 if (orig_memop
& MO_SIGN
) {
2766 tcg_gen_ext16s_i32(val
, val
);
2770 tcg_gen_bswap32_i32(val
, val
);
2773 g_assert_not_reached();
2778 void tcg_gen_qemu_st_i32(TCGv_i32 val
, TCGv addr
, TCGArg idx
, TCGMemOp memop
)
2780 TCGv_i32 swap
= NULL
;
2782 tcg_gen_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2783 memop
= tcg_canonicalize_memop(memop
, 0, 1);
2784 trace_guest_mem_before_tcg(tcg_ctx
->cpu
, cpu_env
,
2785 addr
, trace_mem_get_info(memop
, 1));
2787 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2788 swap
= tcg_temp_new_i32();
2789 switch (memop
& MO_SIZE
) {
2791 tcg_gen_ext16u_i32(swap
, val
);
2792 tcg_gen_bswap16_i32(swap
, swap
);
2795 tcg_gen_bswap32_i32(swap
, val
);
2798 g_assert_not_reached();
2804 gen_ldst_i32(INDEX_op_qemu_st_i32
, val
, addr
, memop
, idx
);
2807 tcg_temp_free_i32(swap
);
2811 void tcg_gen_qemu_ld_i64(TCGv_i64 val
, TCGv addr
, TCGArg idx
, TCGMemOp memop
)
2813 TCGMemOp orig_memop
;
2815 if (TCG_TARGET_REG_BITS
== 32 && (memop
& MO_SIZE
) < MO_64
) {
2816 tcg_gen_qemu_ld_i32(TCGV_LOW(val
), addr
, idx
, memop
);
2817 if (memop
& MO_SIGN
) {
2818 tcg_gen_sari_i32(TCGV_HIGH(val
), TCGV_LOW(val
), 31);
2820 tcg_gen_movi_i32(TCGV_HIGH(val
), 0);
2825 tcg_gen_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2826 memop
= tcg_canonicalize_memop(memop
, 1, 0);
2827 trace_guest_mem_before_tcg(tcg_ctx
->cpu
, cpu_env
,
2828 addr
, trace_mem_get_info(memop
, 0));
2831 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2833 /* The bswap primitive requires zero-extended input. */
2834 if ((memop
& MO_SIGN
) && (memop
& MO_SIZE
) < MO_64
) {
2839 gen_ldst_i64(INDEX_op_qemu_ld_i64
, val
, addr
, memop
, idx
);
2841 if ((orig_memop
^ memop
) & MO_BSWAP
) {
2842 switch (orig_memop
& MO_SIZE
) {
2844 tcg_gen_bswap16_i64(val
, val
);
2845 if (orig_memop
& MO_SIGN
) {
2846 tcg_gen_ext16s_i64(val
, val
);
2850 tcg_gen_bswap32_i64(val
, val
);
2851 if (orig_memop
& MO_SIGN
) {
2852 tcg_gen_ext32s_i64(val
, val
);
2856 tcg_gen_bswap64_i64(val
, val
);
2859 g_assert_not_reached();
2864 void tcg_gen_qemu_st_i64(TCGv_i64 val
, TCGv addr
, TCGArg idx
, TCGMemOp memop
)
2866 TCGv_i64 swap
= NULL
;
2868 if (TCG_TARGET_REG_BITS
== 32 && (memop
& MO_SIZE
) < MO_64
) {
2869 tcg_gen_qemu_st_i32(TCGV_LOW(val
), addr
, idx
, memop
);
2873 tcg_gen_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2874 memop
= tcg_canonicalize_memop(memop
, 1, 1);
2875 trace_guest_mem_before_tcg(tcg_ctx
->cpu
, cpu_env
,
2876 addr
, trace_mem_get_info(memop
, 1));
2878 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2879 swap
= tcg_temp_new_i64();
2880 switch (memop
& MO_SIZE
) {
2882 tcg_gen_ext16u_i64(swap
, val
);
2883 tcg_gen_bswap16_i64(swap
, swap
);
2886 tcg_gen_ext32u_i64(swap
, val
);
2887 tcg_gen_bswap32_i64(swap
, swap
);
2890 tcg_gen_bswap64_i64(swap
, val
);
2893 g_assert_not_reached();
2899 gen_ldst_i64(INDEX_op_qemu_st_i64
, val
, addr
, memop
, idx
);
2902 tcg_temp_free_i64(swap
);
2906 static void tcg_gen_ext_i32(TCGv_i32 ret
, TCGv_i32 val
, TCGMemOp opc
)
2908 switch (opc
& MO_SSIZE
) {
2910 tcg_gen_ext8s_i32(ret
, val
);
2913 tcg_gen_ext8u_i32(ret
, val
);
2916 tcg_gen_ext16s_i32(ret
, val
);
2919 tcg_gen_ext16u_i32(ret
, val
);
2922 tcg_gen_mov_i32(ret
, val
);
2927 static void tcg_gen_ext_i64(TCGv_i64 ret
, TCGv_i64 val
, TCGMemOp opc
)
2929 switch (opc
& MO_SSIZE
) {
2931 tcg_gen_ext8s_i64(ret
, val
);
2934 tcg_gen_ext8u_i64(ret
, val
);
2937 tcg_gen_ext16s_i64(ret
, val
);
2940 tcg_gen_ext16u_i64(ret
, val
);
2943 tcg_gen_ext32s_i64(ret
, val
);
2946 tcg_gen_ext32u_i64(ret
, val
);
2949 tcg_gen_mov_i64(ret
, val
);
2954 #ifdef CONFIG_SOFTMMU
2955 typedef void (*gen_atomic_cx_i32
)(TCGv_i32
, TCGv_env
, TCGv
,
2956 TCGv_i32
, TCGv_i32
, TCGv_i32
);
2957 typedef void (*gen_atomic_cx_i64
)(TCGv_i64
, TCGv_env
, TCGv
,
2958 TCGv_i64
, TCGv_i64
, TCGv_i32
);
2959 typedef void (*gen_atomic_op_i32
)(TCGv_i32
, TCGv_env
, TCGv
,
2960 TCGv_i32
, TCGv_i32
);
2961 typedef void (*gen_atomic_op_i64
)(TCGv_i64
, TCGv_env
, TCGv
,
2962 TCGv_i64
, TCGv_i32
);
2964 typedef void (*gen_atomic_cx_i32
)(TCGv_i32
, TCGv_env
, TCGv
, TCGv_i32
, TCGv_i32
);
2965 typedef void (*gen_atomic_cx_i64
)(TCGv_i64
, TCGv_env
, TCGv
, TCGv_i64
, TCGv_i64
);
2966 typedef void (*gen_atomic_op_i32
)(TCGv_i32
, TCGv_env
, TCGv
, TCGv_i32
);
2967 typedef void (*gen_atomic_op_i64
)(TCGv_i64
, TCGv_env
, TCGv
, TCGv_i64
);
2970 #ifdef CONFIG_ATOMIC64
2971 # define WITH_ATOMIC64(X) X,
2973 # define WITH_ATOMIC64(X)
2976 static void * const table_cmpxchg
[16] = {
2977 [MO_8
] = gen_helper_atomic_cmpxchgb
,
2978 [MO_16
| MO_LE
] = gen_helper_atomic_cmpxchgw_le
,
2979 [MO_16
| MO_BE
] = gen_helper_atomic_cmpxchgw_be
,
2980 [MO_32
| MO_LE
] = gen_helper_atomic_cmpxchgl_le
,
2981 [MO_32
| MO_BE
] = gen_helper_atomic_cmpxchgl_be
,
2982 WITH_ATOMIC64([MO_64
| MO_LE
] = gen_helper_atomic_cmpxchgq_le
)
2983 WITH_ATOMIC64([MO_64
| MO_BE
] = gen_helper_atomic_cmpxchgq_be
)
2986 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv
, TCGv addr
, TCGv_i32 cmpv
,
2987 TCGv_i32 newv
, TCGArg idx
, TCGMemOp memop
)
2989 memop
= tcg_canonicalize_memop(memop
, 0, 0);
2991 if (!(tcg_ctx
->tb_cflags
& CF_PARALLEL
)) {
2992 TCGv_i32 t1
= tcg_temp_new_i32();
2993 TCGv_i32 t2
= tcg_temp_new_i32();
2995 tcg_gen_ext_i32(t2
, cmpv
, memop
& MO_SIZE
);
2997 tcg_gen_qemu_ld_i32(t1
, addr
, idx
, memop
& ~MO_SIGN
);
2998 tcg_gen_movcond_i32(TCG_COND_EQ
, t2
, t1
, t2
, newv
, t1
);
2999 tcg_gen_qemu_st_i32(t2
, addr
, idx
, memop
);
3000 tcg_temp_free_i32(t2
);
3002 if (memop
& MO_SIGN
) {
3003 tcg_gen_ext_i32(retv
, t1
, memop
);
3005 tcg_gen_mov_i32(retv
, t1
);
3007 tcg_temp_free_i32(t1
);
3009 gen_atomic_cx_i32 gen
;
3011 gen
= table_cmpxchg
[memop
& (MO_SIZE
| MO_BSWAP
)];
3012 tcg_debug_assert(gen
!= NULL
);
3014 #ifdef CONFIG_SOFTMMU
3016 TCGv_i32 oi
= tcg_const_i32(make_memop_idx(memop
& ~MO_SIGN
, idx
));
3017 gen(retv
, cpu_env
, addr
, cmpv
, newv
, oi
);
3018 tcg_temp_free_i32(oi
);
3021 gen(retv
, cpu_env
, addr
, cmpv
, newv
);
3024 if (memop
& MO_SIGN
) {
3025 tcg_gen_ext_i32(retv
, retv
, memop
);
3030 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv
, TCGv addr
, TCGv_i64 cmpv
,
3031 TCGv_i64 newv
, TCGArg idx
, TCGMemOp memop
)
3033 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3035 if (!(tcg_ctx
->tb_cflags
& CF_PARALLEL
)) {
3036 TCGv_i64 t1
= tcg_temp_new_i64();
3037 TCGv_i64 t2
= tcg_temp_new_i64();
3039 tcg_gen_ext_i64(t2
, cmpv
, memop
& MO_SIZE
);
3041 tcg_gen_qemu_ld_i64(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3042 tcg_gen_movcond_i64(TCG_COND_EQ
, t2
, t1
, t2
, newv
, t1
);
3043 tcg_gen_qemu_st_i64(t2
, addr
, idx
, memop
);
3044 tcg_temp_free_i64(t2
);
3046 if (memop
& MO_SIGN
) {
3047 tcg_gen_ext_i64(retv
, t1
, memop
);
3049 tcg_gen_mov_i64(retv
, t1
);
3051 tcg_temp_free_i64(t1
);
3052 } else if ((memop
& MO_SIZE
) == MO_64
) {
3053 #ifdef CONFIG_ATOMIC64
3054 gen_atomic_cx_i64 gen
;
3056 gen
= table_cmpxchg
[memop
& (MO_SIZE
| MO_BSWAP
)];
3057 tcg_debug_assert(gen
!= NULL
);
3059 #ifdef CONFIG_SOFTMMU
3061 TCGv_i32 oi
= tcg_const_i32(make_memop_idx(memop
, idx
));
3062 gen(retv
, cpu_env
, addr
, cmpv
, newv
, oi
);
3063 tcg_temp_free_i32(oi
);
3066 gen(retv
, cpu_env
, addr
, cmpv
, newv
);
3069 gen_helper_exit_atomic(cpu_env
);
3070 /* Produce a result, so that we have a well-formed opcode stream
3071 with respect to uses of the result in the (dead) code following. */
3072 tcg_gen_movi_i64(retv
, 0);
3073 #endif /* CONFIG_ATOMIC64 */
3075 TCGv_i32 c32
= tcg_temp_new_i32();
3076 TCGv_i32 n32
= tcg_temp_new_i32();
3077 TCGv_i32 r32
= tcg_temp_new_i32();
3079 tcg_gen_extrl_i64_i32(c32
, cmpv
);
3080 tcg_gen_extrl_i64_i32(n32
, newv
);
3081 tcg_gen_atomic_cmpxchg_i32(r32
, addr
, c32
, n32
, idx
, memop
& ~MO_SIGN
);
3082 tcg_temp_free_i32(c32
);
3083 tcg_temp_free_i32(n32
);
3085 tcg_gen_extu_i32_i64(retv
, r32
);
3086 tcg_temp_free_i32(r32
);
3088 if (memop
& MO_SIGN
) {
3089 tcg_gen_ext_i64(retv
, retv
, memop
);
3094 static void do_nonatomic_op_i32(TCGv_i32 ret
, TCGv addr
, TCGv_i32 val
,
3095 TCGArg idx
, TCGMemOp memop
, bool new_val
,
3096 void (*gen
)(TCGv_i32
, TCGv_i32
, TCGv_i32
))
3098 TCGv_i32 t1
= tcg_temp_new_i32();
3099 TCGv_i32 t2
= tcg_temp_new_i32();
3101 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3103 tcg_gen_qemu_ld_i32(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3105 tcg_gen_qemu_st_i32(t2
, addr
, idx
, memop
);
3107 tcg_gen_ext_i32(ret
, (new_val
? t2
: t1
), memop
);
3108 tcg_temp_free_i32(t1
);
3109 tcg_temp_free_i32(t2
);
3112 static void do_atomic_op_i32(TCGv_i32 ret
, TCGv addr
, TCGv_i32 val
,
3113 TCGArg idx
, TCGMemOp memop
, void * const table
[])
3115 gen_atomic_op_i32 gen
;
3117 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3119 gen
= table
[memop
& (MO_SIZE
| MO_BSWAP
)];
3120 tcg_debug_assert(gen
!= NULL
);
3122 #ifdef CONFIG_SOFTMMU
3124 TCGv_i32 oi
= tcg_const_i32(make_memop_idx(memop
& ~MO_SIGN
, idx
));
3125 gen(ret
, cpu_env
, addr
, val
, oi
);
3126 tcg_temp_free_i32(oi
);
3129 gen(ret
, cpu_env
, addr
, val
);
3132 if (memop
& MO_SIGN
) {
3133 tcg_gen_ext_i32(ret
, ret
, memop
);
3137 static void do_nonatomic_op_i64(TCGv_i64 ret
, TCGv addr
, TCGv_i64 val
,
3138 TCGArg idx
, TCGMemOp memop
, bool new_val
,
3139 void (*gen
)(TCGv_i64
, TCGv_i64
, TCGv_i64
))
3141 TCGv_i64 t1
= tcg_temp_new_i64();
3142 TCGv_i64 t2
= tcg_temp_new_i64();
3144 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3146 tcg_gen_qemu_ld_i64(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3148 tcg_gen_qemu_st_i64(t2
, addr
, idx
, memop
);
3150 tcg_gen_ext_i64(ret
, (new_val
? t2
: t1
), memop
);
3151 tcg_temp_free_i64(t1
);
3152 tcg_temp_free_i64(t2
);
3155 static void do_atomic_op_i64(TCGv_i64 ret
, TCGv addr
, TCGv_i64 val
,
3156 TCGArg idx
, TCGMemOp memop
, void * const table
[])
3158 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3160 if ((memop
& MO_SIZE
) == MO_64
) {
3161 #ifdef CONFIG_ATOMIC64
3162 gen_atomic_op_i64 gen
;
3164 gen
= table
[memop
& (MO_SIZE
| MO_BSWAP
)];
3165 tcg_debug_assert(gen
!= NULL
);
3167 #ifdef CONFIG_SOFTMMU
3169 TCGv_i32 oi
= tcg_const_i32(make_memop_idx(memop
& ~MO_SIGN
, idx
));
3170 gen(ret
, cpu_env
, addr
, val
, oi
);
3171 tcg_temp_free_i32(oi
);
3174 gen(ret
, cpu_env
, addr
, val
);
3177 gen_helper_exit_atomic(cpu_env
);
3178 /* Produce a result, so that we have a well-formed opcode stream
3179 with respect to uses of the result in the (dead) code following. */
3180 tcg_gen_movi_i64(ret
, 0);
3181 #endif /* CONFIG_ATOMIC64 */
3183 TCGv_i32 v32
= tcg_temp_new_i32();
3184 TCGv_i32 r32
= tcg_temp_new_i32();
3186 tcg_gen_extrl_i64_i32(v32
, val
);
3187 do_atomic_op_i32(r32
, addr
, v32
, idx
, memop
& ~MO_SIGN
, table
);
3188 tcg_temp_free_i32(v32
);
3190 tcg_gen_extu_i32_i64(ret
, r32
);
3191 tcg_temp_free_i32(r32
);
3193 if (memop
& MO_SIGN
) {
3194 tcg_gen_ext_i64(ret
, ret
, memop
);
3199 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3200 static void * const table_##NAME[16] = { \
3201 [MO_8] = gen_helper_atomic_##NAME##b, \
3202 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3203 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3204 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3205 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
3206 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3207 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
3209 void tcg_gen_atomic_##NAME##_i32 \
3210 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
3212 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3213 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3215 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3216 tcg_gen_##OP##_i32); \
3219 void tcg_gen_atomic_##NAME##_i64 \
3220 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
3222 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3223 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3225 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3226 tcg_gen_##OP##_i64); \
3230 GEN_ATOMIC_HELPER(fetch_add
, add
, 0)
3231 GEN_ATOMIC_HELPER(fetch_and
, and, 0)
3232 GEN_ATOMIC_HELPER(fetch_or
, or, 0)
3233 GEN_ATOMIC_HELPER(fetch_xor
, xor, 0)
3234 GEN_ATOMIC_HELPER(fetch_smin
, smin
, 0)
3235 GEN_ATOMIC_HELPER(fetch_umin
, umin
, 0)
3236 GEN_ATOMIC_HELPER(fetch_smax
, smax
, 0)
3237 GEN_ATOMIC_HELPER(fetch_umax
, umax
, 0)
3239 GEN_ATOMIC_HELPER(add_fetch
, add
, 1)
3240 GEN_ATOMIC_HELPER(and_fetch
, and, 1)
3241 GEN_ATOMIC_HELPER(or_fetch
, or, 1)
3242 GEN_ATOMIC_HELPER(xor_fetch
, xor, 1)
3243 GEN_ATOMIC_HELPER(smin_fetch
, smin
, 1)
3244 GEN_ATOMIC_HELPER(umin_fetch
, umin
, 1)
3245 GEN_ATOMIC_HELPER(smax_fetch
, smax
, 1)
3246 GEN_ATOMIC_HELPER(umax_fetch
, umax
, 1)
3248 static void tcg_gen_mov2_i32(TCGv_i32 r
, TCGv_i32 a
, TCGv_i32 b
)
3250 tcg_gen_mov_i32(r
, b
);
3253 static void tcg_gen_mov2_i64(TCGv_i64 r
, TCGv_i64 a
, TCGv_i64 b
)
3255 tcg_gen_mov_i64(r
, b
);
3258 GEN_ATOMIC_HELPER(xchg
, mov2
, 0)
3260 #undef GEN_ATOMIC_HELPER