2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2018 Linaro, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-mo.h"
26 /* Reduce the number of ifdefs below. This assumes that all uses of
27 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
28 the compiler can eliminate. */
29 #if TCG_TARGET_REG_BITS == 64
30 extern TCGv_i32
TCGV_LOW_link_error(TCGv_i64
);
31 extern TCGv_i32
TCGV_HIGH_link_error(TCGv_i64
);
32 #define TCGV_LOW TCGV_LOW_link_error
33 #define TCGV_HIGH TCGV_HIGH_link_error
37 * Vector optional opcode tracking.
38 * Except for the basic logical operations (and, or, xor), and
39 * data movement (mov, ld, st, dupi), many vector opcodes are
40 * optional and may not be supported on the host. Thank Intel
41 * for the irregularity in their instruction set.
43 * The gvec expanders allow custom vector operations to be composed,
44 * generally via the .fniv callback in the GVecGen* structures. At
45 * the same time, in deciding whether to use this hook we need to
46 * know if the host supports the required operations. This is
47 * presented as an array of opcodes, terminated by 0. Each opcode
48 * is assumed to be expanded with the given VECE.
50 * For debugging, we want to validate this array. Therefore, when
51 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
52 * will validate that their opcode is present in the list.
54 #ifdef CONFIG_DEBUG_TCG
55 void tcg_assert_listed_vecop(TCGOpcode op
)
57 const TCGOpcode
*p
= tcg_ctx
->vecop_list
;
64 g_assert_not_reached();
69 bool tcg_can_emit_vecop_list(const TCGOpcode
*list
,
70 TCGType type
, unsigned vece
)
76 for (; *list
; ++list
) {
77 TCGOpcode opc
= *list
;
79 #ifdef CONFIG_DEBUG_TCG
81 case INDEX_op_and_vec
:
83 case INDEX_op_xor_vec
:
84 case INDEX_op_mov_vec
:
85 case INDEX_op_dup_vec
:
86 case INDEX_op_dupi_vec
:
87 case INDEX_op_dup2_vec
:
90 case INDEX_op_bitsel_vec
:
91 /* These opcodes are mandatory and should not be listed. */
92 g_assert_not_reached();
93 case INDEX_op_not_vec
:
94 /* These opcodes have generic expansions using the above. */
95 g_assert_not_reached();
101 if (tcg_can_emit_vec_op(opc
, type
, vece
)) {
106 * The opcode list is created by front ends based on what they
107 * actually invoke. We must mirror the logic in the routines
108 * below for generic expansions using other opcodes.
111 case INDEX_op_neg_vec
:
112 if (tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
)) {
116 case INDEX_op_abs_vec
:
117 if (tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
)
118 && (tcg_can_emit_vec_op(INDEX_op_smax_vec
, type
, vece
) > 0
119 || tcg_can_emit_vec_op(INDEX_op_sari_vec
, type
, vece
) > 0
120 || tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
))) {
124 case INDEX_op_cmpsel_vec
:
125 case INDEX_op_smin_vec
:
126 case INDEX_op_smax_vec
:
127 case INDEX_op_umin_vec
:
128 case INDEX_op_umax_vec
:
129 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
)) {
141 void vec_gen_2(TCGOpcode opc
, TCGType type
, unsigned vece
, TCGArg r
, TCGArg a
)
143 TCGOp
*op
= tcg_emit_op(opc
);
144 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
145 TCGOP_VECE(op
) = vece
;
150 void vec_gen_3(TCGOpcode opc
, TCGType type
, unsigned vece
,
151 TCGArg r
, TCGArg a
, TCGArg b
)
153 TCGOp
*op
= tcg_emit_op(opc
);
154 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
155 TCGOP_VECE(op
) = vece
;
161 void vec_gen_4(TCGOpcode opc
, TCGType type
, unsigned vece
,
162 TCGArg r
, TCGArg a
, TCGArg b
, TCGArg c
)
164 TCGOp
*op
= tcg_emit_op(opc
);
165 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
166 TCGOP_VECE(op
) = vece
;
173 static void vec_gen_6(TCGOpcode opc
, TCGType type
, unsigned vece
, TCGArg r
,
174 TCGArg a
, TCGArg b
, TCGArg c
, TCGArg d
, TCGArg e
)
176 TCGOp
*op
= tcg_emit_op(opc
);
177 TCGOP_VECL(op
) = type
- TCG_TYPE_V64
;
178 TCGOP_VECE(op
) = vece
;
187 static void vec_gen_op2(TCGOpcode opc
, unsigned vece
, TCGv_vec r
, TCGv_vec a
)
189 TCGTemp
*rt
= tcgv_vec_temp(r
);
190 TCGTemp
*at
= tcgv_vec_temp(a
);
191 TCGType type
= rt
->base_type
;
193 /* Must enough inputs for the output. */
194 tcg_debug_assert(at
->base_type
>= type
);
195 vec_gen_2(opc
, type
, vece
, temp_arg(rt
), temp_arg(at
));
198 static void vec_gen_op3(TCGOpcode opc
, unsigned vece
,
199 TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
201 TCGTemp
*rt
= tcgv_vec_temp(r
);
202 TCGTemp
*at
= tcgv_vec_temp(a
);
203 TCGTemp
*bt
= tcgv_vec_temp(b
);
204 TCGType type
= rt
->base_type
;
206 /* Must enough inputs for the output. */
207 tcg_debug_assert(at
->base_type
>= type
);
208 tcg_debug_assert(bt
->base_type
>= type
);
209 vec_gen_3(opc
, type
, vece
, temp_arg(rt
), temp_arg(at
), temp_arg(bt
));
212 void tcg_gen_mov_vec(TCGv_vec r
, TCGv_vec a
)
215 vec_gen_op2(INDEX_op_mov_vec
, 0, r
, a
);
219 TCGv_vec
tcg_const_zeros_vec(TCGType type
)
221 TCGv_vec ret
= tcg_temp_new_vec(type
);
222 tcg_gen_dupi_vec(MO_64
, ret
, 0);
226 TCGv_vec
tcg_const_ones_vec(TCGType type
)
228 TCGv_vec ret
= tcg_temp_new_vec(type
);
229 tcg_gen_dupi_vec(MO_64
, ret
, -1);
233 TCGv_vec
tcg_const_zeros_vec_matching(TCGv_vec m
)
235 TCGTemp
*t
= tcgv_vec_temp(m
);
236 return tcg_const_zeros_vec(t
->base_type
);
239 TCGv_vec
tcg_const_ones_vec_matching(TCGv_vec m
)
241 TCGTemp
*t
= tcgv_vec_temp(m
);
242 return tcg_const_ones_vec(t
->base_type
);
245 void tcg_gen_dup64i_vec(TCGv_vec r
, uint64_t a
)
247 tcg_gen_dupi_vec(MO_64
, r
, a
);
250 void tcg_gen_dup32i_vec(TCGv_vec r
, uint32_t a
)
252 tcg_gen_dupi_vec(MO_32
, r
, a
);
255 void tcg_gen_dup16i_vec(TCGv_vec r
, uint32_t a
)
257 tcg_gen_dupi_vec(MO_16
, r
, a
);
260 void tcg_gen_dup8i_vec(TCGv_vec r
, uint32_t a
)
262 tcg_gen_dupi_vec(MO_8
, r
, a
);
265 void tcg_gen_dupi_vec(unsigned vece
, TCGv_vec r
, uint64_t a
)
267 TCGTemp
*rt
= tcgv_vec_temp(r
);
268 tcg_gen_mov_vec(r
, tcg_constant_vec(rt
->base_type
, vece
, a
));
271 void tcg_gen_dup_i64_vec(unsigned vece
, TCGv_vec r
, TCGv_i64 a
)
273 TCGArg ri
= tcgv_vec_arg(r
);
274 TCGTemp
*rt
= arg_temp(ri
);
275 TCGType type
= rt
->base_type
;
277 if (TCG_TARGET_REG_BITS
== 64) {
278 TCGArg ai
= tcgv_i64_arg(a
);
279 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
280 } else if (vece
== MO_64
) {
281 TCGArg al
= tcgv_i32_arg(TCGV_LOW(a
));
282 TCGArg ah
= tcgv_i32_arg(TCGV_HIGH(a
));
283 vec_gen_3(INDEX_op_dup2_vec
, type
, MO_64
, ri
, al
, ah
);
285 TCGArg ai
= tcgv_i32_arg(TCGV_LOW(a
));
286 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
290 void tcg_gen_dup_i32_vec(unsigned vece
, TCGv_vec r
, TCGv_i32 a
)
292 TCGArg ri
= tcgv_vec_arg(r
);
293 TCGArg ai
= tcgv_i32_arg(a
);
294 TCGTemp
*rt
= arg_temp(ri
);
295 TCGType type
= rt
->base_type
;
297 vec_gen_2(INDEX_op_dup_vec
, type
, vece
, ri
, ai
);
300 void tcg_gen_dup_mem_vec(unsigned vece
, TCGv_vec r
, TCGv_ptr b
,
303 TCGArg ri
= tcgv_vec_arg(r
);
304 TCGArg bi
= tcgv_ptr_arg(b
);
305 TCGTemp
*rt
= arg_temp(ri
);
306 TCGType type
= rt
->base_type
;
308 vec_gen_3(INDEX_op_dupm_vec
, type
, vece
, ri
, bi
, ofs
);
311 static void vec_gen_ldst(TCGOpcode opc
, TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
313 TCGArg ri
= tcgv_vec_arg(r
);
314 TCGArg bi
= tcgv_ptr_arg(b
);
315 TCGTemp
*rt
= arg_temp(ri
);
316 TCGType type
= rt
->base_type
;
318 vec_gen_3(opc
, type
, 0, ri
, bi
, o
);
321 void tcg_gen_ld_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
323 vec_gen_ldst(INDEX_op_ld_vec
, r
, b
, o
);
326 void tcg_gen_st_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
)
328 vec_gen_ldst(INDEX_op_st_vec
, r
, b
, o
);
331 void tcg_gen_stl_vec(TCGv_vec r
, TCGv_ptr b
, TCGArg o
, TCGType low_type
)
333 TCGArg ri
= tcgv_vec_arg(r
);
334 TCGArg bi
= tcgv_ptr_arg(b
);
335 TCGTemp
*rt
= arg_temp(ri
);
336 TCGType type
= rt
->base_type
;
338 tcg_debug_assert(low_type
>= TCG_TYPE_V64
);
339 tcg_debug_assert(low_type
<= type
);
340 vec_gen_3(INDEX_op_st_vec
, low_type
, 0, ri
, bi
, o
);
343 void tcg_gen_and_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
345 vec_gen_op3(INDEX_op_and_vec
, 0, r
, a
, b
);
348 void tcg_gen_or_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
350 vec_gen_op3(INDEX_op_or_vec
, 0, r
, a
, b
);
353 void tcg_gen_xor_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
355 vec_gen_op3(INDEX_op_xor_vec
, 0, r
, a
, b
);
358 void tcg_gen_andc_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
360 if (TCG_TARGET_HAS_andc_vec
) {
361 vec_gen_op3(INDEX_op_andc_vec
, 0, r
, a
, b
);
363 TCGv_vec t
= tcg_temp_new_vec_matching(r
);
364 tcg_gen_not_vec(0, t
, b
);
365 tcg_gen_and_vec(0, r
, a
, t
);
366 tcg_temp_free_vec(t
);
370 void tcg_gen_orc_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
372 if (TCG_TARGET_HAS_orc_vec
) {
373 vec_gen_op3(INDEX_op_orc_vec
, 0, r
, a
, b
);
375 TCGv_vec t
= tcg_temp_new_vec_matching(r
);
376 tcg_gen_not_vec(0, t
, b
);
377 tcg_gen_or_vec(0, r
, a
, t
);
378 tcg_temp_free_vec(t
);
382 void tcg_gen_nand_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
384 /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */
385 tcg_gen_and_vec(0, r
, a
, b
);
386 tcg_gen_not_vec(0, r
, r
);
389 void tcg_gen_nor_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
391 /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */
392 tcg_gen_or_vec(0, r
, a
, b
);
393 tcg_gen_not_vec(0, r
, r
);
396 void tcg_gen_eqv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
398 /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */
399 tcg_gen_xor_vec(0, r
, a
, b
);
400 tcg_gen_not_vec(0, r
, r
);
403 static bool do_op2(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGOpcode opc
)
405 TCGTemp
*rt
= tcgv_vec_temp(r
);
406 TCGTemp
*at
= tcgv_vec_temp(a
);
407 TCGArg ri
= temp_arg(rt
);
408 TCGArg ai
= temp_arg(at
);
409 TCGType type
= rt
->base_type
;
412 tcg_debug_assert(at
->base_type
>= type
);
413 tcg_assert_listed_vecop(opc
);
414 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
416 vec_gen_2(opc
, type
, vece
, ri
, ai
);
417 } else if (can
< 0) {
418 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
419 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
);
420 tcg_swap_vecop_list(hold_list
);
427 void tcg_gen_not_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
429 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
431 if (!TCG_TARGET_HAS_not_vec
|| !do_op2(vece
, r
, a
, INDEX_op_not_vec
)) {
432 TCGv_vec t
= tcg_const_ones_vec_matching(r
);
433 tcg_gen_xor_vec(0, r
, a
, t
);
434 tcg_temp_free_vec(t
);
436 tcg_swap_vecop_list(hold_list
);
439 void tcg_gen_neg_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
441 const TCGOpcode
*hold_list
;
443 tcg_assert_listed_vecop(INDEX_op_neg_vec
);
444 hold_list
= tcg_swap_vecop_list(NULL
);
446 if (!TCG_TARGET_HAS_neg_vec
|| !do_op2(vece
, r
, a
, INDEX_op_neg_vec
)) {
447 TCGv_vec t
= tcg_const_zeros_vec_matching(r
);
448 tcg_gen_sub_vec(vece
, r
, t
, a
);
449 tcg_temp_free_vec(t
);
451 tcg_swap_vecop_list(hold_list
);
454 void tcg_gen_abs_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
)
456 const TCGOpcode
*hold_list
;
458 tcg_assert_listed_vecop(INDEX_op_abs_vec
);
459 hold_list
= tcg_swap_vecop_list(NULL
);
461 if (!do_op2(vece
, r
, a
, INDEX_op_abs_vec
)) {
462 TCGType type
= tcgv_vec_temp(r
)->base_type
;
463 TCGv_vec t
= tcg_temp_new_vec(type
);
465 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec
, type
, vece
));
466 if (tcg_can_emit_vec_op(INDEX_op_smax_vec
, type
, vece
) > 0) {
467 tcg_gen_neg_vec(vece
, t
, a
);
468 tcg_gen_smax_vec(vece
, r
, a
, t
);
470 if (tcg_can_emit_vec_op(INDEX_op_sari_vec
, type
, vece
) > 0) {
471 tcg_gen_sari_vec(vece
, t
, a
, (8 << vece
) - 1);
473 tcg_gen_cmp_vec(TCG_COND_LT
, vece
, t
, a
,
474 tcg_constant_vec(type
, vece
, 0));
476 tcg_gen_xor_vec(vece
, r
, a
, t
);
477 tcg_gen_sub_vec(vece
, r
, r
, t
);
480 tcg_temp_free_vec(t
);
482 tcg_swap_vecop_list(hold_list
);
485 static void do_shifti(TCGOpcode opc
, unsigned vece
,
486 TCGv_vec r
, TCGv_vec a
, int64_t i
)
488 TCGTemp
*rt
= tcgv_vec_temp(r
);
489 TCGTemp
*at
= tcgv_vec_temp(a
);
490 TCGArg ri
= temp_arg(rt
);
491 TCGArg ai
= temp_arg(at
);
492 TCGType type
= rt
->base_type
;
495 tcg_debug_assert(at
->base_type
== type
);
496 tcg_debug_assert(i
>= 0 && i
< (8 << vece
));
497 tcg_assert_listed_vecop(opc
);
500 tcg_gen_mov_vec(r
, a
);
504 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
506 vec_gen_3(opc
, type
, vece
, ri
, ai
, i
);
508 /* We leave the choice of expansion via scalar or vector shift
509 to the target. Often, but not always, dupi can feed a vector
510 shift easier than a scalar. */
511 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
512 tcg_debug_assert(can
< 0);
513 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, i
);
514 tcg_swap_vecop_list(hold_list
);
518 void tcg_gen_shli_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
520 do_shifti(INDEX_op_shli_vec
, vece
, r
, a
, i
);
523 void tcg_gen_shri_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
525 do_shifti(INDEX_op_shri_vec
, vece
, r
, a
, i
);
528 void tcg_gen_sari_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
530 do_shifti(INDEX_op_sari_vec
, vece
, r
, a
, i
);
533 void tcg_gen_rotli_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
535 do_shifti(INDEX_op_rotli_vec
, vece
, r
, a
, i
);
538 void tcg_gen_rotri_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, int64_t i
)
540 int bits
= 8 << vece
;
541 tcg_debug_assert(i
>= 0 && i
< bits
);
542 do_shifti(INDEX_op_rotli_vec
, vece
, r
, a
, -i
& (bits
- 1));
545 void tcg_gen_cmp_vec(TCGCond cond
, unsigned vece
,
546 TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
548 TCGTemp
*rt
= tcgv_vec_temp(r
);
549 TCGTemp
*at
= tcgv_vec_temp(a
);
550 TCGTemp
*bt
= tcgv_vec_temp(b
);
551 TCGArg ri
= temp_arg(rt
);
552 TCGArg ai
= temp_arg(at
);
553 TCGArg bi
= temp_arg(bt
);
554 TCGType type
= rt
->base_type
;
557 tcg_debug_assert(at
->base_type
>= type
);
558 tcg_debug_assert(bt
->base_type
>= type
);
559 tcg_assert_listed_vecop(INDEX_op_cmp_vec
);
560 can
= tcg_can_emit_vec_op(INDEX_op_cmp_vec
, type
, vece
);
562 vec_gen_4(INDEX_op_cmp_vec
, type
, vece
, ri
, ai
, bi
, cond
);
564 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
565 tcg_debug_assert(can
< 0);
566 tcg_expand_vec_op(INDEX_op_cmp_vec
, type
, vece
, ri
, ai
, bi
, cond
);
567 tcg_swap_vecop_list(hold_list
);
571 static bool do_op3(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
572 TCGv_vec b
, TCGOpcode opc
)
574 TCGTemp
*rt
= tcgv_vec_temp(r
);
575 TCGTemp
*at
= tcgv_vec_temp(a
);
576 TCGTemp
*bt
= tcgv_vec_temp(b
);
577 TCGArg ri
= temp_arg(rt
);
578 TCGArg ai
= temp_arg(at
);
579 TCGArg bi
= temp_arg(bt
);
580 TCGType type
= rt
->base_type
;
583 tcg_debug_assert(at
->base_type
>= type
);
584 tcg_debug_assert(bt
->base_type
>= type
);
585 tcg_assert_listed_vecop(opc
);
586 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
588 vec_gen_3(opc
, type
, vece
, ri
, ai
, bi
);
589 } else if (can
< 0) {
590 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
591 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, bi
);
592 tcg_swap_vecop_list(hold_list
);
599 static void do_op3_nofail(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
600 TCGv_vec b
, TCGOpcode opc
)
602 bool ok
= do_op3(vece
, r
, a
, b
, opc
);
603 tcg_debug_assert(ok
);
606 void tcg_gen_add_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
608 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_add_vec
);
611 void tcg_gen_sub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
613 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sub_vec
);
616 void tcg_gen_mul_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
618 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_mul_vec
);
621 void tcg_gen_ssadd_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
623 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_ssadd_vec
);
626 void tcg_gen_usadd_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
628 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_usadd_vec
);
631 void tcg_gen_sssub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
633 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sssub_vec
);
636 void tcg_gen_ussub_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
638 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_ussub_vec
);
641 static void do_minmax(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
642 TCGv_vec b
, TCGOpcode opc
, TCGCond cond
)
644 if (!do_op3(vece
, r
, a
, b
, opc
)) {
645 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
646 tcg_gen_cmpsel_vec(cond
, vece
, r
, a
, b
, a
, b
);
647 tcg_swap_vecop_list(hold_list
);
651 void tcg_gen_smin_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
653 do_minmax(vece
, r
, a
, b
, INDEX_op_smin_vec
, TCG_COND_LT
);
656 void tcg_gen_umin_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
658 do_minmax(vece
, r
, a
, b
, INDEX_op_umin_vec
, TCG_COND_LTU
);
661 void tcg_gen_smax_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
663 do_minmax(vece
, r
, a
, b
, INDEX_op_smax_vec
, TCG_COND_GT
);
666 void tcg_gen_umax_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
668 do_minmax(vece
, r
, a
, b
, INDEX_op_umax_vec
, TCG_COND_GTU
);
671 void tcg_gen_shlv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
673 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_shlv_vec
);
676 void tcg_gen_shrv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
678 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_shrv_vec
);
681 void tcg_gen_sarv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
683 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_sarv_vec
);
686 void tcg_gen_rotlv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
688 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_rotlv_vec
);
691 void tcg_gen_rotrv_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_vec b
)
693 do_op3_nofail(vece
, r
, a
, b
, INDEX_op_rotrv_vec
);
696 static void do_shifts(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
697 TCGv_i32 s
, TCGOpcode opc
)
699 TCGTemp
*rt
= tcgv_vec_temp(r
);
700 TCGTemp
*at
= tcgv_vec_temp(a
);
701 TCGTemp
*st
= tcgv_i32_temp(s
);
702 TCGArg ri
= temp_arg(rt
);
703 TCGArg ai
= temp_arg(at
);
704 TCGArg si
= temp_arg(st
);
705 TCGType type
= rt
->base_type
;
708 tcg_debug_assert(at
->base_type
>= type
);
709 tcg_assert_listed_vecop(opc
);
710 can
= tcg_can_emit_vec_op(opc
, type
, vece
);
712 vec_gen_3(opc
, type
, vece
, ri
, ai
, si
);
713 } else if (can
< 0) {
714 const TCGOpcode
*hold_list
= tcg_swap_vecop_list(NULL
);
715 tcg_expand_vec_op(opc
, type
, vece
, ri
, ai
, si
);
716 tcg_swap_vecop_list(hold_list
);
718 g_assert_not_reached();
722 void tcg_gen_shls_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
724 do_shifts(vece
, r
, a
, b
, INDEX_op_shls_vec
);
727 void tcg_gen_shrs_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
729 do_shifts(vece
, r
, a
, b
, INDEX_op_shrs_vec
);
732 void tcg_gen_sars_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 b
)
734 do_shifts(vece
, r
, a
, b
, INDEX_op_sars_vec
);
737 void tcg_gen_rotls_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
, TCGv_i32 s
)
739 do_shifts(vece
, r
, a
, s
, INDEX_op_rotls_vec
);
742 void tcg_gen_bitsel_vec(unsigned vece
, TCGv_vec r
, TCGv_vec a
,
743 TCGv_vec b
, TCGv_vec c
)
745 TCGTemp
*rt
= tcgv_vec_temp(r
);
746 TCGTemp
*at
= tcgv_vec_temp(a
);
747 TCGTemp
*bt
= tcgv_vec_temp(b
);
748 TCGTemp
*ct
= tcgv_vec_temp(c
);
749 TCGType type
= rt
->base_type
;
751 tcg_debug_assert(at
->base_type
>= type
);
752 tcg_debug_assert(bt
->base_type
>= type
);
753 tcg_debug_assert(ct
->base_type
>= type
);
755 if (TCG_TARGET_HAS_bitsel_vec
) {
756 vec_gen_4(INDEX_op_bitsel_vec
, type
, MO_8
,
757 temp_arg(rt
), temp_arg(at
), temp_arg(bt
), temp_arg(ct
));
759 TCGv_vec t
= tcg_temp_new_vec(type
);
760 tcg_gen_and_vec(MO_8
, t
, a
, b
);
761 tcg_gen_andc_vec(MO_8
, r
, c
, a
);
762 tcg_gen_or_vec(MO_8
, r
, r
, t
);
763 tcg_temp_free_vec(t
);
767 void tcg_gen_cmpsel_vec(TCGCond cond
, unsigned vece
, TCGv_vec r
,
768 TCGv_vec a
, TCGv_vec b
, TCGv_vec c
, TCGv_vec d
)
770 TCGTemp
*rt
= tcgv_vec_temp(r
);
771 TCGTemp
*at
= tcgv_vec_temp(a
);
772 TCGTemp
*bt
= tcgv_vec_temp(b
);
773 TCGTemp
*ct
= tcgv_vec_temp(c
);
774 TCGTemp
*dt
= tcgv_vec_temp(d
);
775 TCGArg ri
= temp_arg(rt
);
776 TCGArg ai
= temp_arg(at
);
777 TCGArg bi
= temp_arg(bt
);
778 TCGArg ci
= temp_arg(ct
);
779 TCGArg di
= temp_arg(dt
);
780 TCGType type
= rt
->base_type
;
781 const TCGOpcode
*hold_list
;
784 tcg_debug_assert(at
->base_type
>= type
);
785 tcg_debug_assert(bt
->base_type
>= type
);
786 tcg_debug_assert(ct
->base_type
>= type
);
787 tcg_debug_assert(dt
->base_type
>= type
);
789 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec
);
790 hold_list
= tcg_swap_vecop_list(NULL
);
791 can
= tcg_can_emit_vec_op(INDEX_op_cmpsel_vec
, type
, vece
);
794 vec_gen_6(INDEX_op_cmpsel_vec
, type
, vece
, ri
, ai
, bi
, ci
, di
, cond
);
795 } else if (can
< 0) {
796 tcg_expand_vec_op(INDEX_op_cmpsel_vec
, type
, vece
,
797 ri
, ai
, bi
, ci
, di
, cond
);
799 TCGv_vec t
= tcg_temp_new_vec(type
);
800 tcg_gen_cmp_vec(cond
, vece
, t
, a
, b
);
801 tcg_gen_bitsel_vec(vece
, r
, t
, c
, d
);
802 tcg_temp_free_vec(t
);
804 tcg_swap_vecop_list(hold_list
);