]>
Commit | Line | Data |
---|---|---|
d2fd745f RH |
1 | /* |
2 | * Tiny Code Generator for QEMU | |
3 | * | |
4 | * Copyright (c) 2018 Linaro, Inc. | |
5 | * | |
6 | * This library is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU Lesser General Public | |
8 | * License as published by the Free Software Foundation; either | |
fb0343d5 | 9 | * version 2.1 of the License, or (at your option) any later version. |
d2fd745f RH |
10 | * |
11 | * This library is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * Lesser General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU Lesser General Public | |
17 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | ||
20 | #include "qemu/osdep.h" | |
dcb32f1d | 21 | #include "tcg/tcg.h" |
47f7313d | 22 | #include "tcg/tcg-temp-internal.h" |
ad3d0e4d | 23 | #include "tcg/tcg-op-common.h" |
dcb32f1d | 24 | #include "tcg/tcg-mo.h" |
d56fea79 RH |
25 | #include "tcg-internal.h" |
26 | ||
53229a77 RH |
27 | /* |
28 | * Vector optional opcode tracking. | |
29 | * Except for the basic logical operations (and, or, xor), and | |
30 | * data movement (mov, ld, st, dupi), many vector opcodes are | |
31 | * optional and may not be supported on the host. Thank Intel | |
32 | * for the irregularity in their instruction set. | |
33 | * | |
34 | * The gvec expanders allow custom vector operations to be composed, | |
35 | * generally via the .fniv callback in the GVecGen* structures. At | |
36 | * the same time, in deciding whether to use this hook we need to | |
37 | * know if the host supports the required operations. This is | |
38 | * presented as an array of opcodes, terminated by 0. Each opcode | |
39 | * is assumed to be expanded with the given VECE. | |
40 | * | |
41 | * For debugging, we want to validate this array. Therefore, when | |
42 | * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders | |
43 | * will validate that their opcode is present in the list. | |
44 | */ | |
ec2297bd | 45 | static void tcg_assert_listed_vecop(TCGOpcode op) |
53229a77 | 46 | { |
ec2297bd | 47 | #ifdef CONFIG_DEBUG_TCG |
53229a77 RH |
48 | const TCGOpcode *p = tcg_ctx->vecop_list; |
49 | if (p) { | |
50 | for (; *p; ++p) { | |
51 | if (*p == op) { | |
52 | return; | |
53 | } | |
54 | } | |
55 | g_assert_not_reached(); | |
56 | } | |
53229a77 | 57 | #endif |
ec2297bd | 58 | } |
53229a77 RH |
59 | |
60 | bool tcg_can_emit_vecop_list(const TCGOpcode *list, | |
61 | TCGType type, unsigned vece) | |
62 | { | |
63 | if (list == NULL) { | |
64 | return true; | |
65 | } | |
66 | ||
67 | for (; *list; ++list) { | |
68 | TCGOpcode opc = *list; | |
69 | ||
70 | #ifdef CONFIG_DEBUG_TCG | |
71 | switch (opc) { | |
72 | case INDEX_op_and_vec: | |
73 | case INDEX_op_or_vec: | |
74 | case INDEX_op_xor_vec: | |
75 | case INDEX_op_mov_vec: | |
76 | case INDEX_op_dup_vec: | |
53229a77 RH |
77 | case INDEX_op_dup2_vec: |
78 | case INDEX_op_ld_vec: | |
79 | case INDEX_op_st_vec: | |
38dc1294 | 80 | case INDEX_op_bitsel_vec: |
53229a77 RH |
81 | /* These opcodes are mandatory and should not be listed. */ |
82 | g_assert_not_reached(); | |
11978f6f RH |
83 | case INDEX_op_not_vec: |
84 | /* These opcodes have generic expansions using the above. */ | |
85 | g_assert_not_reached(); | |
53229a77 RH |
86 | default: |
87 | break; | |
88 | } | |
89 | #endif | |
90 | ||
91 | if (tcg_can_emit_vec_op(opc, type, vece)) { | |
92 | continue; | |
93 | } | |
94 | ||
95 | /* | |
96 | * The opcode list is created by front ends based on what they | |
97 | * actually invoke. We must mirror the logic in the routines | |
98 | * below for generic expansions using other opcodes. | |
99 | */ | |
100 | switch (opc) { | |
101 | case INDEX_op_neg_vec: | |
102 | if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) { | |
103 | continue; | |
104 | } | |
105 | break; | |
bcefc902 RH |
106 | case INDEX_op_abs_vec: |
107 | if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece) | |
108 | && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0 | |
109 | || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0 | |
110 | || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) { | |
111 | continue; | |
112 | } | |
113 | break; | |
2552d60e RH |
114 | case INDEX_op_usadd_vec: |
115 | if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) || | |
116 | tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { | |
117 | continue; | |
118 | } | |
119 | break; | |
120 | case INDEX_op_ussub_vec: | |
121 | if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) || | |
122 | tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { | |
123 | continue; | |
124 | } | |
125 | break; | |
f75da298 | 126 | case INDEX_op_cmpsel_vec: |
72b4c792 RH |
127 | case INDEX_op_smin_vec: |
128 | case INDEX_op_smax_vec: | |
129 | case INDEX_op_umin_vec: | |
130 | case INDEX_op_umax_vec: | |
f75da298 RH |
131 | if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { |
132 | continue; | |
133 | } | |
134 | break; | |
53229a77 RH |
135 | default: |
136 | break; | |
137 | } | |
138 | return false; | |
139 | } | |
140 | return true; | |
141 | } | |
142 | ||
d2fd745f RH |
143 | void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) |
144 | { | |
d4478943 | 145 | TCGOp *op = tcg_emit_op(opc, 2); |
d2fd745f RH |
146 | TCGOP_VECL(op) = type - TCG_TYPE_V64; |
147 | TCGOP_VECE(op) = vece; | |
148 | op->args[0] = r; | |
149 | op->args[1] = a; | |
150 | } | |
151 | ||
152 | void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, | |
153 | TCGArg r, TCGArg a, TCGArg b) | |
154 | { | |
d4478943 | 155 | TCGOp *op = tcg_emit_op(opc, 3); |
d2fd745f RH |
156 | TCGOP_VECL(op) = type - TCG_TYPE_V64; |
157 | TCGOP_VECE(op) = vece; | |
158 | op->args[0] = r; | |
159 | op->args[1] = a; | |
160 | op->args[2] = b; | |
161 | } | |
162 | ||
163 | void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, | |
164 | TCGArg r, TCGArg a, TCGArg b, TCGArg c) | |
165 | { | |
d4478943 | 166 | TCGOp *op = tcg_emit_op(opc, 4); |
d2fd745f RH |
167 | TCGOP_VECL(op) = type - TCG_TYPE_V64; |
168 | TCGOP_VECE(op) = vece; | |
169 | op->args[0] = r; | |
170 | op->args[1] = a; | |
171 | op->args[2] = b; | |
172 | op->args[3] = c; | |
173 | } | |
174 | ||
f75da298 RH |
175 | static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, |
176 | TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) | |
177 | { | |
d4478943 | 178 | TCGOp *op = tcg_emit_op(opc, 6); |
f75da298 RH |
179 | TCGOP_VECL(op) = type - TCG_TYPE_V64; |
180 | TCGOP_VECE(op) = vece; | |
181 | op->args[0] = r; | |
182 | op->args[1] = a; | |
183 | op->args[2] = b; | |
184 | op->args[3] = c; | |
185 | op->args[4] = d; | |
186 | op->args[5] = e; | |
187 | } | |
188 | ||
d2fd745f RH |
189 | static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a) |
190 | { | |
191 | TCGTemp *rt = tcgv_vec_temp(r); | |
192 | TCGTemp *at = tcgv_vec_temp(a); | |
193 | TCGType type = rt->base_type; | |
194 | ||
db432672 RH |
195 | /* Must enough inputs for the output. */ |
196 | tcg_debug_assert(at->base_type >= type); | |
d2fd745f RH |
197 | vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at)); |
198 | } | |
199 | ||
200 | static void vec_gen_op3(TCGOpcode opc, unsigned vece, | |
201 | TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
202 | { | |
203 | TCGTemp *rt = tcgv_vec_temp(r); | |
204 | TCGTemp *at = tcgv_vec_temp(a); | |
205 | TCGTemp *bt = tcgv_vec_temp(b); | |
206 | TCGType type = rt->base_type; | |
207 | ||
db432672 RH |
208 | /* Must enough inputs for the output. */ |
209 | tcg_debug_assert(at->base_type >= type); | |
210 | tcg_debug_assert(bt->base_type >= type); | |
d2fd745f RH |
211 | vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt)); |
212 | } | |
213 | ||
214 | void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) | |
215 | { | |
216 | if (r != a) { | |
217 | vec_gen_op2(INDEX_op_mov_vec, 0, r, a); | |
218 | } | |
219 | } | |
220 | ||
db432672 RH |
221 | void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) |
222 | { | |
0b4286dd RH |
223 | TCGTemp *rt = tcgv_vec_temp(r); |
224 | tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a)); | |
d2fd745f RH |
225 | } |
226 | ||
227 | void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) | |
228 | { | |
229 | TCGArg ri = tcgv_vec_arg(r); | |
230 | TCGTemp *rt = arg_temp(ri); | |
231 | TCGType type = rt->base_type; | |
232 | ||
233 | if (TCG_TARGET_REG_BITS == 64) { | |
234 | TCGArg ai = tcgv_i64_arg(a); | |
db432672 | 235 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); |
d2fd745f RH |
236 | } else if (vece == MO_64) { |
237 | TCGArg al = tcgv_i32_arg(TCGV_LOW(a)); | |
238 | TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a)); | |
239 | vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah); | |
240 | } else { | |
241 | TCGArg ai = tcgv_i32_arg(TCGV_LOW(a)); | |
db432672 | 242 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); |
d2fd745f RH |
243 | } |
244 | } | |
245 | ||
246 | void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a) | |
247 | { | |
248 | TCGArg ri = tcgv_vec_arg(r); | |
249 | TCGArg ai = tcgv_i32_arg(a); | |
250 | TCGTemp *rt = arg_temp(ri); | |
251 | TCGType type = rt->base_type; | |
252 | ||
253 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); | |
254 | } | |
255 | ||
37ee55a0 RH |
256 | void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b, |
257 | tcg_target_long ofs) | |
258 | { | |
259 | TCGArg ri = tcgv_vec_arg(r); | |
260 | TCGArg bi = tcgv_ptr_arg(b); | |
261 | TCGTemp *rt = arg_temp(ri); | |
262 | TCGType type = rt->base_type; | |
263 | ||
264 | vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs); | |
265 | } | |
266 | ||
d2fd745f RH |
267 | static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o) |
268 | { | |
269 | TCGArg ri = tcgv_vec_arg(r); | |
270 | TCGArg bi = tcgv_ptr_arg(b); | |
271 | TCGTemp *rt = arg_temp(ri); | |
272 | TCGType type = rt->base_type; | |
273 | ||
274 | vec_gen_3(opc, type, 0, ri, bi, o); | |
275 | } | |
276 | ||
277 | void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) | |
278 | { | |
279 | vec_gen_ldst(INDEX_op_ld_vec, r, b, o); | |
280 | } | |
281 | ||
282 | void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) | |
283 | { | |
284 | vec_gen_ldst(INDEX_op_st_vec, r, b, o); | |
285 | } | |
286 | ||
287 | void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type) | |
288 | { | |
289 | TCGArg ri = tcgv_vec_arg(r); | |
290 | TCGArg bi = tcgv_ptr_arg(b); | |
291 | TCGTemp *rt = arg_temp(ri); | |
292 | TCGType type = rt->base_type; | |
293 | ||
294 | tcg_debug_assert(low_type >= TCG_TYPE_V64); | |
295 | tcg_debug_assert(low_type <= type); | |
296 | vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o); | |
297 | } | |
298 | ||
d2fd745f RH |
299 | void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
300 | { | |
301 | vec_gen_op3(INDEX_op_and_vec, 0, r, a, b); | |
302 | } | |
303 | ||
304 | void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
305 | { | |
306 | vec_gen_op3(INDEX_op_or_vec, 0, r, a, b); | |
307 | } | |
308 | ||
309 | void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
310 | { | |
311 | vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b); | |
312 | } | |
313 | ||
314 | void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
315 | { | |
316 | if (TCG_TARGET_HAS_andc_vec) { | |
317 | vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b); | |
318 | } else { | |
319 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
320 | tcg_gen_not_vec(0, t, b); | |
321 | tcg_gen_and_vec(0, r, a, t); | |
322 | tcg_temp_free_vec(t); | |
323 | } | |
324 | } | |
325 | ||
326 | void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
327 | { | |
328 | if (TCG_TARGET_HAS_orc_vec) { | |
329 | vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b); | |
330 | } else { | |
331 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
332 | tcg_gen_not_vec(0, t, b); | |
333 | tcg_gen_or_vec(0, r, a, t); | |
334 | tcg_temp_free_vec(t); | |
335 | } | |
336 | } | |
337 | ||
f550805d RH |
338 | void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
339 | { | |
ed523473 RH |
340 | if (TCG_TARGET_HAS_nand_vec) { |
341 | vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b); | |
342 | } else { | |
343 | tcg_gen_and_vec(0, r, a, b); | |
344 | tcg_gen_not_vec(0, r, r); | |
345 | } | |
f550805d RH |
346 | } |
347 | ||
348 | void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
349 | { | |
ed523473 RH |
350 | if (TCG_TARGET_HAS_nor_vec) { |
351 | vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b); | |
352 | } else { | |
353 | tcg_gen_or_vec(0, r, a, b); | |
354 | tcg_gen_not_vec(0, r, r); | |
355 | } | |
f550805d RH |
356 | } |
357 | ||
358 | void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
359 | { | |
ed523473 RH |
360 | if (TCG_TARGET_HAS_eqv_vec) { |
361 | vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b); | |
362 | } else { | |
363 | tcg_gen_xor_vec(0, r, a, b); | |
364 | tcg_gen_not_vec(0, r, r); | |
365 | } | |
f550805d RH |
366 | } |
367 | ||
ce27c5d1 | 368 | static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) |
d2fd745f | 369 | { |
ce27c5d1 RH |
370 | TCGTemp *rt = tcgv_vec_temp(r); |
371 | TCGTemp *at = tcgv_vec_temp(a); | |
372 | TCGArg ri = temp_arg(rt); | |
373 | TCGArg ai = temp_arg(at); | |
374 | TCGType type = rt->base_type; | |
375 | int can; | |
376 | ||
377 | tcg_debug_assert(at->base_type >= type); | |
53229a77 | 378 | tcg_assert_listed_vecop(opc); |
ce27c5d1 RH |
379 | can = tcg_can_emit_vec_op(opc, type, vece); |
380 | if (can > 0) { | |
381 | vec_gen_2(opc, type, vece, ri, ai); | |
382 | } else if (can < 0) { | |
53229a77 | 383 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
ce27c5d1 | 384 | tcg_expand_vec_op(opc, type, vece, ri, ai); |
53229a77 | 385 | tcg_swap_vecop_list(hold_list); |
d2fd745f | 386 | } else { |
ce27c5d1 RH |
387 | return false; |
388 | } | |
389 | return true; | |
390 | } | |
391 | ||
392 | void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a) | |
393 | { | |
11978f6f RH |
394 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
395 | ||
ce27c5d1 | 396 | if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) { |
f6ff9c2f | 397 | tcg_gen_xor_vec(0, r, a, tcg_constant_vec_matching(r, 0, -1)); |
d2fd745f | 398 | } |
11978f6f | 399 | tcg_swap_vecop_list(hold_list); |
d2fd745f RH |
400 | } |
401 | ||
402 | void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a) | |
403 | { | |
53229a77 RH |
404 | const TCGOpcode *hold_list; |
405 | ||
406 | tcg_assert_listed_vecop(INDEX_op_neg_vec); | |
407 | hold_list = tcg_swap_vecop_list(NULL); | |
408 | ||
ce27c5d1 | 409 | if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) { |
f6ff9c2f | 410 | tcg_gen_sub_vec(vece, r, tcg_constant_vec_matching(r, vece, 0), a); |
d2fd745f | 411 | } |
53229a77 | 412 | tcg_swap_vecop_list(hold_list); |
d2fd745f | 413 | } |
d0ec9796 | 414 | |
bcefc902 RH |
415 | void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) |
416 | { | |
417 | const TCGOpcode *hold_list; | |
418 | ||
419 | tcg_assert_listed_vecop(INDEX_op_abs_vec); | |
420 | hold_list = tcg_swap_vecop_list(NULL); | |
421 | ||
422 | if (!do_op2(vece, r, a, INDEX_op_abs_vec)) { | |
423 | TCGType type = tcgv_vec_temp(r)->base_type; | |
424 | TCGv_vec t = tcg_temp_new_vec(type); | |
425 | ||
426 | tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)); | |
427 | if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) { | |
428 | tcg_gen_neg_vec(vece, t, a); | |
429 | tcg_gen_smax_vec(vece, r, a, t); | |
430 | } else { | |
431 | if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { | |
432 | tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); | |
433 | } else { | |
0b4286dd RH |
434 | tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, |
435 | tcg_constant_vec(type, vece, 0)); | |
bcefc902 RH |
436 | } |
437 | tcg_gen_xor_vec(vece, r, a, t); | |
438 | tcg_gen_sub_vec(vece, r, r, t); | |
439 | } | |
440 | ||
441 | tcg_temp_free_vec(t); | |
442 | } | |
443 | tcg_swap_vecop_list(hold_list); | |
444 | } | |
445 | ||
d0ec9796 RH |
446 | static void do_shifti(TCGOpcode opc, unsigned vece, |
447 | TCGv_vec r, TCGv_vec a, int64_t i) | |
448 | { | |
449 | TCGTemp *rt = tcgv_vec_temp(r); | |
450 | TCGTemp *at = tcgv_vec_temp(a); | |
451 | TCGArg ri = temp_arg(rt); | |
452 | TCGArg ai = temp_arg(at); | |
453 | TCGType type = rt->base_type; | |
454 | int can; | |
455 | ||
456 | tcg_debug_assert(at->base_type == type); | |
457 | tcg_debug_assert(i >= 0 && i < (8 << vece)); | |
53229a77 | 458 | tcg_assert_listed_vecop(opc); |
d0ec9796 RH |
459 | |
460 | if (i == 0) { | |
461 | tcg_gen_mov_vec(r, a); | |
462 | return; | |
463 | } | |
464 | ||
465 | can = tcg_can_emit_vec_op(opc, type, vece); | |
466 | if (can > 0) { | |
467 | vec_gen_3(opc, type, vece, ri, ai, i); | |
468 | } else { | |
469 | /* We leave the choice of expansion via scalar or vector shift | |
470 | to the target. Often, but not always, dupi can feed a vector | |
471 | shift easier than a scalar. */ | |
53229a77 | 472 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
d0ec9796 RH |
473 | tcg_debug_assert(can < 0); |
474 | tcg_expand_vec_op(opc, type, vece, ri, ai, i); | |
53229a77 | 475 | tcg_swap_vecop_list(hold_list); |
d0ec9796 RH |
476 | } |
477 | } | |
478 | ||
479 | void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
480 | { | |
481 | do_shifti(INDEX_op_shli_vec, vece, r, a, i); | |
482 | } | |
483 | ||
484 | void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
485 | { | |
486 | do_shifti(INDEX_op_shri_vec, vece, r, a, i); | |
487 | } | |
488 | ||
489 | void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
490 | { | |
491 | do_shifti(INDEX_op_sari_vec, vece, r, a, i); | |
492 | } | |
212be173 | 493 | |
b0f7e744 RH |
494 | void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) |
495 | { | |
496 | do_shifti(INDEX_op_rotli_vec, vece, r, a, i); | |
497 | } | |
498 | ||
499 | void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
500 | { | |
501 | int bits = 8 << vece; | |
502 | tcg_debug_assert(i >= 0 && i < bits); | |
503 | do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1)); | |
504 | } | |
505 | ||
212be173 RH |
506 | void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, |
507 | TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
508 | { | |
509 | TCGTemp *rt = tcgv_vec_temp(r); | |
510 | TCGTemp *at = tcgv_vec_temp(a); | |
511 | TCGTemp *bt = tcgv_vec_temp(b); | |
512 | TCGArg ri = temp_arg(rt); | |
513 | TCGArg ai = temp_arg(at); | |
514 | TCGArg bi = temp_arg(bt); | |
515 | TCGType type = rt->base_type; | |
516 | int can; | |
517 | ||
9a938d86 RH |
518 | tcg_debug_assert(at->base_type >= type); |
519 | tcg_debug_assert(bt->base_type >= type); | |
53229a77 | 520 | tcg_assert_listed_vecop(INDEX_op_cmp_vec); |
212be173 RH |
521 | can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece); |
522 | if (can > 0) { | |
523 | vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); | |
524 | } else { | |
53229a77 | 525 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
212be173 RH |
526 | tcg_debug_assert(can < 0); |
527 | tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); | |
53229a77 | 528 | tcg_swap_vecop_list(hold_list); |
212be173 RH |
529 | } |
530 | } | |
3774030a | 531 | |
17f79944 | 532 | static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a, |
8afaf050 | 533 | TCGv_vec b, TCGOpcode opc) |
3774030a RH |
534 | { |
535 | TCGTemp *rt = tcgv_vec_temp(r); | |
536 | TCGTemp *at = tcgv_vec_temp(a); | |
537 | TCGTemp *bt = tcgv_vec_temp(b); | |
538 | TCGArg ri = temp_arg(rt); | |
539 | TCGArg ai = temp_arg(at); | |
540 | TCGArg bi = temp_arg(bt); | |
541 | TCGType type = rt->base_type; | |
542 | int can; | |
543 | ||
9a938d86 RH |
544 | tcg_debug_assert(at->base_type >= type); |
545 | tcg_debug_assert(bt->base_type >= type); | |
53229a77 | 546 | tcg_assert_listed_vecop(opc); |
8afaf050 | 547 | can = tcg_can_emit_vec_op(opc, type, vece); |
3774030a | 548 | if (can > 0) { |
8afaf050 | 549 | vec_gen_3(opc, type, vece, ri, ai, bi); |
17f79944 | 550 | } else if (can < 0) { |
53229a77 | 551 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
8afaf050 | 552 | tcg_expand_vec_op(opc, type, vece, ri, ai, bi); |
53229a77 | 553 | tcg_swap_vecop_list(hold_list); |
17f79944 RH |
554 | } else { |
555 | return false; | |
3774030a | 556 | } |
17f79944 RH |
557 | return true; |
558 | } | |
559 | ||
560 | static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a, | |
561 | TCGv_vec b, TCGOpcode opc) | |
562 | { | |
563 | bool ok = do_op3(vece, r, a, b, opc); | |
564 | tcg_debug_assert(ok); | |
3774030a | 565 | } |
8afaf050 | 566 | |
ce27c5d1 RH |
567 | void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
568 | { | |
17f79944 | 569 | do_op3_nofail(vece, r, a, b, INDEX_op_add_vec); |
ce27c5d1 RH |
570 | } |
571 | ||
572 | void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
573 | { | |
17f79944 | 574 | do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec); |
ce27c5d1 RH |
575 | } |
576 | ||
8afaf050 RH |
577 | void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
578 | { | |
17f79944 | 579 | do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec); |
8afaf050 RH |
580 | } |
581 | ||
582 | void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
583 | { | |
17f79944 | 584 | do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec); |
8afaf050 RH |
585 | } |
586 | ||
587 | void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
588 | { | |
2552d60e RH |
589 | if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) { |
590 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); | |
591 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
592 | ||
593 | /* usadd(a, b) = min(a, ~b) + b */ | |
594 | tcg_gen_not_vec(vece, t, b); | |
595 | tcg_gen_umin_vec(vece, t, t, a); | |
596 | tcg_gen_add_vec(vece, r, t, b); | |
597 | ||
598 | tcg_temp_free_vec(t); | |
599 | tcg_swap_vecop_list(hold_list); | |
600 | } | |
8afaf050 RH |
601 | } |
602 | ||
603 | void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
604 | { | |
17f79944 | 605 | do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec); |
8afaf050 RH |
606 | } |
607 | ||
608 | void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
609 | { | |
2552d60e RH |
610 | if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) { |
611 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); | |
612 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
613 | ||
614 | /* ussub(a, b) = max(a, b) - b */ | |
615 | tcg_gen_umax_vec(vece, t, a, b); | |
616 | tcg_gen_sub_vec(vece, r, t, b); | |
617 | ||
618 | tcg_temp_free_vec(t); | |
619 | tcg_swap_vecop_list(hold_list); | |
620 | } | |
8afaf050 | 621 | } |
dd0a0fcd | 622 | |
72b4c792 RH |
623 | static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a, |
624 | TCGv_vec b, TCGOpcode opc, TCGCond cond) | |
625 | { | |
626 | if (!do_op3(vece, r, a, b, opc)) { | |
69c918d2 | 627 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
72b4c792 | 628 | tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b); |
69c918d2 | 629 | tcg_swap_vecop_list(hold_list); |
72b4c792 RH |
630 | } |
631 | } | |
632 | ||
dd0a0fcd RH |
633 | void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
634 | { | |
72b4c792 | 635 | do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT); |
dd0a0fcd RH |
636 | } |
637 | ||
638 | void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
639 | { | |
72b4c792 | 640 | do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU); |
dd0a0fcd RH |
641 | } |
642 | ||
643 | void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
644 | { | |
72b4c792 | 645 | do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT); |
dd0a0fcd RH |
646 | } |
647 | ||
648 | void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
649 | { | |
72b4c792 | 650 | do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU); |
dd0a0fcd | 651 | } |
5ee5c14c RH |
652 | |
653 | void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
654 | { | |
17f79944 | 655 | do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec); |
5ee5c14c RH |
656 | } |
657 | ||
658 | void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
659 | { | |
17f79944 | 660 | do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec); |
5ee5c14c RH |
661 | } |
662 | ||
663 | void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
664 | { | |
17f79944 | 665 | do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec); |
5ee5c14c | 666 | } |
b4578cd9 | 667 | |
5d0ceda9 RH |
668 | void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
669 | { | |
670 | do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec); | |
671 | } | |
672 | ||
673 | void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
674 | { | |
675 | do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec); | |
676 | } | |
677 | ||
b4578cd9 | 678 | static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a, |
3d5bb2ea | 679 | TCGv_i32 s, TCGOpcode opc) |
b4578cd9 RH |
680 | { |
681 | TCGTemp *rt = tcgv_vec_temp(r); | |
682 | TCGTemp *at = tcgv_vec_temp(a); | |
683 | TCGTemp *st = tcgv_i32_temp(s); | |
684 | TCGArg ri = temp_arg(rt); | |
685 | TCGArg ai = temp_arg(at); | |
686 | TCGArg si = temp_arg(st); | |
687 | TCGType type = rt->base_type; | |
b4578cd9 RH |
688 | int can; |
689 | ||
690 | tcg_debug_assert(at->base_type >= type); | |
3d5bb2ea RH |
691 | tcg_assert_listed_vecop(opc); |
692 | can = tcg_can_emit_vec_op(opc, type, vece); | |
b4578cd9 | 693 | if (can > 0) { |
3d5bb2ea | 694 | vec_gen_3(opc, type, vece, ri, ai, si); |
b4578cd9 | 695 | } else if (can < 0) { |
3d5bb2ea RH |
696 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
697 | tcg_expand_vec_op(opc, type, vece, ri, ai, si); | |
698 | tcg_swap_vecop_list(hold_list); | |
b4578cd9 | 699 | } else { |
3d5bb2ea | 700 | g_assert_not_reached(); |
b4578cd9 | 701 | } |
b4578cd9 RH |
702 | } |
703 | ||
704 | void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
705 | { | |
3d5bb2ea | 706 | do_shifts(vece, r, a, b, INDEX_op_shls_vec); |
b4578cd9 RH |
707 | } |
708 | ||
709 | void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
710 | { | |
3d5bb2ea | 711 | do_shifts(vece, r, a, b, INDEX_op_shrs_vec); |
b4578cd9 RH |
712 | } |
713 | ||
714 | void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
715 | { | |
3d5bb2ea | 716 | do_shifts(vece, r, a, b, INDEX_op_sars_vec); |
b4578cd9 | 717 | } |
38dc1294 | 718 | |
23850a74 RH |
719 | void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s) |
720 | { | |
721 | do_shifts(vece, r, a, s, INDEX_op_rotls_vec); | |
722 | } | |
723 | ||
38dc1294 RH |
724 | void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, |
725 | TCGv_vec b, TCGv_vec c) | |
726 | { | |
727 | TCGTemp *rt = tcgv_vec_temp(r); | |
728 | TCGTemp *at = tcgv_vec_temp(a); | |
729 | TCGTemp *bt = tcgv_vec_temp(b); | |
730 | TCGTemp *ct = tcgv_vec_temp(c); | |
731 | TCGType type = rt->base_type; | |
732 | ||
733 | tcg_debug_assert(at->base_type >= type); | |
734 | tcg_debug_assert(bt->base_type >= type); | |
735 | tcg_debug_assert(ct->base_type >= type); | |
736 | ||
737 | if (TCG_TARGET_HAS_bitsel_vec) { | |
738 | vec_gen_4(INDEX_op_bitsel_vec, type, MO_8, | |
739 | temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct)); | |
740 | } else { | |
741 | TCGv_vec t = tcg_temp_new_vec(type); | |
742 | tcg_gen_and_vec(MO_8, t, a, b); | |
743 | tcg_gen_andc_vec(MO_8, r, c, a); | |
744 | tcg_gen_or_vec(MO_8, r, r, t); | |
745 | tcg_temp_free_vec(t); | |
746 | } | |
747 | } | |
f75da298 RH |
748 | |
749 | void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r, | |
750 | TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d) | |
751 | { | |
752 | TCGTemp *rt = tcgv_vec_temp(r); | |
753 | TCGTemp *at = tcgv_vec_temp(a); | |
754 | TCGTemp *bt = tcgv_vec_temp(b); | |
755 | TCGTemp *ct = tcgv_vec_temp(c); | |
756 | TCGTemp *dt = tcgv_vec_temp(d); | |
757 | TCGArg ri = temp_arg(rt); | |
758 | TCGArg ai = temp_arg(at); | |
759 | TCGArg bi = temp_arg(bt); | |
760 | TCGArg ci = temp_arg(ct); | |
761 | TCGArg di = temp_arg(dt); | |
762 | TCGType type = rt->base_type; | |
763 | const TCGOpcode *hold_list; | |
764 | int can; | |
765 | ||
766 | tcg_debug_assert(at->base_type >= type); | |
767 | tcg_debug_assert(bt->base_type >= type); | |
768 | tcg_debug_assert(ct->base_type >= type); | |
769 | tcg_debug_assert(dt->base_type >= type); | |
770 | ||
771 | tcg_assert_listed_vecop(INDEX_op_cmpsel_vec); | |
772 | hold_list = tcg_swap_vecop_list(NULL); | |
773 | can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece); | |
774 | ||
775 | if (can > 0) { | |
776 | vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond); | |
777 | } else if (can < 0) { | |
778 | tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece, | |
779 | ri, ai, bi, ci, di, cond); | |
780 | } else { | |
781 | TCGv_vec t = tcg_temp_new_vec(type); | |
782 | tcg_gen_cmp_vec(cond, vece, t, a, b); | |
783 | tcg_gen_bitsel_vec(vece, r, t, c, d); | |
784 | tcg_temp_free_vec(t); | |
785 | } | |
786 | tcg_swap_vecop_list(hold_list); | |
787 | } |