]>
Commit | Line | Data |
---|---|---|
d2fd745f RH |
1 | /* |
2 | * Tiny Code Generator for QEMU | |
3 | * | |
4 | * Copyright (c) 2018 Linaro, Inc. | |
5 | * | |
6 | * This library is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU Lesser General Public | |
8 | * License as published by the Free Software Foundation; either | |
fb0343d5 | 9 | * version 2.1 of the License, or (at your option) any later version. |
d2fd745f RH |
10 | * |
11 | * This library is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * Lesser General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU Lesser General Public | |
17 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | ||
20 | #include "qemu/osdep.h" | |
d2fd745f | 21 | #include "cpu.h" |
dcb32f1d PMD |
22 | #include "tcg/tcg.h" |
23 | #include "tcg/tcg-op.h" | |
24 | #include "tcg/tcg-mo.h" | |
d2fd745f RH |
25 | |
26 | /* Reduce the number of ifdefs below. This assumes that all uses of | |
27 | TCGV_HIGH and TCGV_LOW are properly protected by a conditional that | |
28 | the compiler can eliminate. */ | |
29 | #if TCG_TARGET_REG_BITS == 64 | |
30 | extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); | |
31 | extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); | |
32 | #define TCGV_LOW TCGV_LOW_link_error | |
33 | #define TCGV_HIGH TCGV_HIGH_link_error | |
34 | #endif | |
35 | ||
53229a77 RH |
36 | /* |
37 | * Vector optional opcode tracking. | |
38 | * Except for the basic logical operations (and, or, xor), and | |
39 | * data movement (mov, ld, st, dupi), many vector opcodes are | |
40 | * optional and may not be supported on the host. Thank Intel | |
41 | * for the irregularity in their instruction set. | |
42 | * | |
43 | * The gvec expanders allow custom vector operations to be composed, | |
44 | * generally via the .fniv callback in the GVecGen* structures. At | |
45 | * the same time, in deciding whether to use this hook we need to | |
46 | * know if the host supports the required operations. This is | |
47 | * presented as an array of opcodes, terminated by 0. Each opcode | |
48 | * is assumed to be expanded with the given VECE. | |
49 | * | |
50 | * For debugging, we want to validate this array. Therefore, when | |
51 | * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders | |
52 | * will validate that their opcode is present in the list. | |
53 | */ | |
54 | #ifdef CONFIG_DEBUG_TCG | |
55 | void tcg_assert_listed_vecop(TCGOpcode op) | |
56 | { | |
57 | const TCGOpcode *p = tcg_ctx->vecop_list; | |
58 | if (p) { | |
59 | for (; *p; ++p) { | |
60 | if (*p == op) { | |
61 | return; | |
62 | } | |
63 | } | |
64 | g_assert_not_reached(); | |
65 | } | |
66 | } | |
67 | #endif | |
68 | ||
69 | bool tcg_can_emit_vecop_list(const TCGOpcode *list, | |
70 | TCGType type, unsigned vece) | |
71 | { | |
72 | if (list == NULL) { | |
73 | return true; | |
74 | } | |
75 | ||
76 | for (; *list; ++list) { | |
77 | TCGOpcode opc = *list; | |
78 | ||
79 | #ifdef CONFIG_DEBUG_TCG | |
80 | switch (opc) { | |
81 | case INDEX_op_and_vec: | |
82 | case INDEX_op_or_vec: | |
83 | case INDEX_op_xor_vec: | |
84 | case INDEX_op_mov_vec: | |
85 | case INDEX_op_dup_vec: | |
53229a77 RH |
86 | case INDEX_op_dup2_vec: |
87 | case INDEX_op_ld_vec: | |
88 | case INDEX_op_st_vec: | |
38dc1294 | 89 | case INDEX_op_bitsel_vec: |
53229a77 RH |
90 | /* These opcodes are mandatory and should not be listed. */ |
91 | g_assert_not_reached(); | |
11978f6f RH |
92 | case INDEX_op_not_vec: |
93 | /* These opcodes have generic expansions using the above. */ | |
94 | g_assert_not_reached(); | |
53229a77 RH |
95 | default: |
96 | break; | |
97 | } | |
98 | #endif | |
99 | ||
100 | if (tcg_can_emit_vec_op(opc, type, vece)) { | |
101 | continue; | |
102 | } | |
103 | ||
104 | /* | |
105 | * The opcode list is created by front ends based on what they | |
106 | * actually invoke. We must mirror the logic in the routines | |
107 | * below for generic expansions using other opcodes. | |
108 | */ | |
109 | switch (opc) { | |
110 | case INDEX_op_neg_vec: | |
111 | if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) { | |
112 | continue; | |
113 | } | |
114 | break; | |
bcefc902 RH |
115 | case INDEX_op_abs_vec: |
116 | if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece) | |
117 | && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0 | |
118 | || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0 | |
119 | || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) { | |
120 | continue; | |
121 | } | |
122 | break; | |
f75da298 | 123 | case INDEX_op_cmpsel_vec: |
72b4c792 RH |
124 | case INDEX_op_smin_vec: |
125 | case INDEX_op_smax_vec: | |
126 | case INDEX_op_umin_vec: | |
127 | case INDEX_op_umax_vec: | |
f75da298 RH |
128 | if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { |
129 | continue; | |
130 | } | |
131 | break; | |
53229a77 RH |
132 | default: |
133 | break; | |
134 | } | |
135 | return false; | |
136 | } | |
137 | return true; | |
138 | } | |
139 | ||
d2fd745f RH |
140 | void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) |
141 | { | |
142 | TCGOp *op = tcg_emit_op(opc); | |
143 | TCGOP_VECL(op) = type - TCG_TYPE_V64; | |
144 | TCGOP_VECE(op) = vece; | |
145 | op->args[0] = r; | |
146 | op->args[1] = a; | |
147 | } | |
148 | ||
149 | void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, | |
150 | TCGArg r, TCGArg a, TCGArg b) | |
151 | { | |
152 | TCGOp *op = tcg_emit_op(opc); | |
153 | TCGOP_VECL(op) = type - TCG_TYPE_V64; | |
154 | TCGOP_VECE(op) = vece; | |
155 | op->args[0] = r; | |
156 | op->args[1] = a; | |
157 | op->args[2] = b; | |
158 | } | |
159 | ||
160 | void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, | |
161 | TCGArg r, TCGArg a, TCGArg b, TCGArg c) | |
162 | { | |
163 | TCGOp *op = tcg_emit_op(opc); | |
164 | TCGOP_VECL(op) = type - TCG_TYPE_V64; | |
165 | TCGOP_VECE(op) = vece; | |
166 | op->args[0] = r; | |
167 | op->args[1] = a; | |
168 | op->args[2] = b; | |
169 | op->args[3] = c; | |
170 | } | |
171 | ||
f75da298 RH |
172 | static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, |
173 | TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) | |
174 | { | |
175 | TCGOp *op = tcg_emit_op(opc); | |
176 | TCGOP_VECL(op) = type - TCG_TYPE_V64; | |
177 | TCGOP_VECE(op) = vece; | |
178 | op->args[0] = r; | |
179 | op->args[1] = a; | |
180 | op->args[2] = b; | |
181 | op->args[3] = c; | |
182 | op->args[4] = d; | |
183 | op->args[5] = e; | |
184 | } | |
185 | ||
d2fd745f RH |
186 | static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a) |
187 | { | |
188 | TCGTemp *rt = tcgv_vec_temp(r); | |
189 | TCGTemp *at = tcgv_vec_temp(a); | |
190 | TCGType type = rt->base_type; | |
191 | ||
db432672 RH |
192 | /* Must enough inputs for the output. */ |
193 | tcg_debug_assert(at->base_type >= type); | |
d2fd745f RH |
194 | vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at)); |
195 | } | |
196 | ||
197 | static void vec_gen_op3(TCGOpcode opc, unsigned vece, | |
198 | TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
199 | { | |
200 | TCGTemp *rt = tcgv_vec_temp(r); | |
201 | TCGTemp *at = tcgv_vec_temp(a); | |
202 | TCGTemp *bt = tcgv_vec_temp(b); | |
203 | TCGType type = rt->base_type; | |
204 | ||
db432672 RH |
205 | /* Must enough inputs for the output. */ |
206 | tcg_debug_assert(at->base_type >= type); | |
207 | tcg_debug_assert(bt->base_type >= type); | |
d2fd745f RH |
208 | vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt)); |
209 | } | |
210 | ||
211 | void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) | |
212 | { | |
213 | if (r != a) { | |
214 | vec_gen_op2(INDEX_op_mov_vec, 0, r, a); | |
215 | } | |
216 | } | |
217 | ||
d2fd745f RH |
218 | TCGv_vec tcg_const_zeros_vec(TCGType type) |
219 | { | |
220 | TCGv_vec ret = tcg_temp_new_vec(type); | |
0b4286dd | 221 | tcg_gen_dupi_vec(MO_64, ret, 0); |
d2fd745f RH |
222 | return ret; |
223 | } | |
224 | ||
225 | TCGv_vec tcg_const_ones_vec(TCGType type) | |
226 | { | |
227 | TCGv_vec ret = tcg_temp_new_vec(type); | |
0b4286dd | 228 | tcg_gen_dupi_vec(MO_64, ret, -1); |
d2fd745f RH |
229 | return ret; |
230 | } | |
231 | ||
232 | TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m) | |
233 | { | |
234 | TCGTemp *t = tcgv_vec_temp(m); | |
235 | return tcg_const_zeros_vec(t->base_type); | |
236 | } | |
237 | ||
238 | TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m) | |
239 | { | |
240 | TCGTemp *t = tcgv_vec_temp(m); | |
241 | return tcg_const_ones_vec(t->base_type); | |
242 | } | |
243 | ||
db432672 RH |
244 | void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) |
245 | { | |
0b4286dd RH |
246 | TCGTemp *rt = tcgv_vec_temp(r); |
247 | tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a)); | |
d2fd745f RH |
248 | } |
249 | ||
250 | void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) | |
251 | { | |
252 | TCGArg ri = tcgv_vec_arg(r); | |
253 | TCGTemp *rt = arg_temp(ri); | |
254 | TCGType type = rt->base_type; | |
255 | ||
256 | if (TCG_TARGET_REG_BITS == 64) { | |
257 | TCGArg ai = tcgv_i64_arg(a); | |
db432672 | 258 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); |
d2fd745f RH |
259 | } else if (vece == MO_64) { |
260 | TCGArg al = tcgv_i32_arg(TCGV_LOW(a)); | |
261 | TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a)); | |
262 | vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah); | |
263 | } else { | |
264 | TCGArg ai = tcgv_i32_arg(TCGV_LOW(a)); | |
db432672 | 265 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); |
d2fd745f RH |
266 | } |
267 | } | |
268 | ||
269 | void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a) | |
270 | { | |
271 | TCGArg ri = tcgv_vec_arg(r); | |
272 | TCGArg ai = tcgv_i32_arg(a); | |
273 | TCGTemp *rt = arg_temp(ri); | |
274 | TCGType type = rt->base_type; | |
275 | ||
276 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); | |
277 | } | |
278 | ||
37ee55a0 RH |
279 | void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b, |
280 | tcg_target_long ofs) | |
281 | { | |
282 | TCGArg ri = tcgv_vec_arg(r); | |
283 | TCGArg bi = tcgv_ptr_arg(b); | |
284 | TCGTemp *rt = arg_temp(ri); | |
285 | TCGType type = rt->base_type; | |
286 | ||
287 | vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs); | |
288 | } | |
289 | ||
d2fd745f RH |
290 | static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o) |
291 | { | |
292 | TCGArg ri = tcgv_vec_arg(r); | |
293 | TCGArg bi = tcgv_ptr_arg(b); | |
294 | TCGTemp *rt = arg_temp(ri); | |
295 | TCGType type = rt->base_type; | |
296 | ||
297 | vec_gen_3(opc, type, 0, ri, bi, o); | |
298 | } | |
299 | ||
300 | void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) | |
301 | { | |
302 | vec_gen_ldst(INDEX_op_ld_vec, r, b, o); | |
303 | } | |
304 | ||
305 | void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) | |
306 | { | |
307 | vec_gen_ldst(INDEX_op_st_vec, r, b, o); | |
308 | } | |
309 | ||
310 | void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type) | |
311 | { | |
312 | TCGArg ri = tcgv_vec_arg(r); | |
313 | TCGArg bi = tcgv_ptr_arg(b); | |
314 | TCGTemp *rt = arg_temp(ri); | |
315 | TCGType type = rt->base_type; | |
316 | ||
317 | tcg_debug_assert(low_type >= TCG_TYPE_V64); | |
318 | tcg_debug_assert(low_type <= type); | |
319 | vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o); | |
320 | } | |
321 | ||
d2fd745f RH |
322 | void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
323 | { | |
324 | vec_gen_op3(INDEX_op_and_vec, 0, r, a, b); | |
325 | } | |
326 | ||
327 | void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
328 | { | |
329 | vec_gen_op3(INDEX_op_or_vec, 0, r, a, b); | |
330 | } | |
331 | ||
332 | void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
333 | { | |
334 | vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b); | |
335 | } | |
336 | ||
337 | void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
338 | { | |
339 | if (TCG_TARGET_HAS_andc_vec) { | |
340 | vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b); | |
341 | } else { | |
342 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
343 | tcg_gen_not_vec(0, t, b); | |
344 | tcg_gen_and_vec(0, r, a, t); | |
345 | tcg_temp_free_vec(t); | |
346 | } | |
347 | } | |
348 | ||
349 | void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
350 | { | |
351 | if (TCG_TARGET_HAS_orc_vec) { | |
352 | vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b); | |
353 | } else { | |
354 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
355 | tcg_gen_not_vec(0, t, b); | |
356 | tcg_gen_or_vec(0, r, a, t); | |
357 | tcg_temp_free_vec(t); | |
358 | } | |
359 | } | |
360 | ||
f550805d RH |
361 | void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
362 | { | |
363 | /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */ | |
364 | tcg_gen_and_vec(0, r, a, b); | |
365 | tcg_gen_not_vec(0, r, r); | |
366 | } | |
367 | ||
368 | void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
369 | { | |
370 | /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */ | |
371 | tcg_gen_or_vec(0, r, a, b); | |
372 | tcg_gen_not_vec(0, r, r); | |
373 | } | |
374 | ||
375 | void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
376 | { | |
377 | /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */ | |
378 | tcg_gen_xor_vec(0, r, a, b); | |
379 | tcg_gen_not_vec(0, r, r); | |
380 | } | |
381 | ||
ce27c5d1 | 382 | static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) |
d2fd745f | 383 | { |
ce27c5d1 RH |
384 | TCGTemp *rt = tcgv_vec_temp(r); |
385 | TCGTemp *at = tcgv_vec_temp(a); | |
386 | TCGArg ri = temp_arg(rt); | |
387 | TCGArg ai = temp_arg(at); | |
388 | TCGType type = rt->base_type; | |
389 | int can; | |
390 | ||
391 | tcg_debug_assert(at->base_type >= type); | |
53229a77 | 392 | tcg_assert_listed_vecop(opc); |
ce27c5d1 RH |
393 | can = tcg_can_emit_vec_op(opc, type, vece); |
394 | if (can > 0) { | |
395 | vec_gen_2(opc, type, vece, ri, ai); | |
396 | } else if (can < 0) { | |
53229a77 | 397 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
ce27c5d1 | 398 | tcg_expand_vec_op(opc, type, vece, ri, ai); |
53229a77 | 399 | tcg_swap_vecop_list(hold_list); |
d2fd745f | 400 | } else { |
ce27c5d1 RH |
401 | return false; |
402 | } | |
403 | return true; | |
404 | } | |
405 | ||
406 | void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a) | |
407 | { | |
11978f6f RH |
408 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
409 | ||
ce27c5d1 | 410 | if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) { |
d2fd745f RH |
411 | TCGv_vec t = tcg_const_ones_vec_matching(r); |
412 | tcg_gen_xor_vec(0, r, a, t); | |
413 | tcg_temp_free_vec(t); | |
414 | } | |
11978f6f | 415 | tcg_swap_vecop_list(hold_list); |
d2fd745f RH |
416 | } |
417 | ||
418 | void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a) | |
419 | { | |
53229a77 RH |
420 | const TCGOpcode *hold_list; |
421 | ||
422 | tcg_assert_listed_vecop(INDEX_op_neg_vec); | |
423 | hold_list = tcg_swap_vecop_list(NULL); | |
424 | ||
ce27c5d1 | 425 | if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) { |
d2fd745f RH |
426 | TCGv_vec t = tcg_const_zeros_vec_matching(r); |
427 | tcg_gen_sub_vec(vece, r, t, a); | |
428 | tcg_temp_free_vec(t); | |
429 | } | |
53229a77 | 430 | tcg_swap_vecop_list(hold_list); |
d2fd745f | 431 | } |
d0ec9796 | 432 | |
bcefc902 RH |
433 | void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) |
434 | { | |
435 | const TCGOpcode *hold_list; | |
436 | ||
437 | tcg_assert_listed_vecop(INDEX_op_abs_vec); | |
438 | hold_list = tcg_swap_vecop_list(NULL); | |
439 | ||
440 | if (!do_op2(vece, r, a, INDEX_op_abs_vec)) { | |
441 | TCGType type = tcgv_vec_temp(r)->base_type; | |
442 | TCGv_vec t = tcg_temp_new_vec(type); | |
443 | ||
444 | tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)); | |
445 | if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) { | |
446 | tcg_gen_neg_vec(vece, t, a); | |
447 | tcg_gen_smax_vec(vece, r, a, t); | |
448 | } else { | |
449 | if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { | |
450 | tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); | |
451 | } else { | |
0b4286dd RH |
452 | tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, |
453 | tcg_constant_vec(type, vece, 0)); | |
bcefc902 RH |
454 | } |
455 | tcg_gen_xor_vec(vece, r, a, t); | |
456 | tcg_gen_sub_vec(vece, r, r, t); | |
457 | } | |
458 | ||
459 | tcg_temp_free_vec(t); | |
460 | } | |
461 | tcg_swap_vecop_list(hold_list); | |
462 | } | |
463 | ||
d0ec9796 RH |
464 | static void do_shifti(TCGOpcode opc, unsigned vece, |
465 | TCGv_vec r, TCGv_vec a, int64_t i) | |
466 | { | |
467 | TCGTemp *rt = tcgv_vec_temp(r); | |
468 | TCGTemp *at = tcgv_vec_temp(a); | |
469 | TCGArg ri = temp_arg(rt); | |
470 | TCGArg ai = temp_arg(at); | |
471 | TCGType type = rt->base_type; | |
472 | int can; | |
473 | ||
474 | tcg_debug_assert(at->base_type == type); | |
475 | tcg_debug_assert(i >= 0 && i < (8 << vece)); | |
53229a77 | 476 | tcg_assert_listed_vecop(opc); |
d0ec9796 RH |
477 | |
478 | if (i == 0) { | |
479 | tcg_gen_mov_vec(r, a); | |
480 | return; | |
481 | } | |
482 | ||
483 | can = tcg_can_emit_vec_op(opc, type, vece); | |
484 | if (can > 0) { | |
485 | vec_gen_3(opc, type, vece, ri, ai, i); | |
486 | } else { | |
487 | /* We leave the choice of expansion via scalar or vector shift | |
488 | to the target. Often, but not always, dupi can feed a vector | |
489 | shift easier than a scalar. */ | |
53229a77 | 490 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
d0ec9796 RH |
491 | tcg_debug_assert(can < 0); |
492 | tcg_expand_vec_op(opc, type, vece, ri, ai, i); | |
53229a77 | 493 | tcg_swap_vecop_list(hold_list); |
d0ec9796 RH |
494 | } |
495 | } | |
496 | ||
497 | void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
498 | { | |
499 | do_shifti(INDEX_op_shli_vec, vece, r, a, i); | |
500 | } | |
501 | ||
502 | void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
503 | { | |
504 | do_shifti(INDEX_op_shri_vec, vece, r, a, i); | |
505 | } | |
506 | ||
507 | void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
508 | { | |
509 | do_shifti(INDEX_op_sari_vec, vece, r, a, i); | |
510 | } | |
212be173 | 511 | |
b0f7e744 RH |
512 | void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) |
513 | { | |
514 | do_shifti(INDEX_op_rotli_vec, vece, r, a, i); | |
515 | } | |
516 | ||
517 | void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
518 | { | |
519 | int bits = 8 << vece; | |
520 | tcg_debug_assert(i >= 0 && i < bits); | |
521 | do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1)); | |
522 | } | |
523 | ||
212be173 RH |
524 | void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, |
525 | TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
526 | { | |
527 | TCGTemp *rt = tcgv_vec_temp(r); | |
528 | TCGTemp *at = tcgv_vec_temp(a); | |
529 | TCGTemp *bt = tcgv_vec_temp(b); | |
530 | TCGArg ri = temp_arg(rt); | |
531 | TCGArg ai = temp_arg(at); | |
532 | TCGArg bi = temp_arg(bt); | |
533 | TCGType type = rt->base_type; | |
534 | int can; | |
535 | ||
9a938d86 RH |
536 | tcg_debug_assert(at->base_type >= type); |
537 | tcg_debug_assert(bt->base_type >= type); | |
53229a77 | 538 | tcg_assert_listed_vecop(INDEX_op_cmp_vec); |
212be173 RH |
539 | can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece); |
540 | if (can > 0) { | |
541 | vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); | |
542 | } else { | |
53229a77 | 543 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
212be173 RH |
544 | tcg_debug_assert(can < 0); |
545 | tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); | |
53229a77 | 546 | tcg_swap_vecop_list(hold_list); |
212be173 RH |
547 | } |
548 | } | |
3774030a | 549 | |
17f79944 | 550 | static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a, |
8afaf050 | 551 | TCGv_vec b, TCGOpcode opc) |
3774030a RH |
552 | { |
553 | TCGTemp *rt = tcgv_vec_temp(r); | |
554 | TCGTemp *at = tcgv_vec_temp(a); | |
555 | TCGTemp *bt = tcgv_vec_temp(b); | |
556 | TCGArg ri = temp_arg(rt); | |
557 | TCGArg ai = temp_arg(at); | |
558 | TCGArg bi = temp_arg(bt); | |
559 | TCGType type = rt->base_type; | |
560 | int can; | |
561 | ||
9a938d86 RH |
562 | tcg_debug_assert(at->base_type >= type); |
563 | tcg_debug_assert(bt->base_type >= type); | |
53229a77 | 564 | tcg_assert_listed_vecop(opc); |
8afaf050 | 565 | can = tcg_can_emit_vec_op(opc, type, vece); |
3774030a | 566 | if (can > 0) { |
8afaf050 | 567 | vec_gen_3(opc, type, vece, ri, ai, bi); |
17f79944 | 568 | } else if (can < 0) { |
53229a77 | 569 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
8afaf050 | 570 | tcg_expand_vec_op(opc, type, vece, ri, ai, bi); |
53229a77 | 571 | tcg_swap_vecop_list(hold_list); |
17f79944 RH |
572 | } else { |
573 | return false; | |
3774030a | 574 | } |
17f79944 RH |
575 | return true; |
576 | } | |
577 | ||
578 | static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a, | |
579 | TCGv_vec b, TCGOpcode opc) | |
580 | { | |
581 | bool ok = do_op3(vece, r, a, b, opc); | |
582 | tcg_debug_assert(ok); | |
3774030a | 583 | } |
8afaf050 | 584 | |
ce27c5d1 RH |
585 | void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
586 | { | |
17f79944 | 587 | do_op3_nofail(vece, r, a, b, INDEX_op_add_vec); |
ce27c5d1 RH |
588 | } |
589 | ||
590 | void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
591 | { | |
17f79944 | 592 | do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec); |
ce27c5d1 RH |
593 | } |
594 | ||
8afaf050 RH |
595 | void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
596 | { | |
17f79944 | 597 | do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec); |
8afaf050 RH |
598 | } |
599 | ||
600 | void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
601 | { | |
17f79944 | 602 | do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec); |
8afaf050 RH |
603 | } |
604 | ||
605 | void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
606 | { | |
17f79944 | 607 | do_op3_nofail(vece, r, a, b, INDEX_op_usadd_vec); |
8afaf050 RH |
608 | } |
609 | ||
610 | void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
611 | { | |
17f79944 | 612 | do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec); |
8afaf050 RH |
613 | } |
614 | ||
615 | void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
616 | { | |
17f79944 | 617 | do_op3_nofail(vece, r, a, b, INDEX_op_ussub_vec); |
8afaf050 | 618 | } |
dd0a0fcd | 619 | |
72b4c792 RH |
620 | static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a, |
621 | TCGv_vec b, TCGOpcode opc, TCGCond cond) | |
622 | { | |
623 | if (!do_op3(vece, r, a, b, opc)) { | |
69c918d2 | 624 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
72b4c792 | 625 | tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b); |
69c918d2 | 626 | tcg_swap_vecop_list(hold_list); |
72b4c792 RH |
627 | } |
628 | } | |
629 | ||
dd0a0fcd RH |
630 | void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
631 | { | |
72b4c792 | 632 | do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT); |
dd0a0fcd RH |
633 | } |
634 | ||
635 | void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
636 | { | |
72b4c792 | 637 | do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU); |
dd0a0fcd RH |
638 | } |
639 | ||
640 | void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
641 | { | |
72b4c792 | 642 | do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT); |
dd0a0fcd RH |
643 | } |
644 | ||
645 | void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
646 | { | |
72b4c792 | 647 | do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU); |
dd0a0fcd | 648 | } |
5ee5c14c RH |
649 | |
650 | void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
651 | { | |
17f79944 | 652 | do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec); |
5ee5c14c RH |
653 | } |
654 | ||
655 | void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
656 | { | |
17f79944 | 657 | do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec); |
5ee5c14c RH |
658 | } |
659 | ||
660 | void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
661 | { | |
17f79944 | 662 | do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec); |
5ee5c14c | 663 | } |
b4578cd9 | 664 | |
5d0ceda9 RH |
665 | void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
666 | { | |
667 | do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec); | |
668 | } | |
669 | ||
670 | void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
671 | { | |
672 | do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec); | |
673 | } | |
674 | ||
b4578cd9 | 675 | static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a, |
3d5bb2ea | 676 | TCGv_i32 s, TCGOpcode opc) |
b4578cd9 RH |
677 | { |
678 | TCGTemp *rt = tcgv_vec_temp(r); | |
679 | TCGTemp *at = tcgv_vec_temp(a); | |
680 | TCGTemp *st = tcgv_i32_temp(s); | |
681 | TCGArg ri = temp_arg(rt); | |
682 | TCGArg ai = temp_arg(at); | |
683 | TCGArg si = temp_arg(st); | |
684 | TCGType type = rt->base_type; | |
b4578cd9 RH |
685 | int can; |
686 | ||
687 | tcg_debug_assert(at->base_type >= type); | |
3d5bb2ea RH |
688 | tcg_assert_listed_vecop(opc); |
689 | can = tcg_can_emit_vec_op(opc, type, vece); | |
b4578cd9 | 690 | if (can > 0) { |
3d5bb2ea | 691 | vec_gen_3(opc, type, vece, ri, ai, si); |
b4578cd9 | 692 | } else if (can < 0) { |
3d5bb2ea RH |
693 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
694 | tcg_expand_vec_op(opc, type, vece, ri, ai, si); | |
695 | tcg_swap_vecop_list(hold_list); | |
b4578cd9 | 696 | } else { |
3d5bb2ea | 697 | g_assert_not_reached(); |
b4578cd9 | 698 | } |
b4578cd9 RH |
699 | } |
700 | ||
701 | void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
702 | { | |
3d5bb2ea | 703 | do_shifts(vece, r, a, b, INDEX_op_shls_vec); |
b4578cd9 RH |
704 | } |
705 | ||
706 | void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
707 | { | |
3d5bb2ea | 708 | do_shifts(vece, r, a, b, INDEX_op_shrs_vec); |
b4578cd9 RH |
709 | } |
710 | ||
711 | void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
712 | { | |
3d5bb2ea | 713 | do_shifts(vece, r, a, b, INDEX_op_sars_vec); |
b4578cd9 | 714 | } |
38dc1294 | 715 | |
23850a74 RH |
716 | void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s) |
717 | { | |
718 | do_shifts(vece, r, a, s, INDEX_op_rotls_vec); | |
719 | } | |
720 | ||
38dc1294 RH |
721 | void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, |
722 | TCGv_vec b, TCGv_vec c) | |
723 | { | |
724 | TCGTemp *rt = tcgv_vec_temp(r); | |
725 | TCGTemp *at = tcgv_vec_temp(a); | |
726 | TCGTemp *bt = tcgv_vec_temp(b); | |
727 | TCGTemp *ct = tcgv_vec_temp(c); | |
728 | TCGType type = rt->base_type; | |
729 | ||
730 | tcg_debug_assert(at->base_type >= type); | |
731 | tcg_debug_assert(bt->base_type >= type); | |
732 | tcg_debug_assert(ct->base_type >= type); | |
733 | ||
734 | if (TCG_TARGET_HAS_bitsel_vec) { | |
735 | vec_gen_4(INDEX_op_bitsel_vec, type, MO_8, | |
736 | temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct)); | |
737 | } else { | |
738 | TCGv_vec t = tcg_temp_new_vec(type); | |
739 | tcg_gen_and_vec(MO_8, t, a, b); | |
740 | tcg_gen_andc_vec(MO_8, r, c, a); | |
741 | tcg_gen_or_vec(MO_8, r, r, t); | |
742 | tcg_temp_free_vec(t); | |
743 | } | |
744 | } | |
f75da298 RH |
745 | |
746 | void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r, | |
747 | TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d) | |
748 | { | |
749 | TCGTemp *rt = tcgv_vec_temp(r); | |
750 | TCGTemp *at = tcgv_vec_temp(a); | |
751 | TCGTemp *bt = tcgv_vec_temp(b); | |
752 | TCGTemp *ct = tcgv_vec_temp(c); | |
753 | TCGTemp *dt = tcgv_vec_temp(d); | |
754 | TCGArg ri = temp_arg(rt); | |
755 | TCGArg ai = temp_arg(at); | |
756 | TCGArg bi = temp_arg(bt); | |
757 | TCGArg ci = temp_arg(ct); | |
758 | TCGArg di = temp_arg(dt); | |
759 | TCGType type = rt->base_type; | |
760 | const TCGOpcode *hold_list; | |
761 | int can; | |
762 | ||
763 | tcg_debug_assert(at->base_type >= type); | |
764 | tcg_debug_assert(bt->base_type >= type); | |
765 | tcg_debug_assert(ct->base_type >= type); | |
766 | tcg_debug_assert(dt->base_type >= type); | |
767 | ||
768 | tcg_assert_listed_vecop(INDEX_op_cmpsel_vec); | |
769 | hold_list = tcg_swap_vecop_list(NULL); | |
770 | can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece); | |
771 | ||
772 | if (can > 0) { | |
773 | vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond); | |
774 | } else if (can < 0) { | |
775 | tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece, | |
776 | ri, ai, bi, ci, di, cond); | |
777 | } else { | |
778 | TCGv_vec t = tcg_temp_new_vec(type); | |
779 | tcg_gen_cmp_vec(cond, vece, t, a, b); | |
780 | tcg_gen_bitsel_vec(vece, r, t, c, d); | |
781 | tcg_temp_free_vec(t); | |
782 | } | |
783 | tcg_swap_vecop_list(hold_list); | |
784 | } |