]>
Commit | Line | Data |
---|---|---|
d2fd745f RH |
1 | /* |
2 | * Tiny Code Generator for QEMU | |
3 | * | |
4 | * Copyright (c) 2018 Linaro, Inc. | |
5 | * | |
6 | * This library is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU Lesser General Public | |
8 | * License as published by the Free Software Foundation; either | |
fb0343d5 | 9 | * version 2.1 of the License, or (at your option) any later version. |
d2fd745f RH |
10 | * |
11 | * This library is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * Lesser General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU Lesser General Public | |
17 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | ||
20 | #include "qemu/osdep.h" | |
dcb32f1d | 21 | #include "tcg/tcg.h" |
47f7313d | 22 | #include "tcg/tcg-temp-internal.h" |
dcb32f1d PMD |
23 | #include "tcg/tcg-op.h" |
24 | #include "tcg/tcg-mo.h" | |
d56fea79 RH |
25 | #include "tcg-internal.h" |
26 | ||
d2fd745f RH |
27 | |
28 | /* Reduce the number of ifdefs below. This assumes that all uses of | |
29 | TCGV_HIGH and TCGV_LOW are properly protected by a conditional that | |
30 | the compiler can eliminate. */ | |
31 | #if TCG_TARGET_REG_BITS == 64 | |
32 | extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); | |
33 | extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); | |
34 | #define TCGV_LOW TCGV_LOW_link_error | |
35 | #define TCGV_HIGH TCGV_HIGH_link_error | |
36 | #endif | |
37 | ||
53229a77 RH |
38 | /* |
39 | * Vector optional opcode tracking. | |
40 | * Except for the basic logical operations (and, or, xor), and | |
41 | * data movement (mov, ld, st, dupi), many vector opcodes are | |
42 | * optional and may not be supported on the host. Thank Intel | |
43 | * for the irregularity in their instruction set. | |
44 | * | |
45 | * The gvec expanders allow custom vector operations to be composed, | |
46 | * generally via the .fniv callback in the GVecGen* structures. At | |
47 | * the same time, in deciding whether to use this hook we need to | |
48 | * know if the host supports the required operations. This is | |
49 | * presented as an array of opcodes, terminated by 0. Each opcode | |
50 | * is assumed to be expanded with the given VECE. | |
51 | * | |
52 | * For debugging, we want to validate this array. Therefore, when | |
53 | * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders | |
54 | * will validate that their opcode is present in the list. | |
55 | */ | |
56 | #ifdef CONFIG_DEBUG_TCG | |
57 | void tcg_assert_listed_vecop(TCGOpcode op) | |
58 | { | |
59 | const TCGOpcode *p = tcg_ctx->vecop_list; | |
60 | if (p) { | |
61 | for (; *p; ++p) { | |
62 | if (*p == op) { | |
63 | return; | |
64 | } | |
65 | } | |
66 | g_assert_not_reached(); | |
67 | } | |
68 | } | |
69 | #endif | |
70 | ||
71 | bool tcg_can_emit_vecop_list(const TCGOpcode *list, | |
72 | TCGType type, unsigned vece) | |
73 | { | |
74 | if (list == NULL) { | |
75 | return true; | |
76 | } | |
77 | ||
78 | for (; *list; ++list) { | |
79 | TCGOpcode opc = *list; | |
80 | ||
81 | #ifdef CONFIG_DEBUG_TCG | |
82 | switch (opc) { | |
83 | case INDEX_op_and_vec: | |
84 | case INDEX_op_or_vec: | |
85 | case INDEX_op_xor_vec: | |
86 | case INDEX_op_mov_vec: | |
87 | case INDEX_op_dup_vec: | |
53229a77 RH |
88 | case INDEX_op_dup2_vec: |
89 | case INDEX_op_ld_vec: | |
90 | case INDEX_op_st_vec: | |
38dc1294 | 91 | case INDEX_op_bitsel_vec: |
53229a77 RH |
92 | /* These opcodes are mandatory and should not be listed. */ |
93 | g_assert_not_reached(); | |
11978f6f RH |
94 | case INDEX_op_not_vec: |
95 | /* These opcodes have generic expansions using the above. */ | |
96 | g_assert_not_reached(); | |
53229a77 RH |
97 | default: |
98 | break; | |
99 | } | |
100 | #endif | |
101 | ||
102 | if (tcg_can_emit_vec_op(opc, type, vece)) { | |
103 | continue; | |
104 | } | |
105 | ||
106 | /* | |
107 | * The opcode list is created by front ends based on what they | |
108 | * actually invoke. We must mirror the logic in the routines | |
109 | * below for generic expansions using other opcodes. | |
110 | */ | |
111 | switch (opc) { | |
112 | case INDEX_op_neg_vec: | |
113 | if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) { | |
114 | continue; | |
115 | } | |
116 | break; | |
bcefc902 RH |
117 | case INDEX_op_abs_vec: |
118 | if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece) | |
119 | && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0 | |
120 | || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0 | |
121 | || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) { | |
122 | continue; | |
123 | } | |
124 | break; | |
2552d60e RH |
125 | case INDEX_op_usadd_vec: |
126 | if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) || | |
127 | tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { | |
128 | continue; | |
129 | } | |
130 | break; | |
131 | case INDEX_op_ussub_vec: | |
132 | if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) || | |
133 | tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { | |
134 | continue; | |
135 | } | |
136 | break; | |
f75da298 | 137 | case INDEX_op_cmpsel_vec: |
72b4c792 RH |
138 | case INDEX_op_smin_vec: |
139 | case INDEX_op_smax_vec: | |
140 | case INDEX_op_umin_vec: | |
141 | case INDEX_op_umax_vec: | |
f75da298 RH |
142 | if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { |
143 | continue; | |
144 | } | |
145 | break; | |
53229a77 RH |
146 | default: |
147 | break; | |
148 | } | |
149 | return false; | |
150 | } | |
151 | return true; | |
152 | } | |
153 | ||
d2fd745f RH |
154 | void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) |
155 | { | |
d4478943 | 156 | TCGOp *op = tcg_emit_op(opc, 2); |
d2fd745f RH |
157 | TCGOP_VECL(op) = type - TCG_TYPE_V64; |
158 | TCGOP_VECE(op) = vece; | |
159 | op->args[0] = r; | |
160 | op->args[1] = a; | |
161 | } | |
162 | ||
163 | void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, | |
164 | TCGArg r, TCGArg a, TCGArg b) | |
165 | { | |
d4478943 | 166 | TCGOp *op = tcg_emit_op(opc, 3); |
d2fd745f RH |
167 | TCGOP_VECL(op) = type - TCG_TYPE_V64; |
168 | TCGOP_VECE(op) = vece; | |
169 | op->args[0] = r; | |
170 | op->args[1] = a; | |
171 | op->args[2] = b; | |
172 | } | |
173 | ||
174 | void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, | |
175 | TCGArg r, TCGArg a, TCGArg b, TCGArg c) | |
176 | { | |
d4478943 | 177 | TCGOp *op = tcg_emit_op(opc, 4); |
d2fd745f RH |
178 | TCGOP_VECL(op) = type - TCG_TYPE_V64; |
179 | TCGOP_VECE(op) = vece; | |
180 | op->args[0] = r; | |
181 | op->args[1] = a; | |
182 | op->args[2] = b; | |
183 | op->args[3] = c; | |
184 | } | |
185 | ||
f75da298 RH |
186 | static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, |
187 | TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) | |
188 | { | |
d4478943 | 189 | TCGOp *op = tcg_emit_op(opc, 6); |
f75da298 RH |
190 | TCGOP_VECL(op) = type - TCG_TYPE_V64; |
191 | TCGOP_VECE(op) = vece; | |
192 | op->args[0] = r; | |
193 | op->args[1] = a; | |
194 | op->args[2] = b; | |
195 | op->args[3] = c; | |
196 | op->args[4] = d; | |
197 | op->args[5] = e; | |
198 | } | |
199 | ||
d2fd745f RH |
200 | static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a) |
201 | { | |
202 | TCGTemp *rt = tcgv_vec_temp(r); | |
203 | TCGTemp *at = tcgv_vec_temp(a); | |
204 | TCGType type = rt->base_type; | |
205 | ||
db432672 RH |
206 | /* Must enough inputs for the output. */ |
207 | tcg_debug_assert(at->base_type >= type); | |
d2fd745f RH |
208 | vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at)); |
209 | } | |
210 | ||
211 | static void vec_gen_op3(TCGOpcode opc, unsigned vece, | |
212 | TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
213 | { | |
214 | TCGTemp *rt = tcgv_vec_temp(r); | |
215 | TCGTemp *at = tcgv_vec_temp(a); | |
216 | TCGTemp *bt = tcgv_vec_temp(b); | |
217 | TCGType type = rt->base_type; | |
218 | ||
db432672 RH |
219 | /* Must enough inputs for the output. */ |
220 | tcg_debug_assert(at->base_type >= type); | |
221 | tcg_debug_assert(bt->base_type >= type); | |
d2fd745f RH |
222 | vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt)); |
223 | } | |
224 | ||
225 | void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) | |
226 | { | |
227 | if (r != a) { | |
228 | vec_gen_op2(INDEX_op_mov_vec, 0, r, a); | |
229 | } | |
230 | } | |
231 | ||
db432672 RH |
232 | void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) |
233 | { | |
0b4286dd RH |
234 | TCGTemp *rt = tcgv_vec_temp(r); |
235 | tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a)); | |
d2fd745f RH |
236 | } |
237 | ||
238 | void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) | |
239 | { | |
240 | TCGArg ri = tcgv_vec_arg(r); | |
241 | TCGTemp *rt = arg_temp(ri); | |
242 | TCGType type = rt->base_type; | |
243 | ||
244 | if (TCG_TARGET_REG_BITS == 64) { | |
245 | TCGArg ai = tcgv_i64_arg(a); | |
db432672 | 246 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); |
d2fd745f RH |
247 | } else if (vece == MO_64) { |
248 | TCGArg al = tcgv_i32_arg(TCGV_LOW(a)); | |
249 | TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a)); | |
250 | vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah); | |
251 | } else { | |
252 | TCGArg ai = tcgv_i32_arg(TCGV_LOW(a)); | |
db432672 | 253 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); |
d2fd745f RH |
254 | } |
255 | } | |
256 | ||
257 | void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a) | |
258 | { | |
259 | TCGArg ri = tcgv_vec_arg(r); | |
260 | TCGArg ai = tcgv_i32_arg(a); | |
261 | TCGTemp *rt = arg_temp(ri); | |
262 | TCGType type = rt->base_type; | |
263 | ||
264 | vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); | |
265 | } | |
266 | ||
37ee55a0 RH |
267 | void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b, |
268 | tcg_target_long ofs) | |
269 | { | |
270 | TCGArg ri = tcgv_vec_arg(r); | |
271 | TCGArg bi = tcgv_ptr_arg(b); | |
272 | TCGTemp *rt = arg_temp(ri); | |
273 | TCGType type = rt->base_type; | |
274 | ||
275 | vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs); | |
276 | } | |
277 | ||
d2fd745f RH |
278 | static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o) |
279 | { | |
280 | TCGArg ri = tcgv_vec_arg(r); | |
281 | TCGArg bi = tcgv_ptr_arg(b); | |
282 | TCGTemp *rt = arg_temp(ri); | |
283 | TCGType type = rt->base_type; | |
284 | ||
285 | vec_gen_3(opc, type, 0, ri, bi, o); | |
286 | } | |
287 | ||
288 | void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) | |
289 | { | |
290 | vec_gen_ldst(INDEX_op_ld_vec, r, b, o); | |
291 | } | |
292 | ||
293 | void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) | |
294 | { | |
295 | vec_gen_ldst(INDEX_op_st_vec, r, b, o); | |
296 | } | |
297 | ||
298 | void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type) | |
299 | { | |
300 | TCGArg ri = tcgv_vec_arg(r); | |
301 | TCGArg bi = tcgv_ptr_arg(b); | |
302 | TCGTemp *rt = arg_temp(ri); | |
303 | TCGType type = rt->base_type; | |
304 | ||
305 | tcg_debug_assert(low_type >= TCG_TYPE_V64); | |
306 | tcg_debug_assert(low_type <= type); | |
307 | vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o); | |
308 | } | |
309 | ||
d2fd745f RH |
310 | void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
311 | { | |
312 | vec_gen_op3(INDEX_op_and_vec, 0, r, a, b); | |
313 | } | |
314 | ||
315 | void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
316 | { | |
317 | vec_gen_op3(INDEX_op_or_vec, 0, r, a, b); | |
318 | } | |
319 | ||
320 | void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
321 | { | |
322 | vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b); | |
323 | } | |
324 | ||
325 | void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
326 | { | |
327 | if (TCG_TARGET_HAS_andc_vec) { | |
328 | vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b); | |
329 | } else { | |
330 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
331 | tcg_gen_not_vec(0, t, b); | |
332 | tcg_gen_and_vec(0, r, a, t); | |
333 | tcg_temp_free_vec(t); | |
334 | } | |
335 | } | |
336 | ||
337 | void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
338 | { | |
339 | if (TCG_TARGET_HAS_orc_vec) { | |
340 | vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b); | |
341 | } else { | |
342 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
343 | tcg_gen_not_vec(0, t, b); | |
344 | tcg_gen_or_vec(0, r, a, t); | |
345 | tcg_temp_free_vec(t); | |
346 | } | |
347 | } | |
348 | ||
f550805d RH |
349 | void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
350 | { | |
ed523473 RH |
351 | if (TCG_TARGET_HAS_nand_vec) { |
352 | vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b); | |
353 | } else { | |
354 | tcg_gen_and_vec(0, r, a, b); | |
355 | tcg_gen_not_vec(0, r, r); | |
356 | } | |
f550805d RH |
357 | } |
358 | ||
359 | void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
360 | { | |
ed523473 RH |
361 | if (TCG_TARGET_HAS_nor_vec) { |
362 | vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b); | |
363 | } else { | |
364 | tcg_gen_or_vec(0, r, a, b); | |
365 | tcg_gen_not_vec(0, r, r); | |
366 | } | |
f550805d RH |
367 | } |
368 | ||
369 | void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
370 | { | |
ed523473 RH |
371 | if (TCG_TARGET_HAS_eqv_vec) { |
372 | vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b); | |
373 | } else { | |
374 | tcg_gen_xor_vec(0, r, a, b); | |
375 | tcg_gen_not_vec(0, r, r); | |
376 | } | |
f550805d RH |
377 | } |
378 | ||
ce27c5d1 | 379 | static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) |
d2fd745f | 380 | { |
ce27c5d1 RH |
381 | TCGTemp *rt = tcgv_vec_temp(r); |
382 | TCGTemp *at = tcgv_vec_temp(a); | |
383 | TCGArg ri = temp_arg(rt); | |
384 | TCGArg ai = temp_arg(at); | |
385 | TCGType type = rt->base_type; | |
386 | int can; | |
387 | ||
388 | tcg_debug_assert(at->base_type >= type); | |
53229a77 | 389 | tcg_assert_listed_vecop(opc); |
ce27c5d1 RH |
390 | can = tcg_can_emit_vec_op(opc, type, vece); |
391 | if (can > 0) { | |
392 | vec_gen_2(opc, type, vece, ri, ai); | |
393 | } else if (can < 0) { | |
53229a77 | 394 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
ce27c5d1 | 395 | tcg_expand_vec_op(opc, type, vece, ri, ai); |
53229a77 | 396 | tcg_swap_vecop_list(hold_list); |
d2fd745f | 397 | } else { |
ce27c5d1 RH |
398 | return false; |
399 | } | |
400 | return true; | |
401 | } | |
402 | ||
403 | void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a) | |
404 | { | |
11978f6f RH |
405 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
406 | ||
ce27c5d1 | 407 | if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) { |
f6ff9c2f | 408 | tcg_gen_xor_vec(0, r, a, tcg_constant_vec_matching(r, 0, -1)); |
d2fd745f | 409 | } |
11978f6f | 410 | tcg_swap_vecop_list(hold_list); |
d2fd745f RH |
411 | } |
412 | ||
413 | void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a) | |
414 | { | |
53229a77 RH |
415 | const TCGOpcode *hold_list; |
416 | ||
417 | tcg_assert_listed_vecop(INDEX_op_neg_vec); | |
418 | hold_list = tcg_swap_vecop_list(NULL); | |
419 | ||
ce27c5d1 | 420 | if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) { |
f6ff9c2f | 421 | tcg_gen_sub_vec(vece, r, tcg_constant_vec_matching(r, vece, 0), a); |
d2fd745f | 422 | } |
53229a77 | 423 | tcg_swap_vecop_list(hold_list); |
d2fd745f | 424 | } |
d0ec9796 | 425 | |
bcefc902 RH |
426 | void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) |
427 | { | |
428 | const TCGOpcode *hold_list; | |
429 | ||
430 | tcg_assert_listed_vecop(INDEX_op_abs_vec); | |
431 | hold_list = tcg_swap_vecop_list(NULL); | |
432 | ||
433 | if (!do_op2(vece, r, a, INDEX_op_abs_vec)) { | |
434 | TCGType type = tcgv_vec_temp(r)->base_type; | |
435 | TCGv_vec t = tcg_temp_new_vec(type); | |
436 | ||
437 | tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)); | |
438 | if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) { | |
439 | tcg_gen_neg_vec(vece, t, a); | |
440 | tcg_gen_smax_vec(vece, r, a, t); | |
441 | } else { | |
442 | if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { | |
443 | tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); | |
444 | } else { | |
0b4286dd RH |
445 | tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, |
446 | tcg_constant_vec(type, vece, 0)); | |
bcefc902 RH |
447 | } |
448 | tcg_gen_xor_vec(vece, r, a, t); | |
449 | tcg_gen_sub_vec(vece, r, r, t); | |
450 | } | |
451 | ||
452 | tcg_temp_free_vec(t); | |
453 | } | |
454 | tcg_swap_vecop_list(hold_list); | |
455 | } | |
456 | ||
d0ec9796 RH |
457 | static void do_shifti(TCGOpcode opc, unsigned vece, |
458 | TCGv_vec r, TCGv_vec a, int64_t i) | |
459 | { | |
460 | TCGTemp *rt = tcgv_vec_temp(r); | |
461 | TCGTemp *at = tcgv_vec_temp(a); | |
462 | TCGArg ri = temp_arg(rt); | |
463 | TCGArg ai = temp_arg(at); | |
464 | TCGType type = rt->base_type; | |
465 | int can; | |
466 | ||
467 | tcg_debug_assert(at->base_type == type); | |
468 | tcg_debug_assert(i >= 0 && i < (8 << vece)); | |
53229a77 | 469 | tcg_assert_listed_vecop(opc); |
d0ec9796 RH |
470 | |
471 | if (i == 0) { | |
472 | tcg_gen_mov_vec(r, a); | |
473 | return; | |
474 | } | |
475 | ||
476 | can = tcg_can_emit_vec_op(opc, type, vece); | |
477 | if (can > 0) { | |
478 | vec_gen_3(opc, type, vece, ri, ai, i); | |
479 | } else { | |
480 | /* We leave the choice of expansion via scalar or vector shift | |
481 | to the target. Often, but not always, dupi can feed a vector | |
482 | shift easier than a scalar. */ | |
53229a77 | 483 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
d0ec9796 RH |
484 | tcg_debug_assert(can < 0); |
485 | tcg_expand_vec_op(opc, type, vece, ri, ai, i); | |
53229a77 | 486 | tcg_swap_vecop_list(hold_list); |
d0ec9796 RH |
487 | } |
488 | } | |
489 | ||
490 | void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
491 | { | |
492 | do_shifti(INDEX_op_shli_vec, vece, r, a, i); | |
493 | } | |
494 | ||
495 | void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
496 | { | |
497 | do_shifti(INDEX_op_shri_vec, vece, r, a, i); | |
498 | } | |
499 | ||
500 | void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
501 | { | |
502 | do_shifti(INDEX_op_sari_vec, vece, r, a, i); | |
503 | } | |
212be173 | 504 | |
b0f7e744 RH |
505 | void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) |
506 | { | |
507 | do_shifti(INDEX_op_rotli_vec, vece, r, a, i); | |
508 | } | |
509 | ||
510 | void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) | |
511 | { | |
512 | int bits = 8 << vece; | |
513 | tcg_debug_assert(i >= 0 && i < bits); | |
514 | do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1)); | |
515 | } | |
516 | ||
212be173 RH |
517 | void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, |
518 | TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
519 | { | |
520 | TCGTemp *rt = tcgv_vec_temp(r); | |
521 | TCGTemp *at = tcgv_vec_temp(a); | |
522 | TCGTemp *bt = tcgv_vec_temp(b); | |
523 | TCGArg ri = temp_arg(rt); | |
524 | TCGArg ai = temp_arg(at); | |
525 | TCGArg bi = temp_arg(bt); | |
526 | TCGType type = rt->base_type; | |
527 | int can; | |
528 | ||
9a938d86 RH |
529 | tcg_debug_assert(at->base_type >= type); |
530 | tcg_debug_assert(bt->base_type >= type); | |
53229a77 | 531 | tcg_assert_listed_vecop(INDEX_op_cmp_vec); |
212be173 RH |
532 | can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece); |
533 | if (can > 0) { | |
534 | vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); | |
535 | } else { | |
53229a77 | 536 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
212be173 RH |
537 | tcg_debug_assert(can < 0); |
538 | tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); | |
53229a77 | 539 | tcg_swap_vecop_list(hold_list); |
212be173 RH |
540 | } |
541 | } | |
3774030a | 542 | |
17f79944 | 543 | static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a, |
8afaf050 | 544 | TCGv_vec b, TCGOpcode opc) |
3774030a RH |
545 | { |
546 | TCGTemp *rt = tcgv_vec_temp(r); | |
547 | TCGTemp *at = tcgv_vec_temp(a); | |
548 | TCGTemp *bt = tcgv_vec_temp(b); | |
549 | TCGArg ri = temp_arg(rt); | |
550 | TCGArg ai = temp_arg(at); | |
551 | TCGArg bi = temp_arg(bt); | |
552 | TCGType type = rt->base_type; | |
553 | int can; | |
554 | ||
9a938d86 RH |
555 | tcg_debug_assert(at->base_type >= type); |
556 | tcg_debug_assert(bt->base_type >= type); | |
53229a77 | 557 | tcg_assert_listed_vecop(opc); |
8afaf050 | 558 | can = tcg_can_emit_vec_op(opc, type, vece); |
3774030a | 559 | if (can > 0) { |
8afaf050 | 560 | vec_gen_3(opc, type, vece, ri, ai, bi); |
17f79944 | 561 | } else if (can < 0) { |
53229a77 | 562 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
8afaf050 | 563 | tcg_expand_vec_op(opc, type, vece, ri, ai, bi); |
53229a77 | 564 | tcg_swap_vecop_list(hold_list); |
17f79944 RH |
565 | } else { |
566 | return false; | |
3774030a | 567 | } |
17f79944 RH |
568 | return true; |
569 | } | |
570 | ||
571 | static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a, | |
572 | TCGv_vec b, TCGOpcode opc) | |
573 | { | |
574 | bool ok = do_op3(vece, r, a, b, opc); | |
575 | tcg_debug_assert(ok); | |
3774030a | 576 | } |
8afaf050 | 577 | |
ce27c5d1 RH |
578 | void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
579 | { | |
17f79944 | 580 | do_op3_nofail(vece, r, a, b, INDEX_op_add_vec); |
ce27c5d1 RH |
581 | } |
582 | ||
583 | void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
584 | { | |
17f79944 | 585 | do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec); |
ce27c5d1 RH |
586 | } |
587 | ||
8afaf050 RH |
588 | void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
589 | { | |
17f79944 | 590 | do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec); |
8afaf050 RH |
591 | } |
592 | ||
593 | void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
594 | { | |
17f79944 | 595 | do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec); |
8afaf050 RH |
596 | } |
597 | ||
598 | void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
599 | { | |
2552d60e RH |
600 | if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) { |
601 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); | |
602 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
603 | ||
604 | /* usadd(a, b) = min(a, ~b) + b */ | |
605 | tcg_gen_not_vec(vece, t, b); | |
606 | tcg_gen_umin_vec(vece, t, t, a); | |
607 | tcg_gen_add_vec(vece, r, t, b); | |
608 | ||
609 | tcg_temp_free_vec(t); | |
610 | tcg_swap_vecop_list(hold_list); | |
611 | } | |
8afaf050 RH |
612 | } |
613 | ||
614 | void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
615 | { | |
17f79944 | 616 | do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec); |
8afaf050 RH |
617 | } |
618 | ||
619 | void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
620 | { | |
2552d60e RH |
621 | if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) { |
622 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); | |
623 | TCGv_vec t = tcg_temp_new_vec_matching(r); | |
624 | ||
625 | /* ussub(a, b) = max(a, b) - b */ | |
626 | tcg_gen_umax_vec(vece, t, a, b); | |
627 | tcg_gen_sub_vec(vece, r, t, b); | |
628 | ||
629 | tcg_temp_free_vec(t); | |
630 | tcg_swap_vecop_list(hold_list); | |
631 | } | |
8afaf050 | 632 | } |
dd0a0fcd | 633 | |
72b4c792 RH |
634 | static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a, |
635 | TCGv_vec b, TCGOpcode opc, TCGCond cond) | |
636 | { | |
637 | if (!do_op3(vece, r, a, b, opc)) { | |
69c918d2 | 638 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
72b4c792 | 639 | tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b); |
69c918d2 | 640 | tcg_swap_vecop_list(hold_list); |
72b4c792 RH |
641 | } |
642 | } | |
643 | ||
dd0a0fcd RH |
644 | void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
645 | { | |
72b4c792 | 646 | do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT); |
dd0a0fcd RH |
647 | } |
648 | ||
649 | void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
650 | { | |
72b4c792 | 651 | do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU); |
dd0a0fcd RH |
652 | } |
653 | ||
654 | void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
655 | { | |
72b4c792 | 656 | do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT); |
dd0a0fcd RH |
657 | } |
658 | ||
659 | void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
660 | { | |
72b4c792 | 661 | do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU); |
dd0a0fcd | 662 | } |
5ee5c14c RH |
663 | |
664 | void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
665 | { | |
17f79944 | 666 | do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec); |
5ee5c14c RH |
667 | } |
668 | ||
669 | void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
670 | { | |
17f79944 | 671 | do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec); |
5ee5c14c RH |
672 | } |
673 | ||
674 | void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
675 | { | |
17f79944 | 676 | do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec); |
5ee5c14c | 677 | } |
b4578cd9 | 678 | |
5d0ceda9 RH |
679 | void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) |
680 | { | |
681 | do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec); | |
682 | } | |
683 | ||
684 | void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) | |
685 | { | |
686 | do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec); | |
687 | } | |
688 | ||
b4578cd9 | 689 | static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a, |
3d5bb2ea | 690 | TCGv_i32 s, TCGOpcode opc) |
b4578cd9 RH |
691 | { |
692 | TCGTemp *rt = tcgv_vec_temp(r); | |
693 | TCGTemp *at = tcgv_vec_temp(a); | |
694 | TCGTemp *st = tcgv_i32_temp(s); | |
695 | TCGArg ri = temp_arg(rt); | |
696 | TCGArg ai = temp_arg(at); | |
697 | TCGArg si = temp_arg(st); | |
698 | TCGType type = rt->base_type; | |
b4578cd9 RH |
699 | int can; |
700 | ||
701 | tcg_debug_assert(at->base_type >= type); | |
3d5bb2ea RH |
702 | tcg_assert_listed_vecop(opc); |
703 | can = tcg_can_emit_vec_op(opc, type, vece); | |
b4578cd9 | 704 | if (can > 0) { |
3d5bb2ea | 705 | vec_gen_3(opc, type, vece, ri, ai, si); |
b4578cd9 | 706 | } else if (can < 0) { |
3d5bb2ea RH |
707 | const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); |
708 | tcg_expand_vec_op(opc, type, vece, ri, ai, si); | |
709 | tcg_swap_vecop_list(hold_list); | |
b4578cd9 | 710 | } else { |
3d5bb2ea | 711 | g_assert_not_reached(); |
b4578cd9 | 712 | } |
b4578cd9 RH |
713 | } |
714 | ||
715 | void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
716 | { | |
3d5bb2ea | 717 | do_shifts(vece, r, a, b, INDEX_op_shls_vec); |
b4578cd9 RH |
718 | } |
719 | ||
720 | void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
721 | { | |
3d5bb2ea | 722 | do_shifts(vece, r, a, b, INDEX_op_shrs_vec); |
b4578cd9 RH |
723 | } |
724 | ||
725 | void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) | |
726 | { | |
3d5bb2ea | 727 | do_shifts(vece, r, a, b, INDEX_op_sars_vec); |
b4578cd9 | 728 | } |
38dc1294 | 729 | |
23850a74 RH |
730 | void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s) |
731 | { | |
732 | do_shifts(vece, r, a, s, INDEX_op_rotls_vec); | |
733 | } | |
734 | ||
38dc1294 RH |
735 | void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, |
736 | TCGv_vec b, TCGv_vec c) | |
737 | { | |
738 | TCGTemp *rt = tcgv_vec_temp(r); | |
739 | TCGTemp *at = tcgv_vec_temp(a); | |
740 | TCGTemp *bt = tcgv_vec_temp(b); | |
741 | TCGTemp *ct = tcgv_vec_temp(c); | |
742 | TCGType type = rt->base_type; | |
743 | ||
744 | tcg_debug_assert(at->base_type >= type); | |
745 | tcg_debug_assert(bt->base_type >= type); | |
746 | tcg_debug_assert(ct->base_type >= type); | |
747 | ||
748 | if (TCG_TARGET_HAS_bitsel_vec) { | |
749 | vec_gen_4(INDEX_op_bitsel_vec, type, MO_8, | |
750 | temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct)); | |
751 | } else { | |
752 | TCGv_vec t = tcg_temp_new_vec(type); | |
753 | tcg_gen_and_vec(MO_8, t, a, b); | |
754 | tcg_gen_andc_vec(MO_8, r, c, a); | |
755 | tcg_gen_or_vec(MO_8, r, r, t); | |
756 | tcg_temp_free_vec(t); | |
757 | } | |
758 | } | |
f75da298 RH |
759 | |
760 | void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r, | |
761 | TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d) | |
762 | { | |
763 | TCGTemp *rt = tcgv_vec_temp(r); | |
764 | TCGTemp *at = tcgv_vec_temp(a); | |
765 | TCGTemp *bt = tcgv_vec_temp(b); | |
766 | TCGTemp *ct = tcgv_vec_temp(c); | |
767 | TCGTemp *dt = tcgv_vec_temp(d); | |
768 | TCGArg ri = temp_arg(rt); | |
769 | TCGArg ai = temp_arg(at); | |
770 | TCGArg bi = temp_arg(bt); | |
771 | TCGArg ci = temp_arg(ct); | |
772 | TCGArg di = temp_arg(dt); | |
773 | TCGType type = rt->base_type; | |
774 | const TCGOpcode *hold_list; | |
775 | int can; | |
776 | ||
777 | tcg_debug_assert(at->base_type >= type); | |
778 | tcg_debug_assert(bt->base_type >= type); | |
779 | tcg_debug_assert(ct->base_type >= type); | |
780 | tcg_debug_assert(dt->base_type >= type); | |
781 | ||
782 | tcg_assert_listed_vecop(INDEX_op_cmpsel_vec); | |
783 | hold_list = tcg_swap_vecop_list(NULL); | |
784 | can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece); | |
785 | ||
786 | if (can > 0) { | |
787 | vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond); | |
788 | } else if (can < 0) { | |
789 | tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece, | |
790 | ri, ai, bi, ci, di, cond); | |
791 | } else { | |
792 | TCGv_vec t = tcg_temp_new_vec(type); | |
793 | tcg_gen_cmp_vec(cond, vece, t, a, b); | |
794 | tcg_gen_bitsel_vec(vece, r, t, c, d); | |
795 | tcg_temp_free_vec(t); | |
796 | } | |
797 | tcg_swap_vecop_list(hold_list); | |
798 | } |