2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "translate-a64.h"
34 #include "fpu/softfloat.h"
37 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
38 TCGv_i64
, uint32_t, uint32_t);
40 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
42 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
45 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
47 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
50 * Helpers for extracting complex instruction fields.
53 /* See e.g. ASR (immediate, predicated).
54 * Returns -1 for unallocated encoding; diagnose later.
56 static int tszimm_esz(DisasContext
*s
, int x
)
58 x
>>= 3; /* discard imm3 */
62 static int tszimm_shr(DisasContext
*s
, int x
)
64 return (16 << tszimm_esz(s
, x
)) - x
;
67 /* See e.g. LSL (immediate, predicated). */
68 static int tszimm_shl(DisasContext
*s
, int x
)
70 return x
- (8 << tszimm_esz(s
, x
));
73 /* The SH bit is in bit 8. Extract the low 8 and shift. */
74 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
76 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
79 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
81 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
84 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
85 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
87 static inline int msz_dtype(DisasContext
*s
, int msz
)
89 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
94 * Include the generated decoder.
97 #include "decode-sve.c.inc"
100 * Implement all of the translator functions referenced by the decoder.
103 /* Return the offset info CPUARMState of the predicate vector register Pn.
104 * Note for this purpose, FFR is P16.
106 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
108 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
111 /* Return the byte size of the whole predicate register, VL / 64. */
112 static inline int pred_full_reg_size(DisasContext
*s
)
114 return s
->sve_len
>> 3;
117 /* Round up the size of a register to a size allowed by
118 * the tcg vector infrastructure. Any operation which uses this
119 * size may assume that the bits above pred_full_reg_size are zero,
120 * and must leave them the same way.
122 * Note that this is not needed for the vector registers as they
123 * are always properly sized for tcg vectors.
125 static int size_for_gvec(int size
)
130 return QEMU_ALIGN_UP(size
, 16);
134 static int pred_gvec_reg_size(DisasContext
*s
)
136 return size_for_gvec(pred_full_reg_size(s
));
139 /* Invoke an out-of-line helper on 2 Zregs. */
140 static bool gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
141 int rd
, int rn
, int data
)
146 if (sve_access_check(s
)) {
147 unsigned vsz
= vec_full_reg_size(s
);
148 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
149 vec_full_reg_offset(s
, rn
),
155 /* Invoke an out-of-line helper on 3 Zregs. */
156 static bool gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
157 int rd
, int rn
, int rm
, int data
)
162 if (sve_access_check(s
)) {
163 unsigned vsz
= vec_full_reg_size(s
);
164 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
165 vec_full_reg_offset(s
, rn
),
166 vec_full_reg_offset(s
, rm
),
172 static bool gen_gvec_ool_arg_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
173 arg_rrr_esz
*a
, int data
)
175 return gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
178 /* Invoke an out-of-line helper on 4 Zregs. */
179 static bool gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
180 int rd
, int rn
, int rm
, int ra
, int data
)
185 if (sve_access_check(s
)) {
186 unsigned vsz
= vec_full_reg_size(s
);
187 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
188 vec_full_reg_offset(s
, rn
),
189 vec_full_reg_offset(s
, rm
),
190 vec_full_reg_offset(s
, ra
),
196 static bool gen_gvec_ool_arg_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
197 arg_rrrr_esz
*a
, int data
)
199 return gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
202 static bool gen_gvec_ool_arg_zzxz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
205 return gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
208 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
209 static bool gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
210 int rd
, int rn
, int pg
, int data
)
215 if (sve_access_check(s
)) {
216 unsigned vsz
= vec_full_reg_size(s
);
217 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
218 vec_full_reg_offset(s
, rn
),
219 pred_full_reg_offset(s
, pg
),
225 static bool gen_gvec_ool_arg_zpz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
226 arg_rpr_esz
*a
, int data
)
228 return gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, data
);
231 static bool gen_gvec_ool_arg_zpzi(DisasContext
*s
, gen_helper_gvec_3
*fn
,
234 return gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
237 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
238 static bool gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
239 int rd
, int rn
, int rm
, int pg
, int data
)
244 if (sve_access_check(s
)) {
245 unsigned vsz
= vec_full_reg_size(s
);
246 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
247 vec_full_reg_offset(s
, rn
),
248 vec_full_reg_offset(s
, rm
),
249 pred_full_reg_offset(s
, pg
),
255 static bool gen_gvec_ool_arg_zpzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
256 arg_rprr_esz
*a
, int data
)
258 return gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, data
);
261 /* Invoke a vector expander on two Zregs and an immediate. */
262 static bool gen_gvec_fn_zzi(DisasContext
*s
, GVecGen2iFn
*gvec_fn
,
263 int esz
, int rd
, int rn
, uint64_t imm
)
265 if (gvec_fn
== NULL
) {
268 if (sve_access_check(s
)) {
269 unsigned vsz
= vec_full_reg_size(s
);
270 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
271 vec_full_reg_offset(s
, rn
), imm
, vsz
, vsz
);
276 /* Invoke a vector expander on three Zregs. */
277 static bool gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
278 int esz
, int rd
, int rn
, int rm
)
280 if (gvec_fn
== NULL
) {
283 if (sve_access_check(s
)) {
284 unsigned vsz
= vec_full_reg_size(s
);
285 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
286 vec_full_reg_offset(s
, rn
),
287 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
292 static bool gen_gvec_fn_arg_zzz(DisasContext
*s
, GVecGen3Fn
*fn
,
295 return gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
298 /* Invoke a vector expander on four Zregs. */
299 static bool gen_gvec_fn_arg_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
302 if (gvec_fn
== NULL
) {
305 if (sve_access_check(s
)) {
306 unsigned vsz
= vec_full_reg_size(s
);
307 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
308 vec_full_reg_offset(s
, a
->rn
),
309 vec_full_reg_offset(s
, a
->rm
),
310 vec_full_reg_offset(s
, a
->ra
), vsz
, vsz
);
315 /* Invoke a vector move on two Zregs. */
316 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
318 if (sve_access_check(s
)) {
319 unsigned vsz
= vec_full_reg_size(s
);
320 tcg_gen_gvec_mov(MO_8
, vec_full_reg_offset(s
, rd
),
321 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
326 /* Initialize a Zreg with replications of a 64-bit immediate. */
327 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
329 unsigned vsz
= vec_full_reg_size(s
);
330 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
333 /* Invoke a vector expander on three Pregs. */
334 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
335 int rd
, int rn
, int rm
)
337 unsigned psz
= pred_gvec_reg_size(s
);
338 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
339 pred_full_reg_offset(s
, rn
),
340 pred_full_reg_offset(s
, rm
), psz
, psz
);
343 /* Invoke a vector move on two Pregs. */
344 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
346 if (sve_access_check(s
)) {
347 unsigned psz
= pred_gvec_reg_size(s
);
348 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
349 pred_full_reg_offset(s
, rn
), psz
, psz
);
354 /* Set the cpu flags as per a return from an SVE helper. */
355 static void do_pred_flags(TCGv_i32 t
)
357 tcg_gen_mov_i32(cpu_NF
, t
);
358 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
359 tcg_gen_andi_i32(cpu_CF
, t
, 1);
360 tcg_gen_movi_i32(cpu_VF
, 0);
363 /* Subroutines computing the ARM PredTest psuedofunction. */
364 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
366 TCGv_i32 t
= tcg_temp_new_i32();
368 gen_helper_sve_predtest1(t
, d
, g
);
370 tcg_temp_free_i32(t
);
373 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
375 TCGv_ptr dptr
= tcg_temp_new_ptr();
376 TCGv_ptr gptr
= tcg_temp_new_ptr();
377 TCGv_i32 t
= tcg_temp_new_i32();
379 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
380 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
382 gen_helper_sve_predtest(t
, dptr
, gptr
, tcg_constant_i32(words
));
383 tcg_temp_free_ptr(dptr
);
384 tcg_temp_free_ptr(gptr
);
387 tcg_temp_free_i32(t
);
390 /* For each element size, the bits within a predicate word that are active. */
391 const uint64_t pred_esz_masks
[4] = {
392 0xffffffffffffffffull
, 0x5555555555555555ull
,
393 0x1111111111111111ull
, 0x0101010101010101ull
397 *** SVE Logical - Unpredicated Group
400 TRANS_FEAT(AND_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_and
, a
)
401 TRANS_FEAT(ORR_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_or
, a
)
402 TRANS_FEAT(EOR_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_xor
, a
)
403 TRANS_FEAT(BIC_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_andc
, a
)
405 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
407 TCGv_i64 t
= tcg_temp_new_i64();
408 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
410 tcg_gen_xor_i64(t
, n
, m
);
411 tcg_gen_shri_i64(d
, t
, sh
);
412 tcg_gen_shli_i64(t
, t
, 8 - sh
);
413 tcg_gen_andi_i64(d
, d
, mask
);
414 tcg_gen_andi_i64(t
, t
, ~mask
);
415 tcg_gen_or_i64(d
, d
, t
);
416 tcg_temp_free_i64(t
);
419 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
421 TCGv_i64 t
= tcg_temp_new_i64();
422 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
424 tcg_gen_xor_i64(t
, n
, m
);
425 tcg_gen_shri_i64(d
, t
, sh
);
426 tcg_gen_shli_i64(t
, t
, 16 - sh
);
427 tcg_gen_andi_i64(d
, d
, mask
);
428 tcg_gen_andi_i64(t
, t
, ~mask
);
429 tcg_gen_or_i64(d
, d
, t
);
430 tcg_temp_free_i64(t
);
433 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
435 tcg_gen_xor_i32(d
, n
, m
);
436 tcg_gen_rotri_i32(d
, d
, sh
);
439 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
441 tcg_gen_xor_i64(d
, n
, m
);
442 tcg_gen_rotri_i64(d
, d
, sh
);
445 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
446 TCGv_vec m
, int64_t sh
)
448 tcg_gen_xor_vec(vece
, d
, n
, m
);
449 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
452 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
453 uint32_t rm_ofs
, int64_t shift
,
454 uint32_t opr_sz
, uint32_t max_sz
)
456 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
457 static const GVecGen3i ops
[4] = {
458 { .fni8
= gen_xar8_i64
,
460 .fno
= gen_helper_sve2_xar_b
,
463 { .fni8
= gen_xar16_i64
,
465 .fno
= gen_helper_sve2_xar_h
,
468 { .fni4
= gen_xar_i32
,
470 .fno
= gen_helper_sve2_xar_s
,
473 { .fni8
= gen_xar_i64
,
475 .fno
= gen_helper_gvec_xar_d
,
479 int esize
= 8 << vece
;
481 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
482 tcg_debug_assert(shift
>= 0);
483 tcg_debug_assert(shift
<= esize
);
487 /* xar with no rotate devolves to xor. */
488 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
490 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
495 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
497 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
500 if (sve_access_check(s
)) {
501 unsigned vsz
= vec_full_reg_size(s
);
502 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
503 vec_full_reg_offset(s
, a
->rn
),
504 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
509 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
511 tcg_gen_xor_i64(d
, n
, m
);
512 tcg_gen_xor_i64(d
, d
, k
);
515 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
516 TCGv_vec m
, TCGv_vec k
)
518 tcg_gen_xor_vec(vece
, d
, n
, m
);
519 tcg_gen_xor_vec(vece
, d
, d
, k
);
522 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
523 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
525 static const GVecGen4 op
= {
526 .fni8
= gen_eor3_i64
,
527 .fniv
= gen_eor3_vec
,
528 .fno
= gen_helper_sve2_eor3
,
530 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
532 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
535 TRANS_FEAT(EOR3
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_eor3
, a
)
537 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
539 tcg_gen_andc_i64(d
, m
, k
);
540 tcg_gen_xor_i64(d
, d
, n
);
543 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
544 TCGv_vec m
, TCGv_vec k
)
546 tcg_gen_andc_vec(vece
, d
, m
, k
);
547 tcg_gen_xor_vec(vece
, d
, d
, n
);
550 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
551 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
553 static const GVecGen4 op
= {
554 .fni8
= gen_bcax_i64
,
555 .fniv
= gen_bcax_vec
,
556 .fno
= gen_helper_sve2_bcax
,
558 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
560 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
563 TRANS_FEAT(BCAX
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bcax
, a
)
565 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
566 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
568 /* BSL differs from the generic bitsel in argument ordering. */
569 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
572 TRANS_FEAT(BSL
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl
, a
)
574 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
576 tcg_gen_andc_i64(n
, k
, n
);
577 tcg_gen_andc_i64(m
, m
, k
);
578 tcg_gen_or_i64(d
, n
, m
);
581 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
582 TCGv_vec m
, TCGv_vec k
)
584 if (TCG_TARGET_HAS_bitsel_vec
) {
585 tcg_gen_not_vec(vece
, n
, n
);
586 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
588 tcg_gen_andc_vec(vece
, n
, k
, n
);
589 tcg_gen_andc_vec(vece
, m
, m
, k
);
590 tcg_gen_or_vec(vece
, d
, n
, m
);
594 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
595 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
597 static const GVecGen4 op
= {
598 .fni8
= gen_bsl1n_i64
,
599 .fniv
= gen_bsl1n_vec
,
600 .fno
= gen_helper_sve2_bsl1n
,
602 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
604 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
607 TRANS_FEAT(BSL1N
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl1n
, a
)
609 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
612 * Z[dn] = (n & k) | (~m & ~k)
615 tcg_gen_and_i64(n
, n
, k
);
616 if (TCG_TARGET_HAS_orc_i64
) {
617 tcg_gen_or_i64(m
, m
, k
);
618 tcg_gen_orc_i64(d
, n
, m
);
620 tcg_gen_nor_i64(m
, m
, k
);
621 tcg_gen_or_i64(d
, n
, m
);
625 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
626 TCGv_vec m
, TCGv_vec k
)
628 if (TCG_TARGET_HAS_bitsel_vec
) {
629 tcg_gen_not_vec(vece
, m
, m
);
630 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
632 tcg_gen_and_vec(vece
, n
, n
, k
);
633 tcg_gen_or_vec(vece
, m
, m
, k
);
634 tcg_gen_orc_vec(vece
, d
, n
, m
);
638 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
639 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
641 static const GVecGen4 op
= {
642 .fni8
= gen_bsl2n_i64
,
643 .fniv
= gen_bsl2n_vec
,
644 .fno
= gen_helper_sve2_bsl2n
,
646 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
648 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
651 TRANS_FEAT(BSL2N
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl2n
, a
)
653 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
655 tcg_gen_and_i64(n
, n
, k
);
656 tcg_gen_andc_i64(m
, m
, k
);
657 tcg_gen_nor_i64(d
, n
, m
);
660 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
661 TCGv_vec m
, TCGv_vec k
)
663 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
664 tcg_gen_not_vec(vece
, d
, d
);
667 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
668 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
670 static const GVecGen4 op
= {
671 .fni8
= gen_nbsl_i64
,
672 .fniv
= gen_nbsl_vec
,
673 .fno
= gen_helper_sve2_nbsl
,
675 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
677 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
680 TRANS_FEAT(NBSL
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_nbsl
, a
)
683 *** SVE Integer Arithmetic - Unpredicated Group
686 TRANS_FEAT(ADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_add
, a
)
687 TRANS_FEAT(SUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_sub
, a
)
688 TRANS_FEAT(SQADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_ssadd
, a
)
689 TRANS_FEAT(SQSUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_sssub
, a
)
690 TRANS_FEAT(UQADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_usadd
, a
)
691 TRANS_FEAT(UQSUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_ussub
, a
)
694 *** SVE Integer Arithmetic - Binary Predicated Group
697 /* Select active elememnts from Zn and inactive elements from Zm,
698 * storing the result in Zd.
700 static void do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
702 static gen_helper_gvec_4
* const fns
[4] = {
703 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
704 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
706 gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
709 #define DO_ZPZZ(NAME, FEAT, name) \
710 static gen_helper_gvec_4 * const name##_zpzz_fns[4] = { \
711 gen_helper_##name##_zpzz_b, gen_helper_##name##_zpzz_h, \
712 gen_helper_##name##_zpzz_s, gen_helper_##name##_zpzz_d, \
714 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpzz, \
715 name##_zpzz_fns[a->esz], a, 0)
717 DO_ZPZZ(AND_zpzz
, aa64_sve
, sve_and
)
718 DO_ZPZZ(EOR_zpzz
, aa64_sve
, sve_eor
)
719 DO_ZPZZ(ORR_zpzz
, aa64_sve
, sve_orr
)
720 DO_ZPZZ(BIC_zpzz
, aa64_sve
, sve_bic
)
722 DO_ZPZZ(ADD_zpzz
, aa64_sve
, sve_add
)
723 DO_ZPZZ(SUB_zpzz
, aa64_sve
, sve_sub
)
725 DO_ZPZZ(SMAX_zpzz
, aa64_sve
, sve_smax
)
726 DO_ZPZZ(UMAX_zpzz
, aa64_sve
, sve_umax
)
727 DO_ZPZZ(SMIN_zpzz
, aa64_sve
, sve_smin
)
728 DO_ZPZZ(UMIN_zpzz
, aa64_sve
, sve_umin
)
729 DO_ZPZZ(SABD_zpzz
, aa64_sve
, sve_sabd
)
730 DO_ZPZZ(UABD_zpzz
, aa64_sve
, sve_uabd
)
732 DO_ZPZZ(MUL_zpzz
, aa64_sve
, sve_mul
)
733 DO_ZPZZ(SMULH_zpzz
, aa64_sve
, sve_smulh
)
734 DO_ZPZZ(UMULH_zpzz
, aa64_sve
, sve_umulh
)
736 DO_ZPZZ(ASR_zpzz
, aa64_sve
, sve_asr
)
737 DO_ZPZZ(LSR_zpzz
, aa64_sve
, sve_lsr
)
738 DO_ZPZZ(LSL_zpzz
, aa64_sve
, sve_lsl
)
740 static gen_helper_gvec_4
* const sdiv_fns
[4] = {
741 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
743 TRANS_FEAT(SDIV_zpzz
, aa64_sve
, gen_gvec_ool_arg_zpzz
, sdiv_fns
[a
->esz
], a
, 0)
745 static gen_helper_gvec_4
* const udiv_fns
[4] = {
746 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
748 TRANS_FEAT(UDIV_zpzz
, aa64_sve
, gen_gvec_ool_arg_zpzz
, udiv_fns
[a
->esz
], a
, 0)
750 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
752 if (sve_access_check(s
)) {
753 do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
759 *** SVE Integer Arithmetic - Unary Predicated Group
762 #define DO_ZPZ(NAME, FEAT, name) \
763 static gen_helper_gvec_3 * const name##_fns[4] = { \
764 gen_helper_##name##_b, gen_helper_##name##_h, \
765 gen_helper_##name##_s, gen_helper_##name##_d, \
767 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0)
769 DO_ZPZ(CLS
, aa64_sve
, sve_cls
)
770 DO_ZPZ(CLZ
, aa64_sve
, sve_clz
)
771 DO_ZPZ(CNT_zpz
, aa64_sve
, sve_cnt_zpz
)
772 DO_ZPZ(CNOT
, aa64_sve
, sve_cnot
)
773 DO_ZPZ(NOT_zpz
, aa64_sve
, sve_not_zpz
)
774 DO_ZPZ(ABS
, aa64_sve
, sve_abs
)
775 DO_ZPZ(NEG
, aa64_sve
, sve_neg
)
776 DO_ZPZ(RBIT
, aa64_sve
, sve_rbit
)
778 static gen_helper_gvec_3
* const fabs_fns
[4] = {
779 NULL
, gen_helper_sve_fabs_h
,
780 gen_helper_sve_fabs_s
, gen_helper_sve_fabs_d
,
782 TRANS_FEAT(FABS
, aa64_sve
, gen_gvec_ool_arg_zpz
, fabs_fns
[a
->esz
], a
, 0)
784 static gen_helper_gvec_3
* const fneg_fns
[4] = {
785 NULL
, gen_helper_sve_fneg_h
,
786 gen_helper_sve_fneg_s
, gen_helper_sve_fneg_d
,
788 TRANS_FEAT(FNEG
, aa64_sve
, gen_gvec_ool_arg_zpz
, fneg_fns
[a
->esz
], a
, 0)
790 static gen_helper_gvec_3
* const sxtb_fns
[4] = {
791 NULL
, gen_helper_sve_sxtb_h
,
792 gen_helper_sve_sxtb_s
, gen_helper_sve_sxtb_d
,
794 TRANS_FEAT(SXTB
, aa64_sve
, gen_gvec_ool_arg_zpz
, sxtb_fns
[a
->esz
], a
, 0)
796 static gen_helper_gvec_3
* const uxtb_fns
[4] = {
797 NULL
, gen_helper_sve_uxtb_h
,
798 gen_helper_sve_uxtb_s
, gen_helper_sve_uxtb_d
,
800 TRANS_FEAT(UXTB
, aa64_sve
, gen_gvec_ool_arg_zpz
, uxtb_fns
[a
->esz
], a
, 0)
802 static gen_helper_gvec_3
* const sxth_fns
[4] = {
803 NULL
, NULL
, gen_helper_sve_sxth_s
, gen_helper_sve_sxth_d
805 TRANS_FEAT(SXTH
, aa64_sve
, gen_gvec_ool_arg_zpz
, sxth_fns
[a
->esz
], a
, 0)
807 static gen_helper_gvec_3
* const uxth_fns
[4] = {
808 NULL
, NULL
, gen_helper_sve_uxth_s
, gen_helper_sve_uxth_d
810 TRANS_FEAT(UXTH
, aa64_sve
, gen_gvec_ool_arg_zpz
, uxth_fns
[a
->esz
], a
, 0)
812 TRANS_FEAT(SXTW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
813 a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
, a
, 0)
814 TRANS_FEAT(UXTW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
815 a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
, a
, 0)
818 *** SVE Integer Reduction Group
821 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
822 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
823 gen_helper_gvec_reduc
*fn
)
825 unsigned vsz
= vec_full_reg_size(s
);
833 if (!sve_access_check(s
)) {
837 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
838 temp
= tcg_temp_new_i64();
839 t_zn
= tcg_temp_new_ptr();
840 t_pg
= tcg_temp_new_ptr();
842 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
843 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
844 fn(temp
, t_zn
, t_pg
, desc
);
845 tcg_temp_free_ptr(t_zn
);
846 tcg_temp_free_ptr(t_pg
);
848 write_fp_dreg(s
, a
->rd
, temp
);
849 tcg_temp_free_i64(temp
);
853 #define DO_VPZ(NAME, name) \
854 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
856 static gen_helper_gvec_reduc * const fns[4] = { \
857 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
858 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
860 return do_vpz_ool(s, a, fns[a->esz]); \
873 static bool trans_SADDV(DisasContext
*s
, arg_rpr_esz
*a
)
875 static gen_helper_gvec_reduc
* const fns
[4] = {
876 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
877 gen_helper_sve_saddv_s
, NULL
879 return do_vpz_ool(s
, a
, fns
[a
->esz
]);
885 *** SVE Shift by Immediate - Predicated Group
889 * Copy Zn into Zd, storing zeros into inactive elements.
890 * If invert, store zeros into the active elements.
892 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
893 int esz
, bool invert
)
895 static gen_helper_gvec_3
* const fns
[4] = {
896 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
897 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
899 return gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
902 static bool trans_ASR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
904 static gen_helper_gvec_3
* const fns
[4] = {
905 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
906 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
909 /* Invalid tsz encoding -- see tszimm_esz. */
912 /* Shift by element size is architecturally valid. For
913 arithmetic right-shift, it's the same as by one less. */
914 a
->imm
= MIN(a
->imm
, (8 << a
->esz
) - 1);
915 return gen_gvec_ool_arg_zpzi(s
, fns
[a
->esz
], a
);
918 static bool trans_LSR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
920 static gen_helper_gvec_3
* const fns
[4] = {
921 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
922 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
927 /* Shift by element size is architecturally valid.
928 For logical shifts, it is a zeroing operation. */
929 if (a
->imm
>= (8 << a
->esz
)) {
930 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
932 return gen_gvec_ool_arg_zpzi(s
, fns
[a
->esz
], a
);
936 static bool trans_LSL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
938 static gen_helper_gvec_3
* const fns
[4] = {
939 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
940 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
945 /* Shift by element size is architecturally valid.
946 For logical shifts, it is a zeroing operation. */
947 if (a
->imm
>= (8 << a
->esz
)) {
948 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
950 return gen_gvec_ool_arg_zpzi(s
, fns
[a
->esz
], a
);
954 static bool trans_ASRD(DisasContext
*s
, arg_rpri_esz
*a
)
956 static gen_helper_gvec_3
* const fns
[4] = {
957 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
958 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
963 /* Shift by element size is architecturally valid. For arithmetic
964 right shift for division, it is a zeroing operation. */
965 if (a
->imm
>= (8 << a
->esz
)) {
966 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
968 return gen_gvec_ool_arg_zpzi(s
, fns
[a
->esz
], a
);
972 static gen_helper_gvec_3
* const sqshl_zpzi_fns
[4] = {
973 gen_helper_sve2_sqshl_zpzi_b
, gen_helper_sve2_sqshl_zpzi_h
,
974 gen_helper_sve2_sqshl_zpzi_s
, gen_helper_sve2_sqshl_zpzi_d
,
976 TRANS_FEAT(SQSHL_zpzi
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
977 a
->esz
< 0 ? NULL
: sqshl_zpzi_fns
[a
->esz
], a
)
979 static gen_helper_gvec_3
* const uqshl_zpzi_fns
[4] = {
980 gen_helper_sve2_uqshl_zpzi_b
, gen_helper_sve2_uqshl_zpzi_h
,
981 gen_helper_sve2_uqshl_zpzi_s
, gen_helper_sve2_uqshl_zpzi_d
,
983 TRANS_FEAT(UQSHL_zpzi
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
984 a
->esz
< 0 ? NULL
: uqshl_zpzi_fns
[a
->esz
], a
)
986 static gen_helper_gvec_3
* const srshr_fns
[4] = {
987 gen_helper_sve2_srshr_b
, gen_helper_sve2_srshr_h
,
988 gen_helper_sve2_srshr_s
, gen_helper_sve2_srshr_d
,
990 TRANS_FEAT(SRSHR
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
991 a
->esz
< 0 ? NULL
: srshr_fns
[a
->esz
], a
)
993 static gen_helper_gvec_3
* const urshr_fns
[4] = {
994 gen_helper_sve2_urshr_b
, gen_helper_sve2_urshr_h
,
995 gen_helper_sve2_urshr_s
, gen_helper_sve2_urshr_d
,
997 TRANS_FEAT(URSHR
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
998 a
->esz
< 0 ? NULL
: urshr_fns
[a
->esz
], a
)
1000 static gen_helper_gvec_3
* const sqshlu_fns
[4] = {
1001 gen_helper_sve2_sqshlu_b
, gen_helper_sve2_sqshlu_h
,
1002 gen_helper_sve2_sqshlu_s
, gen_helper_sve2_sqshlu_d
,
1004 TRANS_FEAT(SQSHLU
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
1005 a
->esz
< 0 ? NULL
: sqshlu_fns
[a
->esz
], a
)
1008 *** SVE Bitwise Shift - Predicated Group
1011 #define DO_ZPZW(NAME, name) \
1012 static gen_helper_gvec_4 * const name##_zpzw_fns[4] = { \
1013 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1014 gen_helper_sve_##name##_zpzw_s, NULL \
1016 TRANS_FEAT(NAME##_zpzw, aa64_sve, gen_gvec_ool_arg_zpzz, \
1017 a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0)
1026 *** SVE Bitwise Shift - Unpredicated Group
1029 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1030 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1031 int64_t, uint32_t, uint32_t))
1034 /* Invalid tsz encoding -- see tszimm_esz. */
1037 if (sve_access_check(s
)) {
1038 unsigned vsz
= vec_full_reg_size(s
);
1039 /* Shift by element size is architecturally valid. For
1040 arithmetic right-shift, it's the same as by one less.
1041 Otherwise it is a zeroing operation. */
1042 if (a
->imm
>= 8 << a
->esz
) {
1044 a
->imm
= (8 << a
->esz
) - 1;
1046 do_dupi_z(s
, a
->rd
, 0);
1050 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1051 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1056 static bool trans_ASR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1058 return do_shift_imm(s
, a
, true, tcg_gen_gvec_sari
);
1061 static bool trans_LSR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1063 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shri
);
1066 static bool trans_LSL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1068 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shli
);
1071 #define DO_ZZW(NAME, name) \
1072 static gen_helper_gvec_3 * const name##_zzw_fns[4] = { \
1073 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1074 gen_helper_sve_##name##_zzw_s, NULL \
1076 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_arg_zzz, \
1077 name##_zzw_fns[a->esz], a, 0)
1079 DO_ZZW(ASR_zzw
, asr
)
1080 DO_ZZW(LSR_zzw
, lsr
)
1081 DO_ZZW(LSL_zzw
, lsl
)
1086 *** SVE Integer Multiply-Add Group
1089 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1090 gen_helper_gvec_5
*fn
)
1092 if (sve_access_check(s
)) {
1093 unsigned vsz
= vec_full_reg_size(s
);
1094 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1095 vec_full_reg_offset(s
, a
->ra
),
1096 vec_full_reg_offset(s
, a
->rn
),
1097 vec_full_reg_offset(s
, a
->rm
),
1098 pred_full_reg_offset(s
, a
->pg
),
1104 #define DO_ZPZZZ(NAME, name) \
1105 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
1107 static gen_helper_gvec_5 * const fns[4] = { \
1108 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
1109 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1111 return do_zpzzz_ool(s, a, fns[a->esz]); \
1120 *** SVE Index Generation Group
1123 static void do_index(DisasContext
*s
, int esz
, int rd
,
1124 TCGv_i64 start
, TCGv_i64 incr
)
1126 unsigned vsz
= vec_full_reg_size(s
);
1127 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1128 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1130 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1132 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1134 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1135 static index_fn
* const fns
[3] = {
1136 gen_helper_sve_index_b
,
1137 gen_helper_sve_index_h
,
1138 gen_helper_sve_index_s
,
1140 TCGv_i32 s32
= tcg_temp_new_i32();
1141 TCGv_i32 i32
= tcg_temp_new_i32();
1143 tcg_gen_extrl_i64_i32(s32
, start
);
1144 tcg_gen_extrl_i64_i32(i32
, incr
);
1145 fns
[esz
](t_zd
, s32
, i32
, desc
);
1147 tcg_temp_free_i32(s32
);
1148 tcg_temp_free_i32(i32
);
1150 tcg_temp_free_ptr(t_zd
);
1153 static bool trans_INDEX_ii(DisasContext
*s
, arg_INDEX_ii
*a
)
1155 if (sve_access_check(s
)) {
1156 TCGv_i64 start
= tcg_constant_i64(a
->imm1
);
1157 TCGv_i64 incr
= tcg_constant_i64(a
->imm2
);
1158 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1163 static bool trans_INDEX_ir(DisasContext
*s
, arg_INDEX_ir
*a
)
1165 if (sve_access_check(s
)) {
1166 TCGv_i64 start
= tcg_constant_i64(a
->imm
);
1167 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1168 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1173 static bool trans_INDEX_ri(DisasContext
*s
, arg_INDEX_ri
*a
)
1175 if (sve_access_check(s
)) {
1176 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1177 TCGv_i64 incr
= tcg_constant_i64(a
->imm
);
1178 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1183 static bool trans_INDEX_rr(DisasContext
*s
, arg_INDEX_rr
*a
)
1185 if (sve_access_check(s
)) {
1186 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1187 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1188 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1194 *** SVE Stack Allocation Group
1197 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1199 if (sve_access_check(s
)) {
1200 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1201 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1202 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1207 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1209 if (sve_access_check(s
)) {
1210 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1211 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1212 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1217 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1219 if (sve_access_check(s
)) {
1220 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1221 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1227 *** SVE Compute Vector Address Group
1230 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1232 return gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1235 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
1237 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
1240 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
1242 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
1245 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
1247 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
1250 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1252 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1256 *** SVE Integer Misc - Unpredicated Group
1259 static gen_helper_gvec_2
* const fexpa_fns
[4] = {
1260 NULL
, gen_helper_sve_fexpa_h
,
1261 gen_helper_sve_fexpa_s
, gen_helper_sve_fexpa_d
,
1263 TRANS_FEAT(FEXPA
, aa64_sve
, gen_gvec_ool_zz
,
1264 fexpa_fns
[a
->esz
], a
->rd
, a
->rn
, 0)
1266 static gen_helper_gvec_3
* const ftssel_fns
[4] = {
1267 NULL
, gen_helper_sve_ftssel_h
,
1268 gen_helper_sve_ftssel_s
, gen_helper_sve_ftssel_d
,
1270 TRANS_FEAT(FTSSEL
, aa64_sve
, gen_gvec_ool_arg_zzz
, ftssel_fns
[a
->esz
], a
, 0)
1273 *** SVE Predicate Logical Operations Group
1276 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1277 const GVecGen4
*gvec_op
)
1279 if (!sve_access_check(s
)) {
1283 unsigned psz
= pred_gvec_reg_size(s
);
1284 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1285 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1286 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1287 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1290 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1295 /* Do the operation and the flags generation in temps. */
1296 TCGv_i64 pd
= tcg_temp_new_i64();
1297 TCGv_i64 pn
= tcg_temp_new_i64();
1298 TCGv_i64 pm
= tcg_temp_new_i64();
1299 TCGv_i64 pg
= tcg_temp_new_i64();
1301 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1302 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1303 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1305 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1306 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1308 do_predtest1(pd
, pg
);
1310 tcg_temp_free_i64(pd
);
1311 tcg_temp_free_i64(pn
);
1312 tcg_temp_free_i64(pm
);
1313 tcg_temp_free_i64(pg
);
1315 /* The operation and flags generation is large. The computation
1316 * of the flags depends on the original contents of the guarding
1317 * predicate. If the destination overwrites the guarding predicate,
1318 * then the easiest way to get this right is to save a copy.
1321 if (a
->rd
== a
->pg
) {
1322 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1323 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1326 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1327 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1332 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1334 tcg_gen_and_i64(pd
, pn
, pm
);
1335 tcg_gen_and_i64(pd
, pd
, pg
);
1338 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1339 TCGv_vec pm
, TCGv_vec pg
)
1341 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1342 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1345 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1347 static const GVecGen4 op
= {
1348 .fni8
= gen_and_pg_i64
,
1349 .fniv
= gen_and_pg_vec
,
1350 .fno
= gen_helper_sve_and_pppp
,
1351 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1355 if (!sve_access_check(s
)) {
1358 if (a
->rn
== a
->rm
) {
1359 if (a
->pg
== a
->rn
) {
1360 do_mov_p(s
, a
->rd
, a
->rn
);
1362 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1365 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1366 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1370 return do_pppp_flags(s
, a
, &op
);
1373 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1375 tcg_gen_andc_i64(pd
, pn
, pm
);
1376 tcg_gen_and_i64(pd
, pd
, pg
);
1379 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1380 TCGv_vec pm
, TCGv_vec pg
)
1382 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1383 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1386 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1388 static const GVecGen4 op
= {
1389 .fni8
= gen_bic_pg_i64
,
1390 .fniv
= gen_bic_pg_vec
,
1391 .fno
= gen_helper_sve_bic_pppp
,
1392 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1395 if (!a
->s
&& a
->pg
== a
->rn
) {
1396 if (sve_access_check(s
)) {
1397 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1401 return do_pppp_flags(s
, a
, &op
);
1404 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1406 tcg_gen_xor_i64(pd
, pn
, pm
);
1407 tcg_gen_and_i64(pd
, pd
, pg
);
1410 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1411 TCGv_vec pm
, TCGv_vec pg
)
1413 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1414 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1417 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1419 static const GVecGen4 op
= {
1420 .fni8
= gen_eor_pg_i64
,
1421 .fniv
= gen_eor_pg_vec
,
1422 .fno
= gen_helper_sve_eor_pppp
,
1423 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1425 return do_pppp_flags(s
, a
, &op
);
1428 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1433 if (sve_access_check(s
)) {
1434 unsigned psz
= pred_gvec_reg_size(s
);
1435 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1436 pred_full_reg_offset(s
, a
->pg
),
1437 pred_full_reg_offset(s
, a
->rn
),
1438 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1443 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1445 tcg_gen_or_i64(pd
, pn
, pm
);
1446 tcg_gen_and_i64(pd
, pd
, pg
);
1449 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1450 TCGv_vec pm
, TCGv_vec pg
)
1452 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1453 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1456 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1458 static const GVecGen4 op
= {
1459 .fni8
= gen_orr_pg_i64
,
1460 .fniv
= gen_orr_pg_vec
,
1461 .fno
= gen_helper_sve_orr_pppp
,
1462 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1465 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1466 return do_mov_p(s
, a
->rd
, a
->rn
);
1468 return do_pppp_flags(s
, a
, &op
);
1471 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1473 tcg_gen_orc_i64(pd
, pn
, pm
);
1474 tcg_gen_and_i64(pd
, pd
, pg
);
1477 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1478 TCGv_vec pm
, TCGv_vec pg
)
1480 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1481 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1484 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1486 static const GVecGen4 op
= {
1487 .fni8
= gen_orn_pg_i64
,
1488 .fniv
= gen_orn_pg_vec
,
1489 .fno
= gen_helper_sve_orn_pppp
,
1490 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1492 return do_pppp_flags(s
, a
, &op
);
1495 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1497 tcg_gen_or_i64(pd
, pn
, pm
);
1498 tcg_gen_andc_i64(pd
, pg
, pd
);
1501 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1502 TCGv_vec pm
, TCGv_vec pg
)
1504 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1505 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1508 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1510 static const GVecGen4 op
= {
1511 .fni8
= gen_nor_pg_i64
,
1512 .fniv
= gen_nor_pg_vec
,
1513 .fno
= gen_helper_sve_nor_pppp
,
1514 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1516 return do_pppp_flags(s
, a
, &op
);
1519 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1521 tcg_gen_and_i64(pd
, pn
, pm
);
1522 tcg_gen_andc_i64(pd
, pg
, pd
);
1525 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1526 TCGv_vec pm
, TCGv_vec pg
)
1528 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1529 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1532 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1534 static const GVecGen4 op
= {
1535 .fni8
= gen_nand_pg_i64
,
1536 .fniv
= gen_nand_pg_vec
,
1537 .fno
= gen_helper_sve_nand_pppp
,
1538 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1540 return do_pppp_flags(s
, a
, &op
);
1544 *** SVE Predicate Misc Group
1547 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1549 if (sve_access_check(s
)) {
1550 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1551 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1552 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1555 TCGv_i64 pn
= tcg_temp_new_i64();
1556 TCGv_i64 pg
= tcg_temp_new_i64();
1558 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1559 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1560 do_predtest1(pn
, pg
);
1562 tcg_temp_free_i64(pn
);
1563 tcg_temp_free_i64(pg
);
1565 do_predtest(s
, nofs
, gofs
, words
);
1571 /* See the ARM pseudocode DecodePredCount. */
1572 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1574 unsigned elements
= fullsz
>> esz
;
1578 case 0x0: /* POW2 */
1579 return pow2floor(elements
);
1590 case 0x9: /* VL16 */
1591 case 0xa: /* VL32 */
1592 case 0xb: /* VL64 */
1593 case 0xc: /* VL128 */
1594 case 0xd: /* VL256 */
1595 bound
= 16 << (pattern
- 9);
1597 case 0x1d: /* MUL4 */
1598 return elements
- elements
% 4;
1599 case 0x1e: /* MUL3 */
1600 return elements
- elements
% 3;
1601 case 0x1f: /* ALL */
1603 default: /* #uimm5 */
1606 return elements
>= bound
? bound
: 0;
1609 /* This handles all of the predicate initialization instructions,
1610 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1611 * so that decode_pred_count returns 0. For SETFFR, we will have
1612 * set RD == 16 == FFR.
1614 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1616 if (!sve_access_check(s
)) {
1620 unsigned fullsz
= vec_full_reg_size(s
);
1621 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1622 unsigned numelem
, setsz
, i
;
1623 uint64_t word
, lastword
;
1626 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1628 /* Determine what we must store into each bit, and how many. */
1630 lastword
= word
= 0;
1633 setsz
= numelem
<< esz
;
1634 lastword
= word
= pred_esz_masks
[esz
];
1636 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1640 t
= tcg_temp_new_i64();
1642 tcg_gen_movi_i64(t
, lastword
);
1643 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1647 if (word
== lastword
) {
1648 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1649 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1651 if (oprsz
* 8 == setsz
) {
1652 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1660 tcg_gen_movi_i64(t
, word
);
1661 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1662 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1664 if (lastword
!= word
) {
1665 tcg_gen_movi_i64(t
, lastword
);
1666 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1670 tcg_gen_movi_i64(t
, 0);
1671 for (; i
< fullsz
; i
+= 8) {
1672 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1677 tcg_temp_free_i64(t
);
1681 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1682 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1683 tcg_gen_movi_i32(cpu_VF
, 0);
1684 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1689 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1691 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1694 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1696 /* Note pat == 31 is #all, to set all elements. */
1697 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1700 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1702 /* Note pat == 32 is #unimp, to set no elements. */
1703 return do_predset(s
, 0, a
->rd
, 32, false);
1706 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1708 /* The path through do_pppp_flags is complicated enough to want to avoid
1709 * duplication. Frob the arguments into the form of a predicated AND.
1711 arg_rprr_s alt_a
= {
1712 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1713 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1715 return trans_AND_pppp(s
, &alt_a
);
1718 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1720 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1723 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1725 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1728 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1729 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1730 TCGv_ptr
, TCGv_i32
))
1732 if (!sve_access_check(s
)) {
1736 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1737 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1741 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1742 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1744 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1745 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1746 t
= tcg_temp_new_i32();
1748 gen_fn(t
, t_pd
, t_pg
, tcg_constant_i32(desc
));
1749 tcg_temp_free_ptr(t_pd
);
1750 tcg_temp_free_ptr(t_pg
);
1753 tcg_temp_free_i32(t
);
1757 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1759 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1762 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1764 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1768 *** SVE Element Count Group
1771 /* Perform an inline saturating addition of a 32-bit value within
1772 * a 64-bit register. The second operand is known to be positive,
1773 * which halves the comparisions we must perform to bound the result.
1775 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1779 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1781 tcg_gen_ext32u_i64(reg
, reg
);
1783 tcg_gen_ext32s_i64(reg
, reg
);
1786 tcg_gen_sub_i64(reg
, reg
, val
);
1787 ibound
= (u
? 0 : INT32_MIN
);
1788 tcg_gen_smax_i64(reg
, reg
, tcg_constant_i64(ibound
));
1790 tcg_gen_add_i64(reg
, reg
, val
);
1791 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1792 tcg_gen_smin_i64(reg
, reg
, tcg_constant_i64(ibound
));
1796 /* Similarly with 64-bit values. */
1797 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1799 TCGv_i64 t0
= tcg_temp_new_i64();
1804 tcg_gen_sub_i64(t0
, reg
, val
);
1805 t2
= tcg_constant_i64(0);
1806 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t2
, t0
);
1808 tcg_gen_add_i64(t0
, reg
, val
);
1809 t2
= tcg_constant_i64(-1);
1810 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t2
, t0
);
1813 TCGv_i64 t1
= tcg_temp_new_i64();
1815 /* Detect signed overflow for subtraction. */
1816 tcg_gen_xor_i64(t0
, reg
, val
);
1817 tcg_gen_sub_i64(t1
, reg
, val
);
1818 tcg_gen_xor_i64(reg
, reg
, t1
);
1819 tcg_gen_and_i64(t0
, t0
, reg
);
1821 /* Bound the result. */
1822 tcg_gen_movi_i64(reg
, INT64_MIN
);
1823 t2
= tcg_constant_i64(0);
1824 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1826 /* Detect signed overflow for addition. */
1827 tcg_gen_xor_i64(t0
, reg
, val
);
1828 tcg_gen_add_i64(reg
, reg
, val
);
1829 tcg_gen_xor_i64(t1
, reg
, val
);
1830 tcg_gen_andc_i64(t0
, t1
, t0
);
1832 /* Bound the result. */
1833 tcg_gen_movi_i64(t1
, INT64_MAX
);
1834 t2
= tcg_constant_i64(0);
1835 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1837 tcg_temp_free_i64(t1
);
1839 tcg_temp_free_i64(t0
);
1842 /* Similarly with a vector and a scalar operand. */
1843 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1844 TCGv_i64 val
, bool u
, bool d
)
1846 unsigned vsz
= vec_full_reg_size(s
);
1847 TCGv_ptr dptr
, nptr
;
1851 dptr
= tcg_temp_new_ptr();
1852 nptr
= tcg_temp_new_ptr();
1853 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1854 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1855 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1859 t32
= tcg_temp_new_i32();
1860 tcg_gen_extrl_i64_i32(t32
, val
);
1862 tcg_gen_neg_i32(t32
, t32
);
1865 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
1867 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
1869 tcg_temp_free_i32(t32
);
1873 t32
= tcg_temp_new_i32();
1874 tcg_gen_extrl_i64_i32(t32
, val
);
1876 tcg_gen_neg_i32(t32
, t32
);
1879 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
1881 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
1883 tcg_temp_free_i32(t32
);
1887 t64
= tcg_temp_new_i64();
1889 tcg_gen_neg_i64(t64
, val
);
1891 tcg_gen_mov_i64(t64
, val
);
1894 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
1896 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
1898 tcg_temp_free_i64(t64
);
1904 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
1906 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
1909 t64
= tcg_temp_new_i64();
1910 tcg_gen_neg_i64(t64
, val
);
1911 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
1912 tcg_temp_free_i64(t64
);
1914 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
1919 g_assert_not_reached();
1922 tcg_temp_free_ptr(dptr
);
1923 tcg_temp_free_ptr(nptr
);
1926 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
1928 if (sve_access_check(s
)) {
1929 unsigned fullsz
= vec_full_reg_size(s
);
1930 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1931 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
1936 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
1938 if (sve_access_check(s
)) {
1939 unsigned fullsz
= vec_full_reg_size(s
);
1940 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1941 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
1942 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1944 tcg_gen_addi_i64(reg
, reg
, inc
);
1949 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
1951 if (!sve_access_check(s
)) {
1955 unsigned fullsz
= vec_full_reg_size(s
);
1956 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1957 int inc
= numelem
* a
->imm
;
1958 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1960 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1963 tcg_gen_ext32u_i64(reg
, reg
);
1965 tcg_gen_ext32s_i64(reg
, reg
);
1968 do_sat_addsub_32(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
1973 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
1975 if (!sve_access_check(s
)) {
1979 unsigned fullsz
= vec_full_reg_size(s
);
1980 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1981 int inc
= numelem
* a
->imm
;
1982 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1985 do_sat_addsub_64(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
1990 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
1996 unsigned fullsz
= vec_full_reg_size(s
);
1997 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1998 int inc
= numelem
* a
->imm
;
2001 if (sve_access_check(s
)) {
2002 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2003 vec_full_reg_offset(s
, a
->rn
),
2004 tcg_constant_i64(a
->d
? -inc
: inc
),
2008 do_mov_z(s
, a
->rd
, a
->rn
);
2013 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2019 unsigned fullsz
= vec_full_reg_size(s
);
2020 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2021 int inc
= numelem
* a
->imm
;
2024 if (sve_access_check(s
)) {
2025 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
2026 tcg_constant_i64(inc
), a
->u
, a
->d
);
2029 do_mov_z(s
, a
->rd
, a
->rn
);
2035 *** SVE Bitwise Immediate Group
2038 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
2041 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2042 extract32(a
->dbm
, 0, 6),
2043 extract32(a
->dbm
, 6, 6))) {
2046 return gen_gvec_fn_zzi(s
, gvec_fn
, MO_64
, a
->rd
, a
->rn
, imm
);
2049 TRANS_FEAT(AND_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_andi
)
2050 TRANS_FEAT(ORR_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_ori
)
2051 TRANS_FEAT(EOR_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_xori
)
2053 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
2056 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2057 extract32(a
->dbm
, 0, 6),
2058 extract32(a
->dbm
, 6, 6))) {
2061 if (sve_access_check(s
)) {
2062 do_dupi_z(s
, a
->rd
, imm
);
2068 *** SVE Integer Wide Immediate - Predicated Group
2071 /* Implement all merging copies. This is used for CPY (immediate),
2072 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2074 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
2077 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2078 static gen_cpy
* const fns
[4] = {
2079 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
2080 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
2082 unsigned vsz
= vec_full_reg_size(s
);
2083 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
2084 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2085 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2086 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2088 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2089 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2090 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2092 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2094 tcg_temp_free_ptr(t_zd
);
2095 tcg_temp_free_ptr(t_zn
);
2096 tcg_temp_free_ptr(t_pg
);
2099 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2104 if (sve_access_check(s
)) {
2105 /* Decode the VFP immediate. */
2106 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2107 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(imm
));
2112 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2114 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2117 if (sve_access_check(s
)) {
2118 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(a
->imm
));
2123 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2125 static gen_helper_gvec_2i
* const fns
[4] = {
2126 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2127 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2130 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2133 if (sve_access_check(s
)) {
2134 unsigned vsz
= vec_full_reg_size(s
);
2135 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2136 pred_full_reg_offset(s
, a
->pg
),
2137 tcg_constant_i64(a
->imm
),
2138 vsz
, vsz
, 0, fns
[a
->esz
]);
2144 *** SVE Permute Extract Group
2147 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2149 if (!sve_access_check(s
)) {
2153 unsigned vsz
= vec_full_reg_size(s
);
2154 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2155 unsigned n_siz
= vsz
- n_ofs
;
2156 unsigned d
= vec_full_reg_offset(s
, rd
);
2157 unsigned n
= vec_full_reg_offset(s
, rn
);
2158 unsigned m
= vec_full_reg_offset(s
, rm
);
2160 /* Use host vector move insns if we have appropriate sizes
2161 * and no unfortunate overlap.
2164 && n_ofs
== size_for_gvec(n_ofs
)
2165 && n_siz
== size_for_gvec(n_siz
)
2166 && (d
!= n
|| n_siz
<= n_ofs
)) {
2167 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2169 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2172 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2177 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
2179 return do_EXT(s
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
2182 static bool trans_EXT_sve2(DisasContext
*s
, arg_rri
*a
)
2184 if (!dc_isar_feature(aa64_sve2
, s
)) {
2187 return do_EXT(s
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
);
2191 *** SVE Permute - Unpredicated Group
2194 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2196 if (sve_access_check(s
)) {
2197 unsigned vsz
= vec_full_reg_size(s
);
2198 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2199 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2204 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2206 if ((a
->imm
& 0x1f) == 0) {
2209 if (sve_access_check(s
)) {
2210 unsigned vsz
= vec_full_reg_size(s
);
2211 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2212 unsigned esz
, index
;
2214 esz
= ctz32(a
->imm
);
2215 index
= a
->imm
>> (esz
+ 1);
2217 if ((index
<< esz
) < vsz
) {
2218 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2219 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2222 * While dup_mem handles 128-bit elements, dup_imm does not.
2223 * Thankfully element size doesn't matter for splatting zero.
2225 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2231 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2233 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2234 static gen_insr
* const fns
[4] = {
2235 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2236 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2238 unsigned vsz
= vec_full_reg_size(s
);
2239 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
2240 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2241 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2243 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2244 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2246 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2248 tcg_temp_free_ptr(t_zd
);
2249 tcg_temp_free_ptr(t_zn
);
2252 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2254 if (sve_access_check(s
)) {
2255 TCGv_i64 t
= tcg_temp_new_i64();
2256 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2257 do_insr_i64(s
, a
, t
);
2258 tcg_temp_free_i64(t
);
2263 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2265 if (sve_access_check(s
)) {
2266 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2271 static gen_helper_gvec_2
* const rev_fns
[4] = {
2272 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2273 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2275 TRANS_FEAT(REV_v
, aa64_sve
, gen_gvec_ool_zz
, rev_fns
[a
->esz
], a
->rd
, a
->rn
, 0)
2277 static gen_helper_gvec_3
* const sve_tbl_fns
[4] = {
2278 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2279 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2281 TRANS_FEAT(TBL
, aa64_sve
, gen_gvec_ool_arg_zzz
, sve_tbl_fns
[a
->esz
], a
, 0)
2283 static gen_helper_gvec_4
* const sve2_tbl_fns
[4] = {
2284 gen_helper_sve2_tbl_b
, gen_helper_sve2_tbl_h
,
2285 gen_helper_sve2_tbl_s
, gen_helper_sve2_tbl_d
2287 TRANS_FEAT(TBL_sve2
, aa64_sve2
, gen_gvec_ool_zzzz
, sve2_tbl_fns
[a
->esz
],
2288 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->rm
, 0)
2290 static gen_helper_gvec_3
* const tbx_fns
[4] = {
2291 gen_helper_sve2_tbx_b
, gen_helper_sve2_tbx_h
,
2292 gen_helper_sve2_tbx_s
, gen_helper_sve2_tbx_d
2294 TRANS_FEAT(TBX
, aa64_sve2
, gen_gvec_ool_arg_zzz
, tbx_fns
[a
->esz
], a
, 0)
2296 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2298 static gen_helper_gvec_2
* const fns
[4][2] = {
2300 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2301 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2302 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2308 if (sve_access_check(s
)) {
2309 unsigned vsz
= vec_full_reg_size(s
);
2310 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2311 vec_full_reg_offset(s
, a
->rn
)
2312 + (a
->h
? vsz
/ 2 : 0),
2313 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2319 *** SVE Permute - Predicates Group
2322 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2323 gen_helper_gvec_3
*fn
)
2325 if (!sve_access_check(s
)) {
2329 unsigned vsz
= pred_full_reg_size(s
);
2331 TCGv_ptr t_d
= tcg_temp_new_ptr();
2332 TCGv_ptr t_n
= tcg_temp_new_ptr();
2333 TCGv_ptr t_m
= tcg_temp_new_ptr();
2336 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2337 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2338 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2340 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2341 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2342 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2344 fn(t_d
, t_n
, t_m
, tcg_constant_i32(desc
));
2346 tcg_temp_free_ptr(t_d
);
2347 tcg_temp_free_ptr(t_n
);
2348 tcg_temp_free_ptr(t_m
);
2352 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2353 gen_helper_gvec_2
*fn
)
2355 if (!sve_access_check(s
)) {
2359 unsigned vsz
= pred_full_reg_size(s
);
2360 TCGv_ptr t_d
= tcg_temp_new_ptr();
2361 TCGv_ptr t_n
= tcg_temp_new_ptr();
2364 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2365 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2367 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2368 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2369 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2371 fn(t_d
, t_n
, tcg_constant_i32(desc
));
2373 tcg_temp_free_ptr(t_d
);
2374 tcg_temp_free_ptr(t_n
);
2378 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2380 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2383 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2385 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2388 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2390 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2393 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2395 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2398 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2400 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2403 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2405 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2408 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2410 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2413 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2415 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2418 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2420 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2424 *** SVE Permute - Interleaving Group
2427 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2429 static gen_helper_gvec_3
* const fns
[4] = {
2430 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2431 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2434 if (sve_access_check(s
)) {
2435 unsigned vsz
= vec_full_reg_size(s
);
2436 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2437 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2438 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2439 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2440 vsz
, vsz
, 0, fns
[a
->esz
]);
2445 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2447 return do_zip(s
, a
, false);
2450 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2452 return do_zip(s
, a
, true);
2455 static bool do_zip_q(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2457 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2460 if (sve_access_check(s
)) {
2461 unsigned vsz
= vec_full_reg_size(s
);
2462 unsigned high_ofs
= high
? QEMU_ALIGN_DOWN(vsz
, 32) / 2 : 0;
2463 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2464 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2465 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2466 vsz
, vsz
, 0, gen_helper_sve2_zip_q
);
2471 static bool trans_ZIP1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2473 return do_zip_q(s
, a
, false);
2476 static bool trans_ZIP2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2478 return do_zip_q(s
, a
, true);
2481 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2482 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2483 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2486 TRANS_FEAT(UZP1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2487 uzp_fns
[a
->esz
], a
, 0)
2488 TRANS_FEAT(UZP2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2489 uzp_fns
[a
->esz
], a
, 1 << a
->esz
)
2491 TRANS_FEAT(UZP1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2492 gen_helper_sve2_uzp_q
, a
, 0)
2493 TRANS_FEAT(UZP2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2494 gen_helper_sve2_uzp_q
, a
, 16)
2496 static gen_helper_gvec_3
* const trn_fns
[4] = {
2497 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2498 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2501 TRANS_FEAT(TRN1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2502 trn_fns
[a
->esz
], a
, 0)
2503 TRANS_FEAT(TRN2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2504 trn_fns
[a
->esz
], a
, 1 << a
->esz
)
2506 TRANS_FEAT(TRN1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2507 gen_helper_sve2_trn_q
, a
, 0)
2508 TRANS_FEAT(TRN2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2509 gen_helper_sve2_trn_q
, a
, 16)
2512 *** SVE Permute Vector - Predicated Group
2515 static gen_helper_gvec_3
* const compact_fns
[4] = {
2516 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2518 TRANS_FEAT(COMPACT
, aa64_sve
, gen_gvec_ool_arg_zpz
, compact_fns
[a
->esz
], a
, 0)
2520 /* Call the helper that computes the ARM LastActiveElement pseudocode
2521 * function, scaled by the element size. This includes the not found
2522 * indication; e.g. not found for esz=3 is -8.
2524 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2526 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2527 * round up, as we do elsewhere, because we need the exact size.
2529 TCGv_ptr t_p
= tcg_temp_new_ptr();
2532 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2533 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2535 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2537 gen_helper_sve_last_active_element(ret
, t_p
, tcg_constant_i32(desc
));
2539 tcg_temp_free_ptr(t_p
);
2542 /* Increment LAST to the offset of the next element in the vector,
2543 * wrapping around to 0.
2545 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2547 unsigned vsz
= vec_full_reg_size(s
);
2549 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2550 if (is_power_of_2(vsz
)) {
2551 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2553 TCGv_i32 max
= tcg_constant_i32(vsz
);
2554 TCGv_i32 zero
= tcg_constant_i32(0);
2555 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2559 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2560 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2562 unsigned vsz
= vec_full_reg_size(s
);
2564 if (is_power_of_2(vsz
)) {
2565 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2567 TCGv_i32 max
= tcg_constant_i32(vsz
- (1 << esz
));
2568 TCGv_i32 zero
= tcg_constant_i32(0);
2569 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2573 /* Load an unsigned element of ESZ from BASE+OFS. */
2574 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2576 TCGv_i64 r
= tcg_temp_new_i64();
2580 tcg_gen_ld8u_i64(r
, base
, ofs
);
2583 tcg_gen_ld16u_i64(r
, base
, ofs
);
2586 tcg_gen_ld32u_i64(r
, base
, ofs
);
2589 tcg_gen_ld_i64(r
, base
, ofs
);
2592 g_assert_not_reached();
2597 /* Load an unsigned element of ESZ from RM[LAST]. */
2598 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2601 TCGv_ptr p
= tcg_temp_new_ptr();
2604 /* Convert offset into vector into offset into ENV.
2605 * The final adjustment for the vector register base
2606 * is added via constant offset to the load.
2609 /* Adjust for element ordering. See vec_reg_offset. */
2611 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2614 tcg_gen_ext_i32_ptr(p
, last
);
2615 tcg_gen_add_ptr(p
, p
, cpu_env
);
2617 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2618 tcg_temp_free_ptr(p
);
2623 /* Compute CLAST for a Zreg. */
2624 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2629 unsigned vsz
, esz
= a
->esz
;
2631 if (!sve_access_check(s
)) {
2635 last
= tcg_temp_local_new_i32();
2636 over
= gen_new_label();
2638 find_last_active(s
, last
, esz
, a
->pg
);
2640 /* There is of course no movcond for a 2048-bit vector,
2641 * so we must branch over the actual store.
2643 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2646 incr_last_active(s
, last
, esz
);
2649 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2650 tcg_temp_free_i32(last
);
2652 vsz
= vec_full_reg_size(s
);
2653 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2654 tcg_temp_free_i64(ele
);
2656 /* If this insn used MOVPRFX, we may need a second move. */
2657 if (a
->rd
!= a
->rn
) {
2658 TCGLabel
*done
= gen_new_label();
2661 gen_set_label(over
);
2662 do_mov_z(s
, a
->rd
, a
->rn
);
2664 gen_set_label(done
);
2666 gen_set_label(over
);
2671 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2673 return do_clast_vector(s
, a
, false);
2676 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2678 return do_clast_vector(s
, a
, true);
2681 /* Compute CLAST for a scalar. */
2682 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2683 bool before
, TCGv_i64 reg_val
)
2685 TCGv_i32 last
= tcg_temp_new_i32();
2688 find_last_active(s
, last
, esz
, pg
);
2690 /* Extend the original value of last prior to incrementing. */
2691 cmp
= tcg_temp_new_i64();
2692 tcg_gen_ext_i32_i64(cmp
, last
);
2695 incr_last_active(s
, last
, esz
);
2698 /* The conceit here is that while last < 0 indicates not found, after
2699 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2700 * from which we can load garbage. We then discard the garbage with
2701 * a conditional move.
2703 ele
= load_last_active(s
, last
, rm
, esz
);
2704 tcg_temp_free_i32(last
);
2706 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, tcg_constant_i64(0),
2709 tcg_temp_free_i64(cmp
);
2710 tcg_temp_free_i64(ele
);
2713 /* Compute CLAST for a Vreg. */
2714 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2716 if (sve_access_check(s
)) {
2718 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2719 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2721 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2722 write_fp_dreg(s
, a
->rd
, reg
);
2723 tcg_temp_free_i64(reg
);
2728 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2730 return do_clast_fp(s
, a
, false);
2733 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2735 return do_clast_fp(s
, a
, true);
2738 /* Compute CLAST for a Xreg. */
2739 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2743 if (!sve_access_check(s
)) {
2747 reg
= cpu_reg(s
, a
->rd
);
2750 tcg_gen_ext8u_i64(reg
, reg
);
2753 tcg_gen_ext16u_i64(reg
, reg
);
2756 tcg_gen_ext32u_i64(reg
, reg
);
2761 g_assert_not_reached();
2764 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
2768 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2770 return do_clast_general(s
, a
, false);
2773 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2775 return do_clast_general(s
, a
, true);
2778 /* Compute LAST for a scalar. */
2779 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
2780 int pg
, int rm
, bool before
)
2782 TCGv_i32 last
= tcg_temp_new_i32();
2785 find_last_active(s
, last
, esz
, pg
);
2787 wrap_last_active(s
, last
, esz
);
2789 incr_last_active(s
, last
, esz
);
2792 ret
= load_last_active(s
, last
, rm
, esz
);
2793 tcg_temp_free_i32(last
);
2797 /* Compute LAST for a Vreg. */
2798 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2800 if (sve_access_check(s
)) {
2801 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2802 write_fp_dreg(s
, a
->rd
, val
);
2803 tcg_temp_free_i64(val
);
2808 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2810 return do_last_fp(s
, a
, false);
2813 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2815 return do_last_fp(s
, a
, true);
2818 /* Compute LAST for a Xreg. */
2819 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2821 if (sve_access_check(s
)) {
2822 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2823 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
2824 tcg_temp_free_i64(val
);
2829 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2831 return do_last_general(s
, a
, false);
2834 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2836 return do_last_general(s
, a
, true);
2839 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
2841 if (sve_access_check(s
)) {
2842 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
2847 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
2849 if (sve_access_check(s
)) {
2850 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
2851 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
2852 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
2853 tcg_temp_free_i64(t
);
2858 static gen_helper_gvec_3
* const revb_fns
[4] = {
2859 NULL
, gen_helper_sve_revb_h
,
2860 gen_helper_sve_revb_s
, gen_helper_sve_revb_d
,
2862 TRANS_FEAT(REVB
, aa64_sve
, gen_gvec_ool_arg_zpz
, revb_fns
[a
->esz
], a
, 0)
2864 static gen_helper_gvec_3
* const revh_fns
[4] = {
2865 NULL
, NULL
, gen_helper_sve_revh_s
, gen_helper_sve_revh_d
,
2867 TRANS_FEAT(REVH
, aa64_sve
, gen_gvec_ool_arg_zpz
, revh_fns
[a
->esz
], a
, 0)
2869 TRANS_FEAT(REVW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
2870 a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
, a
, 0)
2872 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
2874 return gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
2875 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
2878 static bool trans_SPLICE_sve2(DisasContext
*s
, arg_rpr_esz
*a
)
2880 if (!dc_isar_feature(aa64_sve2
, s
)) {
2883 return gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
2884 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
);
2888 *** SVE Integer Compare - Vectors Group
2891 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
2892 gen_helper_gvec_flags_4
*gen_fn
)
2894 TCGv_ptr pd
, zn
, zm
, pg
;
2898 if (gen_fn
== NULL
) {
2901 if (!sve_access_check(s
)) {
2905 vsz
= vec_full_reg_size(s
);
2906 t
= tcg_temp_new_i32();
2907 pd
= tcg_temp_new_ptr();
2908 zn
= tcg_temp_new_ptr();
2909 zm
= tcg_temp_new_ptr();
2910 pg
= tcg_temp_new_ptr();
2912 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2913 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2914 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
2915 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2917 gen_fn(t
, pd
, zn
, zm
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, 0)));
2919 tcg_temp_free_ptr(pd
);
2920 tcg_temp_free_ptr(zn
);
2921 tcg_temp_free_ptr(zm
);
2922 tcg_temp_free_ptr(pg
);
2926 tcg_temp_free_i32(t
);
2930 #define DO_PPZZ(NAME, name) \
2931 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
2933 static gen_helper_gvec_flags_4 * const fns[4] = { \
2934 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
2935 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
2937 return do_ppzz_flags(s, a, fns[a->esz]); \
2940 DO_PPZZ(CMPEQ
, cmpeq
)
2941 DO_PPZZ(CMPNE
, cmpne
)
2942 DO_PPZZ(CMPGT
, cmpgt
)
2943 DO_PPZZ(CMPGE
, cmpge
)
2944 DO_PPZZ(CMPHI
, cmphi
)
2945 DO_PPZZ(CMPHS
, cmphs
)
2949 #define DO_PPZW(NAME, name) \
2950 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
2952 static gen_helper_gvec_flags_4 * const fns[4] = { \
2953 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
2954 gen_helper_sve_##name##_ppzw_s, NULL \
2956 return do_ppzz_flags(s, a, fns[a->esz]); \
2959 DO_PPZW(CMPEQ
, cmpeq
)
2960 DO_PPZW(CMPNE
, cmpne
)
2961 DO_PPZW(CMPGT
, cmpgt
)
2962 DO_PPZW(CMPGE
, cmpge
)
2963 DO_PPZW(CMPHI
, cmphi
)
2964 DO_PPZW(CMPHS
, cmphs
)
2965 DO_PPZW(CMPLT
, cmplt
)
2966 DO_PPZW(CMPLE
, cmple
)
2967 DO_PPZW(CMPLO
, cmplo
)
2968 DO_PPZW(CMPLS
, cmpls
)
2973 *** SVE Integer Compare - Immediate Groups
2976 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
2977 gen_helper_gvec_flags_3
*gen_fn
)
2979 TCGv_ptr pd
, zn
, pg
;
2983 if (gen_fn
== NULL
) {
2986 if (!sve_access_check(s
)) {
2990 vsz
= vec_full_reg_size(s
);
2991 t
= tcg_temp_new_i32();
2992 pd
= tcg_temp_new_ptr();
2993 zn
= tcg_temp_new_ptr();
2994 pg
= tcg_temp_new_ptr();
2996 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2997 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2998 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3000 gen_fn(t
, pd
, zn
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, a
->imm
)));
3002 tcg_temp_free_ptr(pd
);
3003 tcg_temp_free_ptr(zn
);
3004 tcg_temp_free_ptr(pg
);
3008 tcg_temp_free_i32(t
);
3012 #define DO_PPZI(NAME, name) \
3013 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
3015 static gen_helper_gvec_flags_3 * const fns[4] = { \
3016 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3017 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3019 return do_ppzi_flags(s, a, fns[a->esz]); \
3022 DO_PPZI(CMPEQ
, cmpeq
)
3023 DO_PPZI(CMPNE
, cmpne
)
3024 DO_PPZI(CMPGT
, cmpgt
)
3025 DO_PPZI(CMPGE
, cmpge
)
3026 DO_PPZI(CMPHI
, cmphi
)
3027 DO_PPZI(CMPHS
, cmphs
)
3028 DO_PPZI(CMPLT
, cmplt
)
3029 DO_PPZI(CMPLE
, cmple
)
3030 DO_PPZI(CMPLO
, cmplo
)
3031 DO_PPZI(CMPLS
, cmpls
)
3036 *** SVE Partition Break Group
3039 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
3040 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
3042 if (!sve_access_check(s
)) {
3046 unsigned vsz
= pred_full_reg_size(s
);
3048 /* Predicate sizes may be smaller and cannot use simd_desc. */
3049 TCGv_ptr d
= tcg_temp_new_ptr();
3050 TCGv_ptr n
= tcg_temp_new_ptr();
3051 TCGv_ptr m
= tcg_temp_new_ptr();
3052 TCGv_ptr g
= tcg_temp_new_ptr();
3053 TCGv_i32 desc
= tcg_constant_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3055 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3056 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3057 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
3058 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3061 TCGv_i32 t
= tcg_temp_new_i32();
3062 fn_s(t
, d
, n
, m
, g
, desc
);
3064 tcg_temp_free_i32(t
);
3066 fn(d
, n
, m
, g
, desc
);
3068 tcg_temp_free_ptr(d
);
3069 tcg_temp_free_ptr(n
);
3070 tcg_temp_free_ptr(m
);
3071 tcg_temp_free_ptr(g
);
3075 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
3076 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
3078 if (!sve_access_check(s
)) {
3082 unsigned vsz
= pred_full_reg_size(s
);
3084 /* Predicate sizes may be smaller and cannot use simd_desc. */
3085 TCGv_ptr d
= tcg_temp_new_ptr();
3086 TCGv_ptr n
= tcg_temp_new_ptr();
3087 TCGv_ptr g
= tcg_temp_new_ptr();
3088 TCGv_i32 desc
= tcg_constant_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3090 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3091 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3092 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3095 TCGv_i32 t
= tcg_temp_new_i32();
3096 fn_s(t
, d
, n
, g
, desc
);
3098 tcg_temp_free_i32(t
);
3102 tcg_temp_free_ptr(d
);
3103 tcg_temp_free_ptr(n
);
3104 tcg_temp_free_ptr(g
);
3108 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
3110 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
3113 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
3115 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
3118 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
3120 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
3123 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
3125 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
3128 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
3130 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
3133 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
3135 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
3138 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
3140 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
3144 *** SVE Predicate Count Group
3147 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
3149 unsigned psz
= pred_full_reg_size(s
);
3154 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
3156 TCGv_i64 g
= tcg_temp_new_i64();
3157 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
3158 tcg_gen_and_i64(val
, val
, g
);
3159 tcg_temp_free_i64(g
);
3162 /* Reduce the pred_esz_masks value simply to reduce the
3163 * size of the code generated here.
3165 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
3166 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
3168 tcg_gen_ctpop_i64(val
, val
);
3170 TCGv_ptr t_pn
= tcg_temp_new_ptr();
3171 TCGv_ptr t_pg
= tcg_temp_new_ptr();
3174 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
3175 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
3177 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
3178 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3180 gen_helper_sve_cntp(val
, t_pn
, t_pg
, tcg_constant_i32(desc
));
3181 tcg_temp_free_ptr(t_pn
);
3182 tcg_temp_free_ptr(t_pg
);
3186 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3188 if (sve_access_check(s
)) {
3189 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3194 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3196 if (sve_access_check(s
)) {
3197 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3198 TCGv_i64 val
= tcg_temp_new_i64();
3200 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3202 tcg_gen_sub_i64(reg
, reg
, val
);
3204 tcg_gen_add_i64(reg
, reg
, val
);
3206 tcg_temp_free_i64(val
);
3211 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3216 if (sve_access_check(s
)) {
3217 unsigned vsz
= vec_full_reg_size(s
);
3218 TCGv_i64 val
= tcg_temp_new_i64();
3219 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3221 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3222 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3223 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3228 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3230 if (sve_access_check(s
)) {
3231 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3232 TCGv_i64 val
= tcg_temp_new_i64();
3234 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3235 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3240 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3242 if (sve_access_check(s
)) {
3243 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3244 TCGv_i64 val
= tcg_temp_new_i64();
3246 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3247 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3252 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3257 if (sve_access_check(s
)) {
3258 TCGv_i64 val
= tcg_temp_new_i64();
3259 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3260 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3266 *** SVE Integer Compare Scalars Group
3269 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3271 if (!sve_access_check(s
)) {
3275 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3276 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3277 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3278 TCGv_i64 cmp
= tcg_temp_new_i64();
3280 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3281 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3282 tcg_temp_free_i64(cmp
);
3284 /* VF = !NF & !CF. */
3285 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3286 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3288 /* Both NF and VF actually look at bit 31. */
3289 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3290 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3294 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3296 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3299 unsigned vsz
= vec_full_reg_size(s
);
3303 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3304 bool eq
= a
->eq
== a
->lt
;
3306 /* The greater-than conditions are all SVE2. */
3307 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3310 if (!sve_access_check(s
)) {
3314 op0
= read_cpu_reg(s
, a
->rn
, 1);
3315 op1
= read_cpu_reg(s
, a
->rm
, 1);
3319 tcg_gen_ext32u_i64(op0
, op0
);
3320 tcg_gen_ext32u_i64(op1
, op1
);
3322 tcg_gen_ext32s_i64(op0
, op0
);
3323 tcg_gen_ext32s_i64(op1
, op1
);
3327 /* For the helper, compress the different conditions into a computation
3328 * of how many iterations for which the condition is true.
3330 t0
= tcg_temp_new_i64();
3331 t1
= tcg_temp_new_i64();
3334 tcg_gen_sub_i64(t0
, op1
, op0
);
3336 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3337 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3339 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3340 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3343 tcg_gen_sub_i64(t0
, op0
, op1
);
3346 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3348 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3349 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3353 tmax
= tcg_constant_i64(vsz
>> a
->esz
);
3355 /* Equality means one more iteration. */
3356 tcg_gen_addi_i64(t0
, t0
, 1);
3359 * For the less-than while, if op1 is maxval (and the only time
3360 * the addition above could overflow), then we produce an all-true
3361 * predicate by setting the count to the vector length. This is
3362 * because the pseudocode is described as an increment + compare
3363 * loop, and the maximum integer would always compare true.
3364 * Similarly, the greater-than while has the same issue with the
3365 * minimum integer due to the decrement + compare loop.
3367 tcg_gen_movi_i64(t1
, maxval
);
3368 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3371 /* Bound to the maximum. */
3372 tcg_gen_umin_i64(t0
, t0
, tmax
);
3374 /* Set the count to zero if the condition is false. */
3375 tcg_gen_movi_i64(t1
, 0);
3376 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3377 tcg_temp_free_i64(t1
);
3379 /* Since we're bounded, pass as a 32-bit type. */
3380 t2
= tcg_temp_new_i32();
3381 tcg_gen_extrl_i64_i32(t2
, t0
);
3382 tcg_temp_free_i64(t0
);
3384 /* Scale elements to bits. */
3385 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3387 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3388 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3390 ptr
= tcg_temp_new_ptr();
3391 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3394 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3396 gen_helper_sve_whileg(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3400 tcg_temp_free_ptr(ptr
);
3401 tcg_temp_free_i32(t2
);
3405 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3407 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3410 unsigned vsz
= vec_full_reg_size(s
);
3413 if (!dc_isar_feature(aa64_sve2
, s
)) {
3416 if (!sve_access_check(s
)) {
3420 op0
= read_cpu_reg(s
, a
->rn
, 1);
3421 op1
= read_cpu_reg(s
, a
->rm
, 1);
3423 tmax
= tcg_constant_i64(vsz
);
3424 diff
= tcg_temp_new_i64();
3428 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3429 t1
= tcg_temp_new_i64();
3430 tcg_gen_sub_i64(diff
, op0
, op1
);
3431 tcg_gen_sub_i64(t1
, op1
, op0
);
3432 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3433 tcg_temp_free_i64(t1
);
3434 /* Round down to a multiple of ESIZE. */
3435 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3436 /* If op1 == op0, diff == 0, and the condition is always true. */
3437 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3440 tcg_gen_sub_i64(diff
, op1
, op0
);
3441 /* Round down to a multiple of ESIZE. */
3442 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3443 /* If op0 >= op1, diff <= 0, the condition is always true. */
3444 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3447 /* Bound to the maximum. */
3448 tcg_gen_umin_i64(diff
, diff
, tmax
);
3450 /* Since we're bounded, pass as a 32-bit type. */
3451 t2
= tcg_temp_new_i32();
3452 tcg_gen_extrl_i64_i32(t2
, diff
);
3453 tcg_temp_free_i64(diff
);
3455 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3456 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3458 ptr
= tcg_temp_new_ptr();
3459 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3461 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3464 tcg_temp_free_ptr(ptr
);
3465 tcg_temp_free_i32(t2
);
3470 *** SVE Integer Wide Immediate - Unpredicated Group
3473 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3478 if (sve_access_check(s
)) {
3479 unsigned vsz
= vec_full_reg_size(s
);
3480 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3483 /* Decode the VFP immediate. */
3484 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3485 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3490 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3492 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3495 if (sve_access_check(s
)) {
3496 unsigned vsz
= vec_full_reg_size(s
);
3497 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3499 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3504 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3506 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3509 if (sve_access_check(s
)) {
3510 unsigned vsz
= vec_full_reg_size(s
);
3511 tcg_gen_gvec_addi(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3512 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3517 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3520 return trans_ADD_zzi(s
, a
);
3523 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3525 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3526 static const GVecGen2s op
[4] = {
3527 { .fni8
= tcg_gen_vec_sub8_i64
,
3528 .fniv
= tcg_gen_sub_vec
,
3529 .fno
= gen_helper_sve_subri_b
,
3530 .opt_opc
= vecop_list
,
3532 .scalar_first
= true },
3533 { .fni8
= tcg_gen_vec_sub16_i64
,
3534 .fniv
= tcg_gen_sub_vec
,
3535 .fno
= gen_helper_sve_subri_h
,
3536 .opt_opc
= vecop_list
,
3538 .scalar_first
= true },
3539 { .fni4
= tcg_gen_sub_i32
,
3540 .fniv
= tcg_gen_sub_vec
,
3541 .fno
= gen_helper_sve_subri_s
,
3542 .opt_opc
= vecop_list
,
3544 .scalar_first
= true },
3545 { .fni8
= tcg_gen_sub_i64
,
3546 .fniv
= tcg_gen_sub_vec
,
3547 .fno
= gen_helper_sve_subri_d
,
3548 .opt_opc
= vecop_list
,
3549 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3551 .scalar_first
= true }
3554 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3557 if (sve_access_check(s
)) {
3558 unsigned vsz
= vec_full_reg_size(s
);
3559 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3560 vec_full_reg_offset(s
, a
->rn
),
3561 vsz
, vsz
, tcg_constant_i64(a
->imm
), &op
[a
->esz
]);
3566 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3568 if (sve_access_check(s
)) {
3569 unsigned vsz
= vec_full_reg_size(s
);
3570 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3571 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3576 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3578 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3581 if (sve_access_check(s
)) {
3582 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
3583 tcg_constant_i64(a
->imm
), u
, d
);
3588 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3590 return do_zzi_sat(s
, a
, false, false);
3593 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3595 return do_zzi_sat(s
, a
, true, false);
3598 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3600 return do_zzi_sat(s
, a
, false, true);
3603 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3605 return do_zzi_sat(s
, a
, true, true);
3608 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3610 if (sve_access_check(s
)) {
3611 unsigned vsz
= vec_full_reg_size(s
);
3612 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3613 vec_full_reg_offset(s
, a
->rn
),
3614 tcg_constant_i64(a
->imm
), vsz
, vsz
, 0, fn
);
3619 #define DO_ZZI(NAME, name) \
3620 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3622 static gen_helper_gvec_2i * const fns[4] = { \
3623 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3624 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3626 return do_zzi_ool(s, a, fns[a->esz]); \
3636 static gen_helper_gvec_4
* const dot_fns
[2][2] = {
3637 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3638 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3640 TRANS_FEAT(DOT_zzzz
, aa64_sve
, gen_gvec_ool_zzzz
,
3641 dot_fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0)
3644 * SVE Multiply - Indexed
3647 TRANS_FEAT(SDOT_zzxw_s
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3648 gen_helper_gvec_sdot_idx_b
, a
)
3649 TRANS_FEAT(SDOT_zzxw_d
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3650 gen_helper_gvec_sdot_idx_h
, a
)
3651 TRANS_FEAT(UDOT_zzxw_s
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3652 gen_helper_gvec_udot_idx_b
, a
)
3653 TRANS_FEAT(UDOT_zzxw_d
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3654 gen_helper_gvec_udot_idx_h
, a
)
3656 TRANS_FEAT(SUDOT_zzxw_s
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzxz
,
3657 gen_helper_gvec_sudot_idx_b
, a
)
3658 TRANS_FEAT(USDOT_zzxw_s
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzxz
,
3659 gen_helper_gvec_usdot_idx_b
, a
)
3661 #define DO_SVE2_RRX(NAME, FUNC) \
3662 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3663 a->rd, a->rn, a->rm, a->index)
3665 DO_SVE2_RRX(MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
3666 DO_SVE2_RRX(MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
3667 DO_SVE2_RRX(MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
3669 DO_SVE2_RRX(SQDMULH_zzx_h
, gen_helper_sve2_sqdmulh_idx_h
)
3670 DO_SVE2_RRX(SQDMULH_zzx_s
, gen_helper_sve2_sqdmulh_idx_s
)
3671 DO_SVE2_RRX(SQDMULH_zzx_d
, gen_helper_sve2_sqdmulh_idx_d
)
3673 DO_SVE2_RRX(SQRDMULH_zzx_h
, gen_helper_sve2_sqrdmulh_idx_h
)
3674 DO_SVE2_RRX(SQRDMULH_zzx_s
, gen_helper_sve2_sqrdmulh_idx_s
)
3675 DO_SVE2_RRX(SQRDMULH_zzx_d
, gen_helper_sve2_sqrdmulh_idx_d
)
3679 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3680 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3681 a->rd, a->rn, a->rm, (a->index << 1) | TOP)
3683 DO_SVE2_RRX_TB(SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
3684 DO_SVE2_RRX_TB(SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
3685 DO_SVE2_RRX_TB(SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
3686 DO_SVE2_RRX_TB(SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
3688 DO_SVE2_RRX_TB(SMULLB_zzx_s
, gen_helper_sve2_smull_idx_s
, false)
3689 DO_SVE2_RRX_TB(SMULLB_zzx_d
, gen_helper_sve2_smull_idx_d
, false)
3690 DO_SVE2_RRX_TB(SMULLT_zzx_s
, gen_helper_sve2_smull_idx_s
, true)
3691 DO_SVE2_RRX_TB(SMULLT_zzx_d
, gen_helper_sve2_smull_idx_d
, true)
3693 DO_SVE2_RRX_TB(UMULLB_zzx_s
, gen_helper_sve2_umull_idx_s
, false)
3694 DO_SVE2_RRX_TB(UMULLB_zzx_d
, gen_helper_sve2_umull_idx_d
, false)
3695 DO_SVE2_RRX_TB(UMULLT_zzx_s
, gen_helper_sve2_umull_idx_s
, true)
3696 DO_SVE2_RRX_TB(UMULLT_zzx_d
, gen_helper_sve2_umull_idx_d
, true)
3698 #undef DO_SVE2_RRX_TB
3700 #define DO_SVE2_RRXR(NAME, FUNC) \
3701 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a)
3703 DO_SVE2_RRXR(MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
3704 DO_SVE2_RRXR(MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
3705 DO_SVE2_RRXR(MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
3707 DO_SVE2_RRXR(MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
3708 DO_SVE2_RRXR(MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
3709 DO_SVE2_RRXR(MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
3711 DO_SVE2_RRXR(SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
3712 DO_SVE2_RRXR(SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
3713 DO_SVE2_RRXR(SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
3715 DO_SVE2_RRXR(SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
3716 DO_SVE2_RRXR(SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
3717 DO_SVE2_RRXR(SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
3721 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3722 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3723 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP)
3725 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
3726 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
3727 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
3728 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
3730 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
3731 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
3732 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
3733 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
3735 DO_SVE2_RRXR_TB(SMLALB_zzxw_s
, gen_helper_sve2_smlal_idx_s
, false)
3736 DO_SVE2_RRXR_TB(SMLALB_zzxw_d
, gen_helper_sve2_smlal_idx_d
, false)
3737 DO_SVE2_RRXR_TB(SMLALT_zzxw_s
, gen_helper_sve2_smlal_idx_s
, true)
3738 DO_SVE2_RRXR_TB(SMLALT_zzxw_d
, gen_helper_sve2_smlal_idx_d
, true)
3740 DO_SVE2_RRXR_TB(UMLALB_zzxw_s
, gen_helper_sve2_umlal_idx_s
, false)
3741 DO_SVE2_RRXR_TB(UMLALB_zzxw_d
, gen_helper_sve2_umlal_idx_d
, false)
3742 DO_SVE2_RRXR_TB(UMLALT_zzxw_s
, gen_helper_sve2_umlal_idx_s
, true)
3743 DO_SVE2_RRXR_TB(UMLALT_zzxw_d
, gen_helper_sve2_umlal_idx_d
, true)
3745 DO_SVE2_RRXR_TB(SMLSLB_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, false)
3746 DO_SVE2_RRXR_TB(SMLSLB_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, false)
3747 DO_SVE2_RRXR_TB(SMLSLT_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, true)
3748 DO_SVE2_RRXR_TB(SMLSLT_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, true)
3750 DO_SVE2_RRXR_TB(UMLSLB_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, false)
3751 DO_SVE2_RRXR_TB(UMLSLB_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, false)
3752 DO_SVE2_RRXR_TB(UMLSLT_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, true)
3753 DO_SVE2_RRXR_TB(UMLSLT_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, true)
3755 #undef DO_SVE2_RRXR_TB
3757 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
3758 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3759 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot)
3761 DO_SVE2_RRXR_ROT(CMLA_zzxz_h
, gen_helper_sve2_cmla_idx_h
)
3762 DO_SVE2_RRXR_ROT(CMLA_zzxz_s
, gen_helper_sve2_cmla_idx_s
)
3764 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h
, gen_helper_sve2_sqrdcmlah_idx_h
)
3765 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s
, gen_helper_sve2_sqrdcmlah_idx_s
)
3767 DO_SVE2_RRXR_ROT(CDOT_zzxw_s
, gen_helper_sve2_cdot_idx_s
)
3768 DO_SVE2_RRXR_ROT(CDOT_zzxw_d
, gen_helper_sve2_cdot_idx_d
)
3770 #undef DO_SVE2_RRXR_ROT
3773 *** SVE Floating Point Multiply-Add Indexed Group
3776 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
3778 static gen_helper_gvec_4_ptr
* const fns
[3] = {
3779 gen_helper_gvec_fmla_idx_h
,
3780 gen_helper_gvec_fmla_idx_s
,
3781 gen_helper_gvec_fmla_idx_d
,
3784 if (sve_access_check(s
)) {
3785 unsigned vsz
= vec_full_reg_size(s
);
3786 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3787 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3788 vec_full_reg_offset(s
, a
->rn
),
3789 vec_full_reg_offset(s
, a
->rm
),
3790 vec_full_reg_offset(s
, a
->ra
),
3791 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
3793 tcg_temp_free_ptr(status
);
3798 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3800 return do_FMLA_zzxz(s
, a
, false);
3803 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3805 return do_FMLA_zzxz(s
, a
, true);
3809 *** SVE Floating Point Multiply Indexed Group
3812 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
3814 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3815 gen_helper_gvec_fmul_idx_h
,
3816 gen_helper_gvec_fmul_idx_s
,
3817 gen_helper_gvec_fmul_idx_d
,
3820 if (sve_access_check(s
)) {
3821 unsigned vsz
= vec_full_reg_size(s
);
3822 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3823 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3824 vec_full_reg_offset(s
, a
->rn
),
3825 vec_full_reg_offset(s
, a
->rm
),
3826 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
3827 tcg_temp_free_ptr(status
);
3833 *** SVE Floating Point Fast Reduction Group
3836 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
3837 TCGv_ptr
, TCGv_i32
);
3839 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
3840 gen_helper_fp_reduce
*fn
)
3842 unsigned vsz
= vec_full_reg_size(s
);
3843 unsigned p2vsz
= pow2ceil(vsz
);
3844 TCGv_i32 t_desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, p2vsz
));
3845 TCGv_ptr t_zn
, t_pg
, status
;
3848 temp
= tcg_temp_new_i64();
3849 t_zn
= tcg_temp_new_ptr();
3850 t_pg
= tcg_temp_new_ptr();
3852 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3853 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3854 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3856 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
3857 tcg_temp_free_ptr(t_zn
);
3858 tcg_temp_free_ptr(t_pg
);
3859 tcg_temp_free_ptr(status
);
3861 write_fp_dreg(s
, a
->rd
, temp
);
3862 tcg_temp_free_i64(temp
);
3865 #define DO_VPZ(NAME, name) \
3866 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
3868 static gen_helper_fp_reduce * const fns[3] = { \
3869 gen_helper_sve_##name##_h, \
3870 gen_helper_sve_##name##_s, \
3871 gen_helper_sve_##name##_d, \
3873 if (a->esz == 0) { \
3876 if (sve_access_check(s)) { \
3877 do_reduce(s, a, fns[a->esz - 1]); \
3882 DO_VPZ(FADDV
, faddv
)
3883 DO_VPZ(FMINNMV
, fminnmv
)
3884 DO_VPZ(FMAXNMV
, fmaxnmv
)
3885 DO_VPZ(FMINV
, fminv
)
3886 DO_VPZ(FMAXV
, fmaxv
)
3889 *** SVE Floating Point Unary Operations - Unpredicated Group
3892 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
3894 unsigned vsz
= vec_full_reg_size(s
);
3895 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3897 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
3898 vec_full_reg_offset(s
, a
->rn
),
3899 status
, vsz
, vsz
, 0, fn
);
3900 tcg_temp_free_ptr(status
);
3903 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
3905 static gen_helper_gvec_2_ptr
* const fns
[3] = {
3906 gen_helper_gvec_frecpe_h
,
3907 gen_helper_gvec_frecpe_s
,
3908 gen_helper_gvec_frecpe_d
,
3913 if (sve_access_check(s
)) {
3914 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
3919 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
3921 static gen_helper_gvec_2_ptr
* const fns
[3] = {
3922 gen_helper_gvec_frsqrte_h
,
3923 gen_helper_gvec_frsqrte_s
,
3924 gen_helper_gvec_frsqrte_d
,
3929 if (sve_access_check(s
)) {
3930 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
3936 *** SVE Floating Point Compare with Zero Group
3939 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
3940 gen_helper_gvec_3_ptr
*fn
)
3942 unsigned vsz
= vec_full_reg_size(s
);
3943 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3945 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
3946 vec_full_reg_offset(s
, a
->rn
),
3947 pred_full_reg_offset(s
, a
->pg
),
3948 status
, vsz
, vsz
, 0, fn
);
3949 tcg_temp_free_ptr(status
);
3952 #define DO_PPZ(NAME, name) \
3953 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
3955 static gen_helper_gvec_3_ptr * const fns[3] = { \
3956 gen_helper_sve_##name##_h, \
3957 gen_helper_sve_##name##_s, \
3958 gen_helper_sve_##name##_d, \
3960 if (a->esz == 0) { \
3963 if (sve_access_check(s)) { \
3964 do_ppz_fp(s, a, fns[a->esz - 1]); \
3969 DO_PPZ(FCMGE_ppz0
, fcmge0
)
3970 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
3971 DO_PPZ(FCMLE_ppz0
, fcmle0
)
3972 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
3973 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
3974 DO_PPZ(FCMNE_ppz0
, fcmne0
)
3979 *** SVE floating-point trig multiply-add coefficient
3982 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
3984 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3985 gen_helper_sve_ftmad_h
,
3986 gen_helper_sve_ftmad_s
,
3987 gen_helper_sve_ftmad_d
,
3993 if (sve_access_check(s
)) {
3994 unsigned vsz
= vec_full_reg_size(s
);
3995 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3996 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3997 vec_full_reg_offset(s
, a
->rn
),
3998 vec_full_reg_offset(s
, a
->rm
),
3999 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
4000 tcg_temp_free_ptr(status
);
4006 *** SVE Floating Point Accumulating Reduction Group
4009 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
4011 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
4012 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
4013 static fadda_fn
* const fns
[3] = {
4014 gen_helper_sve_fadda_h
,
4015 gen_helper_sve_fadda_s
,
4016 gen_helper_sve_fadda_d
,
4018 unsigned vsz
= vec_full_reg_size(s
);
4019 TCGv_ptr t_rm
, t_pg
, t_fpst
;
4026 if (!sve_access_check(s
)) {
4030 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
4031 t_rm
= tcg_temp_new_ptr();
4032 t_pg
= tcg_temp_new_ptr();
4033 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
4034 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4035 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4036 t_desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
4038 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
4040 tcg_temp_free_ptr(t_fpst
);
4041 tcg_temp_free_ptr(t_pg
);
4042 tcg_temp_free_ptr(t_rm
);
4044 write_fp_dreg(s
, a
->rd
, t_val
);
4045 tcg_temp_free_i64(t_val
);
4050 *** SVE Floating Point Arithmetic - Unpredicated Group
4053 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
4054 gen_helper_gvec_3_ptr
*fn
)
4059 if (sve_access_check(s
)) {
4060 unsigned vsz
= vec_full_reg_size(s
);
4061 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4062 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4063 vec_full_reg_offset(s
, a
->rn
),
4064 vec_full_reg_offset(s
, a
->rm
),
4065 status
, vsz
, vsz
, 0, fn
);
4066 tcg_temp_free_ptr(status
);
4072 #define DO_FP3(NAME, name) \
4073 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
4075 static gen_helper_gvec_3_ptr * const fns[4] = { \
4076 NULL, gen_helper_gvec_##name##_h, \
4077 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4079 return do_zzz_fp(s, a, fns[a->esz]); \
4082 DO_FP3(FADD_zzz
, fadd
)
4083 DO_FP3(FSUB_zzz
, fsub
)
4084 DO_FP3(FMUL_zzz
, fmul
)
4085 DO_FP3(FTSMUL
, ftsmul
)
4086 DO_FP3(FRECPS
, recps
)
4087 DO_FP3(FRSQRTS
, rsqrts
)
4092 *** SVE Floating Point Arithmetic - Predicated Group
4095 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
4096 gen_helper_gvec_4_ptr
*fn
)
4101 if (sve_access_check(s
)) {
4102 unsigned vsz
= vec_full_reg_size(s
);
4103 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4104 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4105 vec_full_reg_offset(s
, a
->rn
),
4106 vec_full_reg_offset(s
, a
->rm
),
4107 pred_full_reg_offset(s
, a
->pg
),
4108 status
, vsz
, vsz
, 0, fn
);
4109 tcg_temp_free_ptr(status
);
4114 #define DO_FP3(NAME, name) \
4115 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
4117 static gen_helper_gvec_4_ptr * const fns[4] = { \
4118 NULL, gen_helper_sve_##name##_h, \
4119 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4121 return do_zpzz_fp(s, a, fns[a->esz]); \
4124 DO_FP3(FADD_zpzz
, fadd
)
4125 DO_FP3(FSUB_zpzz
, fsub
)
4126 DO_FP3(FMUL_zpzz
, fmul
)
4127 DO_FP3(FMIN_zpzz
, fmin
)
4128 DO_FP3(FMAX_zpzz
, fmax
)
4129 DO_FP3(FMINNM_zpzz
, fminnum
)
4130 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
4132 DO_FP3(FSCALE
, fscalbn
)
4134 DO_FP3(FMULX
, fmulx
)
4138 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
4139 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
4141 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
4142 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
4144 unsigned vsz
= vec_full_reg_size(s
);
4145 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
4148 t_zd
= tcg_temp_new_ptr();
4149 t_zn
= tcg_temp_new_ptr();
4150 t_pg
= tcg_temp_new_ptr();
4151 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
4152 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
4153 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4155 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4156 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
4157 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
4159 tcg_temp_free_ptr(status
);
4160 tcg_temp_free_ptr(t_pg
);
4161 tcg_temp_free_ptr(t_zn
);
4162 tcg_temp_free_ptr(t_zd
);
4165 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
4166 gen_helper_sve_fp2scalar
*fn
)
4168 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4169 tcg_constant_i64(imm
), fn
);
4172 #define DO_FP_IMM(NAME, name, const0, const1) \
4173 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
4175 static gen_helper_sve_fp2scalar * const fns[3] = { \
4176 gen_helper_sve_##name##_h, \
4177 gen_helper_sve_##name##_s, \
4178 gen_helper_sve_##name##_d \
4180 static uint64_t const val[3][2] = { \
4181 { float16_##const0, float16_##const1 }, \
4182 { float32_##const0, float32_##const1 }, \
4183 { float64_##const0, float64_##const1 }, \
4185 if (a->esz == 0) { \
4188 if (sve_access_check(s)) { \
4189 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4194 DO_FP_IMM(FADD
, fadds
, half
, one
)
4195 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
4196 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
4197 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
4198 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
4199 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
4200 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
4201 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
4205 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
4206 gen_helper_gvec_4_ptr
*fn
)
4211 if (sve_access_check(s
)) {
4212 unsigned vsz
= vec_full_reg_size(s
);
4213 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4214 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
4215 vec_full_reg_offset(s
, a
->rn
),
4216 vec_full_reg_offset(s
, a
->rm
),
4217 pred_full_reg_offset(s
, a
->pg
),
4218 status
, vsz
, vsz
, 0, fn
);
4219 tcg_temp_free_ptr(status
);
4224 #define DO_FPCMP(NAME, name) \
4225 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
4227 static gen_helper_gvec_4_ptr * const fns[4] = { \
4228 NULL, gen_helper_sve_##name##_h, \
4229 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4231 return do_fp_cmp(s, a, fns[a->esz]); \
4234 DO_FPCMP(FCMGE
, fcmge
)
4235 DO_FPCMP(FCMGT
, fcmgt
)
4236 DO_FPCMP(FCMEQ
, fcmeq
)
4237 DO_FPCMP(FCMNE
, fcmne
)
4238 DO_FPCMP(FCMUO
, fcmuo
)
4239 DO_FPCMP(FACGE
, facge
)
4240 DO_FPCMP(FACGT
, facgt
)
4244 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
4246 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4247 gen_helper_sve_fcadd_h
,
4248 gen_helper_sve_fcadd_s
,
4249 gen_helper_sve_fcadd_d
4255 if (sve_access_check(s
)) {
4256 unsigned vsz
= vec_full_reg_size(s
);
4257 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4258 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4259 vec_full_reg_offset(s
, a
->rn
),
4260 vec_full_reg_offset(s
, a
->rm
),
4261 pred_full_reg_offset(s
, a
->pg
),
4262 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4263 tcg_temp_free_ptr(status
);
4268 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4269 gen_helper_gvec_5_ptr
*fn
)
4274 if (sve_access_check(s
)) {
4275 unsigned vsz
= vec_full_reg_size(s
);
4276 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4277 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4278 vec_full_reg_offset(s
, a
->rn
),
4279 vec_full_reg_offset(s
, a
->rm
),
4280 vec_full_reg_offset(s
, a
->ra
),
4281 pred_full_reg_offset(s
, a
->pg
),
4282 status
, vsz
, vsz
, 0, fn
);
4283 tcg_temp_free_ptr(status
);
4288 #define DO_FMLA(NAME, name) \
4289 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4291 static gen_helper_gvec_5_ptr * const fns[4] = { \
4292 NULL, gen_helper_sve_##name##_h, \
4293 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4295 return do_fmla(s, a, fns[a->esz]); \
4298 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4299 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4300 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4301 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4305 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4307 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4309 gen_helper_sve_fcmla_zpzzz_h
,
4310 gen_helper_sve_fcmla_zpzzz_s
,
4311 gen_helper_sve_fcmla_zpzzz_d
,
4317 if (sve_access_check(s
)) {
4318 unsigned vsz
= vec_full_reg_size(s
);
4319 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4320 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4321 vec_full_reg_offset(s
, a
->rn
),
4322 vec_full_reg_offset(s
, a
->rm
),
4323 vec_full_reg_offset(s
, a
->ra
),
4324 pred_full_reg_offset(s
, a
->pg
),
4325 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4326 tcg_temp_free_ptr(status
);
4331 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4333 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4334 gen_helper_gvec_fcmlah_idx
,
4335 gen_helper_gvec_fcmlas_idx
,
4338 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4339 tcg_debug_assert(a
->rd
== a
->ra
);
4340 if (sve_access_check(s
)) {
4341 unsigned vsz
= vec_full_reg_size(s
);
4342 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4343 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4344 vec_full_reg_offset(s
, a
->rn
),
4345 vec_full_reg_offset(s
, a
->rm
),
4346 vec_full_reg_offset(s
, a
->ra
),
4348 a
->index
* 4 + a
->rot
,
4350 tcg_temp_free_ptr(status
);
4356 *** SVE Floating Point Unary Operations Predicated Group
4359 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4360 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4362 if (sve_access_check(s
)) {
4363 unsigned vsz
= vec_full_reg_size(s
);
4364 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4365 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4366 vec_full_reg_offset(s
, rn
),
4367 pred_full_reg_offset(s
, pg
),
4368 status
, vsz
, vsz
, 0, fn
);
4369 tcg_temp_free_ptr(status
);
4374 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4376 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4379 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4381 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4384 static bool trans_BFCVT(DisasContext
*s
, arg_rpr_esz
*a
)
4386 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
4389 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvt
);
4392 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4394 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4397 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4399 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4402 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4404 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4407 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4409 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4412 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4414 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4417 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4419 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4422 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4424 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4427 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4429 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4432 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4434 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4437 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4439 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4442 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4444 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4447 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4449 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4452 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4454 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4457 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4459 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4462 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4464 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4467 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4469 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4472 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4474 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4477 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4479 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4482 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4483 gen_helper_sve_frint_h
,
4484 gen_helper_sve_frint_s
,
4485 gen_helper_sve_frint_d
4488 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4493 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4494 frint_fns
[a
->esz
- 1]);
4497 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4499 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4500 gen_helper_sve_frintx_h
,
4501 gen_helper_sve_frintx_s
,
4502 gen_helper_sve_frintx_d
4507 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4510 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
,
4511 int mode
, gen_helper_gvec_3_ptr
*fn
)
4513 if (sve_access_check(s
)) {
4514 unsigned vsz
= vec_full_reg_size(s
);
4515 TCGv_i32 tmode
= tcg_const_i32(mode
);
4516 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4518 gen_helper_set_rmode(tmode
, tmode
, status
);
4520 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4521 vec_full_reg_offset(s
, a
->rn
),
4522 pred_full_reg_offset(s
, a
->pg
),
4523 status
, vsz
, vsz
, 0, fn
);
4525 gen_helper_set_rmode(tmode
, tmode
, status
);
4526 tcg_temp_free_i32(tmode
);
4527 tcg_temp_free_ptr(status
);
4532 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4537 return do_frint_mode(s
, a
, float_round_nearest_even
, frint_fns
[a
->esz
- 1]);
4540 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4545 return do_frint_mode(s
, a
, float_round_up
, frint_fns
[a
->esz
- 1]);
4548 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4553 return do_frint_mode(s
, a
, float_round_down
, frint_fns
[a
->esz
- 1]);
4556 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4561 return do_frint_mode(s
, a
, float_round_to_zero
, frint_fns
[a
->esz
- 1]);
4564 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4569 return do_frint_mode(s
, a
, float_round_ties_away
, frint_fns
[a
->esz
- 1]);
4572 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4574 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4575 gen_helper_sve_frecpx_h
,
4576 gen_helper_sve_frecpx_s
,
4577 gen_helper_sve_frecpx_d
4582 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4585 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4587 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4588 gen_helper_sve_fsqrt_h
,
4589 gen_helper_sve_fsqrt_s
,
4590 gen_helper_sve_fsqrt_d
4595 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4598 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4600 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4603 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4605 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4608 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4610 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4613 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4615 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4618 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4620 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4623 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4625 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4628 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4630 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4633 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4635 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4638 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4640 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4643 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4645 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4648 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4650 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
4653 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4655 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
4658 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4660 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
4663 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4665 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
4669 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4672 /* Subroutine loading a vector register at VOFS of LEN bytes.
4673 * The load should begin at the address Rn + IMM.
4676 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4678 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4679 int len_remain
= len
% 8;
4680 int nparts
= len
/ 8 + ctpop8(len_remain
);
4681 int midx
= get_mem_index(s
);
4682 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
4684 dirty_addr
= tcg_temp_new_i64();
4685 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4686 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4687 tcg_temp_free_i64(dirty_addr
);
4690 * Note that unpredicated load/store of vector/predicate registers
4691 * are defined as a stream of bytes, which equates to little-endian
4692 * operations on larger quantities.
4693 * Attempt to keep code expansion to a minimum by limiting the
4694 * amount of unrolling done.
4699 t0
= tcg_temp_new_i64();
4700 for (i
= 0; i
< len_align
; i
+= 8) {
4701 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4702 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
4703 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4705 tcg_temp_free_i64(t0
);
4707 TCGLabel
*loop
= gen_new_label();
4708 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4710 /* Copy the clean address into a local temp, live across the loop. */
4712 clean_addr
= new_tmp_a64_local(s
);
4713 tcg_gen_mov_i64(clean_addr
, t0
);
4715 gen_set_label(loop
);
4717 t0
= tcg_temp_new_i64();
4718 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4719 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4721 tp
= tcg_temp_new_ptr();
4722 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4723 tcg_gen_addi_ptr(i
, i
, 8);
4724 tcg_gen_st_i64(t0
, tp
, vofs
);
4725 tcg_temp_free_ptr(tp
);
4726 tcg_temp_free_i64(t0
);
4728 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4729 tcg_temp_free_ptr(i
);
4733 * Predicate register loads can be any multiple of 2.
4734 * Note that we still store the entire 64-bit unit into cpu_env.
4737 t0
= tcg_temp_new_i64();
4738 switch (len_remain
) {
4742 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
4743 MO_LE
| ctz32(len_remain
));
4747 t1
= tcg_temp_new_i64();
4748 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4749 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4750 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
4751 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
4752 tcg_temp_free_i64(t1
);
4756 g_assert_not_reached();
4758 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
4759 tcg_temp_free_i64(t0
);
4763 /* Similarly for stores. */
4764 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4766 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4767 int len_remain
= len
% 8;
4768 int nparts
= len
/ 8 + ctpop8(len_remain
);
4769 int midx
= get_mem_index(s
);
4770 TCGv_i64 dirty_addr
, clean_addr
, t0
;
4772 dirty_addr
= tcg_temp_new_i64();
4773 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4774 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4775 tcg_temp_free_i64(dirty_addr
);
4777 /* Note that unpredicated load/store of vector/predicate registers
4778 * are defined as a stream of bytes, which equates to little-endian
4779 * operations on larger quantities. There is no nice way to force
4780 * a little-endian store for aarch64_be-linux-user out of line.
4782 * Attempt to keep code expansion to a minimum by limiting the
4783 * amount of unrolling done.
4788 t0
= tcg_temp_new_i64();
4789 for (i
= 0; i
< len_align
; i
+= 8) {
4790 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
4791 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4792 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4794 tcg_temp_free_i64(t0
);
4796 TCGLabel
*loop
= gen_new_label();
4797 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4799 /* Copy the clean address into a local temp, live across the loop. */
4801 clean_addr
= new_tmp_a64_local(s
);
4802 tcg_gen_mov_i64(clean_addr
, t0
);
4804 gen_set_label(loop
);
4806 t0
= tcg_temp_new_i64();
4807 tp
= tcg_temp_new_ptr();
4808 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4809 tcg_gen_ld_i64(t0
, tp
, vofs
);
4810 tcg_gen_addi_ptr(i
, i
, 8);
4811 tcg_temp_free_ptr(tp
);
4813 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4814 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4815 tcg_temp_free_i64(t0
);
4817 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4818 tcg_temp_free_ptr(i
);
4821 /* Predicate register stores can be any multiple of 2. */
4823 t0
= tcg_temp_new_i64();
4824 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
4826 switch (len_remain
) {
4830 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
4831 MO_LE
| ctz32(len_remain
));
4835 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4836 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4837 tcg_gen_shri_i64(t0
, t0
, 32);
4838 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
4842 g_assert_not_reached();
4844 tcg_temp_free_i64(t0
);
4848 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
4850 if (sve_access_check(s
)) {
4851 int size
= vec_full_reg_size(s
);
4852 int off
= vec_full_reg_offset(s
, a
->rd
);
4853 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
4858 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
4860 if (sve_access_check(s
)) {
4861 int size
= pred_full_reg_size(s
);
4862 int off
= pred_full_reg_offset(s
, a
->rd
);
4863 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
4868 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
4870 if (sve_access_check(s
)) {
4871 int size
= vec_full_reg_size(s
);
4872 int off
= vec_full_reg_offset(s
, a
->rd
);
4873 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
4878 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
4880 if (sve_access_check(s
)) {
4881 int size
= pred_full_reg_size(s
);
4882 int off
= pred_full_reg_offset(s
, a
->rd
);
4883 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
4889 *** SVE Memory - Contiguous Load Group
4892 /* The memory mode of the dtype. */
4893 static const MemOp dtype_mop
[16] = {
4894 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
4895 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
4896 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
4897 MO_SB
, MO_SB
, MO_SB
, MO_UQ
4900 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
4902 /* The vector element size of dtype. */
4903 static const uint8_t dtype_esz
[16] = {
4910 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
4911 int dtype
, uint32_t mte_n
, bool is_write
,
4912 gen_helper_gvec_mem
*fn
)
4914 unsigned vsz
= vec_full_reg_size(s
);
4919 * For e.g. LD4, there are not enough arguments to pass all 4
4920 * registers as pointers, so encode the regno into the data field.
4921 * For consistency, do this even for LD1.
4923 if (s
->mte_active
[0]) {
4924 int msz
= dtype_msz(dtype
);
4926 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
4927 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
4928 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
4929 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
4930 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
4931 desc
<<= SVE_MTEDESC_SHIFT
;
4933 addr
= clean_data_tbi(s
, addr
);
4936 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
4937 t_pg
= tcg_temp_new_ptr();
4939 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4940 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(desc
));
4942 tcg_temp_free_ptr(t_pg
);
4945 /* Indexed by [mte][be][dtype][nreg] */
4946 static gen_helper_gvec_mem
* const ldr_fns
[2][2][16][4] = {
4947 { /* mte inactive, little-endian */
4948 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
4949 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
4950 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
4951 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
4952 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
4954 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
4955 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
4956 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
4957 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
4958 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
4960 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
4961 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
4962 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
4963 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
4964 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
4966 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
4967 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
4968 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
4969 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
4970 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
4972 /* mte inactive, big-endian */
4973 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
4974 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
4975 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
4976 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
4977 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
4979 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
4980 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
4981 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
4982 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
4983 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
4985 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
4986 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
4987 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
4988 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
4989 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
4991 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
4992 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
4993 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
4994 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
4995 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
4997 { /* mte active, little-endian */
4998 { { gen_helper_sve_ld1bb_r_mte
,
4999 gen_helper_sve_ld2bb_r_mte
,
5000 gen_helper_sve_ld3bb_r_mte
,
5001 gen_helper_sve_ld4bb_r_mte
},
5002 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5003 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5004 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5006 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
5007 { gen_helper_sve_ld1hh_le_r_mte
,
5008 gen_helper_sve_ld2hh_le_r_mte
,
5009 gen_helper_sve_ld3hh_le_r_mte
,
5010 gen_helper_sve_ld4hh_le_r_mte
},
5011 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
5012 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
5014 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
5015 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
5016 { gen_helper_sve_ld1ss_le_r_mte
,
5017 gen_helper_sve_ld2ss_le_r_mte
,
5018 gen_helper_sve_ld3ss_le_r_mte
,
5019 gen_helper_sve_ld4ss_le_r_mte
},
5020 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
5022 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5023 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5024 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5025 { gen_helper_sve_ld1dd_le_r_mte
,
5026 gen_helper_sve_ld2dd_le_r_mte
,
5027 gen_helper_sve_ld3dd_le_r_mte
,
5028 gen_helper_sve_ld4dd_le_r_mte
} },
5030 /* mte active, big-endian */
5031 { { gen_helper_sve_ld1bb_r_mte
,
5032 gen_helper_sve_ld2bb_r_mte
,
5033 gen_helper_sve_ld3bb_r_mte
,
5034 gen_helper_sve_ld4bb_r_mte
},
5035 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5036 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5037 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5039 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
5040 { gen_helper_sve_ld1hh_be_r_mte
,
5041 gen_helper_sve_ld2hh_be_r_mte
,
5042 gen_helper_sve_ld3hh_be_r_mte
,
5043 gen_helper_sve_ld4hh_be_r_mte
},
5044 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
5045 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
5047 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
5048 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
5049 { gen_helper_sve_ld1ss_be_r_mte
,
5050 gen_helper_sve_ld2ss_be_r_mte
,
5051 gen_helper_sve_ld3ss_be_r_mte
,
5052 gen_helper_sve_ld4ss_be_r_mte
},
5053 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
5055 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5056 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5057 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5058 { gen_helper_sve_ld1dd_be_r_mte
,
5059 gen_helper_sve_ld2dd_be_r_mte
,
5060 gen_helper_sve_ld3dd_be_r_mte
,
5061 gen_helper_sve_ld4dd_be_r_mte
} } },
5064 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
5065 TCGv_i64 addr
, int dtype
, int nreg
)
5067 gen_helper_gvec_mem
*fn
5068 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
5071 * While there are holes in the table, they are not
5072 * accessible via the instruction encoding.
5075 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
5078 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5083 if (sve_access_check(s
)) {
5084 TCGv_i64 addr
= new_tmp_a64(s
);
5085 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5086 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5087 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5092 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5094 if (sve_access_check(s
)) {
5095 int vsz
= vec_full_reg_size(s
);
5096 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5097 TCGv_i64 addr
= new_tmp_a64(s
);
5099 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5100 (a
->imm
* elements
* (a
->nreg
+ 1))
5101 << dtype_msz(a
->dtype
));
5102 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5107 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5109 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5110 { /* mte inactive, little-endian */
5111 { gen_helper_sve_ldff1bb_r
,
5112 gen_helper_sve_ldff1bhu_r
,
5113 gen_helper_sve_ldff1bsu_r
,
5114 gen_helper_sve_ldff1bdu_r
,
5116 gen_helper_sve_ldff1sds_le_r
,
5117 gen_helper_sve_ldff1hh_le_r
,
5118 gen_helper_sve_ldff1hsu_le_r
,
5119 gen_helper_sve_ldff1hdu_le_r
,
5121 gen_helper_sve_ldff1hds_le_r
,
5122 gen_helper_sve_ldff1hss_le_r
,
5123 gen_helper_sve_ldff1ss_le_r
,
5124 gen_helper_sve_ldff1sdu_le_r
,
5126 gen_helper_sve_ldff1bds_r
,
5127 gen_helper_sve_ldff1bss_r
,
5128 gen_helper_sve_ldff1bhs_r
,
5129 gen_helper_sve_ldff1dd_le_r
},
5131 /* mte inactive, big-endian */
5132 { gen_helper_sve_ldff1bb_r
,
5133 gen_helper_sve_ldff1bhu_r
,
5134 gen_helper_sve_ldff1bsu_r
,
5135 gen_helper_sve_ldff1bdu_r
,
5137 gen_helper_sve_ldff1sds_be_r
,
5138 gen_helper_sve_ldff1hh_be_r
,
5139 gen_helper_sve_ldff1hsu_be_r
,
5140 gen_helper_sve_ldff1hdu_be_r
,
5142 gen_helper_sve_ldff1hds_be_r
,
5143 gen_helper_sve_ldff1hss_be_r
,
5144 gen_helper_sve_ldff1ss_be_r
,
5145 gen_helper_sve_ldff1sdu_be_r
,
5147 gen_helper_sve_ldff1bds_r
,
5148 gen_helper_sve_ldff1bss_r
,
5149 gen_helper_sve_ldff1bhs_r
,
5150 gen_helper_sve_ldff1dd_be_r
} },
5152 { /* mte active, little-endian */
5153 { gen_helper_sve_ldff1bb_r_mte
,
5154 gen_helper_sve_ldff1bhu_r_mte
,
5155 gen_helper_sve_ldff1bsu_r_mte
,
5156 gen_helper_sve_ldff1bdu_r_mte
,
5158 gen_helper_sve_ldff1sds_le_r_mte
,
5159 gen_helper_sve_ldff1hh_le_r_mte
,
5160 gen_helper_sve_ldff1hsu_le_r_mte
,
5161 gen_helper_sve_ldff1hdu_le_r_mte
,
5163 gen_helper_sve_ldff1hds_le_r_mte
,
5164 gen_helper_sve_ldff1hss_le_r_mte
,
5165 gen_helper_sve_ldff1ss_le_r_mte
,
5166 gen_helper_sve_ldff1sdu_le_r_mte
,
5168 gen_helper_sve_ldff1bds_r_mte
,
5169 gen_helper_sve_ldff1bss_r_mte
,
5170 gen_helper_sve_ldff1bhs_r_mte
,
5171 gen_helper_sve_ldff1dd_le_r_mte
},
5173 /* mte active, big-endian */
5174 { gen_helper_sve_ldff1bb_r_mte
,
5175 gen_helper_sve_ldff1bhu_r_mte
,
5176 gen_helper_sve_ldff1bsu_r_mte
,
5177 gen_helper_sve_ldff1bdu_r_mte
,
5179 gen_helper_sve_ldff1sds_be_r_mte
,
5180 gen_helper_sve_ldff1hh_be_r_mte
,
5181 gen_helper_sve_ldff1hsu_be_r_mte
,
5182 gen_helper_sve_ldff1hdu_be_r_mte
,
5184 gen_helper_sve_ldff1hds_be_r_mte
,
5185 gen_helper_sve_ldff1hss_be_r_mte
,
5186 gen_helper_sve_ldff1ss_be_r_mte
,
5187 gen_helper_sve_ldff1sdu_be_r_mte
,
5189 gen_helper_sve_ldff1bds_r_mte
,
5190 gen_helper_sve_ldff1bss_r_mte
,
5191 gen_helper_sve_ldff1bhs_r_mte
,
5192 gen_helper_sve_ldff1dd_be_r_mte
} },
5195 if (sve_access_check(s
)) {
5196 TCGv_i64 addr
= new_tmp_a64(s
);
5197 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5198 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5199 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5200 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5205 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5207 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5208 { /* mte inactive, little-endian */
5209 { gen_helper_sve_ldnf1bb_r
,
5210 gen_helper_sve_ldnf1bhu_r
,
5211 gen_helper_sve_ldnf1bsu_r
,
5212 gen_helper_sve_ldnf1bdu_r
,
5214 gen_helper_sve_ldnf1sds_le_r
,
5215 gen_helper_sve_ldnf1hh_le_r
,
5216 gen_helper_sve_ldnf1hsu_le_r
,
5217 gen_helper_sve_ldnf1hdu_le_r
,
5219 gen_helper_sve_ldnf1hds_le_r
,
5220 gen_helper_sve_ldnf1hss_le_r
,
5221 gen_helper_sve_ldnf1ss_le_r
,
5222 gen_helper_sve_ldnf1sdu_le_r
,
5224 gen_helper_sve_ldnf1bds_r
,
5225 gen_helper_sve_ldnf1bss_r
,
5226 gen_helper_sve_ldnf1bhs_r
,
5227 gen_helper_sve_ldnf1dd_le_r
},
5229 /* mte inactive, big-endian */
5230 { gen_helper_sve_ldnf1bb_r
,
5231 gen_helper_sve_ldnf1bhu_r
,
5232 gen_helper_sve_ldnf1bsu_r
,
5233 gen_helper_sve_ldnf1bdu_r
,
5235 gen_helper_sve_ldnf1sds_be_r
,
5236 gen_helper_sve_ldnf1hh_be_r
,
5237 gen_helper_sve_ldnf1hsu_be_r
,
5238 gen_helper_sve_ldnf1hdu_be_r
,
5240 gen_helper_sve_ldnf1hds_be_r
,
5241 gen_helper_sve_ldnf1hss_be_r
,
5242 gen_helper_sve_ldnf1ss_be_r
,
5243 gen_helper_sve_ldnf1sdu_be_r
,
5245 gen_helper_sve_ldnf1bds_r
,
5246 gen_helper_sve_ldnf1bss_r
,
5247 gen_helper_sve_ldnf1bhs_r
,
5248 gen_helper_sve_ldnf1dd_be_r
} },
5250 { /* mte inactive, little-endian */
5251 { gen_helper_sve_ldnf1bb_r_mte
,
5252 gen_helper_sve_ldnf1bhu_r_mte
,
5253 gen_helper_sve_ldnf1bsu_r_mte
,
5254 gen_helper_sve_ldnf1bdu_r_mte
,
5256 gen_helper_sve_ldnf1sds_le_r_mte
,
5257 gen_helper_sve_ldnf1hh_le_r_mte
,
5258 gen_helper_sve_ldnf1hsu_le_r_mte
,
5259 gen_helper_sve_ldnf1hdu_le_r_mte
,
5261 gen_helper_sve_ldnf1hds_le_r_mte
,
5262 gen_helper_sve_ldnf1hss_le_r_mte
,
5263 gen_helper_sve_ldnf1ss_le_r_mte
,
5264 gen_helper_sve_ldnf1sdu_le_r_mte
,
5266 gen_helper_sve_ldnf1bds_r_mte
,
5267 gen_helper_sve_ldnf1bss_r_mte
,
5268 gen_helper_sve_ldnf1bhs_r_mte
,
5269 gen_helper_sve_ldnf1dd_le_r_mte
},
5271 /* mte inactive, big-endian */
5272 { gen_helper_sve_ldnf1bb_r_mte
,
5273 gen_helper_sve_ldnf1bhu_r_mte
,
5274 gen_helper_sve_ldnf1bsu_r_mte
,
5275 gen_helper_sve_ldnf1bdu_r_mte
,
5277 gen_helper_sve_ldnf1sds_be_r_mte
,
5278 gen_helper_sve_ldnf1hh_be_r_mte
,
5279 gen_helper_sve_ldnf1hsu_be_r_mte
,
5280 gen_helper_sve_ldnf1hdu_be_r_mte
,
5282 gen_helper_sve_ldnf1hds_be_r_mte
,
5283 gen_helper_sve_ldnf1hss_be_r_mte
,
5284 gen_helper_sve_ldnf1ss_be_r_mte
,
5285 gen_helper_sve_ldnf1sdu_be_r_mte
,
5287 gen_helper_sve_ldnf1bds_r_mte
,
5288 gen_helper_sve_ldnf1bss_r_mte
,
5289 gen_helper_sve_ldnf1bhs_r_mte
,
5290 gen_helper_sve_ldnf1dd_be_r_mte
} },
5293 if (sve_access_check(s
)) {
5294 int vsz
= vec_full_reg_size(s
);
5295 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5296 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5297 TCGv_i64 addr
= new_tmp_a64(s
);
5299 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5300 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5301 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5306 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5308 unsigned vsz
= vec_full_reg_size(s
);
5312 /* Load the first quadword using the normal predicated load helpers. */
5313 poff
= pred_full_reg_offset(s
, pg
);
5316 * Zero-extend the first 16 bits of the predicate into a temporary.
5317 * This avoids triggering an assert making sure we don't have bits
5318 * set within a predicate beyond VQ, but we have lowered VQ to 1
5319 * for this load operation.
5321 TCGv_i64 tmp
= tcg_temp_new_i64();
5325 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5327 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5328 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5329 tcg_temp_free_i64(tmp
);
5332 t_pg
= tcg_temp_new_ptr();
5333 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5335 gen_helper_gvec_mem
*fn
5336 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5337 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(16, 16, zt
)));
5339 tcg_temp_free_ptr(t_pg
);
5341 /* Replicate that first quadword. */
5343 int doff
= vec_full_reg_offset(s
, zt
);
5344 tcg_gen_gvec_dup_mem(4, doff
+ 16, doff
, vsz
- 16, vsz
- 16);
5348 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5353 if (sve_access_check(s
)) {
5354 int msz
= dtype_msz(a
->dtype
);
5355 TCGv_i64 addr
= new_tmp_a64(s
);
5356 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5357 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5358 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5363 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5365 if (sve_access_check(s
)) {
5366 TCGv_i64 addr
= new_tmp_a64(s
);
5367 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5368 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5373 static void do_ldro(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5375 unsigned vsz
= vec_full_reg_size(s
);
5382 * Note that this UNDEFINED check comes after CheckSVEEnabled()
5383 * in the ARM pseudocode, which is the sve_access_check() done
5384 * in our caller. We should not now return false from the caller.
5386 unallocated_encoding(s
);
5390 /* Load the first octaword using the normal predicated load helpers. */
5392 poff
= pred_full_reg_offset(s
, pg
);
5395 * Zero-extend the first 32 bits of the predicate into a temporary.
5396 * This avoids triggering an assert making sure we don't have bits
5397 * set within a predicate beyond VQ, but we have lowered VQ to 2
5398 * for this load operation.
5400 TCGv_i64 tmp
= tcg_temp_new_i64();
5404 tcg_gen_ld32u_i64(tmp
, cpu_env
, poff
);
5406 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5407 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5408 tcg_temp_free_i64(tmp
);
5411 t_pg
= tcg_temp_new_ptr();
5412 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5414 gen_helper_gvec_mem
*fn
5415 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5416 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(32, 32, zt
)));
5418 tcg_temp_free_ptr(t_pg
);
5421 * Replicate that first octaword.
5422 * The replication happens in units of 32; if the full vector size
5423 * is not a multiple of 32, the final bits are zeroed.
5425 doff
= vec_full_reg_offset(s
, zt
);
5426 vsz_r32
= QEMU_ALIGN_DOWN(vsz
, 32);
5428 tcg_gen_gvec_dup_mem(5, doff
+ 32, doff
, vsz_r32
- 32, vsz_r32
- 32);
5432 tcg_gen_gvec_dup_imm(MO_64
, doff
+ vsz_r32
, vsz
, vsz
, 0);
5436 static bool trans_LD1RO_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5438 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5444 if (sve_access_check(s
)) {
5445 TCGv_i64 addr
= new_tmp_a64(s
);
5446 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5447 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5448 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5453 static bool trans_LD1RO_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5455 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5458 if (sve_access_check(s
)) {
5459 TCGv_i64 addr
= new_tmp_a64(s
);
5460 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 32);
5461 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5466 /* Load and broadcast element. */
5467 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5469 unsigned vsz
= vec_full_reg_size(s
);
5470 unsigned psz
= pred_full_reg_size(s
);
5471 unsigned esz
= dtype_esz
[a
->dtype
];
5472 unsigned msz
= dtype_msz(a
->dtype
);
5474 TCGv_i64 temp
, clean_addr
;
5476 if (!sve_access_check(s
)) {
5480 over
= gen_new_label();
5482 /* If the guarding predicate has no bits set, no load occurs. */
5484 /* Reduce the pred_esz_masks value simply to reduce the
5485 * size of the code generated here.
5487 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5488 temp
= tcg_temp_new_i64();
5489 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5490 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5491 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5492 tcg_temp_free_i64(temp
);
5494 TCGv_i32 t32
= tcg_temp_new_i32();
5495 find_last_active(s
, t32
, esz
, a
->pg
);
5496 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5497 tcg_temp_free_i32(t32
);
5500 /* Load the data. */
5501 temp
= tcg_temp_new_i64();
5502 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5503 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5505 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5506 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5508 /* Broadcast to *all* elements. */
5509 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5511 tcg_temp_free_i64(temp
);
5513 /* Zero the inactive elements. */
5514 gen_set_label(over
);
5515 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5518 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5519 int msz
, int esz
, int nreg
)
5521 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5522 { { { gen_helper_sve_st1bb_r
,
5523 gen_helper_sve_st1bh_r
,
5524 gen_helper_sve_st1bs_r
,
5525 gen_helper_sve_st1bd_r
},
5527 gen_helper_sve_st1hh_le_r
,
5528 gen_helper_sve_st1hs_le_r
,
5529 gen_helper_sve_st1hd_le_r
},
5531 gen_helper_sve_st1ss_le_r
,
5532 gen_helper_sve_st1sd_le_r
},
5534 gen_helper_sve_st1dd_le_r
} },
5535 { { gen_helper_sve_st1bb_r
,
5536 gen_helper_sve_st1bh_r
,
5537 gen_helper_sve_st1bs_r
,
5538 gen_helper_sve_st1bd_r
},
5540 gen_helper_sve_st1hh_be_r
,
5541 gen_helper_sve_st1hs_be_r
,
5542 gen_helper_sve_st1hd_be_r
},
5544 gen_helper_sve_st1ss_be_r
,
5545 gen_helper_sve_st1sd_be_r
},
5547 gen_helper_sve_st1dd_be_r
} } },
5549 { { { gen_helper_sve_st1bb_r_mte
,
5550 gen_helper_sve_st1bh_r_mte
,
5551 gen_helper_sve_st1bs_r_mte
,
5552 gen_helper_sve_st1bd_r_mte
},
5554 gen_helper_sve_st1hh_le_r_mte
,
5555 gen_helper_sve_st1hs_le_r_mte
,
5556 gen_helper_sve_st1hd_le_r_mte
},
5558 gen_helper_sve_st1ss_le_r_mte
,
5559 gen_helper_sve_st1sd_le_r_mte
},
5561 gen_helper_sve_st1dd_le_r_mte
} },
5562 { { gen_helper_sve_st1bb_r_mte
,
5563 gen_helper_sve_st1bh_r_mte
,
5564 gen_helper_sve_st1bs_r_mte
,
5565 gen_helper_sve_st1bd_r_mte
},
5567 gen_helper_sve_st1hh_be_r_mte
,
5568 gen_helper_sve_st1hs_be_r_mte
,
5569 gen_helper_sve_st1hd_be_r_mte
},
5571 gen_helper_sve_st1ss_be_r_mte
,
5572 gen_helper_sve_st1sd_be_r_mte
},
5574 gen_helper_sve_st1dd_be_r_mte
} } },
5576 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5577 { { { gen_helper_sve_st2bb_r
,
5578 gen_helper_sve_st2hh_le_r
,
5579 gen_helper_sve_st2ss_le_r
,
5580 gen_helper_sve_st2dd_le_r
},
5581 { gen_helper_sve_st3bb_r
,
5582 gen_helper_sve_st3hh_le_r
,
5583 gen_helper_sve_st3ss_le_r
,
5584 gen_helper_sve_st3dd_le_r
},
5585 { gen_helper_sve_st4bb_r
,
5586 gen_helper_sve_st4hh_le_r
,
5587 gen_helper_sve_st4ss_le_r
,
5588 gen_helper_sve_st4dd_le_r
} },
5589 { { gen_helper_sve_st2bb_r
,
5590 gen_helper_sve_st2hh_be_r
,
5591 gen_helper_sve_st2ss_be_r
,
5592 gen_helper_sve_st2dd_be_r
},
5593 { gen_helper_sve_st3bb_r
,
5594 gen_helper_sve_st3hh_be_r
,
5595 gen_helper_sve_st3ss_be_r
,
5596 gen_helper_sve_st3dd_be_r
},
5597 { gen_helper_sve_st4bb_r
,
5598 gen_helper_sve_st4hh_be_r
,
5599 gen_helper_sve_st4ss_be_r
,
5600 gen_helper_sve_st4dd_be_r
} } },
5601 { { { gen_helper_sve_st2bb_r_mte
,
5602 gen_helper_sve_st2hh_le_r_mte
,
5603 gen_helper_sve_st2ss_le_r_mte
,
5604 gen_helper_sve_st2dd_le_r_mte
},
5605 { gen_helper_sve_st3bb_r_mte
,
5606 gen_helper_sve_st3hh_le_r_mte
,
5607 gen_helper_sve_st3ss_le_r_mte
,
5608 gen_helper_sve_st3dd_le_r_mte
},
5609 { gen_helper_sve_st4bb_r_mte
,
5610 gen_helper_sve_st4hh_le_r_mte
,
5611 gen_helper_sve_st4ss_le_r_mte
,
5612 gen_helper_sve_st4dd_le_r_mte
} },
5613 { { gen_helper_sve_st2bb_r_mte
,
5614 gen_helper_sve_st2hh_be_r_mte
,
5615 gen_helper_sve_st2ss_be_r_mte
,
5616 gen_helper_sve_st2dd_be_r_mte
},
5617 { gen_helper_sve_st3bb_r_mte
,
5618 gen_helper_sve_st3hh_be_r_mte
,
5619 gen_helper_sve_st3ss_be_r_mte
,
5620 gen_helper_sve_st3dd_be_r_mte
},
5621 { gen_helper_sve_st4bb_r_mte
,
5622 gen_helper_sve_st4hh_be_r_mte
,
5623 gen_helper_sve_st4ss_be_r_mte
,
5624 gen_helper_sve_st4dd_be_r_mte
} } },
5626 gen_helper_gvec_mem
*fn
;
5627 int be
= s
->be_data
== MO_BE
;
5631 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5634 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5636 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5639 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5642 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5644 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
5647 if (sve_access_check(s
)) {
5648 TCGv_i64 addr
= new_tmp_a64(s
);
5649 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
5650 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5651 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5656 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
5658 if (a
->msz
> a
->esz
) {
5661 if (sve_access_check(s
)) {
5662 int vsz
= vec_full_reg_size(s
);
5663 int elements
= vsz
>> a
->esz
;
5664 TCGv_i64 addr
= new_tmp_a64(s
);
5666 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5667 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
5668 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5674 *** SVE gather loads / scatter stores
5677 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
5678 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
5679 gen_helper_gvec_mem_scatter
*fn
)
5681 unsigned vsz
= vec_full_reg_size(s
);
5682 TCGv_ptr t_zm
= tcg_temp_new_ptr();
5683 TCGv_ptr t_pg
= tcg_temp_new_ptr();
5684 TCGv_ptr t_zt
= tcg_temp_new_ptr();
5687 if (s
->mte_active
[0]) {
5688 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5689 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5690 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5691 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5692 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
5693 desc
<<= SVE_MTEDESC_SHIFT
;
5695 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
5697 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5698 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
5699 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
5700 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, tcg_constant_i32(desc
));
5702 tcg_temp_free_ptr(t_zt
);
5703 tcg_temp_free_ptr(t_zm
);
5704 tcg_temp_free_ptr(t_pg
);
5707 /* Indexed by [mte][be][ff][xs][u][msz]. */
5708 static gen_helper_gvec_mem_scatter
* const
5709 gather_load_fn32
[2][2][2][2][2][3] = {
5710 { /* MTE Inactive */
5711 { /* Little-endian */
5712 { { { gen_helper_sve_ldbss_zsu
,
5713 gen_helper_sve_ldhss_le_zsu
,
5715 { gen_helper_sve_ldbsu_zsu
,
5716 gen_helper_sve_ldhsu_le_zsu
,
5717 gen_helper_sve_ldss_le_zsu
, } },
5718 { { gen_helper_sve_ldbss_zss
,
5719 gen_helper_sve_ldhss_le_zss
,
5721 { gen_helper_sve_ldbsu_zss
,
5722 gen_helper_sve_ldhsu_le_zss
,
5723 gen_helper_sve_ldss_le_zss
, } } },
5726 { { { gen_helper_sve_ldffbss_zsu
,
5727 gen_helper_sve_ldffhss_le_zsu
,
5729 { gen_helper_sve_ldffbsu_zsu
,
5730 gen_helper_sve_ldffhsu_le_zsu
,
5731 gen_helper_sve_ldffss_le_zsu
, } },
5732 { { gen_helper_sve_ldffbss_zss
,
5733 gen_helper_sve_ldffhss_le_zss
,
5735 { gen_helper_sve_ldffbsu_zss
,
5736 gen_helper_sve_ldffhsu_le_zss
,
5737 gen_helper_sve_ldffss_le_zss
, } } } },
5740 { { { gen_helper_sve_ldbss_zsu
,
5741 gen_helper_sve_ldhss_be_zsu
,
5743 { gen_helper_sve_ldbsu_zsu
,
5744 gen_helper_sve_ldhsu_be_zsu
,
5745 gen_helper_sve_ldss_be_zsu
, } },
5746 { { gen_helper_sve_ldbss_zss
,
5747 gen_helper_sve_ldhss_be_zss
,
5749 { gen_helper_sve_ldbsu_zss
,
5750 gen_helper_sve_ldhsu_be_zss
,
5751 gen_helper_sve_ldss_be_zss
, } } },
5754 { { { gen_helper_sve_ldffbss_zsu
,
5755 gen_helper_sve_ldffhss_be_zsu
,
5757 { gen_helper_sve_ldffbsu_zsu
,
5758 gen_helper_sve_ldffhsu_be_zsu
,
5759 gen_helper_sve_ldffss_be_zsu
, } },
5760 { { gen_helper_sve_ldffbss_zss
,
5761 gen_helper_sve_ldffhss_be_zss
,
5763 { gen_helper_sve_ldffbsu_zss
,
5764 gen_helper_sve_ldffhsu_be_zss
,
5765 gen_helper_sve_ldffss_be_zss
, } } } } },
5767 { /* Little-endian */
5768 { { { gen_helper_sve_ldbss_zsu_mte
,
5769 gen_helper_sve_ldhss_le_zsu_mte
,
5771 { gen_helper_sve_ldbsu_zsu_mte
,
5772 gen_helper_sve_ldhsu_le_zsu_mte
,
5773 gen_helper_sve_ldss_le_zsu_mte
, } },
5774 { { gen_helper_sve_ldbss_zss_mte
,
5775 gen_helper_sve_ldhss_le_zss_mte
,
5777 { gen_helper_sve_ldbsu_zss_mte
,
5778 gen_helper_sve_ldhsu_le_zss_mte
,
5779 gen_helper_sve_ldss_le_zss_mte
, } } },
5782 { { { gen_helper_sve_ldffbss_zsu_mte
,
5783 gen_helper_sve_ldffhss_le_zsu_mte
,
5785 { gen_helper_sve_ldffbsu_zsu_mte
,
5786 gen_helper_sve_ldffhsu_le_zsu_mte
,
5787 gen_helper_sve_ldffss_le_zsu_mte
, } },
5788 { { gen_helper_sve_ldffbss_zss_mte
,
5789 gen_helper_sve_ldffhss_le_zss_mte
,
5791 { gen_helper_sve_ldffbsu_zss_mte
,
5792 gen_helper_sve_ldffhsu_le_zss_mte
,
5793 gen_helper_sve_ldffss_le_zss_mte
, } } } },
5796 { { { gen_helper_sve_ldbss_zsu_mte
,
5797 gen_helper_sve_ldhss_be_zsu_mte
,
5799 { gen_helper_sve_ldbsu_zsu_mte
,
5800 gen_helper_sve_ldhsu_be_zsu_mte
,
5801 gen_helper_sve_ldss_be_zsu_mte
, } },
5802 { { gen_helper_sve_ldbss_zss_mte
,
5803 gen_helper_sve_ldhss_be_zss_mte
,
5805 { gen_helper_sve_ldbsu_zss_mte
,
5806 gen_helper_sve_ldhsu_be_zss_mte
,
5807 gen_helper_sve_ldss_be_zss_mte
, } } },
5810 { { { gen_helper_sve_ldffbss_zsu_mte
,
5811 gen_helper_sve_ldffhss_be_zsu_mte
,
5813 { gen_helper_sve_ldffbsu_zsu_mte
,
5814 gen_helper_sve_ldffhsu_be_zsu_mte
,
5815 gen_helper_sve_ldffss_be_zsu_mte
, } },
5816 { { gen_helper_sve_ldffbss_zss_mte
,
5817 gen_helper_sve_ldffhss_be_zss_mte
,
5819 { gen_helper_sve_ldffbsu_zss_mte
,
5820 gen_helper_sve_ldffhsu_be_zss_mte
,
5821 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
5824 /* Note that we overload xs=2 to indicate 64-bit offset. */
5825 static gen_helper_gvec_mem_scatter
* const
5826 gather_load_fn64
[2][2][2][3][2][4] = {
5827 { /* MTE Inactive */
5828 { /* Little-endian */
5829 { { { gen_helper_sve_ldbds_zsu
,
5830 gen_helper_sve_ldhds_le_zsu
,
5831 gen_helper_sve_ldsds_le_zsu
,
5833 { gen_helper_sve_ldbdu_zsu
,
5834 gen_helper_sve_ldhdu_le_zsu
,
5835 gen_helper_sve_ldsdu_le_zsu
,
5836 gen_helper_sve_lddd_le_zsu
, } },
5837 { { gen_helper_sve_ldbds_zss
,
5838 gen_helper_sve_ldhds_le_zss
,
5839 gen_helper_sve_ldsds_le_zss
,
5841 { gen_helper_sve_ldbdu_zss
,
5842 gen_helper_sve_ldhdu_le_zss
,
5843 gen_helper_sve_ldsdu_le_zss
,
5844 gen_helper_sve_lddd_le_zss
, } },
5845 { { gen_helper_sve_ldbds_zd
,
5846 gen_helper_sve_ldhds_le_zd
,
5847 gen_helper_sve_ldsds_le_zd
,
5849 { gen_helper_sve_ldbdu_zd
,
5850 gen_helper_sve_ldhdu_le_zd
,
5851 gen_helper_sve_ldsdu_le_zd
,
5852 gen_helper_sve_lddd_le_zd
, } } },
5855 { { { gen_helper_sve_ldffbds_zsu
,
5856 gen_helper_sve_ldffhds_le_zsu
,
5857 gen_helper_sve_ldffsds_le_zsu
,
5859 { gen_helper_sve_ldffbdu_zsu
,
5860 gen_helper_sve_ldffhdu_le_zsu
,
5861 gen_helper_sve_ldffsdu_le_zsu
,
5862 gen_helper_sve_ldffdd_le_zsu
, } },
5863 { { gen_helper_sve_ldffbds_zss
,
5864 gen_helper_sve_ldffhds_le_zss
,
5865 gen_helper_sve_ldffsds_le_zss
,
5867 { gen_helper_sve_ldffbdu_zss
,
5868 gen_helper_sve_ldffhdu_le_zss
,
5869 gen_helper_sve_ldffsdu_le_zss
,
5870 gen_helper_sve_ldffdd_le_zss
, } },
5871 { { gen_helper_sve_ldffbds_zd
,
5872 gen_helper_sve_ldffhds_le_zd
,
5873 gen_helper_sve_ldffsds_le_zd
,
5875 { gen_helper_sve_ldffbdu_zd
,
5876 gen_helper_sve_ldffhdu_le_zd
,
5877 gen_helper_sve_ldffsdu_le_zd
,
5878 gen_helper_sve_ldffdd_le_zd
, } } } },
5880 { { { gen_helper_sve_ldbds_zsu
,
5881 gen_helper_sve_ldhds_be_zsu
,
5882 gen_helper_sve_ldsds_be_zsu
,
5884 { gen_helper_sve_ldbdu_zsu
,
5885 gen_helper_sve_ldhdu_be_zsu
,
5886 gen_helper_sve_ldsdu_be_zsu
,
5887 gen_helper_sve_lddd_be_zsu
, } },
5888 { { gen_helper_sve_ldbds_zss
,
5889 gen_helper_sve_ldhds_be_zss
,
5890 gen_helper_sve_ldsds_be_zss
,
5892 { gen_helper_sve_ldbdu_zss
,
5893 gen_helper_sve_ldhdu_be_zss
,
5894 gen_helper_sve_ldsdu_be_zss
,
5895 gen_helper_sve_lddd_be_zss
, } },
5896 { { gen_helper_sve_ldbds_zd
,
5897 gen_helper_sve_ldhds_be_zd
,
5898 gen_helper_sve_ldsds_be_zd
,
5900 { gen_helper_sve_ldbdu_zd
,
5901 gen_helper_sve_ldhdu_be_zd
,
5902 gen_helper_sve_ldsdu_be_zd
,
5903 gen_helper_sve_lddd_be_zd
, } } },
5906 { { { gen_helper_sve_ldffbds_zsu
,
5907 gen_helper_sve_ldffhds_be_zsu
,
5908 gen_helper_sve_ldffsds_be_zsu
,
5910 { gen_helper_sve_ldffbdu_zsu
,
5911 gen_helper_sve_ldffhdu_be_zsu
,
5912 gen_helper_sve_ldffsdu_be_zsu
,
5913 gen_helper_sve_ldffdd_be_zsu
, } },
5914 { { gen_helper_sve_ldffbds_zss
,
5915 gen_helper_sve_ldffhds_be_zss
,
5916 gen_helper_sve_ldffsds_be_zss
,
5918 { gen_helper_sve_ldffbdu_zss
,
5919 gen_helper_sve_ldffhdu_be_zss
,
5920 gen_helper_sve_ldffsdu_be_zss
,
5921 gen_helper_sve_ldffdd_be_zss
, } },
5922 { { gen_helper_sve_ldffbds_zd
,
5923 gen_helper_sve_ldffhds_be_zd
,
5924 gen_helper_sve_ldffsds_be_zd
,
5926 { gen_helper_sve_ldffbdu_zd
,
5927 gen_helper_sve_ldffhdu_be_zd
,
5928 gen_helper_sve_ldffsdu_be_zd
,
5929 gen_helper_sve_ldffdd_be_zd
, } } } } },
5931 { /* Little-endian */
5932 { { { gen_helper_sve_ldbds_zsu_mte
,
5933 gen_helper_sve_ldhds_le_zsu_mte
,
5934 gen_helper_sve_ldsds_le_zsu_mte
,
5936 { gen_helper_sve_ldbdu_zsu_mte
,
5937 gen_helper_sve_ldhdu_le_zsu_mte
,
5938 gen_helper_sve_ldsdu_le_zsu_mte
,
5939 gen_helper_sve_lddd_le_zsu_mte
, } },
5940 { { gen_helper_sve_ldbds_zss_mte
,
5941 gen_helper_sve_ldhds_le_zss_mte
,
5942 gen_helper_sve_ldsds_le_zss_mte
,
5944 { gen_helper_sve_ldbdu_zss_mte
,
5945 gen_helper_sve_ldhdu_le_zss_mte
,
5946 gen_helper_sve_ldsdu_le_zss_mte
,
5947 gen_helper_sve_lddd_le_zss_mte
, } },
5948 { { gen_helper_sve_ldbds_zd_mte
,
5949 gen_helper_sve_ldhds_le_zd_mte
,
5950 gen_helper_sve_ldsds_le_zd_mte
,
5952 { gen_helper_sve_ldbdu_zd_mte
,
5953 gen_helper_sve_ldhdu_le_zd_mte
,
5954 gen_helper_sve_ldsdu_le_zd_mte
,
5955 gen_helper_sve_lddd_le_zd_mte
, } } },
5958 { { { gen_helper_sve_ldffbds_zsu_mte
,
5959 gen_helper_sve_ldffhds_le_zsu_mte
,
5960 gen_helper_sve_ldffsds_le_zsu_mte
,
5962 { gen_helper_sve_ldffbdu_zsu_mte
,
5963 gen_helper_sve_ldffhdu_le_zsu_mte
,
5964 gen_helper_sve_ldffsdu_le_zsu_mte
,
5965 gen_helper_sve_ldffdd_le_zsu_mte
, } },
5966 { { gen_helper_sve_ldffbds_zss_mte
,
5967 gen_helper_sve_ldffhds_le_zss_mte
,
5968 gen_helper_sve_ldffsds_le_zss_mte
,
5970 { gen_helper_sve_ldffbdu_zss_mte
,
5971 gen_helper_sve_ldffhdu_le_zss_mte
,
5972 gen_helper_sve_ldffsdu_le_zss_mte
,
5973 gen_helper_sve_ldffdd_le_zss_mte
, } },
5974 { { gen_helper_sve_ldffbds_zd_mte
,
5975 gen_helper_sve_ldffhds_le_zd_mte
,
5976 gen_helper_sve_ldffsds_le_zd_mte
,
5978 { gen_helper_sve_ldffbdu_zd_mte
,
5979 gen_helper_sve_ldffhdu_le_zd_mte
,
5980 gen_helper_sve_ldffsdu_le_zd_mte
,
5981 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
5983 { { { gen_helper_sve_ldbds_zsu_mte
,
5984 gen_helper_sve_ldhds_be_zsu_mte
,
5985 gen_helper_sve_ldsds_be_zsu_mte
,
5987 { gen_helper_sve_ldbdu_zsu_mte
,
5988 gen_helper_sve_ldhdu_be_zsu_mte
,
5989 gen_helper_sve_ldsdu_be_zsu_mte
,
5990 gen_helper_sve_lddd_be_zsu_mte
, } },
5991 { { gen_helper_sve_ldbds_zss_mte
,
5992 gen_helper_sve_ldhds_be_zss_mte
,
5993 gen_helper_sve_ldsds_be_zss_mte
,
5995 { gen_helper_sve_ldbdu_zss_mte
,
5996 gen_helper_sve_ldhdu_be_zss_mte
,
5997 gen_helper_sve_ldsdu_be_zss_mte
,
5998 gen_helper_sve_lddd_be_zss_mte
, } },
5999 { { gen_helper_sve_ldbds_zd_mte
,
6000 gen_helper_sve_ldhds_be_zd_mte
,
6001 gen_helper_sve_ldsds_be_zd_mte
,
6003 { gen_helper_sve_ldbdu_zd_mte
,
6004 gen_helper_sve_ldhdu_be_zd_mte
,
6005 gen_helper_sve_ldsdu_be_zd_mte
,
6006 gen_helper_sve_lddd_be_zd_mte
, } } },
6009 { { { gen_helper_sve_ldffbds_zsu_mte
,
6010 gen_helper_sve_ldffhds_be_zsu_mte
,
6011 gen_helper_sve_ldffsds_be_zsu_mte
,
6013 { gen_helper_sve_ldffbdu_zsu_mte
,
6014 gen_helper_sve_ldffhdu_be_zsu_mte
,
6015 gen_helper_sve_ldffsdu_be_zsu_mte
,
6016 gen_helper_sve_ldffdd_be_zsu_mte
, } },
6017 { { gen_helper_sve_ldffbds_zss_mte
,
6018 gen_helper_sve_ldffhds_be_zss_mte
,
6019 gen_helper_sve_ldffsds_be_zss_mte
,
6021 { gen_helper_sve_ldffbdu_zss_mte
,
6022 gen_helper_sve_ldffhdu_be_zss_mte
,
6023 gen_helper_sve_ldffsdu_be_zss_mte
,
6024 gen_helper_sve_ldffdd_be_zss_mte
, } },
6025 { { gen_helper_sve_ldffbds_zd_mte
,
6026 gen_helper_sve_ldffhds_be_zd_mte
,
6027 gen_helper_sve_ldffsds_be_zd_mte
,
6029 { gen_helper_sve_ldffbdu_zd_mte
,
6030 gen_helper_sve_ldffhdu_be_zd_mte
,
6031 gen_helper_sve_ldffsdu_be_zd_mte
,
6032 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
6035 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6037 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6038 bool be
= s
->be_data
== MO_BE
;
6039 bool mte
= s
->mte_active
[0];
6041 if (!sve_access_check(s
)) {
6047 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6050 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6055 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6056 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
6060 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
6062 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6063 bool be
= s
->be_data
== MO_BE
;
6064 bool mte
= s
->mte_active
[0];
6066 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
6069 if (!sve_access_check(s
)) {
6075 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
6078 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
6083 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6084 * by loading the immediate into the scalar parameter.
6086 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6087 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, false, fn
);
6091 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6093 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6094 bool be
= s
->be_data
== MO_BE
;
6095 bool mte
= s
->mte_active
[0];
6097 if (a
->esz
< a
->msz
+ !a
->u
) {
6100 if (!dc_isar_feature(aa64_sve2
, s
)) {
6103 if (!sve_access_check(s
)) {
6109 fn
= gather_load_fn32
[mte
][be
][0][0][a
->u
][a
->msz
];
6112 fn
= gather_load_fn64
[mte
][be
][0][2][a
->u
][a
->msz
];
6117 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6118 cpu_reg(s
, a
->rm
), a
->msz
, false, fn
);
6122 /* Indexed by [mte][be][xs][msz]. */
6123 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
6124 { /* MTE Inactive */
6125 { /* Little-endian */
6126 { gen_helper_sve_stbs_zsu
,
6127 gen_helper_sve_sths_le_zsu
,
6128 gen_helper_sve_stss_le_zsu
, },
6129 { gen_helper_sve_stbs_zss
,
6130 gen_helper_sve_sths_le_zss
,
6131 gen_helper_sve_stss_le_zss
, } },
6133 { gen_helper_sve_stbs_zsu
,
6134 gen_helper_sve_sths_be_zsu
,
6135 gen_helper_sve_stss_be_zsu
, },
6136 { gen_helper_sve_stbs_zss
,
6137 gen_helper_sve_sths_be_zss
,
6138 gen_helper_sve_stss_be_zss
, } } },
6140 { /* Little-endian */
6141 { gen_helper_sve_stbs_zsu_mte
,
6142 gen_helper_sve_sths_le_zsu_mte
,
6143 gen_helper_sve_stss_le_zsu_mte
, },
6144 { gen_helper_sve_stbs_zss_mte
,
6145 gen_helper_sve_sths_le_zss_mte
,
6146 gen_helper_sve_stss_le_zss_mte
, } },
6148 { gen_helper_sve_stbs_zsu_mte
,
6149 gen_helper_sve_sths_be_zsu_mte
,
6150 gen_helper_sve_stss_be_zsu_mte
, },
6151 { gen_helper_sve_stbs_zss_mte
,
6152 gen_helper_sve_sths_be_zss_mte
,
6153 gen_helper_sve_stss_be_zss_mte
, } } },
6156 /* Note that we overload xs=2 to indicate 64-bit offset. */
6157 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
6158 { /* MTE Inactive */
6159 { /* Little-endian */
6160 { gen_helper_sve_stbd_zsu
,
6161 gen_helper_sve_sthd_le_zsu
,
6162 gen_helper_sve_stsd_le_zsu
,
6163 gen_helper_sve_stdd_le_zsu
, },
6164 { gen_helper_sve_stbd_zss
,
6165 gen_helper_sve_sthd_le_zss
,
6166 gen_helper_sve_stsd_le_zss
,
6167 gen_helper_sve_stdd_le_zss
, },
6168 { gen_helper_sve_stbd_zd
,
6169 gen_helper_sve_sthd_le_zd
,
6170 gen_helper_sve_stsd_le_zd
,
6171 gen_helper_sve_stdd_le_zd
, } },
6173 { gen_helper_sve_stbd_zsu
,
6174 gen_helper_sve_sthd_be_zsu
,
6175 gen_helper_sve_stsd_be_zsu
,
6176 gen_helper_sve_stdd_be_zsu
, },
6177 { gen_helper_sve_stbd_zss
,
6178 gen_helper_sve_sthd_be_zss
,
6179 gen_helper_sve_stsd_be_zss
,
6180 gen_helper_sve_stdd_be_zss
, },
6181 { gen_helper_sve_stbd_zd
,
6182 gen_helper_sve_sthd_be_zd
,
6183 gen_helper_sve_stsd_be_zd
,
6184 gen_helper_sve_stdd_be_zd
, } } },
6185 { /* MTE Inactive */
6186 { /* Little-endian */
6187 { gen_helper_sve_stbd_zsu_mte
,
6188 gen_helper_sve_sthd_le_zsu_mte
,
6189 gen_helper_sve_stsd_le_zsu_mte
,
6190 gen_helper_sve_stdd_le_zsu_mte
, },
6191 { gen_helper_sve_stbd_zss_mte
,
6192 gen_helper_sve_sthd_le_zss_mte
,
6193 gen_helper_sve_stsd_le_zss_mte
,
6194 gen_helper_sve_stdd_le_zss_mte
, },
6195 { gen_helper_sve_stbd_zd_mte
,
6196 gen_helper_sve_sthd_le_zd_mte
,
6197 gen_helper_sve_stsd_le_zd_mte
,
6198 gen_helper_sve_stdd_le_zd_mte
, } },
6200 { gen_helper_sve_stbd_zsu_mte
,
6201 gen_helper_sve_sthd_be_zsu_mte
,
6202 gen_helper_sve_stsd_be_zsu_mte
,
6203 gen_helper_sve_stdd_be_zsu_mte
, },
6204 { gen_helper_sve_stbd_zss_mte
,
6205 gen_helper_sve_sthd_be_zss_mte
,
6206 gen_helper_sve_stsd_be_zss_mte
,
6207 gen_helper_sve_stdd_be_zss_mte
, },
6208 { gen_helper_sve_stbd_zd_mte
,
6209 gen_helper_sve_sthd_be_zd_mte
,
6210 gen_helper_sve_stsd_be_zd_mte
,
6211 gen_helper_sve_stdd_be_zd_mte
, } } },
6214 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6216 gen_helper_gvec_mem_scatter
*fn
;
6217 bool be
= s
->be_data
== MO_BE
;
6218 bool mte
= s
->mte_active
[0];
6220 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
6223 if (!sve_access_check(s
)) {
6228 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
6231 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
6234 g_assert_not_reached();
6236 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6237 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
6241 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
6243 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6244 bool be
= s
->be_data
== MO_BE
;
6245 bool mte
= s
->mte_active
[0];
6247 if (a
->esz
< a
->msz
) {
6250 if (!sve_access_check(s
)) {
6256 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6259 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6264 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6265 * by loading the immediate into the scalar parameter.
6267 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6268 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, true, fn
);
6272 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6274 gen_helper_gvec_mem_scatter
*fn
;
6275 bool be
= s
->be_data
== MO_BE
;
6276 bool mte
= s
->mte_active
[0];
6278 if (a
->esz
< a
->msz
) {
6281 if (!dc_isar_feature(aa64_sve2
, s
)) {
6284 if (!sve_access_check(s
)) {
6290 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6293 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6296 g_assert_not_reached();
6299 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6300 cpu_reg(s
, a
->rm
), a
->msz
, true, fn
);
6308 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6310 /* Prefetch is a nop within QEMU. */
6311 (void)sve_access_check(s
);
6315 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6320 /* Prefetch is a nop within QEMU. */
6321 (void)sve_access_check(s
);
6328 * TODO: The implementation so far could handle predicated merging movprfx.
6329 * The helper functions as written take an extra source register to
6330 * use in the operation, but the result is only written when predication
6331 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6332 * to allow the final write back to the destination to be unconditional.
6333 * For predicated zeroing movprfx, we need to rearrange the helpers to
6334 * allow the final write back to zero inactives.
6336 * In the meantime, just emit the moves.
6339 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6341 return do_mov_z(s
, a
->rd
, a
->rn
);
6344 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6346 if (sve_access_check(s
)) {
6347 do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6352 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6354 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6358 * SVE2 Integer Multiply - Unpredicated
6361 TRANS_FEAT(MUL_zzz
, aa64_sve2
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_mul
, a
)
6363 static gen_helper_gvec_3
* const smulh_zzz_fns
[4] = {
6364 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6365 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6367 TRANS_FEAT(SMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6368 smulh_zzz_fns
[a
->esz
], a
, 0)
6370 static gen_helper_gvec_3
* const umulh_zzz_fns
[4] = {
6371 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6372 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6374 TRANS_FEAT(UMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6375 umulh_zzz_fns
[a
->esz
], a
, 0)
6377 TRANS_FEAT(PMUL_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6378 gen_helper_gvec_pmul_b
, a
, 0)
6380 static gen_helper_gvec_3
* const sqdmulh_zzz_fns
[4] = {
6381 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6382 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6384 TRANS_FEAT(SQDMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6385 sqdmulh_zzz_fns
[a
->esz
], a
, 0)
6387 static gen_helper_gvec_3
* const sqrdmulh_zzz_fns
[4] = {
6388 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6389 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6391 TRANS_FEAT(SQRDMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6392 sqrdmulh_zzz_fns
[a
->esz
], a
, 0)
6395 * SVE2 Integer - Predicated
6398 static gen_helper_gvec_4
* const sadlp_fns
[4] = {
6399 NULL
, gen_helper_sve2_sadalp_zpzz_h
,
6400 gen_helper_sve2_sadalp_zpzz_s
, gen_helper_sve2_sadalp_zpzz_d
,
6402 TRANS_FEAT(SADALP_zpzz
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
6403 sadlp_fns
[a
->esz
], a
, 0)
6405 static gen_helper_gvec_4
* const uadlp_fns
[4] = {
6406 NULL
, gen_helper_sve2_uadalp_zpzz_h
,
6407 gen_helper_sve2_uadalp_zpzz_s
, gen_helper_sve2_uadalp_zpzz_d
,
6409 TRANS_FEAT(UADALP_zpzz
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
6410 uadlp_fns
[a
->esz
], a
, 0)
6413 * SVE2 integer unary operations (predicated)
6416 TRANS_FEAT(URECPE
, aa64_sve2
, gen_gvec_ool_arg_zpz
,
6417 a
->esz
== 2 ? gen_helper_sve2_urecpe_s
: NULL
, a
, 0)
6419 TRANS_FEAT(URSQRTE
, aa64_sve2
, gen_gvec_ool_arg_zpz
,
6420 a
->esz
== 2 ? gen_helper_sve2_ursqrte_s
: NULL
, a
, 0)
6422 static gen_helper_gvec_3
* const sqabs_fns
[4] = {
6423 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6424 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6426 TRANS_FEAT(SQABS
, aa64_sve2
, gen_gvec_ool_arg_zpz
, sqabs_fns
[a
->esz
], a
, 0)
6428 static gen_helper_gvec_3
* const sqneg_fns
[4] = {
6429 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6430 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6432 TRANS_FEAT(SQNEG
, aa64_sve2
, gen_gvec_ool_arg_zpz
, sqneg_fns
[a
->esz
], a
, 0)
6434 DO_ZPZZ(SQSHL
, aa64_sve2
, sve2_sqshl
)
6435 DO_ZPZZ(SQRSHL
, aa64_sve2
, sve2_sqrshl
)
6436 DO_ZPZZ(SRSHL
, aa64_sve2
, sve2_srshl
)
6438 DO_ZPZZ(UQSHL
, aa64_sve2
, sve2_uqshl
)
6439 DO_ZPZZ(UQRSHL
, aa64_sve2
, sve2_uqrshl
)
6440 DO_ZPZZ(URSHL
, aa64_sve2
, sve2_urshl
)
6442 DO_ZPZZ(SHADD
, aa64_sve2
, sve2_shadd
)
6443 DO_ZPZZ(SRHADD
, aa64_sve2
, sve2_srhadd
)
6444 DO_ZPZZ(SHSUB
, aa64_sve2
, sve2_shsub
)
6446 DO_ZPZZ(UHADD
, aa64_sve2
, sve2_uhadd
)
6447 DO_ZPZZ(URHADD
, aa64_sve2
, sve2_urhadd
)
6448 DO_ZPZZ(UHSUB
, aa64_sve2
, sve2_uhsub
)
6450 DO_ZPZZ(ADDP
, aa64_sve2
, sve2_addp
)
6451 DO_ZPZZ(SMAXP
, aa64_sve2
, sve2_smaxp
)
6452 DO_ZPZZ(UMAXP
, aa64_sve2
, sve2_umaxp
)
6453 DO_ZPZZ(SMINP
, aa64_sve2
, sve2_sminp
)
6454 DO_ZPZZ(UMINP
, aa64_sve2
, sve2_uminp
)
6456 DO_ZPZZ(SQADD_zpzz
, aa64_sve2
, sve2_sqadd
)
6457 DO_ZPZZ(UQADD_zpzz
, aa64_sve2
, sve2_uqadd
)
6458 DO_ZPZZ(SQSUB_zpzz
, aa64_sve2
, sve2_sqsub
)
6459 DO_ZPZZ(UQSUB_zpzz
, aa64_sve2
, sve2_uqsub
)
6460 DO_ZPZZ(SUQADD
, aa64_sve2
, sve2_suqadd
)
6461 DO_ZPZZ(USQADD
, aa64_sve2
, sve2_usqadd
)
6464 * SVE2 Widening Integer Arithmetic
6467 static gen_helper_gvec_3
* const saddl_fns
[4] = {
6468 NULL
, gen_helper_sve2_saddl_h
,
6469 gen_helper_sve2_saddl_s
, gen_helper_sve2_saddl_d
,
6471 TRANS_FEAT(SADDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6472 saddl_fns
[a
->esz
], a
, 0)
6473 TRANS_FEAT(SADDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6474 saddl_fns
[a
->esz
], a
, 3)
6475 TRANS_FEAT(SADDLBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6476 saddl_fns
[a
->esz
], a
, 2)
6478 static gen_helper_gvec_3
* const ssubl_fns
[4] = {
6479 NULL
, gen_helper_sve2_ssubl_h
,
6480 gen_helper_sve2_ssubl_s
, gen_helper_sve2_ssubl_d
,
6482 TRANS_FEAT(SSUBLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6483 ssubl_fns
[a
->esz
], a
, 0)
6484 TRANS_FEAT(SSUBLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6485 ssubl_fns
[a
->esz
], a
, 3)
6486 TRANS_FEAT(SSUBLBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6487 ssubl_fns
[a
->esz
], a
, 2)
6488 TRANS_FEAT(SSUBLTB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6489 ssubl_fns
[a
->esz
], a
, 1)
6491 static gen_helper_gvec_3
* const sabdl_fns
[4] = {
6492 NULL
, gen_helper_sve2_sabdl_h
,
6493 gen_helper_sve2_sabdl_s
, gen_helper_sve2_sabdl_d
,
6495 TRANS_FEAT(SABDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6496 sabdl_fns
[a
->esz
], a
, 0)
6497 TRANS_FEAT(SABDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6498 sabdl_fns
[a
->esz
], a
, 3)
6500 static gen_helper_gvec_3
* const uaddl_fns
[4] = {
6501 NULL
, gen_helper_sve2_uaddl_h
,
6502 gen_helper_sve2_uaddl_s
, gen_helper_sve2_uaddl_d
,
6504 TRANS_FEAT(UADDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6505 uaddl_fns
[a
->esz
], a
, 0)
6506 TRANS_FEAT(UADDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6507 uaddl_fns
[a
->esz
], a
, 3)
6509 static gen_helper_gvec_3
* const usubl_fns
[4] = {
6510 NULL
, gen_helper_sve2_usubl_h
,
6511 gen_helper_sve2_usubl_s
, gen_helper_sve2_usubl_d
,
6513 TRANS_FEAT(USUBLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6514 usubl_fns
[a
->esz
], a
, 0)
6515 TRANS_FEAT(USUBLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6516 usubl_fns
[a
->esz
], a
, 3)
6518 static gen_helper_gvec_3
* const uabdl_fns
[4] = {
6519 NULL
, gen_helper_sve2_uabdl_h
,
6520 gen_helper_sve2_uabdl_s
, gen_helper_sve2_uabdl_d
,
6522 TRANS_FEAT(UABDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6523 uabdl_fns
[a
->esz
], a
, 0)
6524 TRANS_FEAT(UABDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6525 uabdl_fns
[a
->esz
], a
, 3)
6527 static gen_helper_gvec_3
* const sqdmull_fns
[4] = {
6528 NULL
, gen_helper_sve2_sqdmull_zzz_h
,
6529 gen_helper_sve2_sqdmull_zzz_s
, gen_helper_sve2_sqdmull_zzz_d
,
6531 TRANS_FEAT(SQDMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6532 sqdmull_fns
[a
->esz
], a
, 0)
6533 TRANS_FEAT(SQDMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6534 sqdmull_fns
[a
->esz
], a
, 3)
6536 static gen_helper_gvec_3
* const smull_fns
[4] = {
6537 NULL
, gen_helper_sve2_smull_zzz_h
,
6538 gen_helper_sve2_smull_zzz_s
, gen_helper_sve2_smull_zzz_d
,
6540 TRANS_FEAT(SMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6541 smull_fns
[a
->esz
], a
, 0)
6542 TRANS_FEAT(SMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6543 smull_fns
[a
->esz
], a
, 3)
6545 static gen_helper_gvec_3
* const umull_fns
[4] = {
6546 NULL
, gen_helper_sve2_umull_zzz_h
,
6547 gen_helper_sve2_umull_zzz_s
, gen_helper_sve2_umull_zzz_d
,
6549 TRANS_FEAT(UMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6550 umull_fns
[a
->esz
], a
, 0)
6551 TRANS_FEAT(UMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6552 umull_fns
[a
->esz
], a
, 3)
6554 static gen_helper_gvec_3
* const eoril_fns
[4] = {
6555 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6556 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6558 TRANS_FEAT(EORBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, eoril_fns
[a
->esz
], a
, 2)
6559 TRANS_FEAT(EORTB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, eoril_fns
[a
->esz
], a
, 1)
6561 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6563 static gen_helper_gvec_3
* const fns
[4] = {
6564 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6565 NULL
, gen_helper_sve2_pmull_d
,
6567 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6570 return gen_gvec_ool_arg_zzz(s
, fns
[a
->esz
], a
, sel
);
6573 TRANS_FEAT(PMULLB
, aa64_sve2
, do_trans_pmull
, a
, false)
6574 TRANS_FEAT(PMULLT
, aa64_sve2
, do_trans_pmull
, a
, true)
6576 static gen_helper_gvec_3
* const saddw_fns
[4] = {
6577 NULL
, gen_helper_sve2_saddw_h
,
6578 gen_helper_sve2_saddw_s
, gen_helper_sve2_saddw_d
,
6580 TRANS_FEAT(SADDWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, saddw_fns
[a
->esz
], a
, 0)
6581 TRANS_FEAT(SADDWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, saddw_fns
[a
->esz
], a
, 1)
6583 static gen_helper_gvec_3
* const ssubw_fns
[4] = {
6584 NULL
, gen_helper_sve2_ssubw_h
,
6585 gen_helper_sve2_ssubw_s
, gen_helper_sve2_ssubw_d
,
6587 TRANS_FEAT(SSUBWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, ssubw_fns
[a
->esz
], a
, 0)
6588 TRANS_FEAT(SSUBWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, ssubw_fns
[a
->esz
], a
, 1)
6590 static gen_helper_gvec_3
* const uaddw_fns
[4] = {
6591 NULL
, gen_helper_sve2_uaddw_h
,
6592 gen_helper_sve2_uaddw_s
, gen_helper_sve2_uaddw_d
,
6594 TRANS_FEAT(UADDWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, uaddw_fns
[a
->esz
], a
, 0)
6595 TRANS_FEAT(UADDWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, uaddw_fns
[a
->esz
], a
, 1)
6597 static gen_helper_gvec_3
* const usubw_fns
[4] = {
6598 NULL
, gen_helper_sve2_usubw_h
,
6599 gen_helper_sve2_usubw_s
, gen_helper_sve2_usubw_d
,
6601 TRANS_FEAT(USUBWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, usubw_fns
[a
->esz
], a
, 0)
6602 TRANS_FEAT(USUBWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, usubw_fns
[a
->esz
], a
, 1)
6604 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6608 int halfbits
= 4 << vece
;
6611 if (shl
== halfbits
) {
6612 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6613 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6614 tcg_gen_and_vec(vece
, d
, n
, t
);
6615 tcg_temp_free_vec(t
);
6617 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
6618 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6621 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6622 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
6626 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
6628 int halfbits
= 4 << vece
;
6630 int shl
= (imm
>> 1);
6634 mask
= MAKE_64BIT_MASK(0, halfbits
);
6636 mask
= dup_const(vece
, mask
);
6638 shift
= shl
- top
* halfbits
;
6640 tcg_gen_shri_i64(d
, n
, -shift
);
6642 tcg_gen_shli_i64(d
, n
, shift
);
6644 tcg_gen_andi_i64(d
, d
, mask
);
6647 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6649 gen_ushll_i64(MO_16
, d
, n
, imm
);
6652 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6654 gen_ushll_i64(MO_32
, d
, n
, imm
);
6657 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6659 gen_ushll_i64(MO_64
, d
, n
, imm
);
6662 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6664 int halfbits
= 4 << vece
;
6669 if (shl
== halfbits
) {
6670 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6671 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6672 tcg_gen_and_vec(vece
, d
, n
, t
);
6673 tcg_temp_free_vec(t
);
6675 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
6676 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6680 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6681 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6682 tcg_gen_and_vec(vece
, d
, n
, t
);
6683 tcg_temp_free_vec(t
);
6685 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6686 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
6691 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
6694 static const TCGOpcode sshll_list
[] = {
6695 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
6697 static const TCGOpcode ushll_list
[] = {
6698 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
6700 static const GVecGen2i ops
[2][3] = {
6701 { { .fniv
= gen_sshll_vec
,
6702 .opt_opc
= sshll_list
,
6703 .fno
= gen_helper_sve2_sshll_h
,
6705 { .fniv
= gen_sshll_vec
,
6706 .opt_opc
= sshll_list
,
6707 .fno
= gen_helper_sve2_sshll_s
,
6709 { .fniv
= gen_sshll_vec
,
6710 .opt_opc
= sshll_list
,
6711 .fno
= gen_helper_sve2_sshll_d
,
6713 { { .fni8
= gen_ushll16_i64
,
6714 .fniv
= gen_ushll_vec
,
6715 .opt_opc
= ushll_list
,
6716 .fno
= gen_helper_sve2_ushll_h
,
6718 { .fni8
= gen_ushll32_i64
,
6719 .fniv
= gen_ushll_vec
,
6720 .opt_opc
= ushll_list
,
6721 .fno
= gen_helper_sve2_ushll_s
,
6723 { .fni8
= gen_ushll64_i64
,
6724 .fniv
= gen_ushll_vec
,
6725 .opt_opc
= ushll_list
,
6726 .fno
= gen_helper_sve2_ushll_d
,
6730 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
6733 if (sve_access_check(s
)) {
6734 unsigned vsz
= vec_full_reg_size(s
);
6735 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6736 vec_full_reg_offset(s
, a
->rn
),
6737 vsz
, vsz
, (a
->imm
<< 1) | sel
,
6743 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6745 return do_sve2_shll_tb(s
, a
, false, false);
6748 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6750 return do_sve2_shll_tb(s
, a
, true, false);
6753 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6755 return do_sve2_shll_tb(s
, a
, false, true);
6758 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6760 return do_sve2_shll_tb(s
, a
, true, true);
6763 static gen_helper_gvec_3
* const bext_fns
[4] = {
6764 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
6765 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
6767 TRANS_FEAT(BEXT
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6768 bext_fns
[a
->esz
], a
, 0)
6770 static gen_helper_gvec_3
* const bdep_fns
[4] = {
6771 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
6772 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
6774 TRANS_FEAT(BDEP
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6775 bdep_fns
[a
->esz
], a
, 0)
6777 static gen_helper_gvec_3
* const bgrp_fns
[4] = {
6778 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
6779 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
6781 TRANS_FEAT(BGRP
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6782 bgrp_fns
[a
->esz
], a
, 0)
6784 static gen_helper_gvec_3
* const cadd_fns
[4] = {
6785 gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
6786 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
,
6788 TRANS_FEAT(CADD_rot90
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6789 cadd_fns
[a
->esz
], a
, 0)
6790 TRANS_FEAT(CADD_rot270
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6791 cadd_fns
[a
->esz
], a
, 1)
6793 static gen_helper_gvec_3
* const sqcadd_fns
[4] = {
6794 gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
6795 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
,
6797 TRANS_FEAT(SQCADD_rot90
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6798 sqcadd_fns
[a
->esz
], a
, 0)
6799 TRANS_FEAT(SQCADD_rot270
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6800 sqcadd_fns
[a
->esz
], a
, 1)
6802 static gen_helper_gvec_4
* const sabal_fns
[4] = {
6803 NULL
, gen_helper_sve2_sabal_h
,
6804 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
,
6806 TRANS_FEAT(SABALB
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, sabal_fns
[a
->esz
], a
, 0)
6807 TRANS_FEAT(SABALT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, sabal_fns
[a
->esz
], a
, 1)
6809 static gen_helper_gvec_4
* const uabal_fns
[4] = {
6810 NULL
, gen_helper_sve2_uabal_h
,
6811 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
,
6813 TRANS_FEAT(UABALB
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, uabal_fns
[a
->esz
], a
, 0)
6814 TRANS_FEAT(UABALT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, uabal_fns
[a
->esz
], a
, 1)
6816 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
6818 static gen_helper_gvec_4
* const fns
[2] = {
6819 gen_helper_sve2_adcl_s
,
6820 gen_helper_sve2_adcl_d
,
6823 * Note that in this case the ESZ field encodes both size and sign.
6824 * Split out 'subtract' into bit 1 of the data field for the helper.
6826 return gen_gvec_ool_arg_zzzz(s
, fns
[a
->esz
& 1], a
, (a
->esz
& 2) | sel
);
6829 TRANS_FEAT(ADCLB
, aa64_sve2
, do_adcl
, a
, false)
6830 TRANS_FEAT(ADCLT
, aa64_sve2
, do_adcl
, a
, true)
6832 static bool do_sve2_fn2i(DisasContext
*s
, arg_rri_esz
*a
, GVecGen2iFn
*fn
)
6834 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
6837 return gen_gvec_fn_zzi(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->imm
);
6840 static bool trans_SSRA(DisasContext
*s
, arg_rri_esz
*a
)
6842 return do_sve2_fn2i(s
, a
, gen_gvec_ssra
);
6845 static bool trans_USRA(DisasContext
*s
, arg_rri_esz
*a
)
6847 return do_sve2_fn2i(s
, a
, gen_gvec_usra
);
6850 static bool trans_SRSRA(DisasContext
*s
, arg_rri_esz
*a
)
6852 return do_sve2_fn2i(s
, a
, gen_gvec_srsra
);
6855 static bool trans_URSRA(DisasContext
*s
, arg_rri_esz
*a
)
6857 return do_sve2_fn2i(s
, a
, gen_gvec_ursra
);
6860 static bool trans_SRI(DisasContext
*s
, arg_rri_esz
*a
)
6862 return do_sve2_fn2i(s
, a
, gen_gvec_sri
);
6865 static bool trans_SLI(DisasContext
*s
, arg_rri_esz
*a
)
6867 return do_sve2_fn2i(s
, a
, gen_gvec_sli
);
6870 TRANS_FEAT(SABA
, aa64_sve2
, gen_gvec_fn_arg_zzz
, gen_gvec_saba
, a
)
6871 TRANS_FEAT(UABA
, aa64_sve2
, gen_gvec_fn_arg_zzz
, gen_gvec_uaba
, a
)
6873 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
6874 const GVecGen2 ops
[3])
6876 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
6877 !dc_isar_feature(aa64_sve2
, s
)) {
6880 if (sve_access_check(s
)) {
6881 unsigned vsz
= vec_full_reg_size(s
);
6882 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
6883 vec_full_reg_offset(s
, a
->rn
),
6884 vsz
, vsz
, &ops
[a
->esz
]);
6889 static const TCGOpcode sqxtn_list
[] = {
6890 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
6893 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6895 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6896 int halfbits
= 4 << vece
;
6897 int64_t mask
= (1ull << halfbits
) - 1;
6898 int64_t min
= -1ull << (halfbits
- 1);
6899 int64_t max
= -min
- 1;
6901 tcg_gen_dupi_vec(vece
, t
, min
);
6902 tcg_gen_smax_vec(vece
, d
, n
, t
);
6903 tcg_gen_dupi_vec(vece
, t
, max
);
6904 tcg_gen_smin_vec(vece
, d
, d
, t
);
6905 tcg_gen_dupi_vec(vece
, t
, mask
);
6906 tcg_gen_and_vec(vece
, d
, d
, t
);
6907 tcg_temp_free_vec(t
);
6910 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
6912 static const GVecGen2 ops
[3] = {
6913 { .fniv
= gen_sqxtnb_vec
,
6914 .opt_opc
= sqxtn_list
,
6915 .fno
= gen_helper_sve2_sqxtnb_h
,
6917 { .fniv
= gen_sqxtnb_vec
,
6918 .opt_opc
= sqxtn_list
,
6919 .fno
= gen_helper_sve2_sqxtnb_s
,
6921 { .fniv
= gen_sqxtnb_vec
,
6922 .opt_opc
= sqxtn_list
,
6923 .fno
= gen_helper_sve2_sqxtnb_d
,
6926 return do_sve2_narrow_extract(s
, a
, ops
);
6929 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6931 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6932 int halfbits
= 4 << vece
;
6933 int64_t mask
= (1ull << halfbits
) - 1;
6934 int64_t min
= -1ull << (halfbits
- 1);
6935 int64_t max
= -min
- 1;
6937 tcg_gen_dupi_vec(vece
, t
, min
);
6938 tcg_gen_smax_vec(vece
, n
, n
, t
);
6939 tcg_gen_dupi_vec(vece
, t
, max
);
6940 tcg_gen_smin_vec(vece
, n
, n
, t
);
6941 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
6942 tcg_gen_dupi_vec(vece
, t
, mask
);
6943 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
6944 tcg_temp_free_vec(t
);
6947 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
6949 static const GVecGen2 ops
[3] = {
6950 { .fniv
= gen_sqxtnt_vec
,
6951 .opt_opc
= sqxtn_list
,
6953 .fno
= gen_helper_sve2_sqxtnt_h
,
6955 { .fniv
= gen_sqxtnt_vec
,
6956 .opt_opc
= sqxtn_list
,
6958 .fno
= gen_helper_sve2_sqxtnt_s
,
6960 { .fniv
= gen_sqxtnt_vec
,
6961 .opt_opc
= sqxtn_list
,
6963 .fno
= gen_helper_sve2_sqxtnt_d
,
6966 return do_sve2_narrow_extract(s
, a
, ops
);
6969 static const TCGOpcode uqxtn_list
[] = {
6970 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
6973 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6975 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6976 int halfbits
= 4 << vece
;
6977 int64_t max
= (1ull << halfbits
) - 1;
6979 tcg_gen_dupi_vec(vece
, t
, max
);
6980 tcg_gen_umin_vec(vece
, d
, n
, t
);
6981 tcg_temp_free_vec(t
);
6984 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
6986 static const GVecGen2 ops
[3] = {
6987 { .fniv
= gen_uqxtnb_vec
,
6988 .opt_opc
= uqxtn_list
,
6989 .fno
= gen_helper_sve2_uqxtnb_h
,
6991 { .fniv
= gen_uqxtnb_vec
,
6992 .opt_opc
= uqxtn_list
,
6993 .fno
= gen_helper_sve2_uqxtnb_s
,
6995 { .fniv
= gen_uqxtnb_vec
,
6996 .opt_opc
= uqxtn_list
,
6997 .fno
= gen_helper_sve2_uqxtnb_d
,
7000 return do_sve2_narrow_extract(s
, a
, ops
);
7003 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7005 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7006 int halfbits
= 4 << vece
;
7007 int64_t max
= (1ull << halfbits
) - 1;
7009 tcg_gen_dupi_vec(vece
, t
, max
);
7010 tcg_gen_umin_vec(vece
, n
, n
, t
);
7011 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7012 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7013 tcg_temp_free_vec(t
);
7016 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7018 static const GVecGen2 ops
[3] = {
7019 { .fniv
= gen_uqxtnt_vec
,
7020 .opt_opc
= uqxtn_list
,
7022 .fno
= gen_helper_sve2_uqxtnt_h
,
7024 { .fniv
= gen_uqxtnt_vec
,
7025 .opt_opc
= uqxtn_list
,
7027 .fno
= gen_helper_sve2_uqxtnt_s
,
7029 { .fniv
= gen_uqxtnt_vec
,
7030 .opt_opc
= uqxtn_list
,
7032 .fno
= gen_helper_sve2_uqxtnt_d
,
7035 return do_sve2_narrow_extract(s
, a
, ops
);
7038 static const TCGOpcode sqxtun_list
[] = {
7039 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
7042 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7044 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7045 int halfbits
= 4 << vece
;
7046 int64_t max
= (1ull << halfbits
) - 1;
7048 tcg_gen_dupi_vec(vece
, t
, 0);
7049 tcg_gen_smax_vec(vece
, d
, n
, t
);
7050 tcg_gen_dupi_vec(vece
, t
, max
);
7051 tcg_gen_umin_vec(vece
, d
, d
, t
);
7052 tcg_temp_free_vec(t
);
7055 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
7057 static const GVecGen2 ops
[3] = {
7058 { .fniv
= gen_sqxtunb_vec
,
7059 .opt_opc
= sqxtun_list
,
7060 .fno
= gen_helper_sve2_sqxtunb_h
,
7062 { .fniv
= gen_sqxtunb_vec
,
7063 .opt_opc
= sqxtun_list
,
7064 .fno
= gen_helper_sve2_sqxtunb_s
,
7066 { .fniv
= gen_sqxtunb_vec
,
7067 .opt_opc
= sqxtun_list
,
7068 .fno
= gen_helper_sve2_sqxtunb_d
,
7071 return do_sve2_narrow_extract(s
, a
, ops
);
7074 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7076 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7077 int halfbits
= 4 << vece
;
7078 int64_t max
= (1ull << halfbits
) - 1;
7080 tcg_gen_dupi_vec(vece
, t
, 0);
7081 tcg_gen_smax_vec(vece
, n
, n
, t
);
7082 tcg_gen_dupi_vec(vece
, t
, max
);
7083 tcg_gen_umin_vec(vece
, n
, n
, t
);
7084 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7085 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7086 tcg_temp_free_vec(t
);
7089 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
7091 static const GVecGen2 ops
[3] = {
7092 { .fniv
= gen_sqxtunt_vec
,
7093 .opt_opc
= sqxtun_list
,
7095 .fno
= gen_helper_sve2_sqxtunt_h
,
7097 { .fniv
= gen_sqxtunt_vec
,
7098 .opt_opc
= sqxtun_list
,
7100 .fno
= gen_helper_sve2_sqxtunt_s
,
7102 { .fniv
= gen_sqxtunt_vec
,
7103 .opt_opc
= sqxtun_list
,
7105 .fno
= gen_helper_sve2_sqxtunt_d
,
7108 return do_sve2_narrow_extract(s
, a
, ops
);
7111 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
7112 const GVecGen2i ops
[3])
7114 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
7117 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
7118 if (sve_access_check(s
)) {
7119 unsigned vsz
= vec_full_reg_size(s
);
7120 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7121 vec_full_reg_offset(s
, a
->rn
),
7122 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
7127 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7129 int halfbits
= 4 << vece
;
7130 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7132 tcg_gen_shri_i64(d
, n
, shr
);
7133 tcg_gen_andi_i64(d
, d
, mask
);
7136 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7138 gen_shrnb_i64(MO_16
, d
, n
, shr
);
7141 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7143 gen_shrnb_i64(MO_32
, d
, n
, shr
);
7146 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7148 gen_shrnb_i64(MO_64
, d
, n
, shr
);
7151 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7153 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7154 int halfbits
= 4 << vece
;
7155 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7157 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7158 tcg_gen_dupi_vec(vece
, t
, mask
);
7159 tcg_gen_and_vec(vece
, d
, n
, t
);
7160 tcg_temp_free_vec(t
);
7163 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7165 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
7166 static const GVecGen2i ops
[3] = {
7167 { .fni8
= gen_shrnb16_i64
,
7168 .fniv
= gen_shrnb_vec
,
7169 .opt_opc
= vec_list
,
7170 .fno
= gen_helper_sve2_shrnb_h
,
7172 { .fni8
= gen_shrnb32_i64
,
7173 .fniv
= gen_shrnb_vec
,
7174 .opt_opc
= vec_list
,
7175 .fno
= gen_helper_sve2_shrnb_s
,
7177 { .fni8
= gen_shrnb64_i64
,
7178 .fniv
= gen_shrnb_vec
,
7179 .opt_opc
= vec_list
,
7180 .fno
= gen_helper_sve2_shrnb_d
,
7183 return do_sve2_shr_narrow(s
, a
, ops
);
7186 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7188 int halfbits
= 4 << vece
;
7189 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7191 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
7192 tcg_gen_andi_i64(n
, n
, ~mask
);
7193 tcg_gen_andi_i64(d
, d
, mask
);
7194 tcg_gen_or_i64(d
, d
, n
);
7197 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7199 gen_shrnt_i64(MO_16
, d
, n
, shr
);
7202 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7204 gen_shrnt_i64(MO_32
, d
, n
, shr
);
7207 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7209 tcg_gen_shri_i64(n
, n
, shr
);
7210 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
7213 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7215 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7216 int halfbits
= 4 << vece
;
7217 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7219 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
7220 tcg_gen_dupi_vec(vece
, t
, mask
);
7221 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7222 tcg_temp_free_vec(t
);
7225 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7227 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
7228 static const GVecGen2i ops
[3] = {
7229 { .fni8
= gen_shrnt16_i64
,
7230 .fniv
= gen_shrnt_vec
,
7231 .opt_opc
= vec_list
,
7233 .fno
= gen_helper_sve2_shrnt_h
,
7235 { .fni8
= gen_shrnt32_i64
,
7236 .fniv
= gen_shrnt_vec
,
7237 .opt_opc
= vec_list
,
7239 .fno
= gen_helper_sve2_shrnt_s
,
7241 { .fni8
= gen_shrnt64_i64
,
7242 .fniv
= gen_shrnt_vec
,
7243 .opt_opc
= vec_list
,
7245 .fno
= gen_helper_sve2_shrnt_d
,
7248 return do_sve2_shr_narrow(s
, a
, ops
);
7251 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7253 static const GVecGen2i ops
[3] = {
7254 { .fno
= gen_helper_sve2_rshrnb_h
},
7255 { .fno
= gen_helper_sve2_rshrnb_s
},
7256 { .fno
= gen_helper_sve2_rshrnb_d
},
7258 return do_sve2_shr_narrow(s
, a
, ops
);
7261 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7263 static const GVecGen2i ops
[3] = {
7264 { .fno
= gen_helper_sve2_rshrnt_h
},
7265 { .fno
= gen_helper_sve2_rshrnt_s
},
7266 { .fno
= gen_helper_sve2_rshrnt_d
},
7268 return do_sve2_shr_narrow(s
, a
, ops
);
7271 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
7272 TCGv_vec n
, int64_t shr
)
7274 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7275 int halfbits
= 4 << vece
;
7277 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7278 tcg_gen_dupi_vec(vece
, t
, 0);
7279 tcg_gen_smax_vec(vece
, n
, n
, t
);
7280 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7281 tcg_gen_umin_vec(vece
, d
, n
, t
);
7282 tcg_temp_free_vec(t
);
7285 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7287 static const TCGOpcode vec_list
[] = {
7288 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7290 static const GVecGen2i ops
[3] = {
7291 { .fniv
= gen_sqshrunb_vec
,
7292 .opt_opc
= vec_list
,
7293 .fno
= gen_helper_sve2_sqshrunb_h
,
7295 { .fniv
= gen_sqshrunb_vec
,
7296 .opt_opc
= vec_list
,
7297 .fno
= gen_helper_sve2_sqshrunb_s
,
7299 { .fniv
= gen_sqshrunb_vec
,
7300 .opt_opc
= vec_list
,
7301 .fno
= gen_helper_sve2_sqshrunb_d
,
7304 return do_sve2_shr_narrow(s
, a
, ops
);
7307 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7308 TCGv_vec n
, int64_t shr
)
7310 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7311 int halfbits
= 4 << vece
;
7313 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7314 tcg_gen_dupi_vec(vece
, t
, 0);
7315 tcg_gen_smax_vec(vece
, n
, n
, t
);
7316 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7317 tcg_gen_umin_vec(vece
, n
, n
, t
);
7318 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7319 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7320 tcg_temp_free_vec(t
);
7323 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7325 static const TCGOpcode vec_list
[] = {
7326 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7327 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7329 static const GVecGen2i ops
[3] = {
7330 { .fniv
= gen_sqshrunt_vec
,
7331 .opt_opc
= vec_list
,
7333 .fno
= gen_helper_sve2_sqshrunt_h
,
7335 { .fniv
= gen_sqshrunt_vec
,
7336 .opt_opc
= vec_list
,
7338 .fno
= gen_helper_sve2_sqshrunt_s
,
7340 { .fniv
= gen_sqshrunt_vec
,
7341 .opt_opc
= vec_list
,
7343 .fno
= gen_helper_sve2_sqshrunt_d
,
7346 return do_sve2_shr_narrow(s
, a
, ops
);
7349 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7351 static const GVecGen2i ops
[3] = {
7352 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7353 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7354 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7356 return do_sve2_shr_narrow(s
, a
, ops
);
7359 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7361 static const GVecGen2i ops
[3] = {
7362 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7363 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7364 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7366 return do_sve2_shr_narrow(s
, a
, ops
);
7369 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7370 TCGv_vec n
, int64_t shr
)
7372 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7373 int halfbits
= 4 << vece
;
7374 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7375 int64_t min
= -max
- 1;
7377 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7378 tcg_gen_dupi_vec(vece
, t
, min
);
7379 tcg_gen_smax_vec(vece
, n
, n
, t
);
7380 tcg_gen_dupi_vec(vece
, t
, max
);
7381 tcg_gen_smin_vec(vece
, n
, n
, t
);
7382 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7383 tcg_gen_and_vec(vece
, d
, n
, t
);
7384 tcg_temp_free_vec(t
);
7387 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7389 static const TCGOpcode vec_list
[] = {
7390 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7392 static const GVecGen2i ops
[3] = {
7393 { .fniv
= gen_sqshrnb_vec
,
7394 .opt_opc
= vec_list
,
7395 .fno
= gen_helper_sve2_sqshrnb_h
,
7397 { .fniv
= gen_sqshrnb_vec
,
7398 .opt_opc
= vec_list
,
7399 .fno
= gen_helper_sve2_sqshrnb_s
,
7401 { .fniv
= gen_sqshrnb_vec
,
7402 .opt_opc
= vec_list
,
7403 .fno
= gen_helper_sve2_sqshrnb_d
,
7406 return do_sve2_shr_narrow(s
, a
, ops
);
7409 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7410 TCGv_vec n
, int64_t shr
)
7412 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7413 int halfbits
= 4 << vece
;
7414 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7415 int64_t min
= -max
- 1;
7417 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7418 tcg_gen_dupi_vec(vece
, t
, min
);
7419 tcg_gen_smax_vec(vece
, n
, n
, t
);
7420 tcg_gen_dupi_vec(vece
, t
, max
);
7421 tcg_gen_smin_vec(vece
, n
, n
, t
);
7422 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7423 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7424 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7425 tcg_temp_free_vec(t
);
7428 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7430 static const TCGOpcode vec_list
[] = {
7431 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7432 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7434 static const GVecGen2i ops
[3] = {
7435 { .fniv
= gen_sqshrnt_vec
,
7436 .opt_opc
= vec_list
,
7438 .fno
= gen_helper_sve2_sqshrnt_h
,
7440 { .fniv
= gen_sqshrnt_vec
,
7441 .opt_opc
= vec_list
,
7443 .fno
= gen_helper_sve2_sqshrnt_s
,
7445 { .fniv
= gen_sqshrnt_vec
,
7446 .opt_opc
= vec_list
,
7448 .fno
= gen_helper_sve2_sqshrnt_d
,
7451 return do_sve2_shr_narrow(s
, a
, ops
);
7454 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7456 static const GVecGen2i ops
[3] = {
7457 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7458 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7459 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7461 return do_sve2_shr_narrow(s
, a
, ops
);
7464 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7466 static const GVecGen2i ops
[3] = {
7467 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7468 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7469 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7471 return do_sve2_shr_narrow(s
, a
, ops
);
7474 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7475 TCGv_vec n
, int64_t shr
)
7477 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7478 int halfbits
= 4 << vece
;
7480 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7481 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7482 tcg_gen_umin_vec(vece
, d
, n
, t
);
7483 tcg_temp_free_vec(t
);
7486 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7488 static const TCGOpcode vec_list
[] = {
7489 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7491 static const GVecGen2i ops
[3] = {
7492 { .fniv
= gen_uqshrnb_vec
,
7493 .opt_opc
= vec_list
,
7494 .fno
= gen_helper_sve2_uqshrnb_h
,
7496 { .fniv
= gen_uqshrnb_vec
,
7497 .opt_opc
= vec_list
,
7498 .fno
= gen_helper_sve2_uqshrnb_s
,
7500 { .fniv
= gen_uqshrnb_vec
,
7501 .opt_opc
= vec_list
,
7502 .fno
= gen_helper_sve2_uqshrnb_d
,
7505 return do_sve2_shr_narrow(s
, a
, ops
);
7508 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
7509 TCGv_vec n
, int64_t shr
)
7511 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7512 int halfbits
= 4 << vece
;
7514 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7515 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7516 tcg_gen_umin_vec(vece
, n
, n
, t
);
7517 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7518 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7519 tcg_temp_free_vec(t
);
7522 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7524 static const TCGOpcode vec_list
[] = {
7525 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7527 static const GVecGen2i ops
[3] = {
7528 { .fniv
= gen_uqshrnt_vec
,
7529 .opt_opc
= vec_list
,
7531 .fno
= gen_helper_sve2_uqshrnt_h
,
7533 { .fniv
= gen_uqshrnt_vec
,
7534 .opt_opc
= vec_list
,
7536 .fno
= gen_helper_sve2_uqshrnt_s
,
7538 { .fniv
= gen_uqshrnt_vec
,
7539 .opt_opc
= vec_list
,
7541 .fno
= gen_helper_sve2_uqshrnt_d
,
7544 return do_sve2_shr_narrow(s
, a
, ops
);
7547 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7549 static const GVecGen2i ops
[3] = {
7550 { .fno
= gen_helper_sve2_uqrshrnb_h
},
7551 { .fno
= gen_helper_sve2_uqrshrnb_s
},
7552 { .fno
= gen_helper_sve2_uqrshrnb_d
},
7554 return do_sve2_shr_narrow(s
, a
, ops
);
7557 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7559 static const GVecGen2i ops
[3] = {
7560 { .fno
= gen_helper_sve2_uqrshrnt_h
},
7561 { .fno
= gen_helper_sve2_uqrshrnt_s
},
7562 { .fno
= gen_helper_sve2_uqrshrnt_d
},
7564 return do_sve2_shr_narrow(s
, a
, ops
);
7567 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
7568 static gen_helper_gvec_3 * const name##_fns[4] = { \
7569 NULL, gen_helper_sve2_##name##_h, \
7570 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7572 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzz, \
7573 name##_fns[a->esz], a, 0)
7575 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
7576 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
7577 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
7578 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
7580 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
7581 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
7582 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
7583 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
7585 static bool do_sve2_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
7586 gen_helper_gvec_flags_4
*fn
)
7588 if (!dc_isar_feature(aa64_sve2
, s
)) {
7591 return do_ppzz_flags(s
, a
, fn
);
7594 #define DO_SVE2_PPZZ_MATCH(NAME, name) \
7595 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7597 static gen_helper_gvec_flags_4 * const fns[4] = { \
7598 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
7601 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
7604 DO_SVE2_PPZZ_MATCH(MATCH
, match
)
7605 DO_SVE2_PPZZ_MATCH(NMATCH
, nmatch
)
7607 static gen_helper_gvec_4
* const histcnt_fns
[4] = {
7608 NULL
, NULL
, gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
7610 TRANS_FEAT(HISTCNT
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
7611 histcnt_fns
[a
->esz
], a
, 0)
7613 TRANS_FEAT(HISTSEG
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
7614 a
->esz
== 0 ? gen_helper_sve2_histseg
: NULL
, a
, 0)
7616 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
7617 gen_helper_gvec_4_ptr
*fn
)
7619 if (!dc_isar_feature(aa64_sve2
, s
)) {
7622 return do_zpzz_fp(s
, a
, fn
);
7625 #define DO_SVE2_ZPZZ_FP(NAME, name) \
7626 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7628 static gen_helper_gvec_4_ptr * const fns[4] = { \
7629 NULL, gen_helper_sve2_##name##_zpzz_h, \
7630 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
7632 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
7635 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
7636 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
7637 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
7638 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
7639 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
7642 * SVE Integer Multiply-Add (unpredicated)
7645 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
7647 gen_helper_gvec_4_ptr
*fn
;
7651 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
7654 fn
= gen_helper_fmmla_s
;
7657 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
7660 fn
= gen_helper_fmmla_d
;
7666 if (sve_access_check(s
)) {
7667 unsigned vsz
= vec_full_reg_size(s
);
7668 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7669 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7670 vec_full_reg_offset(s
, a
->rn
),
7671 vec_full_reg_offset(s
, a
->rm
),
7672 vec_full_reg_offset(s
, a
->ra
),
7673 status
, vsz
, vsz
, 0, fn
);
7674 tcg_temp_free_ptr(status
);
7679 static gen_helper_gvec_4
* const sqdmlal_zzzw_fns
[] = {
7680 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
7681 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
7683 TRANS_FEAT(SQDMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7684 sqdmlal_zzzw_fns
[a
->esz
], a
, 0)
7685 TRANS_FEAT(SQDMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7686 sqdmlal_zzzw_fns
[a
->esz
], a
, 3)
7687 TRANS_FEAT(SQDMLALBT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7688 sqdmlal_zzzw_fns
[a
->esz
], a
, 2)
7690 static gen_helper_gvec_4
* const sqdmlsl_zzzw_fns
[] = {
7691 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
7692 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
7694 TRANS_FEAT(SQDMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7695 sqdmlsl_zzzw_fns
[a
->esz
], a
, 0)
7696 TRANS_FEAT(SQDMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7697 sqdmlsl_zzzw_fns
[a
->esz
], a
, 3)
7698 TRANS_FEAT(SQDMLSLBT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7699 sqdmlsl_zzzw_fns
[a
->esz
], a
, 2)
7701 static gen_helper_gvec_4
* const sqrdmlah_fns
[] = {
7702 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
7703 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
7705 TRANS_FEAT(SQRDMLAH_zzzz
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7706 sqrdmlah_fns
[a
->esz
], a
, 0)
7708 static gen_helper_gvec_4
* const sqrdmlsh_fns
[] = {
7709 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
7710 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
7712 TRANS_FEAT(SQRDMLSH_zzzz
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7713 sqrdmlsh_fns
[a
->esz
], a
, 0)
7715 static gen_helper_gvec_4
* const smlal_zzzw_fns
[] = {
7716 NULL
, gen_helper_sve2_smlal_zzzw_h
,
7717 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
7719 TRANS_FEAT(SMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7720 smlal_zzzw_fns
[a
->esz
], a
, 0)
7721 TRANS_FEAT(SMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7722 smlal_zzzw_fns
[a
->esz
], a
, 1)
7724 static gen_helper_gvec_4
* const umlal_zzzw_fns
[] = {
7725 NULL
, gen_helper_sve2_umlal_zzzw_h
,
7726 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
7728 TRANS_FEAT(UMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7729 umlal_zzzw_fns
[a
->esz
], a
, 0)
7730 TRANS_FEAT(UMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7731 umlal_zzzw_fns
[a
->esz
], a
, 1)
7733 static gen_helper_gvec_4
* const smlsl_zzzw_fns
[] = {
7734 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
7735 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
7737 TRANS_FEAT(SMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7738 smlsl_zzzw_fns
[a
->esz
], a
, 0)
7739 TRANS_FEAT(SMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7740 smlsl_zzzw_fns
[a
->esz
], a
, 1)
7742 static gen_helper_gvec_4
* const umlsl_zzzw_fns
[] = {
7743 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
7744 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
7746 TRANS_FEAT(UMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7747 umlsl_zzzw_fns
[a
->esz
], a
, 0)
7748 TRANS_FEAT(UMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7749 umlsl_zzzw_fns
[a
->esz
], a
, 1)
7751 static gen_helper_gvec_4
* const cmla_fns
[] = {
7752 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
7753 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
7755 TRANS_FEAT(CMLA_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7756 cmla_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7758 static gen_helper_gvec_4
* const cdot_fns
[] = {
7759 NULL
, NULL
, gen_helper_sve2_cdot_zzzz_s
, gen_helper_sve2_cdot_zzzz_d
7761 TRANS_FEAT(CDOT_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7762 cdot_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7764 static gen_helper_gvec_4
* const sqrdcmlah_fns
[] = {
7765 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
7766 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
7768 TRANS_FEAT(SQRDCMLAH_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7769 sqrdcmlah_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7771 TRANS_FEAT(USDOT_zzzz
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7772 a
->esz
== 2 ? gen_helper_gvec_usdot_b
: NULL
, a
, 0)
7774 TRANS_FEAT(AESMC
, aa64_sve2_aes
, gen_gvec_ool_zz
,
7775 gen_helper_crypto_aesmc
, a
->rd
, a
->rd
, a
->decrypt
)
7777 TRANS_FEAT(AESE
, aa64_sve2_aes
, gen_gvec_ool_arg_zzz
,
7778 gen_helper_crypto_aese
, a
, false)
7779 TRANS_FEAT(AESD
, aa64_sve2_aes
, gen_gvec_ool_arg_zzz
,
7780 gen_helper_crypto_aese
, a
, true)
7782 TRANS_FEAT(SM4E
, aa64_sve2_sm4
, gen_gvec_ool_arg_zzz
,
7783 gen_helper_crypto_sm4e
, a
, 0)
7784 TRANS_FEAT(SM4EKEY
, aa64_sve2_sm4
, gen_gvec_ool_arg_zzz
,
7785 gen_helper_crypto_sm4ekey
, a
, 0)
7787 TRANS_FEAT(RAX1
, aa64_sve2_sha3
, gen_gvec_fn_arg_zzz
, gen_gvec_rax1
, a
)
7789 static bool trans_FCVTNT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
7791 if (!dc_isar_feature(aa64_sve2
, s
)) {
7794 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_sh
);
7797 static bool trans_BFCVTNT(DisasContext
*s
, arg_rpr_esz
*a
)
7799 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7802 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvtnt
);
7805 static bool trans_FCVTNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7807 if (!dc_isar_feature(aa64_sve2
, s
)) {
7810 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_ds
);
7813 static bool trans_FCVTLT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
7815 if (!dc_isar_feature(aa64_sve2
, s
)) {
7818 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_hs
);
7821 static bool trans_FCVTLT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
7823 if (!dc_isar_feature(aa64_sve2
, s
)) {
7826 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_sd
);
7829 static bool trans_FCVTX_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7831 if (!dc_isar_feature(aa64_sve2
, s
)) {
7834 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve_fcvt_ds
);
7837 static bool trans_FCVTXNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7839 if (!dc_isar_feature(aa64_sve2
, s
)) {
7842 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve2_fcvtnt_ds
);
7845 static bool trans_FLOGB(DisasContext
*s
, arg_rpr_esz
*a
)
7847 static gen_helper_gvec_3_ptr
* const fns
[] = {
7848 NULL
, gen_helper_flogb_h
,
7849 gen_helper_flogb_s
, gen_helper_flogb_d
7852 if (!dc_isar_feature(aa64_sve2
, s
) || fns
[a
->esz
] == NULL
) {
7855 if (sve_access_check(s
)) {
7857 fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
7858 unsigned vsz
= vec_full_reg_size(s
);
7860 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
7861 vec_full_reg_offset(s
, a
->rn
),
7862 pred_full_reg_offset(s
, a
->pg
),
7863 status
, vsz
, vsz
, 0, fns
[a
->esz
]);
7864 tcg_temp_free_ptr(status
);
7869 static bool do_FMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sub
, bool sel
)
7871 if (!dc_isar_feature(aa64_sve2
, s
)) {
7874 if (sve_access_check(s
)) {
7875 unsigned vsz
= vec_full_reg_size(s
);
7876 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7877 vec_full_reg_offset(s
, a
->rn
),
7878 vec_full_reg_offset(s
, a
->rm
),
7879 vec_full_reg_offset(s
, a
->ra
),
7880 cpu_env
, vsz
, vsz
, (sel
<< 1) | sub
,
7881 gen_helper_sve2_fmlal_zzzw_s
);
7886 static bool trans_FMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7888 return do_FMLAL_zzzw(s
, a
, false, false);
7891 static bool trans_FMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7893 return do_FMLAL_zzzw(s
, a
, false, true);
7896 static bool trans_FMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7898 return do_FMLAL_zzzw(s
, a
, true, false);
7901 static bool trans_FMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7903 return do_FMLAL_zzzw(s
, a
, true, true);
7906 static bool do_FMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
, bool sel
)
7908 if (!dc_isar_feature(aa64_sve2
, s
)) {
7911 if (sve_access_check(s
)) {
7912 unsigned vsz
= vec_full_reg_size(s
);
7913 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7914 vec_full_reg_offset(s
, a
->rn
),
7915 vec_full_reg_offset(s
, a
->rm
),
7916 vec_full_reg_offset(s
, a
->ra
),
7918 (a
->index
<< 2) | (sel
<< 1) | sub
,
7919 gen_helper_sve2_fmlal_zzxw_s
);
7924 static bool trans_FMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7926 return do_FMLAL_zzxw(s
, a
, false, false);
7929 static bool trans_FMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7931 return do_FMLAL_zzxw(s
, a
, false, true);
7934 static bool trans_FMLSLB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7936 return do_FMLAL_zzxw(s
, a
, true, false);
7939 static bool trans_FMLSLT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7941 return do_FMLAL_zzxw(s
, a
, true, true);
7944 TRANS_FEAT(SMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7945 gen_helper_gvec_smmla_b
, a
, 0)
7946 TRANS_FEAT(USMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7947 gen_helper_gvec_usmmla_b
, a
, 0)
7948 TRANS_FEAT(UMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7949 gen_helper_gvec_ummla_b
, a
, 0)
7951 TRANS_FEAT(BFDOT_zzzz
, aa64_sve_bf16
, gen_gvec_ool_arg_zzzz
,
7952 gen_helper_gvec_bfdot
, a
, 0)
7953 TRANS_FEAT(BFDOT_zzxz
, aa64_sve_bf16
, gen_gvec_ool_arg_zzxz
,
7954 gen_helper_gvec_bfdot_idx
, a
)
7956 TRANS_FEAT(BFMMLA
, aa64_sve_bf16
, gen_gvec_ool_arg_zzzz
,
7957 gen_helper_gvec_bfmmla
, a
, 0)
7959 static bool do_BFMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7961 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7964 if (sve_access_check(s
)) {
7965 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7966 unsigned vsz
= vec_full_reg_size(s
);
7968 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7969 vec_full_reg_offset(s
, a
->rn
),
7970 vec_full_reg_offset(s
, a
->rm
),
7971 vec_full_reg_offset(s
, a
->ra
),
7972 status
, vsz
, vsz
, sel
,
7973 gen_helper_gvec_bfmlal
);
7974 tcg_temp_free_ptr(status
);
7979 static bool trans_BFMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7981 return do_BFMLAL_zzzw(s
, a
, false);
7984 static bool trans_BFMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7986 return do_BFMLAL_zzzw(s
, a
, true);
7989 static bool do_BFMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sel
)
7991 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7994 if (sve_access_check(s
)) {
7995 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7996 unsigned vsz
= vec_full_reg_size(s
);
7998 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7999 vec_full_reg_offset(s
, a
->rn
),
8000 vec_full_reg_offset(s
, a
->rm
),
8001 vec_full_reg_offset(s
, a
->ra
),
8002 status
, vsz
, vsz
, (a
->index
<< 1) | sel
,
8003 gen_helper_gvec_bfmlal_idx
);
8004 tcg_temp_free_ptr(status
);
8009 static bool trans_BFMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8011 return do_BFMLAL_zzxw(s
, a
, false);
8014 static bool trans_BFMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8016 return do_BFMLAL_zzxw(s
, a
, true);