]> git.proxmox.com Git - mirror_qemu.git/blame - target/arm/translate-sve.c
target/arm: Implement SVE2 saturating multiply-add (indexed)
[mirror_qemu.git] / target / arm / translate-sve.c
CommitLineData
38388f7e
RH
1/*
2 * AArch64 SVE translation
3 *
4 * Copyright (c) 2018 Linaro, Ltd
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
50f57e09 9 * version 2.1 of the License, or (at your option) any later version.
38388f7e
RH
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include "qemu/osdep.h"
21#include "cpu.h"
22#include "exec/exec-all.h"
dcb32f1d
PMD
23#include "tcg/tcg-op.h"
24#include "tcg/tcg-op-gvec.h"
25#include "tcg/tcg-gvec-desc.h"
38388f7e
RH
26#include "qemu/log.h"
27#include "arm_ldst.h"
28#include "translate.h"
29#include "internals.h"
30#include "exec/helper-proto.h"
31#include "exec/helper-gen.h"
32#include "exec/log.h"
33#include "trace-tcg.h"
34#include "translate-a64.h"
cc48affe 35#include "fpu/softfloat.h"
38388f7e 36
757f9cff 37
9ee3a611
RH
38typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
39 TCGv_i64, uint32_t, uint32_t);
40
38cadeba
RH
41typedef void gen_helper_gvec_flags_3(TCGv_i32, TCGv_ptr, TCGv_ptr,
42 TCGv_ptr, TCGv_i32);
757f9cff
RH
43typedef void gen_helper_gvec_flags_4(TCGv_i32, TCGv_ptr, TCGv_ptr,
44 TCGv_ptr, TCGv_ptr, TCGv_i32);
45
c4e7c493 46typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i32);
f6dbf62a
RH
47typedef void gen_helper_gvec_mem_scatter(TCGv_env, TCGv_ptr, TCGv_ptr,
48 TCGv_ptr, TCGv_i64, TCGv_i32);
c4e7c493 49
ccd841c3
RH
50/*
51 * Helpers for extracting complex instruction fields.
52 */
53
54/* See e.g. ASR (immediate, predicated).
55 * Returns -1 for unallocated encoding; diagnose later.
56 */
451e4ffd 57static int tszimm_esz(DisasContext *s, int x)
ccd841c3
RH
58{
59 x >>= 3; /* discard imm3 */
60 return 31 - clz32(x);
61}
62
451e4ffd 63static int tszimm_shr(DisasContext *s, int x)
ccd841c3 64{
451e4ffd 65 return (16 << tszimm_esz(s, x)) - x;
ccd841c3
RH
66}
67
68/* See e.g. LSL (immediate, predicated). */
451e4ffd 69static int tszimm_shl(DisasContext *s, int x)
ccd841c3 70{
451e4ffd 71 return x - (8 << tszimm_esz(s, x));
ccd841c3
RH
72}
73
451e4ffd 74static inline int plus1(DisasContext *s, int x)
24e82e68
RH
75{
76 return x + 1;
77}
78
f25a2361 79/* The SH bit is in bit 8. Extract the low 8 and shift. */
451e4ffd 80static inline int expand_imm_sh8s(DisasContext *s, int x)
f25a2361
RH
81{
82 return (int8_t)x << (x & 0x100 ? 8 : 0);
83}
84
451e4ffd 85static inline int expand_imm_sh8u(DisasContext *s, int x)
6e6a157d
RH
86{
87 return (uint8_t)x << (x & 0x100 ? 8 : 0);
88}
89
c4e7c493
RH
90/* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
91 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
92 */
451e4ffd 93static inline int msz_dtype(DisasContext *s, int msz)
c4e7c493
RH
94{
95 static const uint8_t dtype[4] = { 0, 5, 10, 15 };
96 return dtype[msz];
97}
98
38388f7e
RH
99/*
100 * Include the generated decoder.
101 */
102
139c1837 103#include "decode-sve.c.inc"
38388f7e
RH
104
105/*
106 * Implement all of the translator functions referenced by the decoder.
107 */
108
d1822297
RH
109/* Return the offset info CPUARMState of the predicate vector register Pn.
110 * Note for this purpose, FFR is P16.
111 */
112static inline int pred_full_reg_offset(DisasContext *s, int regno)
113{
114 return offsetof(CPUARMState, vfp.pregs[regno]);
115}
116
117/* Return the byte size of the whole predicate register, VL / 64. */
118static inline int pred_full_reg_size(DisasContext *s)
119{
120 return s->sve_len >> 3;
121}
122
516e246a
RH
123/* Round up the size of a register to a size allowed by
124 * the tcg vector infrastructure. Any operation which uses this
125 * size may assume that the bits above pred_full_reg_size are zero,
126 * and must leave them the same way.
127 *
128 * Note that this is not needed for the vector registers as they
129 * are always properly sized for tcg vectors.
130 */
131static int size_for_gvec(int size)
132{
133 if (size <= 8) {
134 return 8;
135 } else {
136 return QEMU_ALIGN_UP(size, 16);
137 }
138}
139
140static int pred_gvec_reg_size(DisasContext *s)
141{
142 return size_for_gvec(pred_full_reg_size(s));
143}
144
40e32e5a
RH
145/* Invoke an out-of-line helper on 2 Zregs. */
146static void gen_gvec_ool_zz(DisasContext *s, gen_helper_gvec_2 *fn,
147 int rd, int rn, int data)
148{
149 unsigned vsz = vec_full_reg_size(s);
150 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
151 vec_full_reg_offset(s, rn),
152 vsz, vsz, data, fn);
153}
154
e645d1a1
RH
155/* Invoke an out-of-line helper on 3 Zregs. */
156static void gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn,
157 int rd, int rn, int rm, int data)
158{
159 unsigned vsz = vec_full_reg_size(s);
160 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
161 vec_full_reg_offset(s, rn),
162 vec_full_reg_offset(s, rm),
163 vsz, vsz, data, fn);
164}
165
38650638
RH
166/* Invoke an out-of-line helper on 4 Zregs. */
167static void gen_gvec_ool_zzzz(DisasContext *s, gen_helper_gvec_4 *fn,
168 int rd, int rn, int rm, int ra, int data)
169{
170 unsigned vsz = vec_full_reg_size(s);
171 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
172 vec_full_reg_offset(s, rn),
173 vec_full_reg_offset(s, rm),
174 vec_full_reg_offset(s, ra),
175 vsz, vsz, data, fn);
176}
177
96a461f7
RH
178/* Invoke an out-of-line helper on 2 Zregs and a predicate. */
179static void gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn,
180 int rd, int rn, int pg, int data)
181{
182 unsigned vsz = vec_full_reg_size(s);
183 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
184 vec_full_reg_offset(s, rn),
185 pred_full_reg_offset(s, pg),
186 vsz, vsz, data, fn);
187}
188
36cbb7a8
RH
189/* Invoke an out-of-line helper on 3 Zregs and a predicate. */
190static void gen_gvec_ool_zzzp(DisasContext *s, gen_helper_gvec_4 *fn,
191 int rd, int rn, int rm, int pg, int data)
192{
193 unsigned vsz = vec_full_reg_size(s);
194 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
195 vec_full_reg_offset(s, rn),
196 vec_full_reg_offset(s, rm),
197 pred_full_reg_offset(s, pg),
198 vsz, vsz, data, fn);
199}
f7d79c41 200
36cbb7a8 201/* Invoke a vector expander on two Zregs. */
f7d79c41
RH
202static void gen_gvec_fn_zz(DisasContext *s, GVecGen2Fn *gvec_fn,
203 int esz, int rd, int rn)
38388f7e 204{
f7d79c41
RH
205 unsigned vsz = vec_full_reg_size(s);
206 gvec_fn(esz, vec_full_reg_offset(s, rd),
207 vec_full_reg_offset(s, rn), vsz, vsz);
38388f7e
RH
208}
209
39eea561 210/* Invoke a vector expander on three Zregs. */
28c4da31
RH
211static void gen_gvec_fn_zzz(DisasContext *s, GVecGen3Fn *gvec_fn,
212 int esz, int rd, int rn, int rm)
38388f7e 213{
28c4da31
RH
214 unsigned vsz = vec_full_reg_size(s);
215 gvec_fn(esz, vec_full_reg_offset(s, rd),
216 vec_full_reg_offset(s, rn),
217 vec_full_reg_offset(s, rm), vsz, vsz);
38388f7e
RH
218}
219
911cdc6d
RH
220/* Invoke a vector expander on four Zregs. */
221static void gen_gvec_fn_zzzz(DisasContext *s, GVecGen4Fn *gvec_fn,
222 int esz, int rd, int rn, int rm, int ra)
223{
224 unsigned vsz = vec_full_reg_size(s);
225 gvec_fn(esz, vec_full_reg_offset(s, rd),
226 vec_full_reg_offset(s, rn),
227 vec_full_reg_offset(s, rm),
228 vec_full_reg_offset(s, ra), vsz, vsz);
229}
230
39eea561
RH
231/* Invoke a vector move on two Zregs. */
232static bool do_mov_z(DisasContext *s, int rd, int rn)
38388f7e 233{
f7d79c41
RH
234 if (sve_access_check(s)) {
235 gen_gvec_fn_zz(s, tcg_gen_gvec_mov, MO_8, rd, rn);
236 }
237 return true;
38388f7e
RH
238}
239
d9d78dcc
RH
240/* Initialize a Zreg with replications of a 64-bit immediate. */
241static void do_dupi_z(DisasContext *s, int rd, uint64_t word)
242{
243 unsigned vsz = vec_full_reg_size(s);
8711e71f 244 tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), vsz, vsz, word);
d9d78dcc
RH
245}
246
516e246a 247/* Invoke a vector expander on three Pregs. */
dd81a8d7
RH
248static void gen_gvec_fn_ppp(DisasContext *s, GVecGen3Fn *gvec_fn,
249 int rd, int rn, int rm)
516e246a 250{
dd81a8d7
RH
251 unsigned psz = pred_gvec_reg_size(s);
252 gvec_fn(MO_64, pred_full_reg_offset(s, rd),
253 pred_full_reg_offset(s, rn),
254 pred_full_reg_offset(s, rm), psz, psz);
516e246a
RH
255}
256
257/* Invoke a vector move on two Pregs. */
258static bool do_mov_p(DisasContext *s, int rd, int rn)
259{
d0b2df5a
RH
260 if (sve_access_check(s)) {
261 unsigned psz = pred_gvec_reg_size(s);
262 tcg_gen_gvec_mov(MO_8, pred_full_reg_offset(s, rd),
263 pred_full_reg_offset(s, rn), psz, psz);
264 }
265 return true;
516e246a
RH
266}
267
9e18d7a6
RH
268/* Set the cpu flags as per a return from an SVE helper. */
269static void do_pred_flags(TCGv_i32 t)
270{
271 tcg_gen_mov_i32(cpu_NF, t);
272 tcg_gen_andi_i32(cpu_ZF, t, 2);
273 tcg_gen_andi_i32(cpu_CF, t, 1);
274 tcg_gen_movi_i32(cpu_VF, 0);
275}
276
277/* Subroutines computing the ARM PredTest psuedofunction. */
278static void do_predtest1(TCGv_i64 d, TCGv_i64 g)
279{
280 TCGv_i32 t = tcg_temp_new_i32();
281
282 gen_helper_sve_predtest1(t, d, g);
283 do_pred_flags(t);
284 tcg_temp_free_i32(t);
285}
286
287static void do_predtest(DisasContext *s, int dofs, int gofs, int words)
288{
289 TCGv_ptr dptr = tcg_temp_new_ptr();
290 TCGv_ptr gptr = tcg_temp_new_ptr();
291 TCGv_i32 t;
292
293 tcg_gen_addi_ptr(dptr, cpu_env, dofs);
294 tcg_gen_addi_ptr(gptr, cpu_env, gofs);
295 t = tcg_const_i32(words);
296
297 gen_helper_sve_predtest(t, dptr, gptr, t);
298 tcg_temp_free_ptr(dptr);
299 tcg_temp_free_ptr(gptr);
300
301 do_pred_flags(t);
302 tcg_temp_free_i32(t);
303}
304
028e2a7b
RH
305/* For each element size, the bits within a predicate word that are active. */
306const uint64_t pred_esz_masks[4] = {
307 0xffffffffffffffffull, 0x5555555555555555ull,
308 0x1111111111111111ull, 0x0101010101010101ull
309};
310
39eea561
RH
311/*
312 *** SVE Logical - Unpredicated Group
313 */
314
28c4da31
RH
315static bool do_zzz_fn(DisasContext *s, arg_rrr_esz *a, GVecGen3Fn *gvec_fn)
316{
317 if (sve_access_check(s)) {
318 gen_gvec_fn_zzz(s, gvec_fn, a->esz, a->rd, a->rn, a->rm);
319 }
320 return true;
321}
322
3a7be554 323static bool trans_AND_zzz(DisasContext *s, arg_rrr_esz *a)
39eea561 324{
28c4da31 325 return do_zzz_fn(s, a, tcg_gen_gvec_and);
39eea561
RH
326}
327
3a7be554 328static bool trans_ORR_zzz(DisasContext *s, arg_rrr_esz *a)
39eea561 329{
28c4da31 330 return do_zzz_fn(s, a, tcg_gen_gvec_or);
39eea561
RH
331}
332
3a7be554 333static bool trans_EOR_zzz(DisasContext *s, arg_rrr_esz *a)
39eea561 334{
28c4da31 335 return do_zzz_fn(s, a, tcg_gen_gvec_xor);
39eea561
RH
336}
337
3a7be554 338static bool trans_BIC_zzz(DisasContext *s, arg_rrr_esz *a)
38388f7e 339{
28c4da31 340 return do_zzz_fn(s, a, tcg_gen_gvec_andc);
38388f7e 341}
d1822297 342
e6eba6e5
RH
343static void gen_xar8_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
344{
345 TCGv_i64 t = tcg_temp_new_i64();
346 uint64_t mask = dup_const(MO_8, 0xff >> sh);
347
348 tcg_gen_xor_i64(t, n, m);
349 tcg_gen_shri_i64(d, t, sh);
350 tcg_gen_shli_i64(t, t, 8 - sh);
351 tcg_gen_andi_i64(d, d, mask);
352 tcg_gen_andi_i64(t, t, ~mask);
353 tcg_gen_or_i64(d, d, t);
354 tcg_temp_free_i64(t);
355}
356
357static void gen_xar16_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
358{
359 TCGv_i64 t = tcg_temp_new_i64();
360 uint64_t mask = dup_const(MO_16, 0xffff >> sh);
361
362 tcg_gen_xor_i64(t, n, m);
363 tcg_gen_shri_i64(d, t, sh);
364 tcg_gen_shli_i64(t, t, 16 - sh);
365 tcg_gen_andi_i64(d, d, mask);
366 tcg_gen_andi_i64(t, t, ~mask);
367 tcg_gen_or_i64(d, d, t);
368 tcg_temp_free_i64(t);
369}
370
371static void gen_xar_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, int32_t sh)
372{
373 tcg_gen_xor_i32(d, n, m);
374 tcg_gen_rotri_i32(d, d, sh);
375}
376
377static void gen_xar_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
378{
379 tcg_gen_xor_i64(d, n, m);
380 tcg_gen_rotri_i64(d, d, sh);
381}
382
383static void gen_xar_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
384 TCGv_vec m, int64_t sh)
385{
386 tcg_gen_xor_vec(vece, d, n, m);
387 tcg_gen_rotri_vec(vece, d, d, sh);
388}
389
390void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
391 uint32_t rm_ofs, int64_t shift,
392 uint32_t opr_sz, uint32_t max_sz)
393{
394 static const TCGOpcode vecop[] = { INDEX_op_rotli_vec, 0 };
395 static const GVecGen3i ops[4] = {
396 { .fni8 = gen_xar8_i64,
397 .fniv = gen_xar_vec,
398 .fno = gen_helper_sve2_xar_b,
399 .opt_opc = vecop,
400 .vece = MO_8 },
401 { .fni8 = gen_xar16_i64,
402 .fniv = gen_xar_vec,
403 .fno = gen_helper_sve2_xar_h,
404 .opt_opc = vecop,
405 .vece = MO_16 },
406 { .fni4 = gen_xar_i32,
407 .fniv = gen_xar_vec,
408 .fno = gen_helper_sve2_xar_s,
409 .opt_opc = vecop,
410 .vece = MO_32 },
411 { .fni8 = gen_xar_i64,
412 .fniv = gen_xar_vec,
413 .fno = gen_helper_gvec_xar_d,
414 .opt_opc = vecop,
415 .vece = MO_64 }
416 };
417 int esize = 8 << vece;
418
419 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
420 tcg_debug_assert(shift >= 0);
421 tcg_debug_assert(shift <= esize);
422 shift &= esize - 1;
423
424 if (shift == 0) {
425 /* xar with no rotate devolves to xor. */
426 tcg_gen_gvec_xor(vece, rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz);
427 } else {
428 tcg_gen_gvec_3i(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz,
429 shift, &ops[vece]);
430 }
431}
432
433static bool trans_XAR(DisasContext *s, arg_rrri_esz *a)
434{
435 if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
436 return false;
437 }
438 if (sve_access_check(s)) {
439 unsigned vsz = vec_full_reg_size(s);
440 gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd),
441 vec_full_reg_offset(s, a->rn),
442 vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz);
443 }
444 return true;
445}
446
911cdc6d
RH
447static bool do_sve2_zzzz_fn(DisasContext *s, arg_rrrr_esz *a, GVecGen4Fn *fn)
448{
449 if (!dc_isar_feature(aa64_sve2, s)) {
450 return false;
451 }
452 if (sve_access_check(s)) {
453 gen_gvec_fn_zzzz(s, fn, a->esz, a->rd, a->rn, a->rm, a->ra);
454 }
455 return true;
456}
457
458static void gen_eor3_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
459{
460 tcg_gen_xor_i64(d, n, m);
461 tcg_gen_xor_i64(d, d, k);
462}
463
464static void gen_eor3_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
465 TCGv_vec m, TCGv_vec k)
466{
467 tcg_gen_xor_vec(vece, d, n, m);
468 tcg_gen_xor_vec(vece, d, d, k);
469}
470
471static void gen_eor3(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
472 uint32_t a, uint32_t oprsz, uint32_t maxsz)
473{
474 static const GVecGen4 op = {
475 .fni8 = gen_eor3_i64,
476 .fniv = gen_eor3_vec,
477 .fno = gen_helper_sve2_eor3,
478 .vece = MO_64,
479 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
480 };
481 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
482}
483
484static bool trans_EOR3(DisasContext *s, arg_rrrr_esz *a)
485{
486 return do_sve2_zzzz_fn(s, a, gen_eor3);
487}
488
489static void gen_bcax_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
490{
491 tcg_gen_andc_i64(d, m, k);
492 tcg_gen_xor_i64(d, d, n);
493}
494
495static void gen_bcax_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
496 TCGv_vec m, TCGv_vec k)
497{
498 tcg_gen_andc_vec(vece, d, m, k);
499 tcg_gen_xor_vec(vece, d, d, n);
500}
501
502static void gen_bcax(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
503 uint32_t a, uint32_t oprsz, uint32_t maxsz)
504{
505 static const GVecGen4 op = {
506 .fni8 = gen_bcax_i64,
507 .fniv = gen_bcax_vec,
508 .fno = gen_helper_sve2_bcax,
509 .vece = MO_64,
510 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
511 };
512 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
513}
514
515static bool trans_BCAX(DisasContext *s, arg_rrrr_esz *a)
516{
517 return do_sve2_zzzz_fn(s, a, gen_bcax);
518}
519
520static void gen_bsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
521 uint32_t a, uint32_t oprsz, uint32_t maxsz)
522{
523 /* BSL differs from the generic bitsel in argument ordering. */
524 tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz);
525}
526
527static bool trans_BSL(DisasContext *s, arg_rrrr_esz *a)
528{
529 return do_sve2_zzzz_fn(s, a, gen_bsl);
530}
531
532static void gen_bsl1n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
533{
534 tcg_gen_andc_i64(n, k, n);
535 tcg_gen_andc_i64(m, m, k);
536 tcg_gen_or_i64(d, n, m);
537}
538
539static void gen_bsl1n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
540 TCGv_vec m, TCGv_vec k)
541{
542 if (TCG_TARGET_HAS_bitsel_vec) {
543 tcg_gen_not_vec(vece, n, n);
544 tcg_gen_bitsel_vec(vece, d, k, n, m);
545 } else {
546 tcg_gen_andc_vec(vece, n, k, n);
547 tcg_gen_andc_vec(vece, m, m, k);
548 tcg_gen_or_vec(vece, d, n, m);
549 }
550}
551
552static void gen_bsl1n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
553 uint32_t a, uint32_t oprsz, uint32_t maxsz)
554{
555 static const GVecGen4 op = {
556 .fni8 = gen_bsl1n_i64,
557 .fniv = gen_bsl1n_vec,
558 .fno = gen_helper_sve2_bsl1n,
559 .vece = MO_64,
560 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
561 };
562 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
563}
564
565static bool trans_BSL1N(DisasContext *s, arg_rrrr_esz *a)
566{
567 return do_sve2_zzzz_fn(s, a, gen_bsl1n);
568}
569
570static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
571{
572 /*
573 * Z[dn] = (n & k) | (~m & ~k)
574 * = | ~(m | k)
575 */
576 tcg_gen_and_i64(n, n, k);
577 if (TCG_TARGET_HAS_orc_i64) {
578 tcg_gen_or_i64(m, m, k);
579 tcg_gen_orc_i64(d, n, m);
580 } else {
581 tcg_gen_nor_i64(m, m, k);
582 tcg_gen_or_i64(d, n, m);
583 }
584}
585
586static void gen_bsl2n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
587 TCGv_vec m, TCGv_vec k)
588{
589 if (TCG_TARGET_HAS_bitsel_vec) {
590 tcg_gen_not_vec(vece, m, m);
591 tcg_gen_bitsel_vec(vece, d, k, n, m);
592 } else {
593 tcg_gen_and_vec(vece, n, n, k);
594 tcg_gen_or_vec(vece, m, m, k);
595 tcg_gen_orc_vec(vece, d, n, m);
596 }
597}
598
599static void gen_bsl2n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
600 uint32_t a, uint32_t oprsz, uint32_t maxsz)
601{
602 static const GVecGen4 op = {
603 .fni8 = gen_bsl2n_i64,
604 .fniv = gen_bsl2n_vec,
605 .fno = gen_helper_sve2_bsl2n,
606 .vece = MO_64,
607 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
608 };
609 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
610}
611
612static bool trans_BSL2N(DisasContext *s, arg_rrrr_esz *a)
613{
614 return do_sve2_zzzz_fn(s, a, gen_bsl2n);
615}
616
617static void gen_nbsl_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
618{
619 tcg_gen_and_i64(n, n, k);
620 tcg_gen_andc_i64(m, m, k);
621 tcg_gen_nor_i64(d, n, m);
622}
623
624static void gen_nbsl_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
625 TCGv_vec m, TCGv_vec k)
626{
627 tcg_gen_bitsel_vec(vece, d, k, n, m);
628 tcg_gen_not_vec(vece, d, d);
629}
630
631static void gen_nbsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
632 uint32_t a, uint32_t oprsz, uint32_t maxsz)
633{
634 static const GVecGen4 op = {
635 .fni8 = gen_nbsl_i64,
636 .fniv = gen_nbsl_vec,
637 .fno = gen_helper_sve2_nbsl,
638 .vece = MO_64,
639 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
640 };
641 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
642}
643
644static bool trans_NBSL(DisasContext *s, arg_rrrr_esz *a)
645{
646 return do_sve2_zzzz_fn(s, a, gen_nbsl);
647}
648
fea98f9c
RH
649/*
650 *** SVE Integer Arithmetic - Unpredicated Group
651 */
652
3a7be554 653static bool trans_ADD_zzz(DisasContext *s, arg_rrr_esz *a)
fea98f9c 654{
28c4da31 655 return do_zzz_fn(s, a, tcg_gen_gvec_add);
fea98f9c
RH
656}
657
3a7be554 658static bool trans_SUB_zzz(DisasContext *s, arg_rrr_esz *a)
fea98f9c 659{
28c4da31 660 return do_zzz_fn(s, a, tcg_gen_gvec_sub);
fea98f9c
RH
661}
662
3a7be554 663static bool trans_SQADD_zzz(DisasContext *s, arg_rrr_esz *a)
fea98f9c 664{
28c4da31 665 return do_zzz_fn(s, a, tcg_gen_gvec_ssadd);
fea98f9c
RH
666}
667
3a7be554 668static bool trans_SQSUB_zzz(DisasContext *s, arg_rrr_esz *a)
fea98f9c 669{
28c4da31 670 return do_zzz_fn(s, a, tcg_gen_gvec_sssub);
fea98f9c
RH
671}
672
3a7be554 673static bool trans_UQADD_zzz(DisasContext *s, arg_rrr_esz *a)
fea98f9c 674{
28c4da31 675 return do_zzz_fn(s, a, tcg_gen_gvec_usadd);
fea98f9c
RH
676}
677
3a7be554 678static bool trans_UQSUB_zzz(DisasContext *s, arg_rrr_esz *a)
fea98f9c 679{
28c4da31 680 return do_zzz_fn(s, a, tcg_gen_gvec_ussub);
fea98f9c
RH
681}
682
f97cfd59
RH
683/*
684 *** SVE Integer Arithmetic - Binary Predicated Group
685 */
686
687static bool do_zpzz_ool(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4 *fn)
688{
f97cfd59
RH
689 if (fn == NULL) {
690 return false;
691 }
692 if (sve_access_check(s)) {
36cbb7a8 693 gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0);
f97cfd59
RH
694 }
695 return true;
696}
697
a2103582
RH
698/* Select active elememnts from Zn and inactive elements from Zm,
699 * storing the result in Zd.
700 */
701static void do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz)
702{
703 static gen_helper_gvec_4 * const fns[4] = {
704 gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
705 gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d
706 };
36cbb7a8 707 gen_gvec_ool_zzzp(s, fns[esz], rd, rn, rm, pg, 0);
a2103582
RH
708}
709
f97cfd59 710#define DO_ZPZZ(NAME, name) \
3a7be554 711static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a) \
f97cfd59
RH
712{ \
713 static gen_helper_gvec_4 * const fns[4] = { \
714 gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h, \
715 gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d, \
716 }; \
717 return do_zpzz_ool(s, a, fns[a->esz]); \
718}
719
720DO_ZPZZ(AND, and)
721DO_ZPZZ(EOR, eor)
722DO_ZPZZ(ORR, orr)
723DO_ZPZZ(BIC, bic)
724
725DO_ZPZZ(ADD, add)
726DO_ZPZZ(SUB, sub)
727
728DO_ZPZZ(SMAX, smax)
729DO_ZPZZ(UMAX, umax)
730DO_ZPZZ(SMIN, smin)
731DO_ZPZZ(UMIN, umin)
732DO_ZPZZ(SABD, sabd)
733DO_ZPZZ(UABD, uabd)
734
735DO_ZPZZ(MUL, mul)
736DO_ZPZZ(SMULH, smulh)
737DO_ZPZZ(UMULH, umulh)
738
27721dbb
RH
739DO_ZPZZ(ASR, asr)
740DO_ZPZZ(LSR, lsr)
741DO_ZPZZ(LSL, lsl)
742
3a7be554 743static bool trans_SDIV_zpzz(DisasContext *s, arg_rprr_esz *a)
f97cfd59
RH
744{
745 static gen_helper_gvec_4 * const fns[4] = {
746 NULL, NULL, gen_helper_sve_sdiv_zpzz_s, gen_helper_sve_sdiv_zpzz_d
747 };
748 return do_zpzz_ool(s, a, fns[a->esz]);
749}
750
3a7be554 751static bool trans_UDIV_zpzz(DisasContext *s, arg_rprr_esz *a)
f97cfd59
RH
752{
753 static gen_helper_gvec_4 * const fns[4] = {
754 NULL, NULL, gen_helper_sve_udiv_zpzz_s, gen_helper_sve_udiv_zpzz_d
755 };
756 return do_zpzz_ool(s, a, fns[a->esz]);
757}
758
3a7be554 759static bool trans_SEL_zpzz(DisasContext *s, arg_rprr_esz *a)
a2103582
RH
760{
761 if (sve_access_check(s)) {
762 do_sel_z(s, a->rd, a->rn, a->rm, a->pg, a->esz);
763 }
764 return true;
765}
d3fe4a29 766
f97cfd59
RH
767#undef DO_ZPZZ
768
afac6d04
RH
769/*
770 *** SVE Integer Arithmetic - Unary Predicated Group
771 */
772
773static bool do_zpz_ool(DisasContext *s, arg_rpr_esz *a, gen_helper_gvec_3 *fn)
774{
775 if (fn == NULL) {
776 return false;
777 }
778 if (sve_access_check(s)) {
96a461f7 779 gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, 0);
afac6d04
RH
780 }
781 return true;
782}
783
784#define DO_ZPZ(NAME, name) \
3a7be554 785static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
afac6d04
RH
786{ \
787 static gen_helper_gvec_3 * const fns[4] = { \
788 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
789 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
790 }; \
791 return do_zpz_ool(s, a, fns[a->esz]); \
792}
793
794DO_ZPZ(CLS, cls)
795DO_ZPZ(CLZ, clz)
796DO_ZPZ(CNT_zpz, cnt_zpz)
797DO_ZPZ(CNOT, cnot)
798DO_ZPZ(NOT_zpz, not_zpz)
799DO_ZPZ(ABS, abs)
800DO_ZPZ(NEG, neg)
801
3a7be554 802static bool trans_FABS(DisasContext *s, arg_rpr_esz *a)
afac6d04
RH
803{
804 static gen_helper_gvec_3 * const fns[4] = {
805 NULL,
806 gen_helper_sve_fabs_h,
807 gen_helper_sve_fabs_s,
808 gen_helper_sve_fabs_d
809 };
810 return do_zpz_ool(s, a, fns[a->esz]);
811}
812
3a7be554 813static bool trans_FNEG(DisasContext *s, arg_rpr_esz *a)
afac6d04
RH
814{
815 static gen_helper_gvec_3 * const fns[4] = {
816 NULL,
817 gen_helper_sve_fneg_h,
818 gen_helper_sve_fneg_s,
819 gen_helper_sve_fneg_d
820 };
821 return do_zpz_ool(s, a, fns[a->esz]);
822}
823
3a7be554 824static bool trans_SXTB(DisasContext *s, arg_rpr_esz *a)
afac6d04
RH
825{
826 static gen_helper_gvec_3 * const fns[4] = {
827 NULL,
828 gen_helper_sve_sxtb_h,
829 gen_helper_sve_sxtb_s,
830 gen_helper_sve_sxtb_d
831 };
832 return do_zpz_ool(s, a, fns[a->esz]);
833}
834
3a7be554 835static bool trans_UXTB(DisasContext *s, arg_rpr_esz *a)
afac6d04
RH
836{
837 static gen_helper_gvec_3 * const fns[4] = {
838 NULL,
839 gen_helper_sve_uxtb_h,
840 gen_helper_sve_uxtb_s,
841 gen_helper_sve_uxtb_d
842 };
843 return do_zpz_ool(s, a, fns[a->esz]);
844}
845
3a7be554 846static bool trans_SXTH(DisasContext *s, arg_rpr_esz *a)
afac6d04
RH
847{
848 static gen_helper_gvec_3 * const fns[4] = {
849 NULL, NULL,
850 gen_helper_sve_sxth_s,
851 gen_helper_sve_sxth_d
852 };
853 return do_zpz_ool(s, a, fns[a->esz]);
854}
855
3a7be554 856static bool trans_UXTH(DisasContext *s, arg_rpr_esz *a)
afac6d04
RH
857{
858 static gen_helper_gvec_3 * const fns[4] = {
859 NULL, NULL,
860 gen_helper_sve_uxth_s,
861 gen_helper_sve_uxth_d
862 };
863 return do_zpz_ool(s, a, fns[a->esz]);
864}
865
3a7be554 866static bool trans_SXTW(DisasContext *s, arg_rpr_esz *a)
afac6d04
RH
867{
868 return do_zpz_ool(s, a, a->esz == 3 ? gen_helper_sve_sxtw_d : NULL);
869}
870
3a7be554 871static bool trans_UXTW(DisasContext *s, arg_rpr_esz *a)
afac6d04
RH
872{
873 return do_zpz_ool(s, a, a->esz == 3 ? gen_helper_sve_uxtw_d : NULL);
874}
875
876#undef DO_ZPZ
877
047cec97
RH
878/*
879 *** SVE Integer Reduction Group
880 */
881
882typedef void gen_helper_gvec_reduc(TCGv_i64, TCGv_ptr, TCGv_ptr, TCGv_i32);
883static bool do_vpz_ool(DisasContext *s, arg_rpr_esz *a,
884 gen_helper_gvec_reduc *fn)
885{
886 unsigned vsz = vec_full_reg_size(s);
887 TCGv_ptr t_zn, t_pg;
888 TCGv_i32 desc;
889 TCGv_i64 temp;
890
891 if (fn == NULL) {
892 return false;
893 }
894 if (!sve_access_check(s)) {
895 return true;
896 }
897
898 desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
899 temp = tcg_temp_new_i64();
900 t_zn = tcg_temp_new_ptr();
901 t_pg = tcg_temp_new_ptr();
902
903 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn));
904 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg));
905 fn(temp, t_zn, t_pg, desc);
906 tcg_temp_free_ptr(t_zn);
907 tcg_temp_free_ptr(t_pg);
908 tcg_temp_free_i32(desc);
909
910 write_fp_dreg(s, a->rd, temp);
911 tcg_temp_free_i64(temp);
912 return true;
913}
914
915#define DO_VPZ(NAME, name) \
3a7be554 916static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
047cec97
RH
917{ \
918 static gen_helper_gvec_reduc * const fns[4] = { \
919 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
920 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
921 }; \
922 return do_vpz_ool(s, a, fns[a->esz]); \
923}
924
925DO_VPZ(ORV, orv)
926DO_VPZ(ANDV, andv)
927DO_VPZ(EORV, eorv)
928
929DO_VPZ(UADDV, uaddv)
930DO_VPZ(SMAXV, smaxv)
931DO_VPZ(UMAXV, umaxv)
932DO_VPZ(SMINV, sminv)
933DO_VPZ(UMINV, uminv)
934
3a7be554 935static bool trans_SADDV(DisasContext *s, arg_rpr_esz *a)
047cec97
RH
936{
937 static gen_helper_gvec_reduc * const fns[4] = {
938 gen_helper_sve_saddv_b, gen_helper_sve_saddv_h,
939 gen_helper_sve_saddv_s, NULL
940 };
941 return do_vpz_ool(s, a, fns[a->esz]);
942}
943
944#undef DO_VPZ
945
ccd841c3
RH
946/*
947 *** SVE Shift by Immediate - Predicated Group
948 */
949
60245996
RH
950/*
951 * Copy Zn into Zd, storing zeros into inactive elements.
952 * If invert, store zeros into the active elements.
ccd841c3 953 */
60245996
RH
954static bool do_movz_zpz(DisasContext *s, int rd, int rn, int pg,
955 int esz, bool invert)
ccd841c3 956{
60245996
RH
957 static gen_helper_gvec_3 * const fns[4] = {
958 gen_helper_sve_movz_b, gen_helper_sve_movz_h,
959 gen_helper_sve_movz_s, gen_helper_sve_movz_d,
ccd841c3 960 };
60245996 961
ccd841c3 962 if (sve_access_check(s)) {
96a461f7 963 gen_gvec_ool_zzp(s, fns[esz], rd, rn, pg, invert);
ccd841c3
RH
964 }
965 return true;
966}
967
968static bool do_zpzi_ool(DisasContext *s, arg_rpri_esz *a,
969 gen_helper_gvec_3 *fn)
970{
971 if (sve_access_check(s)) {
96a461f7 972 gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm);
ccd841c3
RH
973 }
974 return true;
975}
976
3a7be554 977static bool trans_ASR_zpzi(DisasContext *s, arg_rpri_esz *a)
ccd841c3
RH
978{
979 static gen_helper_gvec_3 * const fns[4] = {
980 gen_helper_sve_asr_zpzi_b, gen_helper_sve_asr_zpzi_h,
981 gen_helper_sve_asr_zpzi_s, gen_helper_sve_asr_zpzi_d,
982 };
983 if (a->esz < 0) {
984 /* Invalid tsz encoding -- see tszimm_esz. */
985 return false;
986 }
987 /* Shift by element size is architecturally valid. For
988 arithmetic right-shift, it's the same as by one less. */
989 a->imm = MIN(a->imm, (8 << a->esz) - 1);
990 return do_zpzi_ool(s, a, fns[a->esz]);
991}
992
3a7be554 993static bool trans_LSR_zpzi(DisasContext *s, arg_rpri_esz *a)
ccd841c3
RH
994{
995 static gen_helper_gvec_3 * const fns[4] = {
996 gen_helper_sve_lsr_zpzi_b, gen_helper_sve_lsr_zpzi_h,
997 gen_helper_sve_lsr_zpzi_s, gen_helper_sve_lsr_zpzi_d,
998 };
999 if (a->esz < 0) {
1000 return false;
1001 }
1002 /* Shift by element size is architecturally valid.
1003 For logical shifts, it is a zeroing operation. */
1004 if (a->imm >= (8 << a->esz)) {
60245996 1005 return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true);
ccd841c3
RH
1006 } else {
1007 return do_zpzi_ool(s, a, fns[a->esz]);
1008 }
1009}
1010
3a7be554 1011static bool trans_LSL_zpzi(DisasContext *s, arg_rpri_esz *a)
ccd841c3
RH
1012{
1013 static gen_helper_gvec_3 * const fns[4] = {
1014 gen_helper_sve_lsl_zpzi_b, gen_helper_sve_lsl_zpzi_h,
1015 gen_helper_sve_lsl_zpzi_s, gen_helper_sve_lsl_zpzi_d,
1016 };
1017 if (a->esz < 0) {
1018 return false;
1019 }
1020 /* Shift by element size is architecturally valid.
1021 For logical shifts, it is a zeroing operation. */
1022 if (a->imm >= (8 << a->esz)) {
60245996 1023 return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true);
ccd841c3
RH
1024 } else {
1025 return do_zpzi_ool(s, a, fns[a->esz]);
1026 }
1027}
1028
3a7be554 1029static bool trans_ASRD(DisasContext *s, arg_rpri_esz *a)
ccd841c3
RH
1030{
1031 static gen_helper_gvec_3 * const fns[4] = {
1032 gen_helper_sve_asrd_b, gen_helper_sve_asrd_h,
1033 gen_helper_sve_asrd_s, gen_helper_sve_asrd_d,
1034 };
1035 if (a->esz < 0) {
1036 return false;
1037 }
1038 /* Shift by element size is architecturally valid. For arithmetic
1039 right shift for division, it is a zeroing operation. */
1040 if (a->imm >= (8 << a->esz)) {
60245996 1041 return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true);
ccd841c3
RH
1042 } else {
1043 return do_zpzi_ool(s, a, fns[a->esz]);
1044 }
1045}
1046
fe7f8dfb
RH
1047/*
1048 *** SVE Bitwise Shift - Predicated Group
1049 */
1050
1051#define DO_ZPZW(NAME, name) \
3a7be554 1052static bool trans_##NAME##_zpzw(DisasContext *s, arg_rprr_esz *a) \
fe7f8dfb
RH
1053{ \
1054 static gen_helper_gvec_4 * const fns[3] = { \
1055 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1056 gen_helper_sve_##name##_zpzw_s, \
1057 }; \
1058 if (a->esz < 0 || a->esz >= 3) { \
1059 return false; \
1060 } \
1061 return do_zpzz_ool(s, a, fns[a->esz]); \
1062}
1063
1064DO_ZPZW(ASR, asr)
1065DO_ZPZW(LSR, lsr)
1066DO_ZPZW(LSL, lsl)
1067
1068#undef DO_ZPZW
1069
d9d78dcc
RH
1070/*
1071 *** SVE Bitwise Shift - Unpredicated Group
1072 */
1073
1074static bool do_shift_imm(DisasContext *s, arg_rri_esz *a, bool asr,
1075 void (*gvec_fn)(unsigned, uint32_t, uint32_t,
1076 int64_t, uint32_t, uint32_t))
1077{
1078 if (a->esz < 0) {
1079 /* Invalid tsz encoding -- see tszimm_esz. */
1080 return false;
1081 }
1082 if (sve_access_check(s)) {
1083 unsigned vsz = vec_full_reg_size(s);
1084 /* Shift by element size is architecturally valid. For
1085 arithmetic right-shift, it's the same as by one less.
1086 Otherwise it is a zeroing operation. */
1087 if (a->imm >= 8 << a->esz) {
1088 if (asr) {
1089 a->imm = (8 << a->esz) - 1;
1090 } else {
1091 do_dupi_z(s, a->rd, 0);
1092 return true;
1093 }
1094 }
1095 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
1096 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
1097 }
1098 return true;
1099}
1100
3a7be554 1101static bool trans_ASR_zzi(DisasContext *s, arg_rri_esz *a)
d9d78dcc
RH
1102{
1103 return do_shift_imm(s, a, true, tcg_gen_gvec_sari);
1104}
1105
3a7be554 1106static bool trans_LSR_zzi(DisasContext *s, arg_rri_esz *a)
d9d78dcc
RH
1107{
1108 return do_shift_imm(s, a, false, tcg_gen_gvec_shri);
1109}
1110
3a7be554 1111static bool trans_LSL_zzi(DisasContext *s, arg_rri_esz *a)
d9d78dcc
RH
1112{
1113 return do_shift_imm(s, a, false, tcg_gen_gvec_shli);
1114}
1115
1116static bool do_zzw_ool(DisasContext *s, arg_rrr_esz *a, gen_helper_gvec_3 *fn)
1117{
1118 if (fn == NULL) {
1119 return false;
1120 }
1121 if (sve_access_check(s)) {
e645d1a1 1122 gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, 0);
d9d78dcc
RH
1123 }
1124 return true;
1125}
1126
1127#define DO_ZZW(NAME, name) \
3a7be554 1128static bool trans_##NAME##_zzw(DisasContext *s, arg_rrr_esz *a) \
d9d78dcc
RH
1129{ \
1130 static gen_helper_gvec_3 * const fns[4] = { \
1131 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1132 gen_helper_sve_##name##_zzw_s, NULL \
1133 }; \
1134 return do_zzw_ool(s, a, fns[a->esz]); \
1135}
1136
1137DO_ZZW(ASR, asr)
1138DO_ZZW(LSR, lsr)
1139DO_ZZW(LSL, lsl)
1140
1141#undef DO_ZZW
1142
96a36e4a
RH
1143/*
1144 *** SVE Integer Multiply-Add Group
1145 */
1146
1147static bool do_zpzzz_ool(DisasContext *s, arg_rprrr_esz *a,
1148 gen_helper_gvec_5 *fn)
1149{
1150 if (sve_access_check(s)) {
1151 unsigned vsz = vec_full_reg_size(s);
1152 tcg_gen_gvec_5_ool(vec_full_reg_offset(s, a->rd),
1153 vec_full_reg_offset(s, a->ra),
1154 vec_full_reg_offset(s, a->rn),
1155 vec_full_reg_offset(s, a->rm),
1156 pred_full_reg_offset(s, a->pg),
1157 vsz, vsz, 0, fn);
1158 }
1159 return true;
1160}
1161
1162#define DO_ZPZZZ(NAME, name) \
3a7be554 1163static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
96a36e4a
RH
1164{ \
1165 static gen_helper_gvec_5 * const fns[4] = { \
1166 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
1167 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1168 }; \
1169 return do_zpzzz_ool(s, a, fns[a->esz]); \
1170}
1171
1172DO_ZPZZZ(MLA, mla)
1173DO_ZPZZZ(MLS, mls)
1174
1175#undef DO_ZPZZZ
1176
9a56c9c3
RH
1177/*
1178 *** SVE Index Generation Group
1179 */
1180
1181static void do_index(DisasContext *s, int esz, int rd,
1182 TCGv_i64 start, TCGv_i64 incr)
1183{
1184 unsigned vsz = vec_full_reg_size(s);
1185 TCGv_i32 desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
1186 TCGv_ptr t_zd = tcg_temp_new_ptr();
1187
1188 tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, rd));
1189 if (esz == 3) {
1190 gen_helper_sve_index_d(t_zd, start, incr, desc);
1191 } else {
1192 typedef void index_fn(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32);
1193 static index_fn * const fns[3] = {
1194 gen_helper_sve_index_b,
1195 gen_helper_sve_index_h,
1196 gen_helper_sve_index_s,
1197 };
1198 TCGv_i32 s32 = tcg_temp_new_i32();
1199 TCGv_i32 i32 = tcg_temp_new_i32();
1200
1201 tcg_gen_extrl_i64_i32(s32, start);
1202 tcg_gen_extrl_i64_i32(i32, incr);
1203 fns[esz](t_zd, s32, i32, desc);
1204
1205 tcg_temp_free_i32(s32);
1206 tcg_temp_free_i32(i32);
1207 }
1208 tcg_temp_free_ptr(t_zd);
1209 tcg_temp_free_i32(desc);
1210}
1211
3a7be554 1212static bool trans_INDEX_ii(DisasContext *s, arg_INDEX_ii *a)
9a56c9c3
RH
1213{
1214 if (sve_access_check(s)) {
1215 TCGv_i64 start = tcg_const_i64(a->imm1);
1216 TCGv_i64 incr = tcg_const_i64(a->imm2);
1217 do_index(s, a->esz, a->rd, start, incr);
1218 tcg_temp_free_i64(start);
1219 tcg_temp_free_i64(incr);
1220 }
1221 return true;
1222}
1223
3a7be554 1224static bool trans_INDEX_ir(DisasContext *s, arg_INDEX_ir *a)
9a56c9c3
RH
1225{
1226 if (sve_access_check(s)) {
1227 TCGv_i64 start = tcg_const_i64(a->imm);
1228 TCGv_i64 incr = cpu_reg(s, a->rm);
1229 do_index(s, a->esz, a->rd, start, incr);
1230 tcg_temp_free_i64(start);
1231 }
1232 return true;
1233}
1234
3a7be554 1235static bool trans_INDEX_ri(DisasContext *s, arg_INDEX_ri *a)
9a56c9c3
RH
1236{
1237 if (sve_access_check(s)) {
1238 TCGv_i64 start = cpu_reg(s, a->rn);
1239 TCGv_i64 incr = tcg_const_i64(a->imm);
1240 do_index(s, a->esz, a->rd, start, incr);
1241 tcg_temp_free_i64(incr);
1242 }
1243 return true;
1244}
1245
3a7be554 1246static bool trans_INDEX_rr(DisasContext *s, arg_INDEX_rr *a)
9a56c9c3
RH
1247{
1248 if (sve_access_check(s)) {
1249 TCGv_i64 start = cpu_reg(s, a->rn);
1250 TCGv_i64 incr = cpu_reg(s, a->rm);
1251 do_index(s, a->esz, a->rd, start, incr);
1252 }
1253 return true;
1254}
1255
96f922cc
RH
1256/*
1257 *** SVE Stack Allocation Group
1258 */
1259
3a7be554 1260static bool trans_ADDVL(DisasContext *s, arg_ADDVL *a)
96f922cc 1261{
5de56742
AC
1262 if (sve_access_check(s)) {
1263 TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1264 TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1265 tcg_gen_addi_i64(rd, rn, a->imm * vec_full_reg_size(s));
1266 }
96f922cc
RH
1267 return true;
1268}
1269
3a7be554 1270static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a)
96f922cc 1271{
5de56742
AC
1272 if (sve_access_check(s)) {
1273 TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1274 TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1275 tcg_gen_addi_i64(rd, rn, a->imm * pred_full_reg_size(s));
1276 }
96f922cc
RH
1277 return true;
1278}
1279
3a7be554 1280static bool trans_RDVL(DisasContext *s, arg_RDVL *a)
96f922cc 1281{
5de56742
AC
1282 if (sve_access_check(s)) {
1283 TCGv_i64 reg = cpu_reg(s, a->rd);
1284 tcg_gen_movi_i64(reg, a->imm * vec_full_reg_size(s));
1285 }
96f922cc
RH
1286 return true;
1287}
1288
4b242d9c
RH
1289/*
1290 *** SVE Compute Vector Address Group
1291 */
1292
1293static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn)
1294{
1295 if (sve_access_check(s)) {
e645d1a1 1296 gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm);
4b242d9c
RH
1297 }
1298 return true;
1299}
1300
3a7be554 1301static bool trans_ADR_p32(DisasContext *s, arg_rrri *a)
4b242d9c
RH
1302{
1303 return do_adr(s, a, gen_helper_sve_adr_p32);
1304}
1305
3a7be554 1306static bool trans_ADR_p64(DisasContext *s, arg_rrri *a)
4b242d9c
RH
1307{
1308 return do_adr(s, a, gen_helper_sve_adr_p64);
1309}
1310
3a7be554 1311static bool trans_ADR_s32(DisasContext *s, arg_rrri *a)
4b242d9c
RH
1312{
1313 return do_adr(s, a, gen_helper_sve_adr_s32);
1314}
1315
3a7be554 1316static bool trans_ADR_u32(DisasContext *s, arg_rrri *a)
4b242d9c
RH
1317{
1318 return do_adr(s, a, gen_helper_sve_adr_u32);
1319}
1320
0762cd42
RH
1321/*
1322 *** SVE Integer Misc - Unpredicated Group
1323 */
1324
3a7be554 1325static bool trans_FEXPA(DisasContext *s, arg_rr_esz *a)
0762cd42
RH
1326{
1327 static gen_helper_gvec_2 * const fns[4] = {
1328 NULL,
1329 gen_helper_sve_fexpa_h,
1330 gen_helper_sve_fexpa_s,
1331 gen_helper_sve_fexpa_d,
1332 };
1333 if (a->esz == 0) {
1334 return false;
1335 }
1336 if (sve_access_check(s)) {
40e32e5a 1337 gen_gvec_ool_zz(s, fns[a->esz], a->rd, a->rn, 0);
0762cd42
RH
1338 }
1339 return true;
1340}
1341
3a7be554 1342static bool trans_FTSSEL(DisasContext *s, arg_rrr_esz *a)
a1f233f2
RH
1343{
1344 static gen_helper_gvec_3 * const fns[4] = {
1345 NULL,
1346 gen_helper_sve_ftssel_h,
1347 gen_helper_sve_ftssel_s,
1348 gen_helper_sve_ftssel_d,
1349 };
1350 if (a->esz == 0) {
1351 return false;
1352 }
1353 if (sve_access_check(s)) {
e645d1a1 1354 gen_gvec_ool_zzz(s, fns[a->esz], a->rd, a->rn, a->rm, 0);
a1f233f2
RH
1355 }
1356 return true;
1357}
1358
516e246a
RH
1359/*
1360 *** SVE Predicate Logical Operations Group
1361 */
1362
1363static bool do_pppp_flags(DisasContext *s, arg_rprr_s *a,
1364 const GVecGen4 *gvec_op)
1365{
1366 if (!sve_access_check(s)) {
1367 return true;
1368 }
1369
1370 unsigned psz = pred_gvec_reg_size(s);
1371 int dofs = pred_full_reg_offset(s, a->rd);
1372 int nofs = pred_full_reg_offset(s, a->rn);
1373 int mofs = pred_full_reg_offset(s, a->rm);
1374 int gofs = pred_full_reg_offset(s, a->pg);
1375
dd81a8d7
RH
1376 if (!a->s) {
1377 tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op);
1378 return true;
1379 }
1380
516e246a
RH
1381 if (psz == 8) {
1382 /* Do the operation and the flags generation in temps. */
1383 TCGv_i64 pd = tcg_temp_new_i64();
1384 TCGv_i64 pn = tcg_temp_new_i64();
1385 TCGv_i64 pm = tcg_temp_new_i64();
1386 TCGv_i64 pg = tcg_temp_new_i64();
1387
1388 tcg_gen_ld_i64(pn, cpu_env, nofs);
1389 tcg_gen_ld_i64(pm, cpu_env, mofs);
1390 tcg_gen_ld_i64(pg, cpu_env, gofs);
1391
1392 gvec_op->fni8(pd, pn, pm, pg);
1393 tcg_gen_st_i64(pd, cpu_env, dofs);
1394
1395 do_predtest1(pd, pg);
1396
1397 tcg_temp_free_i64(pd);
1398 tcg_temp_free_i64(pn);
1399 tcg_temp_free_i64(pm);
1400 tcg_temp_free_i64(pg);
1401 } else {
1402 /* The operation and flags generation is large. The computation
1403 * of the flags depends on the original contents of the guarding
1404 * predicate. If the destination overwrites the guarding predicate,
1405 * then the easiest way to get this right is to save a copy.
1406 */
1407 int tofs = gofs;
1408 if (a->rd == a->pg) {
1409 tofs = offsetof(CPUARMState, vfp.preg_tmp);
1410 tcg_gen_gvec_mov(0, tofs, gofs, psz, psz);
1411 }
1412
1413 tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op);
1414 do_predtest(s, dofs, tofs, psz / 8);
1415 }
1416 return true;
1417}
1418
1419static void gen_and_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1420{
1421 tcg_gen_and_i64(pd, pn, pm);
1422 tcg_gen_and_i64(pd, pd, pg);
1423}
1424
1425static void gen_and_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1426 TCGv_vec pm, TCGv_vec pg)
1427{
1428 tcg_gen_and_vec(vece, pd, pn, pm);
1429 tcg_gen_and_vec(vece, pd, pd, pg);
1430}
1431
3a7be554 1432static bool trans_AND_pppp(DisasContext *s, arg_rprr_s *a)
516e246a
RH
1433{
1434 static const GVecGen4 op = {
1435 .fni8 = gen_and_pg_i64,
1436 .fniv = gen_and_pg_vec,
1437 .fno = gen_helper_sve_and_pppp,
1438 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1439 };
dd81a8d7
RH
1440
1441 if (!a->s) {
1442 if (!sve_access_check(s)) {
1443 return true;
1444 }
1445 if (a->rn == a->rm) {
1446 if (a->pg == a->rn) {
1447 do_mov_p(s, a->rd, a->rn);
1448 } else {
1449 gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg);
1450 }
1451 return true;
1452 } else if (a->pg == a->rn || a->pg == a->rm) {
1453 gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm);
1454 return true;
516e246a 1455 }
516e246a 1456 }
dd81a8d7 1457 return do_pppp_flags(s, a, &op);
516e246a
RH
1458}
1459
1460static void gen_bic_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1461{
1462 tcg_gen_andc_i64(pd, pn, pm);
1463 tcg_gen_and_i64(pd, pd, pg);
1464}
1465
1466static void gen_bic_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1467 TCGv_vec pm, TCGv_vec pg)
1468{
1469 tcg_gen_andc_vec(vece, pd, pn, pm);
1470 tcg_gen_and_vec(vece, pd, pd, pg);
1471}
1472
3a7be554 1473static bool trans_BIC_pppp(DisasContext *s, arg_rprr_s *a)
516e246a
RH
1474{
1475 static const GVecGen4 op = {
1476 .fni8 = gen_bic_pg_i64,
1477 .fniv = gen_bic_pg_vec,
1478 .fno = gen_helper_sve_bic_pppp,
1479 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1480 };
dd81a8d7
RH
1481
1482 if (!a->s && a->pg == a->rn) {
1483 if (sve_access_check(s)) {
1484 gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm);
1485 }
1486 return true;
516e246a 1487 }
dd81a8d7 1488 return do_pppp_flags(s, a, &op);
516e246a
RH
1489}
1490
1491static void gen_eor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1492{
1493 tcg_gen_xor_i64(pd, pn, pm);
1494 tcg_gen_and_i64(pd, pd, pg);
1495}
1496
1497static void gen_eor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1498 TCGv_vec pm, TCGv_vec pg)
1499{
1500 tcg_gen_xor_vec(vece, pd, pn, pm);
1501 tcg_gen_and_vec(vece, pd, pd, pg);
1502}
1503
3a7be554 1504static bool trans_EOR_pppp(DisasContext *s, arg_rprr_s *a)
516e246a
RH
1505{
1506 static const GVecGen4 op = {
1507 .fni8 = gen_eor_pg_i64,
1508 .fniv = gen_eor_pg_vec,
1509 .fno = gen_helper_sve_eor_pppp,
1510 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1511 };
dd81a8d7 1512 return do_pppp_flags(s, a, &op);
516e246a
RH
1513}
1514
3a7be554 1515static bool trans_SEL_pppp(DisasContext *s, arg_rprr_s *a)
516e246a 1516{
516e246a
RH
1517 if (a->s) {
1518 return false;
516e246a 1519 }
d4bc6232
RH
1520 if (sve_access_check(s)) {
1521 unsigned psz = pred_gvec_reg_size(s);
1522 tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd),
1523 pred_full_reg_offset(s, a->pg),
1524 pred_full_reg_offset(s, a->rn),
1525 pred_full_reg_offset(s, a->rm), psz, psz);
1526 }
1527 return true;
516e246a
RH
1528}
1529
1530static void gen_orr_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1531{
1532 tcg_gen_or_i64(pd, pn, pm);
1533 tcg_gen_and_i64(pd, pd, pg);
1534}
1535
1536static void gen_orr_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1537 TCGv_vec pm, TCGv_vec pg)
1538{
1539 tcg_gen_or_vec(vece, pd, pn, pm);
1540 tcg_gen_and_vec(vece, pd, pd, pg);
1541}
1542
3a7be554 1543static bool trans_ORR_pppp(DisasContext *s, arg_rprr_s *a)
516e246a
RH
1544{
1545 static const GVecGen4 op = {
1546 .fni8 = gen_orr_pg_i64,
1547 .fniv = gen_orr_pg_vec,
1548 .fno = gen_helper_sve_orr_pppp,
1549 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1550 };
dd81a8d7
RH
1551
1552 if (!a->s && a->pg == a->rn && a->rn == a->rm) {
516e246a 1553 return do_mov_p(s, a->rd, a->rn);
516e246a 1554 }
dd81a8d7 1555 return do_pppp_flags(s, a, &op);
516e246a
RH
1556}
1557
1558static void gen_orn_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1559{
1560 tcg_gen_orc_i64(pd, pn, pm);
1561 tcg_gen_and_i64(pd, pd, pg);
1562}
1563
1564static void gen_orn_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1565 TCGv_vec pm, TCGv_vec pg)
1566{
1567 tcg_gen_orc_vec(vece, pd, pn, pm);
1568 tcg_gen_and_vec(vece, pd, pd, pg);
1569}
1570
3a7be554 1571static bool trans_ORN_pppp(DisasContext *s, arg_rprr_s *a)
516e246a
RH
1572{
1573 static const GVecGen4 op = {
1574 .fni8 = gen_orn_pg_i64,
1575 .fniv = gen_orn_pg_vec,
1576 .fno = gen_helper_sve_orn_pppp,
1577 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1578 };
dd81a8d7 1579 return do_pppp_flags(s, a, &op);
516e246a
RH
1580}
1581
1582static void gen_nor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1583{
1584 tcg_gen_or_i64(pd, pn, pm);
1585 tcg_gen_andc_i64(pd, pg, pd);
1586}
1587
1588static void gen_nor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1589 TCGv_vec pm, TCGv_vec pg)
1590{
1591 tcg_gen_or_vec(vece, pd, pn, pm);
1592 tcg_gen_andc_vec(vece, pd, pg, pd);
1593}
1594
3a7be554 1595static bool trans_NOR_pppp(DisasContext *s, arg_rprr_s *a)
516e246a
RH
1596{
1597 static const GVecGen4 op = {
1598 .fni8 = gen_nor_pg_i64,
1599 .fniv = gen_nor_pg_vec,
1600 .fno = gen_helper_sve_nor_pppp,
1601 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1602 };
dd81a8d7 1603 return do_pppp_flags(s, a, &op);
516e246a
RH
1604}
1605
1606static void gen_nand_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1607{
1608 tcg_gen_and_i64(pd, pn, pm);
1609 tcg_gen_andc_i64(pd, pg, pd);
1610}
1611
1612static void gen_nand_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1613 TCGv_vec pm, TCGv_vec pg)
1614{
1615 tcg_gen_and_vec(vece, pd, pn, pm);
1616 tcg_gen_andc_vec(vece, pd, pg, pd);
1617}
1618
3a7be554 1619static bool trans_NAND_pppp(DisasContext *s, arg_rprr_s *a)
516e246a
RH
1620{
1621 static const GVecGen4 op = {
1622 .fni8 = gen_nand_pg_i64,
1623 .fniv = gen_nand_pg_vec,
1624 .fno = gen_helper_sve_nand_pppp,
1625 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1626 };
dd81a8d7 1627 return do_pppp_flags(s, a, &op);
516e246a
RH
1628}
1629
9e18d7a6
RH
1630/*
1631 *** SVE Predicate Misc Group
1632 */
1633
3a7be554 1634static bool trans_PTEST(DisasContext *s, arg_PTEST *a)
9e18d7a6
RH
1635{
1636 if (sve_access_check(s)) {
1637 int nofs = pred_full_reg_offset(s, a->rn);
1638 int gofs = pred_full_reg_offset(s, a->pg);
1639 int words = DIV_ROUND_UP(pred_full_reg_size(s), 8);
1640
1641 if (words == 1) {
1642 TCGv_i64 pn = tcg_temp_new_i64();
1643 TCGv_i64 pg = tcg_temp_new_i64();
1644
1645 tcg_gen_ld_i64(pn, cpu_env, nofs);
1646 tcg_gen_ld_i64(pg, cpu_env, gofs);
1647 do_predtest1(pn, pg);
1648
1649 tcg_temp_free_i64(pn);
1650 tcg_temp_free_i64(pg);
1651 } else {
1652 do_predtest(s, nofs, gofs, words);
1653 }
1654 }
1655 return true;
1656}
1657
028e2a7b
RH
1658/* See the ARM pseudocode DecodePredCount. */
1659static unsigned decode_pred_count(unsigned fullsz, int pattern, int esz)
1660{
1661 unsigned elements = fullsz >> esz;
1662 unsigned bound;
1663
1664 switch (pattern) {
1665 case 0x0: /* POW2 */
1666 return pow2floor(elements);
1667 case 0x1: /* VL1 */
1668 case 0x2: /* VL2 */
1669 case 0x3: /* VL3 */
1670 case 0x4: /* VL4 */
1671 case 0x5: /* VL5 */
1672 case 0x6: /* VL6 */
1673 case 0x7: /* VL7 */
1674 case 0x8: /* VL8 */
1675 bound = pattern;
1676 break;
1677 case 0x9: /* VL16 */
1678 case 0xa: /* VL32 */
1679 case 0xb: /* VL64 */
1680 case 0xc: /* VL128 */
1681 case 0xd: /* VL256 */
1682 bound = 16 << (pattern - 9);
1683 break;
1684 case 0x1d: /* MUL4 */
1685 return elements - elements % 4;
1686 case 0x1e: /* MUL3 */
1687 return elements - elements % 3;
1688 case 0x1f: /* ALL */
1689 return elements;
1690 default: /* #uimm5 */
1691 return 0;
1692 }
1693 return elements >= bound ? bound : 0;
1694}
1695
1696/* This handles all of the predicate initialization instructions,
1697 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1698 * so that decode_pred_count returns 0. For SETFFR, we will have
1699 * set RD == 16 == FFR.
1700 */
1701static bool do_predset(DisasContext *s, int esz, int rd, int pat, bool setflag)
1702{
1703 if (!sve_access_check(s)) {
1704 return true;
1705 }
1706
1707 unsigned fullsz = vec_full_reg_size(s);
1708 unsigned ofs = pred_full_reg_offset(s, rd);
1709 unsigned numelem, setsz, i;
1710 uint64_t word, lastword;
1711 TCGv_i64 t;
1712
1713 numelem = decode_pred_count(fullsz, pat, esz);
1714
1715 /* Determine what we must store into each bit, and how many. */
1716 if (numelem == 0) {
1717 lastword = word = 0;
1718 setsz = fullsz;
1719 } else {
1720 setsz = numelem << esz;
1721 lastword = word = pred_esz_masks[esz];
1722 if (setsz % 64) {
973558a3 1723 lastword &= MAKE_64BIT_MASK(0, setsz % 64);
028e2a7b
RH
1724 }
1725 }
1726
1727 t = tcg_temp_new_i64();
1728 if (fullsz <= 64) {
1729 tcg_gen_movi_i64(t, lastword);
1730 tcg_gen_st_i64(t, cpu_env, ofs);
1731 goto done;
1732 }
1733
1734 if (word == lastword) {
1735 unsigned maxsz = size_for_gvec(fullsz / 8);
1736 unsigned oprsz = size_for_gvec(setsz / 8);
1737
1738 if (oprsz * 8 == setsz) {
8711e71f 1739 tcg_gen_gvec_dup_imm(MO_64, ofs, oprsz, maxsz, word);
028e2a7b
RH
1740 goto done;
1741 }
028e2a7b
RH
1742 }
1743
1744 setsz /= 8;
1745 fullsz /= 8;
1746
1747 tcg_gen_movi_i64(t, word);
973558a3 1748 for (i = 0; i < QEMU_ALIGN_DOWN(setsz, 8); i += 8) {
028e2a7b
RH
1749 tcg_gen_st_i64(t, cpu_env, ofs + i);
1750 }
1751 if (lastword != word) {
1752 tcg_gen_movi_i64(t, lastword);
1753 tcg_gen_st_i64(t, cpu_env, ofs + i);
1754 i += 8;
1755 }
1756 if (i < fullsz) {
1757 tcg_gen_movi_i64(t, 0);
1758 for (; i < fullsz; i += 8) {
1759 tcg_gen_st_i64(t, cpu_env, ofs + i);
1760 }
1761 }
1762
1763 done:
1764 tcg_temp_free_i64(t);
1765
1766 /* PTRUES */
1767 if (setflag) {
1768 tcg_gen_movi_i32(cpu_NF, -(word != 0));
1769 tcg_gen_movi_i32(cpu_CF, word == 0);
1770 tcg_gen_movi_i32(cpu_VF, 0);
1771 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1772 }
1773 return true;
1774}
1775
3a7be554 1776static bool trans_PTRUE(DisasContext *s, arg_PTRUE *a)
028e2a7b
RH
1777{
1778 return do_predset(s, a->esz, a->rd, a->pat, a->s);
1779}
1780
3a7be554 1781static bool trans_SETFFR(DisasContext *s, arg_SETFFR *a)
028e2a7b
RH
1782{
1783 /* Note pat == 31 is #all, to set all elements. */
1784 return do_predset(s, 0, FFR_PRED_NUM, 31, false);
1785}
1786
3a7be554 1787static bool trans_PFALSE(DisasContext *s, arg_PFALSE *a)
028e2a7b
RH
1788{
1789 /* Note pat == 32 is #unimp, to set no elements. */
1790 return do_predset(s, 0, a->rd, 32, false);
1791}
1792
3a7be554 1793static bool trans_RDFFR_p(DisasContext *s, arg_RDFFR_p *a)
028e2a7b
RH
1794{
1795 /* The path through do_pppp_flags is complicated enough to want to avoid
1796 * duplication. Frob the arguments into the form of a predicated AND.
1797 */
1798 arg_rprr_s alt_a = {
1799 .rd = a->rd, .pg = a->pg, .s = a->s,
1800 .rn = FFR_PRED_NUM, .rm = FFR_PRED_NUM,
1801 };
3a7be554 1802 return trans_AND_pppp(s, &alt_a);
028e2a7b
RH
1803}
1804
3a7be554 1805static bool trans_RDFFR(DisasContext *s, arg_RDFFR *a)
028e2a7b
RH
1806{
1807 return do_mov_p(s, a->rd, FFR_PRED_NUM);
1808}
1809
3a7be554 1810static bool trans_WRFFR(DisasContext *s, arg_WRFFR *a)
028e2a7b
RH
1811{
1812 return do_mov_p(s, FFR_PRED_NUM, a->rn);
1813}
1814
1815static bool do_pfirst_pnext(DisasContext *s, arg_rr_esz *a,
1816 void (*gen_fn)(TCGv_i32, TCGv_ptr,
1817 TCGv_ptr, TCGv_i32))
1818{
1819 if (!sve_access_check(s)) {
1820 return true;
1821 }
1822
1823 TCGv_ptr t_pd = tcg_temp_new_ptr();
1824 TCGv_ptr t_pg = tcg_temp_new_ptr();
1825 TCGv_i32 t;
86300b5d 1826 unsigned desc = 0;
028e2a7b 1827
86300b5d
RH
1828 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
1829 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
028e2a7b
RH
1830
1831 tcg_gen_addi_ptr(t_pd, cpu_env, pred_full_reg_offset(s, a->rd));
1832 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->rn));
1833 t = tcg_const_i32(desc);
1834
1835 gen_fn(t, t_pd, t_pg, t);
1836 tcg_temp_free_ptr(t_pd);
1837 tcg_temp_free_ptr(t_pg);
1838
1839 do_pred_flags(t);
1840 tcg_temp_free_i32(t);
1841 return true;
1842}
1843
3a7be554 1844static bool trans_PFIRST(DisasContext *s, arg_rr_esz *a)
028e2a7b
RH
1845{
1846 return do_pfirst_pnext(s, a, gen_helper_sve_pfirst);
1847}
1848
3a7be554 1849static bool trans_PNEXT(DisasContext *s, arg_rr_esz *a)
028e2a7b
RH
1850{
1851 return do_pfirst_pnext(s, a, gen_helper_sve_pnext);
1852}
1853
24e82e68
RH
1854/*
1855 *** SVE Element Count Group
1856 */
1857
1858/* Perform an inline saturating addition of a 32-bit value within
1859 * a 64-bit register. The second operand is known to be positive,
1860 * which halves the comparisions we must perform to bound the result.
1861 */
1862static void do_sat_addsub_32(TCGv_i64 reg, TCGv_i64 val, bool u, bool d)
1863{
1864 int64_t ibound;
1865 TCGv_i64 bound;
1866 TCGCond cond;
1867
1868 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1869 if (u) {
1870 tcg_gen_ext32u_i64(reg, reg);
1871 } else {
1872 tcg_gen_ext32s_i64(reg, reg);
1873 }
1874 if (d) {
1875 tcg_gen_sub_i64(reg, reg, val);
1876 ibound = (u ? 0 : INT32_MIN);
1877 cond = TCG_COND_LT;
1878 } else {
1879 tcg_gen_add_i64(reg, reg, val);
1880 ibound = (u ? UINT32_MAX : INT32_MAX);
1881 cond = TCG_COND_GT;
1882 }
1883 bound = tcg_const_i64(ibound);
1884 tcg_gen_movcond_i64(cond, reg, reg, bound, bound, reg);
1885 tcg_temp_free_i64(bound);
1886}
1887
1888/* Similarly with 64-bit values. */
1889static void do_sat_addsub_64(TCGv_i64 reg, TCGv_i64 val, bool u, bool d)
1890{
1891 TCGv_i64 t0 = tcg_temp_new_i64();
1892 TCGv_i64 t1 = tcg_temp_new_i64();
1893 TCGv_i64 t2;
1894
1895 if (u) {
1896 if (d) {
1897 tcg_gen_sub_i64(t0, reg, val);
1898 tcg_gen_movi_i64(t1, 0);
1899 tcg_gen_movcond_i64(TCG_COND_LTU, reg, reg, val, t1, t0);
1900 } else {
1901 tcg_gen_add_i64(t0, reg, val);
1902 tcg_gen_movi_i64(t1, -1);
1903 tcg_gen_movcond_i64(TCG_COND_LTU, reg, t0, reg, t1, t0);
1904 }
1905 } else {
1906 if (d) {
1907 /* Detect signed overflow for subtraction. */
1908 tcg_gen_xor_i64(t0, reg, val);
1909 tcg_gen_sub_i64(t1, reg, val);
7a31e0c6 1910 tcg_gen_xor_i64(reg, reg, t1);
24e82e68
RH
1911 tcg_gen_and_i64(t0, t0, reg);
1912
1913 /* Bound the result. */
1914 tcg_gen_movi_i64(reg, INT64_MIN);
1915 t2 = tcg_const_i64(0);
1916 tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, reg, t1);
1917 } else {
1918 /* Detect signed overflow for addition. */
1919 tcg_gen_xor_i64(t0, reg, val);
1920 tcg_gen_add_i64(reg, reg, val);
1921 tcg_gen_xor_i64(t1, reg, val);
1922 tcg_gen_andc_i64(t0, t1, t0);
1923
1924 /* Bound the result. */
1925 tcg_gen_movi_i64(t1, INT64_MAX);
1926 t2 = tcg_const_i64(0);
1927 tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, t1, reg);
1928 }
1929 tcg_temp_free_i64(t2);
1930 }
1931 tcg_temp_free_i64(t0);
1932 tcg_temp_free_i64(t1);
1933}
1934
1935/* Similarly with a vector and a scalar operand. */
1936static void do_sat_addsub_vec(DisasContext *s, int esz, int rd, int rn,
1937 TCGv_i64 val, bool u, bool d)
1938{
1939 unsigned vsz = vec_full_reg_size(s);
1940 TCGv_ptr dptr, nptr;
1941 TCGv_i32 t32, desc;
1942 TCGv_i64 t64;
1943
1944 dptr = tcg_temp_new_ptr();
1945 nptr = tcg_temp_new_ptr();
1946 tcg_gen_addi_ptr(dptr, cpu_env, vec_full_reg_offset(s, rd));
1947 tcg_gen_addi_ptr(nptr, cpu_env, vec_full_reg_offset(s, rn));
1948 desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
1949
1950 switch (esz) {
1951 case MO_8:
1952 t32 = tcg_temp_new_i32();
1953 tcg_gen_extrl_i64_i32(t32, val);
1954 if (d) {
1955 tcg_gen_neg_i32(t32, t32);
1956 }
1957 if (u) {
1958 gen_helper_sve_uqaddi_b(dptr, nptr, t32, desc);
1959 } else {
1960 gen_helper_sve_sqaddi_b(dptr, nptr, t32, desc);
1961 }
1962 tcg_temp_free_i32(t32);
1963 break;
1964
1965 case MO_16:
1966 t32 = tcg_temp_new_i32();
1967 tcg_gen_extrl_i64_i32(t32, val);
1968 if (d) {
1969 tcg_gen_neg_i32(t32, t32);
1970 }
1971 if (u) {
1972 gen_helper_sve_uqaddi_h(dptr, nptr, t32, desc);
1973 } else {
1974 gen_helper_sve_sqaddi_h(dptr, nptr, t32, desc);
1975 }
1976 tcg_temp_free_i32(t32);
1977 break;
1978
1979 case MO_32:
1980 t64 = tcg_temp_new_i64();
1981 if (d) {
1982 tcg_gen_neg_i64(t64, val);
1983 } else {
1984 tcg_gen_mov_i64(t64, val);
1985 }
1986 if (u) {
1987 gen_helper_sve_uqaddi_s(dptr, nptr, t64, desc);
1988 } else {
1989 gen_helper_sve_sqaddi_s(dptr, nptr, t64, desc);
1990 }
1991 tcg_temp_free_i64(t64);
1992 break;
1993
1994 case MO_64:
1995 if (u) {
1996 if (d) {
1997 gen_helper_sve_uqsubi_d(dptr, nptr, val, desc);
1998 } else {
1999 gen_helper_sve_uqaddi_d(dptr, nptr, val, desc);
2000 }
2001 } else if (d) {
2002 t64 = tcg_temp_new_i64();
2003 tcg_gen_neg_i64(t64, val);
2004 gen_helper_sve_sqaddi_d(dptr, nptr, t64, desc);
2005 tcg_temp_free_i64(t64);
2006 } else {
2007 gen_helper_sve_sqaddi_d(dptr, nptr, val, desc);
2008 }
2009 break;
2010
2011 default:
2012 g_assert_not_reached();
2013 }
2014
2015 tcg_temp_free_ptr(dptr);
2016 tcg_temp_free_ptr(nptr);
2017 tcg_temp_free_i32(desc);
2018}
2019
3a7be554 2020static bool trans_CNT_r(DisasContext *s, arg_CNT_r *a)
24e82e68
RH
2021{
2022 if (sve_access_check(s)) {
2023 unsigned fullsz = vec_full_reg_size(s);
2024 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2025 tcg_gen_movi_i64(cpu_reg(s, a->rd), numelem * a->imm);
2026 }
2027 return true;
2028}
2029
3a7be554 2030static bool trans_INCDEC_r(DisasContext *s, arg_incdec_cnt *a)
24e82e68
RH
2031{
2032 if (sve_access_check(s)) {
2033 unsigned fullsz = vec_full_reg_size(s);
2034 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2035 int inc = numelem * a->imm * (a->d ? -1 : 1);
2036 TCGv_i64 reg = cpu_reg(s, a->rd);
2037
2038 tcg_gen_addi_i64(reg, reg, inc);
2039 }
2040 return true;
2041}
2042
3a7be554 2043static bool trans_SINCDEC_r_32(DisasContext *s, arg_incdec_cnt *a)
24e82e68
RH
2044{
2045 if (!sve_access_check(s)) {
2046 return true;
2047 }
2048
2049 unsigned fullsz = vec_full_reg_size(s);
2050 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2051 int inc = numelem * a->imm;
2052 TCGv_i64 reg = cpu_reg(s, a->rd);
2053
2054 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
2055 if (inc == 0) {
2056 if (a->u) {
2057 tcg_gen_ext32u_i64(reg, reg);
2058 } else {
2059 tcg_gen_ext32s_i64(reg, reg);
2060 }
2061 } else {
2062 TCGv_i64 t = tcg_const_i64(inc);
2063 do_sat_addsub_32(reg, t, a->u, a->d);
2064 tcg_temp_free_i64(t);
2065 }
2066 return true;
2067}
2068
3a7be554 2069static bool trans_SINCDEC_r_64(DisasContext *s, arg_incdec_cnt *a)
24e82e68
RH
2070{
2071 if (!sve_access_check(s)) {
2072 return true;
2073 }
2074
2075 unsigned fullsz = vec_full_reg_size(s);
2076 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2077 int inc = numelem * a->imm;
2078 TCGv_i64 reg = cpu_reg(s, a->rd);
2079
2080 if (inc != 0) {
2081 TCGv_i64 t = tcg_const_i64(inc);
2082 do_sat_addsub_64(reg, t, a->u, a->d);
2083 tcg_temp_free_i64(t);
2084 }
2085 return true;
2086}
2087
3a7be554 2088static bool trans_INCDEC_v(DisasContext *s, arg_incdec2_cnt *a)
24e82e68
RH
2089{
2090 if (a->esz == 0) {
2091 return false;
2092 }
2093
2094 unsigned fullsz = vec_full_reg_size(s);
2095 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2096 int inc = numelem * a->imm;
2097
2098 if (inc != 0) {
2099 if (sve_access_check(s)) {
2100 TCGv_i64 t = tcg_const_i64(a->d ? -inc : inc);
2101 tcg_gen_gvec_adds(a->esz, vec_full_reg_offset(s, a->rd),
2102 vec_full_reg_offset(s, a->rn),
2103 t, fullsz, fullsz);
2104 tcg_temp_free_i64(t);
2105 }
2106 } else {
2107 do_mov_z(s, a->rd, a->rn);
2108 }
2109 return true;
2110}
2111
3a7be554 2112static bool trans_SINCDEC_v(DisasContext *s, arg_incdec2_cnt *a)
24e82e68
RH
2113{
2114 if (a->esz == 0) {
2115 return false;
2116 }
2117
2118 unsigned fullsz = vec_full_reg_size(s);
2119 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2120 int inc = numelem * a->imm;
2121
2122 if (inc != 0) {
2123 if (sve_access_check(s)) {
2124 TCGv_i64 t = tcg_const_i64(inc);
2125 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, t, a->u, a->d);
2126 tcg_temp_free_i64(t);
2127 }
2128 } else {
2129 do_mov_z(s, a->rd, a->rn);
2130 }
2131 return true;
2132}
2133
e1fa1164
RH
2134/*
2135 *** SVE Bitwise Immediate Group
2136 */
2137
2138static bool do_zz_dbm(DisasContext *s, arg_rr_dbm *a, GVecGen2iFn *gvec_fn)
2139{
2140 uint64_t imm;
2141 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
2142 extract32(a->dbm, 0, 6),
2143 extract32(a->dbm, 6, 6))) {
2144 return false;
2145 }
2146 if (sve_access_check(s)) {
2147 unsigned vsz = vec_full_reg_size(s);
2148 gvec_fn(MO_64, vec_full_reg_offset(s, a->rd),
2149 vec_full_reg_offset(s, a->rn), imm, vsz, vsz);
2150 }
2151 return true;
2152}
2153
3a7be554 2154static bool trans_AND_zzi(DisasContext *s, arg_rr_dbm *a)
e1fa1164
RH
2155{
2156 return do_zz_dbm(s, a, tcg_gen_gvec_andi);
2157}
2158
3a7be554 2159static bool trans_ORR_zzi(DisasContext *s, arg_rr_dbm *a)
e1fa1164
RH
2160{
2161 return do_zz_dbm(s, a, tcg_gen_gvec_ori);
2162}
2163
3a7be554 2164static bool trans_EOR_zzi(DisasContext *s, arg_rr_dbm *a)
e1fa1164
RH
2165{
2166 return do_zz_dbm(s, a, tcg_gen_gvec_xori);
2167}
2168
3a7be554 2169static bool trans_DUPM(DisasContext *s, arg_DUPM *a)
e1fa1164
RH
2170{
2171 uint64_t imm;
2172 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
2173 extract32(a->dbm, 0, 6),
2174 extract32(a->dbm, 6, 6))) {
2175 return false;
2176 }
2177 if (sve_access_check(s)) {
2178 do_dupi_z(s, a->rd, imm);
2179 }
2180 return true;
2181}
2182
f25a2361
RH
2183/*
2184 *** SVE Integer Wide Immediate - Predicated Group
2185 */
2186
2187/* Implement all merging copies. This is used for CPY (immediate),
2188 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2189 */
2190static void do_cpy_m(DisasContext *s, int esz, int rd, int rn, int pg,
2191 TCGv_i64 val)
2192{
2193 typedef void gen_cpy(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
2194 static gen_cpy * const fns[4] = {
2195 gen_helper_sve_cpy_m_b, gen_helper_sve_cpy_m_h,
2196 gen_helper_sve_cpy_m_s, gen_helper_sve_cpy_m_d,
2197 };
2198 unsigned vsz = vec_full_reg_size(s);
2199 TCGv_i32 desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
2200 TCGv_ptr t_zd = tcg_temp_new_ptr();
2201 TCGv_ptr t_zn = tcg_temp_new_ptr();
2202 TCGv_ptr t_pg = tcg_temp_new_ptr();
2203
2204 tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, rd));
2205 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, rn));
2206 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
2207
2208 fns[esz](t_zd, t_zn, t_pg, val, desc);
2209
2210 tcg_temp_free_ptr(t_zd);
2211 tcg_temp_free_ptr(t_zn);
2212 tcg_temp_free_ptr(t_pg);
2213 tcg_temp_free_i32(desc);
2214}
2215
3a7be554 2216static bool trans_FCPY(DisasContext *s, arg_FCPY *a)
f25a2361
RH
2217{
2218 if (a->esz == 0) {
2219 return false;
2220 }
2221 if (sve_access_check(s)) {
2222 /* Decode the VFP immediate. */
2223 uint64_t imm = vfp_expand_imm(a->esz, a->imm);
2224 TCGv_i64 t_imm = tcg_const_i64(imm);
2225 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, t_imm);
2226 tcg_temp_free_i64(t_imm);
2227 }
2228 return true;
2229}
2230
3a7be554 2231static bool trans_CPY_m_i(DisasContext *s, arg_rpri_esz *a)
f25a2361 2232{
3a7be554 2233 if (a->esz == 0 && extract32(s->insn, 13, 1)) {
f25a2361
RH
2234 return false;
2235 }
2236 if (sve_access_check(s)) {
2237 TCGv_i64 t_imm = tcg_const_i64(a->imm);
2238 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, t_imm);
2239 tcg_temp_free_i64(t_imm);
2240 }
2241 return true;
2242}
2243
3a7be554 2244static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a)
f25a2361
RH
2245{
2246 static gen_helper_gvec_2i * const fns[4] = {
2247 gen_helper_sve_cpy_z_b, gen_helper_sve_cpy_z_h,
2248 gen_helper_sve_cpy_z_s, gen_helper_sve_cpy_z_d,
2249 };
2250
3a7be554 2251 if (a->esz == 0 && extract32(s->insn, 13, 1)) {
f25a2361
RH
2252 return false;
2253 }
2254 if (sve_access_check(s)) {
2255 unsigned vsz = vec_full_reg_size(s);
2256 TCGv_i64 t_imm = tcg_const_i64(a->imm);
2257 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd),
2258 pred_full_reg_offset(s, a->pg),
2259 t_imm, vsz, vsz, 0, fns[a->esz]);
2260 tcg_temp_free_i64(t_imm);
2261 }
2262 return true;
2263}
2264
b94f8f60
RH
2265/*
2266 *** SVE Permute Extract Group
2267 */
2268
75114792 2269static bool do_EXT(DisasContext *s, int rd, int rn, int rm, int imm)
b94f8f60
RH
2270{
2271 if (!sve_access_check(s)) {
2272 return true;
2273 }
2274
2275 unsigned vsz = vec_full_reg_size(s);
75114792 2276 unsigned n_ofs = imm >= vsz ? 0 : imm;
b94f8f60 2277 unsigned n_siz = vsz - n_ofs;
75114792
SL
2278 unsigned d = vec_full_reg_offset(s, rd);
2279 unsigned n = vec_full_reg_offset(s, rn);
2280 unsigned m = vec_full_reg_offset(s, rm);
b94f8f60
RH
2281
2282 /* Use host vector move insns if we have appropriate sizes
2283 * and no unfortunate overlap.
2284 */
2285 if (m != d
2286 && n_ofs == size_for_gvec(n_ofs)
2287 && n_siz == size_for_gvec(n_siz)
2288 && (d != n || n_siz <= n_ofs)) {
2289 tcg_gen_gvec_mov(0, d, n + n_ofs, n_siz, n_siz);
2290 if (n_ofs != 0) {
2291 tcg_gen_gvec_mov(0, d + n_siz, m, n_ofs, n_ofs);
2292 }
2293 } else {
2294 tcg_gen_gvec_3_ool(d, n, m, vsz, vsz, n_ofs, gen_helper_sve_ext);
2295 }
2296 return true;
2297}
2298
75114792
SL
2299static bool trans_EXT(DisasContext *s, arg_EXT *a)
2300{
2301 return do_EXT(s, a->rd, a->rn, a->rm, a->imm);
2302}
2303
2304static bool trans_EXT_sve2(DisasContext *s, arg_rri *a)
2305{
2306 if (!dc_isar_feature(aa64_sve2, s)) {
2307 return false;
2308 }
2309 return do_EXT(s, a->rd, a->rn, (a->rn + 1) % 32, a->imm);
2310}
2311
30562ab7
RH
2312/*
2313 *** SVE Permute - Unpredicated Group
2314 */
2315
3a7be554 2316static bool trans_DUP_s(DisasContext *s, arg_DUP_s *a)
30562ab7
RH
2317{
2318 if (sve_access_check(s)) {
2319 unsigned vsz = vec_full_reg_size(s);
2320 tcg_gen_gvec_dup_i64(a->esz, vec_full_reg_offset(s, a->rd),
2321 vsz, vsz, cpu_reg_sp(s, a->rn));
2322 }
2323 return true;
2324}
2325
3a7be554 2326static bool trans_DUP_x(DisasContext *s, arg_DUP_x *a)
30562ab7
RH
2327{
2328 if ((a->imm & 0x1f) == 0) {
2329 return false;
2330 }
2331 if (sve_access_check(s)) {
2332 unsigned vsz = vec_full_reg_size(s);
2333 unsigned dofs = vec_full_reg_offset(s, a->rd);
2334 unsigned esz, index;
2335
2336 esz = ctz32(a->imm);
2337 index = a->imm >> (esz + 1);
2338
2339 if ((index << esz) < vsz) {
2340 unsigned nofs = vec_reg_offset(s, a->rn, index, esz);
2341 tcg_gen_gvec_dup_mem(esz, dofs, nofs, vsz, vsz);
2342 } else {
7e17d50e
RH
2343 /*
2344 * While dup_mem handles 128-bit elements, dup_imm does not.
2345 * Thankfully element size doesn't matter for splatting zero.
2346 */
2347 tcg_gen_gvec_dup_imm(MO_64, dofs, vsz, vsz, 0);
30562ab7
RH
2348 }
2349 }
2350 return true;
2351}
2352
2353static void do_insr_i64(DisasContext *s, arg_rrr_esz *a, TCGv_i64 val)
2354{
2355 typedef void gen_insr(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
2356 static gen_insr * const fns[4] = {
2357 gen_helper_sve_insr_b, gen_helper_sve_insr_h,
2358 gen_helper_sve_insr_s, gen_helper_sve_insr_d,
2359 };
2360 unsigned vsz = vec_full_reg_size(s);
2361 TCGv_i32 desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
2362 TCGv_ptr t_zd = tcg_temp_new_ptr();
2363 TCGv_ptr t_zn = tcg_temp_new_ptr();
2364
2365 tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, a->rd));
2366 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn));
2367
2368 fns[a->esz](t_zd, t_zn, val, desc);
2369
2370 tcg_temp_free_ptr(t_zd);
2371 tcg_temp_free_ptr(t_zn);
2372 tcg_temp_free_i32(desc);
2373}
2374
3a7be554 2375static bool trans_INSR_f(DisasContext *s, arg_rrr_esz *a)
30562ab7
RH
2376{
2377 if (sve_access_check(s)) {
2378 TCGv_i64 t = tcg_temp_new_i64();
2379 tcg_gen_ld_i64(t, cpu_env, vec_reg_offset(s, a->rm, 0, MO_64));
2380 do_insr_i64(s, a, t);
2381 tcg_temp_free_i64(t);
2382 }
2383 return true;
2384}
2385
3a7be554 2386static bool trans_INSR_r(DisasContext *s, arg_rrr_esz *a)
30562ab7
RH
2387{
2388 if (sve_access_check(s)) {
2389 do_insr_i64(s, a, cpu_reg(s, a->rm));
2390 }
2391 return true;
2392}
2393
3a7be554 2394static bool trans_REV_v(DisasContext *s, arg_rr_esz *a)
30562ab7
RH
2395{
2396 static gen_helper_gvec_2 * const fns[4] = {
2397 gen_helper_sve_rev_b, gen_helper_sve_rev_h,
2398 gen_helper_sve_rev_s, gen_helper_sve_rev_d
2399 };
2400
2401 if (sve_access_check(s)) {
40e32e5a 2402 gen_gvec_ool_zz(s, fns[a->esz], a->rd, a->rn, 0);
30562ab7
RH
2403 }
2404 return true;
2405}
2406
3a7be554 2407static bool trans_TBL(DisasContext *s, arg_rrr_esz *a)
30562ab7
RH
2408{
2409 static gen_helper_gvec_3 * const fns[4] = {
2410 gen_helper_sve_tbl_b, gen_helper_sve_tbl_h,
2411 gen_helper_sve_tbl_s, gen_helper_sve_tbl_d
2412 };
2413
2414 if (sve_access_check(s)) {
e645d1a1 2415 gen_gvec_ool_zzz(s, fns[a->esz], a->rd, a->rn, a->rm, 0);
30562ab7
RH
2416 }
2417 return true;
2418}
2419
3a7be554 2420static bool trans_UNPK(DisasContext *s, arg_UNPK *a)
30562ab7
RH
2421{
2422 static gen_helper_gvec_2 * const fns[4][2] = {
2423 { NULL, NULL },
2424 { gen_helper_sve_sunpk_h, gen_helper_sve_uunpk_h },
2425 { gen_helper_sve_sunpk_s, gen_helper_sve_uunpk_s },
2426 { gen_helper_sve_sunpk_d, gen_helper_sve_uunpk_d },
2427 };
2428
2429 if (a->esz == 0) {
2430 return false;
2431 }
2432 if (sve_access_check(s)) {
2433 unsigned vsz = vec_full_reg_size(s);
2434 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd),
2435 vec_full_reg_offset(s, a->rn)
2436 + (a->h ? vsz / 2 : 0),
2437 vsz, vsz, 0, fns[a->esz][a->u]);
2438 }
2439 return true;
2440}
2441
d731d8cb
RH
2442/*
2443 *** SVE Permute - Predicates Group
2444 */
2445
2446static bool do_perm_pred3(DisasContext *s, arg_rrr_esz *a, bool high_odd,
2447 gen_helper_gvec_3 *fn)
2448{
2449 if (!sve_access_check(s)) {
2450 return true;
2451 }
2452
2453 unsigned vsz = pred_full_reg_size(s);
2454
d731d8cb
RH
2455 TCGv_ptr t_d = tcg_temp_new_ptr();
2456 TCGv_ptr t_n = tcg_temp_new_ptr();
2457 TCGv_ptr t_m = tcg_temp_new_ptr();
2458 TCGv_i32 t_desc;
f9b0fcce 2459 uint32_t desc = 0;
d731d8cb 2460
f9b0fcce
RH
2461 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz);
2462 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
2463 desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd);
d731d8cb
RH
2464
2465 tcg_gen_addi_ptr(t_d, cpu_env, pred_full_reg_offset(s, a->rd));
2466 tcg_gen_addi_ptr(t_n, cpu_env, pred_full_reg_offset(s, a->rn));
2467 tcg_gen_addi_ptr(t_m, cpu_env, pred_full_reg_offset(s, a->rm));
2468 t_desc = tcg_const_i32(desc);
2469
2470 fn(t_d, t_n, t_m, t_desc);
2471
2472 tcg_temp_free_ptr(t_d);
2473 tcg_temp_free_ptr(t_n);
2474 tcg_temp_free_ptr(t_m);
2475 tcg_temp_free_i32(t_desc);
2476 return true;
2477}
2478
2479static bool do_perm_pred2(DisasContext *s, arg_rr_esz *a, bool high_odd,
2480 gen_helper_gvec_2 *fn)
2481{
2482 if (!sve_access_check(s)) {
2483 return true;
2484 }
2485
2486 unsigned vsz = pred_full_reg_size(s);
2487 TCGv_ptr t_d = tcg_temp_new_ptr();
2488 TCGv_ptr t_n = tcg_temp_new_ptr();
2489 TCGv_i32 t_desc;
70acaafe 2490 uint32_t desc = 0;
d731d8cb
RH
2491
2492 tcg_gen_addi_ptr(t_d, cpu_env, pred_full_reg_offset(s, a->rd));
2493 tcg_gen_addi_ptr(t_n, cpu_env, pred_full_reg_offset(s, a->rn));
2494
70acaafe
RH
2495 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz);
2496 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
2497 desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd);
d731d8cb
RH
2498 t_desc = tcg_const_i32(desc);
2499
2500 fn(t_d, t_n, t_desc);
2501
2502 tcg_temp_free_i32(t_desc);
2503 tcg_temp_free_ptr(t_d);
2504 tcg_temp_free_ptr(t_n);
2505 return true;
2506}
2507
3a7be554 2508static bool trans_ZIP1_p(DisasContext *s, arg_rrr_esz *a)
d731d8cb
RH
2509{
2510 return do_perm_pred3(s, a, 0, gen_helper_sve_zip_p);
2511}
2512
3a7be554 2513static bool trans_ZIP2_p(DisasContext *s, arg_rrr_esz *a)
d731d8cb
RH
2514{
2515 return do_perm_pred3(s, a, 1, gen_helper_sve_zip_p);
2516}
2517
3a7be554 2518static bool trans_UZP1_p(DisasContext *s, arg_rrr_esz *a)
d731d8cb
RH
2519{
2520 return do_perm_pred3(s, a, 0, gen_helper_sve_uzp_p);
2521}
2522
3a7be554 2523static bool trans_UZP2_p(DisasContext *s, arg_rrr_esz *a)
d731d8cb
RH
2524{
2525 return do_perm_pred3(s, a, 1, gen_helper_sve_uzp_p);
2526}
2527
3a7be554 2528static bool trans_TRN1_p(DisasContext *s, arg_rrr_esz *a)
d731d8cb
RH
2529{
2530 return do_perm_pred3(s, a, 0, gen_helper_sve_trn_p);
2531}
2532
3a7be554 2533static bool trans_TRN2_p(DisasContext *s, arg_rrr_esz *a)
d731d8cb
RH
2534{
2535 return do_perm_pred3(s, a, 1, gen_helper_sve_trn_p);
2536}
2537
3a7be554 2538static bool trans_REV_p(DisasContext *s, arg_rr_esz *a)
d731d8cb
RH
2539{
2540 return do_perm_pred2(s, a, 0, gen_helper_sve_rev_p);
2541}
2542
3a7be554 2543static bool trans_PUNPKLO(DisasContext *s, arg_PUNPKLO *a)
d731d8cb
RH
2544{
2545 return do_perm_pred2(s, a, 0, gen_helper_sve_punpk_p);
2546}
2547
3a7be554 2548static bool trans_PUNPKHI(DisasContext *s, arg_PUNPKHI *a)
d731d8cb
RH
2549{
2550 return do_perm_pred2(s, a, 1, gen_helper_sve_punpk_p);
2551}
2552
234b48e9
RH
2553/*
2554 *** SVE Permute - Interleaving Group
2555 */
2556
2557static bool do_zip(DisasContext *s, arg_rrr_esz *a, bool high)
2558{
2559 static gen_helper_gvec_3 * const fns[4] = {
2560 gen_helper_sve_zip_b, gen_helper_sve_zip_h,
2561 gen_helper_sve_zip_s, gen_helper_sve_zip_d,
2562 };
2563
2564 if (sve_access_check(s)) {
2565 unsigned vsz = vec_full_reg_size(s);
2566 unsigned high_ofs = high ? vsz / 2 : 0;
2567 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
2568 vec_full_reg_offset(s, a->rn) + high_ofs,
2569 vec_full_reg_offset(s, a->rm) + high_ofs,
2570 vsz, vsz, 0, fns[a->esz]);
2571 }
2572 return true;
2573}
2574
2575static bool do_zzz_data_ool(DisasContext *s, arg_rrr_esz *a, int data,
2576 gen_helper_gvec_3 *fn)
2577{
2578 if (sve_access_check(s)) {
e645d1a1 2579 gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data);
234b48e9
RH
2580 }
2581 return true;
2582}
2583
3a7be554 2584static bool trans_ZIP1_z(DisasContext *s, arg_rrr_esz *a)
234b48e9
RH
2585{
2586 return do_zip(s, a, false);
2587}
2588
3a7be554 2589static bool trans_ZIP2_z(DisasContext *s, arg_rrr_esz *a)
234b48e9
RH
2590{
2591 return do_zip(s, a, true);
2592}
2593
2594static gen_helper_gvec_3 * const uzp_fns[4] = {
2595 gen_helper_sve_uzp_b, gen_helper_sve_uzp_h,
2596 gen_helper_sve_uzp_s, gen_helper_sve_uzp_d,
2597};
2598
3a7be554 2599static bool trans_UZP1_z(DisasContext *s, arg_rrr_esz *a)
234b48e9
RH
2600{
2601 return do_zzz_data_ool(s, a, 0, uzp_fns[a->esz]);
2602}
2603
3a7be554 2604static bool trans_UZP2_z(DisasContext *s, arg_rrr_esz *a)
234b48e9
RH
2605{
2606 return do_zzz_data_ool(s, a, 1 << a->esz, uzp_fns[a->esz]);
2607}
2608
2609static gen_helper_gvec_3 * const trn_fns[4] = {
2610 gen_helper_sve_trn_b, gen_helper_sve_trn_h,
2611 gen_helper_sve_trn_s, gen_helper_sve_trn_d,
2612};
2613
3a7be554 2614static bool trans_TRN1_z(DisasContext *s, arg_rrr_esz *a)
234b48e9
RH
2615{
2616 return do_zzz_data_ool(s, a, 0, trn_fns[a->esz]);
2617}
2618
3a7be554 2619static bool trans_TRN2_z(DisasContext *s, arg_rrr_esz *a)
234b48e9
RH
2620{
2621 return do_zzz_data_ool(s, a, 1 << a->esz, trn_fns[a->esz]);
2622}
2623
3ca879ae
RH
2624/*
2625 *** SVE Permute Vector - Predicated Group
2626 */
2627
3a7be554 2628static bool trans_COMPACT(DisasContext *s, arg_rpr_esz *a)
3ca879ae
RH
2629{
2630 static gen_helper_gvec_3 * const fns[4] = {
2631 NULL, NULL, gen_helper_sve_compact_s, gen_helper_sve_compact_d
2632 };
2633 return do_zpz_ool(s, a, fns[a->esz]);
2634}
2635
ef23cb72
RH
2636/* Call the helper that computes the ARM LastActiveElement pseudocode
2637 * function, scaled by the element size. This includes the not found
2638 * indication; e.g. not found for esz=3 is -8.
2639 */
2640static void find_last_active(DisasContext *s, TCGv_i32 ret, int esz, int pg)
2641{
2642 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2643 * round up, as we do elsewhere, because we need the exact size.
2644 */
2645 TCGv_ptr t_p = tcg_temp_new_ptr();
2646 TCGv_i32 t_desc;
2acbfbe4 2647 unsigned desc = 0;
ef23cb72 2648
2acbfbe4
RH
2649 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
2650 desc = FIELD_DP32(desc, PREDDESC, ESZ, esz);
ef23cb72
RH
2651
2652 tcg_gen_addi_ptr(t_p, cpu_env, pred_full_reg_offset(s, pg));
2653 t_desc = tcg_const_i32(desc);
2654
2655 gen_helper_sve_last_active_element(ret, t_p, t_desc);
2656
2657 tcg_temp_free_i32(t_desc);
2658 tcg_temp_free_ptr(t_p);
2659}
2660
2661/* Increment LAST to the offset of the next element in the vector,
2662 * wrapping around to 0.
2663 */
2664static void incr_last_active(DisasContext *s, TCGv_i32 last, int esz)
2665{
2666 unsigned vsz = vec_full_reg_size(s);
2667
2668 tcg_gen_addi_i32(last, last, 1 << esz);
2669 if (is_power_of_2(vsz)) {
2670 tcg_gen_andi_i32(last, last, vsz - 1);
2671 } else {
2672 TCGv_i32 max = tcg_const_i32(vsz);
2673 TCGv_i32 zero = tcg_const_i32(0);
2674 tcg_gen_movcond_i32(TCG_COND_GEU, last, last, max, zero, last);
2675 tcg_temp_free_i32(max);
2676 tcg_temp_free_i32(zero);
2677 }
2678}
2679
2680/* If LAST < 0, set LAST to the offset of the last element in the vector. */
2681static void wrap_last_active(DisasContext *s, TCGv_i32 last, int esz)
2682{
2683 unsigned vsz = vec_full_reg_size(s);
2684
2685 if (is_power_of_2(vsz)) {
2686 tcg_gen_andi_i32(last, last, vsz - 1);
2687 } else {
2688 TCGv_i32 max = tcg_const_i32(vsz - (1 << esz));
2689 TCGv_i32 zero = tcg_const_i32(0);
2690 tcg_gen_movcond_i32(TCG_COND_LT, last, last, zero, max, last);
2691 tcg_temp_free_i32(max);
2692 tcg_temp_free_i32(zero);
2693 }
2694}
2695
2696/* Load an unsigned element of ESZ from BASE+OFS. */
2697static TCGv_i64 load_esz(TCGv_ptr base, int ofs, int esz)
2698{
2699 TCGv_i64 r = tcg_temp_new_i64();
2700
2701 switch (esz) {
2702 case 0:
2703 tcg_gen_ld8u_i64(r, base, ofs);
2704 break;
2705 case 1:
2706 tcg_gen_ld16u_i64(r, base, ofs);
2707 break;
2708 case 2:
2709 tcg_gen_ld32u_i64(r, base, ofs);
2710 break;
2711 case 3:
2712 tcg_gen_ld_i64(r, base, ofs);
2713 break;
2714 default:
2715 g_assert_not_reached();
2716 }
2717 return r;
2718}
2719
2720/* Load an unsigned element of ESZ from RM[LAST]. */
2721static TCGv_i64 load_last_active(DisasContext *s, TCGv_i32 last,
2722 int rm, int esz)
2723{
2724 TCGv_ptr p = tcg_temp_new_ptr();
2725 TCGv_i64 r;
2726
2727 /* Convert offset into vector into offset into ENV.
2728 * The final adjustment for the vector register base
2729 * is added via constant offset to the load.
2730 */
2731#ifdef HOST_WORDS_BIGENDIAN
2732 /* Adjust for element ordering. See vec_reg_offset. */
2733 if (esz < 3) {
2734 tcg_gen_xori_i32(last, last, 8 - (1 << esz));
2735 }
2736#endif
2737 tcg_gen_ext_i32_ptr(p, last);
2738 tcg_gen_add_ptr(p, p, cpu_env);
2739
2740 r = load_esz(p, vec_full_reg_offset(s, rm), esz);
2741 tcg_temp_free_ptr(p);
2742
2743 return r;
2744}
2745
2746/* Compute CLAST for a Zreg. */
2747static bool do_clast_vector(DisasContext *s, arg_rprr_esz *a, bool before)
2748{
2749 TCGv_i32 last;
2750 TCGLabel *over;
2751 TCGv_i64 ele;
2752 unsigned vsz, esz = a->esz;
2753
2754 if (!sve_access_check(s)) {
2755 return true;
2756 }
2757
2758 last = tcg_temp_local_new_i32();
2759 over = gen_new_label();
2760
2761 find_last_active(s, last, esz, a->pg);
2762
2763 /* There is of course no movcond for a 2048-bit vector,
2764 * so we must branch over the actual store.
2765 */
2766 tcg_gen_brcondi_i32(TCG_COND_LT, last, 0, over);
2767
2768 if (!before) {
2769 incr_last_active(s, last, esz);
2770 }
2771
2772 ele = load_last_active(s, last, a->rm, esz);
2773 tcg_temp_free_i32(last);
2774
2775 vsz = vec_full_reg_size(s);
2776 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), vsz, vsz, ele);
2777 tcg_temp_free_i64(ele);
2778
2779 /* If this insn used MOVPRFX, we may need a second move. */
2780 if (a->rd != a->rn) {
2781 TCGLabel *done = gen_new_label();
2782 tcg_gen_br(done);
2783
2784 gen_set_label(over);
2785 do_mov_z(s, a->rd, a->rn);
2786
2787 gen_set_label(done);
2788 } else {
2789 gen_set_label(over);
2790 }
2791 return true;
2792}
2793
3a7be554 2794static bool trans_CLASTA_z(DisasContext *s, arg_rprr_esz *a)
ef23cb72
RH
2795{
2796 return do_clast_vector(s, a, false);
2797}
2798
3a7be554 2799static bool trans_CLASTB_z(DisasContext *s, arg_rprr_esz *a)
ef23cb72
RH
2800{
2801 return do_clast_vector(s, a, true);
2802}
2803
2804/* Compute CLAST for a scalar. */
2805static void do_clast_scalar(DisasContext *s, int esz, int pg, int rm,
2806 bool before, TCGv_i64 reg_val)
2807{
2808 TCGv_i32 last = tcg_temp_new_i32();
2809 TCGv_i64 ele, cmp, zero;
2810
2811 find_last_active(s, last, esz, pg);
2812
2813 /* Extend the original value of last prior to incrementing. */
2814 cmp = tcg_temp_new_i64();
2815 tcg_gen_ext_i32_i64(cmp, last);
2816
2817 if (!before) {
2818 incr_last_active(s, last, esz);
2819 }
2820
2821 /* The conceit here is that while last < 0 indicates not found, after
2822 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2823 * from which we can load garbage. We then discard the garbage with
2824 * a conditional move.
2825 */
2826 ele = load_last_active(s, last, rm, esz);
2827 tcg_temp_free_i32(last);
2828
2829 zero = tcg_const_i64(0);
2830 tcg_gen_movcond_i64(TCG_COND_GE, reg_val, cmp, zero, ele, reg_val);
2831
2832 tcg_temp_free_i64(zero);
2833 tcg_temp_free_i64(cmp);
2834 tcg_temp_free_i64(ele);
2835}
2836
2837/* Compute CLAST for a Vreg. */
2838static bool do_clast_fp(DisasContext *s, arg_rpr_esz *a, bool before)
2839{
2840 if (sve_access_check(s)) {
2841 int esz = a->esz;
2842 int ofs = vec_reg_offset(s, a->rd, 0, esz);
2843 TCGv_i64 reg = load_esz(cpu_env, ofs, esz);
2844
2845 do_clast_scalar(s, esz, a->pg, a->rn, before, reg);
2846 write_fp_dreg(s, a->rd, reg);
2847 tcg_temp_free_i64(reg);
2848 }
2849 return true;
2850}
2851
3a7be554 2852static bool trans_CLASTA_v(DisasContext *s, arg_rpr_esz *a)
ef23cb72
RH
2853{
2854 return do_clast_fp(s, a, false);
2855}
2856
3a7be554 2857static bool trans_CLASTB_v(DisasContext *s, arg_rpr_esz *a)
ef23cb72
RH
2858{
2859 return do_clast_fp(s, a, true);
2860}
2861
2862/* Compute CLAST for a Xreg. */
2863static bool do_clast_general(DisasContext *s, arg_rpr_esz *a, bool before)
2864{
2865 TCGv_i64 reg;
2866
2867 if (!sve_access_check(s)) {
2868 return true;
2869 }
2870
2871 reg = cpu_reg(s, a->rd);
2872 switch (a->esz) {
2873 case 0:
2874 tcg_gen_ext8u_i64(reg, reg);
2875 break;
2876 case 1:
2877 tcg_gen_ext16u_i64(reg, reg);
2878 break;
2879 case 2:
2880 tcg_gen_ext32u_i64(reg, reg);
2881 break;
2882 case 3:
2883 break;
2884 default:
2885 g_assert_not_reached();
2886 }
2887
2888 do_clast_scalar(s, a->esz, a->pg, a->rn, before, reg);
2889 return true;
2890}
2891
3a7be554 2892static bool trans_CLASTA_r(DisasContext *s, arg_rpr_esz *a)
ef23cb72
RH
2893{
2894 return do_clast_general(s, a, false);
2895}
2896
3a7be554 2897static bool trans_CLASTB_r(DisasContext *s, arg_rpr_esz *a)
ef23cb72
RH
2898{
2899 return do_clast_general(s, a, true);
2900}
2901
2902/* Compute LAST for a scalar. */
2903static TCGv_i64 do_last_scalar(DisasContext *s, int esz,
2904 int pg, int rm, bool before)
2905{
2906 TCGv_i32 last = tcg_temp_new_i32();
2907 TCGv_i64 ret;
2908
2909 find_last_active(s, last, esz, pg);
2910 if (before) {
2911 wrap_last_active(s, last, esz);
2912 } else {
2913 incr_last_active(s, last, esz);
2914 }
2915
2916 ret = load_last_active(s, last, rm, esz);
2917 tcg_temp_free_i32(last);
2918 return ret;
2919}
2920
2921/* Compute LAST for a Vreg. */
2922static bool do_last_fp(DisasContext *s, arg_rpr_esz *a, bool before)
2923{
2924 if (sve_access_check(s)) {
2925 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before);
2926 write_fp_dreg(s, a->rd, val);
2927 tcg_temp_free_i64(val);
2928 }
2929 return true;
2930}
2931
3a7be554 2932static bool trans_LASTA_v(DisasContext *s, arg_rpr_esz *a)
ef23cb72
RH
2933{
2934 return do_last_fp(s, a, false);
2935}
2936
3a7be554 2937static bool trans_LASTB_v(DisasContext *s, arg_rpr_esz *a)
ef23cb72
RH
2938{
2939 return do_last_fp(s, a, true);
2940}
2941
2942/* Compute LAST for a Xreg. */
2943static bool do_last_general(DisasContext *s, arg_rpr_esz *a, bool before)
2944{
2945 if (sve_access_check(s)) {
2946 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before);
2947 tcg_gen_mov_i64(cpu_reg(s, a->rd), val);
2948 tcg_temp_free_i64(val);
2949 }
2950 return true;
2951}
2952
3a7be554 2953static bool trans_LASTA_r(DisasContext *s, arg_rpr_esz *a)
ef23cb72
RH
2954{
2955 return do_last_general(s, a, false);
2956}
2957
3a7be554 2958static bool trans_LASTB_r(DisasContext *s, arg_rpr_esz *a)
ef23cb72
RH
2959{
2960 return do_last_general(s, a, true);
2961}
2962
3a7be554 2963static bool trans_CPY_m_r(DisasContext *s, arg_rpr_esz *a)
792a5578
RH
2964{
2965 if (sve_access_check(s)) {
2966 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, cpu_reg_sp(s, a->rn));
2967 }
2968 return true;
2969}
2970
3a7be554 2971static bool trans_CPY_m_v(DisasContext *s, arg_rpr_esz *a)
792a5578
RH
2972{
2973 if (sve_access_check(s)) {
2974 int ofs = vec_reg_offset(s, a->rn, 0, a->esz);
2975 TCGv_i64 t = load_esz(cpu_env, ofs, a->esz);
2976 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, t);
2977 tcg_temp_free_i64(t);
2978 }
2979 return true;
2980}
2981
3a7be554 2982static bool trans_REVB(DisasContext *s, arg_rpr_esz *a)
dae8fb90
RH
2983{
2984 static gen_helper_gvec_3 * const fns[4] = {
2985 NULL,
2986 gen_helper_sve_revb_h,
2987 gen_helper_sve_revb_s,
2988 gen_helper_sve_revb_d,
2989 };
2990 return do_zpz_ool(s, a, fns[a->esz]);
2991}
2992
3a7be554 2993static bool trans_REVH(DisasContext *s, arg_rpr_esz *a)
dae8fb90
RH
2994{
2995 static gen_helper_gvec_3 * const fns[4] = {
2996 NULL,
2997 NULL,
2998 gen_helper_sve_revh_s,
2999 gen_helper_sve_revh_d,
3000 };
3001 return do_zpz_ool(s, a, fns[a->esz]);
3002}
3003
3a7be554 3004static bool trans_REVW(DisasContext *s, arg_rpr_esz *a)
dae8fb90
RH
3005{
3006 return do_zpz_ool(s, a, a->esz == 3 ? gen_helper_sve_revw_d : NULL);
3007}
3008
3a7be554 3009static bool trans_RBIT(DisasContext *s, arg_rpr_esz *a)
dae8fb90
RH
3010{
3011 static gen_helper_gvec_3 * const fns[4] = {
3012 gen_helper_sve_rbit_b,
3013 gen_helper_sve_rbit_h,
3014 gen_helper_sve_rbit_s,
3015 gen_helper_sve_rbit_d,
3016 };
3017 return do_zpz_ool(s, a, fns[a->esz]);
3018}
3019
3a7be554 3020static bool trans_SPLICE(DisasContext *s, arg_rprr_esz *a)
b48ff240
RH
3021{
3022 if (sve_access_check(s)) {
36cbb7a8 3023 gen_gvec_ool_zzzp(s, gen_helper_sve_splice,
dd701faf 3024 a->rd, a->rn, a->rm, a->pg, a->esz);
b48ff240
RH
3025 }
3026 return true;
3027}
3028
75114792
SL
3029static bool trans_SPLICE_sve2(DisasContext *s, arg_rpr_esz *a)
3030{
3031 if (!dc_isar_feature(aa64_sve2, s)) {
3032 return false;
3033 }
3034 if (sve_access_check(s)) {
3035 gen_gvec_ool_zzzp(s, gen_helper_sve_splice,
3036 a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz);
3037 }
3038 return true;
3039}
3040
757f9cff
RH
3041/*
3042 *** SVE Integer Compare - Vectors Group
3043 */
3044
3045static bool do_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
3046 gen_helper_gvec_flags_4 *gen_fn)
3047{
3048 TCGv_ptr pd, zn, zm, pg;
3049 unsigned vsz;
3050 TCGv_i32 t;
3051
3052 if (gen_fn == NULL) {
3053 return false;
3054 }
3055 if (!sve_access_check(s)) {
3056 return true;
3057 }
3058
3059 vsz = vec_full_reg_size(s);
3060 t = tcg_const_i32(simd_desc(vsz, vsz, 0));
3061 pd = tcg_temp_new_ptr();
3062 zn = tcg_temp_new_ptr();
3063 zm = tcg_temp_new_ptr();
3064 pg = tcg_temp_new_ptr();
3065
3066 tcg_gen_addi_ptr(pd, cpu_env, pred_full_reg_offset(s, a->rd));
3067 tcg_gen_addi_ptr(zn, cpu_env, vec_full_reg_offset(s, a->rn));
3068 tcg_gen_addi_ptr(zm, cpu_env, vec_full_reg_offset(s, a->rm));
3069 tcg_gen_addi_ptr(pg, cpu_env, pred_full_reg_offset(s, a->pg));
3070
3071 gen_fn(t, pd, zn, zm, pg, t);
3072
3073 tcg_temp_free_ptr(pd);
3074 tcg_temp_free_ptr(zn);
3075 tcg_temp_free_ptr(zm);
3076 tcg_temp_free_ptr(pg);
3077
3078 do_pred_flags(t);
3079
3080 tcg_temp_free_i32(t);
3081 return true;
3082}
3083
3084#define DO_PPZZ(NAME, name) \
3a7be554 3085static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
757f9cff
RH
3086{ \
3087 static gen_helper_gvec_flags_4 * const fns[4] = { \
3088 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
3089 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
3090 }; \
3091 return do_ppzz_flags(s, a, fns[a->esz]); \
3092}
3093
3094DO_PPZZ(CMPEQ, cmpeq)
3095DO_PPZZ(CMPNE, cmpne)
3096DO_PPZZ(CMPGT, cmpgt)
3097DO_PPZZ(CMPGE, cmpge)
3098DO_PPZZ(CMPHI, cmphi)
3099DO_PPZZ(CMPHS, cmphs)
3100
3101#undef DO_PPZZ
3102
3103#define DO_PPZW(NAME, name) \
3a7be554 3104static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
757f9cff
RH
3105{ \
3106 static gen_helper_gvec_flags_4 * const fns[4] = { \
3107 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
3108 gen_helper_sve_##name##_ppzw_s, NULL \
3109 }; \
3110 return do_ppzz_flags(s, a, fns[a->esz]); \
3111}
3112
3113DO_PPZW(CMPEQ, cmpeq)
3114DO_PPZW(CMPNE, cmpne)
3115DO_PPZW(CMPGT, cmpgt)
3116DO_PPZW(CMPGE, cmpge)
3117DO_PPZW(CMPHI, cmphi)
3118DO_PPZW(CMPHS, cmphs)
3119DO_PPZW(CMPLT, cmplt)
3120DO_PPZW(CMPLE, cmple)
3121DO_PPZW(CMPLO, cmplo)
3122DO_PPZW(CMPLS, cmpls)
3123
3124#undef DO_PPZW
3125
38cadeba
RH
3126/*
3127 *** SVE Integer Compare - Immediate Groups
3128 */
3129
3130static bool do_ppzi_flags(DisasContext *s, arg_rpri_esz *a,
3131 gen_helper_gvec_flags_3 *gen_fn)
3132{
3133 TCGv_ptr pd, zn, pg;
3134 unsigned vsz;
3135 TCGv_i32 t;
3136
3137 if (gen_fn == NULL) {
3138 return false;
3139 }
3140 if (!sve_access_check(s)) {
3141 return true;
3142 }
3143
3144 vsz = vec_full_reg_size(s);
3145 t = tcg_const_i32(simd_desc(vsz, vsz, a->imm));
3146 pd = tcg_temp_new_ptr();
3147 zn = tcg_temp_new_ptr();
3148 pg = tcg_temp_new_ptr();
3149
3150 tcg_gen_addi_ptr(pd, cpu_env, pred_full_reg_offset(s, a->rd));
3151 tcg_gen_addi_ptr(zn, cpu_env, vec_full_reg_offset(s, a->rn));
3152 tcg_gen_addi_ptr(pg, cpu_env, pred_full_reg_offset(s, a->pg));
3153
3154 gen_fn(t, pd, zn, pg, t);
3155
3156 tcg_temp_free_ptr(pd);
3157 tcg_temp_free_ptr(zn);
3158 tcg_temp_free_ptr(pg);
3159
3160 do_pred_flags(t);
3161
3162 tcg_temp_free_i32(t);
3163 return true;
3164}
3165
3166#define DO_PPZI(NAME, name) \
3a7be554 3167static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
38cadeba
RH
3168{ \
3169 static gen_helper_gvec_flags_3 * const fns[4] = { \
3170 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3171 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3172 }; \
3173 return do_ppzi_flags(s, a, fns[a->esz]); \
3174}
3175
3176DO_PPZI(CMPEQ, cmpeq)
3177DO_PPZI(CMPNE, cmpne)
3178DO_PPZI(CMPGT, cmpgt)
3179DO_PPZI(CMPGE, cmpge)
3180DO_PPZI(CMPHI, cmphi)
3181DO_PPZI(CMPHS, cmphs)
3182DO_PPZI(CMPLT, cmplt)
3183DO_PPZI(CMPLE, cmple)
3184DO_PPZI(CMPLO, cmplo)
3185DO_PPZI(CMPLS, cmpls)
3186
3187#undef DO_PPZI
3188
35da316f
RH
3189/*
3190 *** SVE Partition Break Group
3191 */
3192
3193static bool do_brk3(DisasContext *s, arg_rprr_s *a,
3194 gen_helper_gvec_4 *fn, gen_helper_gvec_flags_4 *fn_s)
3195{
3196 if (!sve_access_check(s)) {
3197 return true;
3198 }
3199
3200 unsigned vsz = pred_full_reg_size(s);
3201
3202 /* Predicate sizes may be smaller and cannot use simd_desc. */
3203 TCGv_ptr d = tcg_temp_new_ptr();
3204 TCGv_ptr n = tcg_temp_new_ptr();
3205 TCGv_ptr m = tcg_temp_new_ptr();
3206 TCGv_ptr g = tcg_temp_new_ptr();
04c774a2 3207 TCGv_i32 t = tcg_const_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz));
35da316f
RH
3208
3209 tcg_gen_addi_ptr(d, cpu_env, pred_full_reg_offset(s, a->rd));
3210 tcg_gen_addi_ptr(n, cpu_env, pred_full_reg_offset(s, a->rn));
3211 tcg_gen_addi_ptr(m, cpu_env, pred_full_reg_offset(s, a->rm));
3212 tcg_gen_addi_ptr(g, cpu_env, pred_full_reg_offset(s, a->pg));
3213
3214 if (a->s) {
3215 fn_s(t, d, n, m, g, t);
3216 do_pred_flags(t);
3217 } else {
3218 fn(d, n, m, g, t);
3219 }
3220 tcg_temp_free_ptr(d);
3221 tcg_temp_free_ptr(n);
3222 tcg_temp_free_ptr(m);
3223 tcg_temp_free_ptr(g);
3224 tcg_temp_free_i32(t);
3225 return true;
3226}
3227
3228static bool do_brk2(DisasContext *s, arg_rpr_s *a,
3229 gen_helper_gvec_3 *fn, gen_helper_gvec_flags_3 *fn_s)
3230{
3231 if (!sve_access_check(s)) {
3232 return true;
3233 }
3234
3235 unsigned vsz = pred_full_reg_size(s);
3236
3237 /* Predicate sizes may be smaller and cannot use simd_desc. */
3238 TCGv_ptr d = tcg_temp_new_ptr();
3239 TCGv_ptr n = tcg_temp_new_ptr();
3240 TCGv_ptr g = tcg_temp_new_ptr();
04c774a2 3241 TCGv_i32 t = tcg_const_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz));
35da316f
RH
3242
3243 tcg_gen_addi_ptr(d, cpu_env, pred_full_reg_offset(s, a->rd));
3244 tcg_gen_addi_ptr(n, cpu_env, pred_full_reg_offset(s, a->rn));
3245 tcg_gen_addi_ptr(g, cpu_env, pred_full_reg_offset(s, a->pg));
3246
3247 if (a->s) {
3248 fn_s(t, d, n, g, t);
3249 do_pred_flags(t);
3250 } else {
3251 fn(d, n, g, t);
3252 }
3253 tcg_temp_free_ptr(d);
3254 tcg_temp_free_ptr(n);
3255 tcg_temp_free_ptr(g);
3256 tcg_temp_free_i32(t);
3257 return true;
3258}
3259
3a7be554 3260static bool trans_BRKPA(DisasContext *s, arg_rprr_s *a)
35da316f
RH
3261{
3262 return do_brk3(s, a, gen_helper_sve_brkpa, gen_helper_sve_brkpas);
3263}
3264
3a7be554 3265static bool trans_BRKPB(DisasContext *s, arg_rprr_s *a)
35da316f
RH
3266{
3267 return do_brk3(s, a, gen_helper_sve_brkpb, gen_helper_sve_brkpbs);
3268}
3269
3a7be554 3270static bool trans_BRKA_m(DisasContext *s, arg_rpr_s *a)
35da316f
RH
3271{
3272 return do_brk2(s, a, gen_helper_sve_brka_m, gen_helper_sve_brkas_m);
3273}
3274
3a7be554 3275static bool trans_BRKB_m(DisasContext *s, arg_rpr_s *a)
35da316f
RH
3276{
3277 return do_brk2(s, a, gen_helper_sve_brkb_m, gen_helper_sve_brkbs_m);
3278}
3279
3a7be554 3280static bool trans_BRKA_z(DisasContext *s, arg_rpr_s *a)
35da316f
RH
3281{
3282 return do_brk2(s, a, gen_helper_sve_brka_z, gen_helper_sve_brkas_z);
3283}
3284
3a7be554 3285static bool trans_BRKB_z(DisasContext *s, arg_rpr_s *a)
35da316f
RH
3286{
3287 return do_brk2(s, a, gen_helper_sve_brkb_z, gen_helper_sve_brkbs_z);
3288}
3289
3a7be554 3290static bool trans_BRKN(DisasContext *s, arg_rpr_s *a)
35da316f
RH
3291{
3292 return do_brk2(s, a, gen_helper_sve_brkn, gen_helper_sve_brkns);
3293}
3294
9ee3a611
RH
3295/*
3296 *** SVE Predicate Count Group
3297 */
3298
3299static void do_cntp(DisasContext *s, TCGv_i64 val, int esz, int pn, int pg)
3300{
3301 unsigned psz = pred_full_reg_size(s);
3302
3303 if (psz <= 8) {
3304 uint64_t psz_mask;
3305
3306 tcg_gen_ld_i64(val, cpu_env, pred_full_reg_offset(s, pn));
3307 if (pn != pg) {
3308 TCGv_i64 g = tcg_temp_new_i64();
3309 tcg_gen_ld_i64(g, cpu_env, pred_full_reg_offset(s, pg));
3310 tcg_gen_and_i64(val, val, g);
3311 tcg_temp_free_i64(g);
3312 }
3313
3314 /* Reduce the pred_esz_masks value simply to reduce the
3315 * size of the code generated here.
3316 */
3317 psz_mask = MAKE_64BIT_MASK(0, psz * 8);
3318 tcg_gen_andi_i64(val, val, pred_esz_masks[esz] & psz_mask);
3319
3320 tcg_gen_ctpop_i64(val, val);
3321 } else {
3322 TCGv_ptr t_pn = tcg_temp_new_ptr();
3323 TCGv_ptr t_pg = tcg_temp_new_ptr();
f556a201 3324 unsigned desc = 0;
9ee3a611
RH
3325 TCGv_i32 t_desc;
3326
f556a201
RH
3327 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, psz);
3328 desc = FIELD_DP32(desc, PREDDESC, ESZ, esz);
9ee3a611
RH
3329
3330 tcg_gen_addi_ptr(t_pn, cpu_env, pred_full_reg_offset(s, pn));
3331 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
3332 t_desc = tcg_const_i32(desc);
3333
3334 gen_helper_sve_cntp(val, t_pn, t_pg, t_desc);
3335 tcg_temp_free_ptr(t_pn);
3336 tcg_temp_free_ptr(t_pg);
3337 tcg_temp_free_i32(t_desc);
3338 }
3339}
3340
3a7be554 3341static bool trans_CNTP(DisasContext *s, arg_CNTP *a)
9ee3a611
RH
3342{
3343 if (sve_access_check(s)) {
3344 do_cntp(s, cpu_reg(s, a->rd), a->esz, a->rn, a->pg);
3345 }
3346 return true;
3347}
3348
3a7be554 3349static bool trans_INCDECP_r(DisasContext *s, arg_incdec_pred *a)
9ee3a611
RH
3350{
3351 if (sve_access_check(s)) {
3352 TCGv_i64 reg = cpu_reg(s, a->rd);
3353 TCGv_i64 val = tcg_temp_new_i64();
3354
3355 do_cntp(s, val, a->esz, a->pg, a->pg);
3356 if (a->d) {
3357 tcg_gen_sub_i64(reg, reg, val);
3358 } else {
3359 tcg_gen_add_i64(reg, reg, val);
3360 }
3361 tcg_temp_free_i64(val);
3362 }
3363 return true;
3364}
3365
3a7be554 3366static bool trans_INCDECP_z(DisasContext *s, arg_incdec2_pred *a)
9ee3a611
RH
3367{
3368 if (a->esz == 0) {
3369 return false;
3370 }
3371 if (sve_access_check(s)) {
3372 unsigned vsz = vec_full_reg_size(s);
3373 TCGv_i64 val = tcg_temp_new_i64();
3374 GVecGen2sFn *gvec_fn = a->d ? tcg_gen_gvec_subs : tcg_gen_gvec_adds;
3375
3376 do_cntp(s, val, a->esz, a->pg, a->pg);
3377 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
3378 vec_full_reg_offset(s, a->rn), val, vsz, vsz);
3379 }
3380 return true;
3381}
3382
3a7be554 3383static bool trans_SINCDECP_r_32(DisasContext *s, arg_incdec_pred *a)
9ee3a611
RH
3384{
3385 if (sve_access_check(s)) {
3386 TCGv_i64 reg = cpu_reg(s, a->rd);
3387 TCGv_i64 val = tcg_temp_new_i64();
3388
3389 do_cntp(s, val, a->esz, a->pg, a->pg);
3390 do_sat_addsub_32(reg, val, a->u, a->d);
3391 }
3392 return true;
3393}
3394
3a7be554 3395static bool trans_SINCDECP_r_64(DisasContext *s, arg_incdec_pred *a)
9ee3a611
RH
3396{
3397 if (sve_access_check(s)) {
3398 TCGv_i64 reg = cpu_reg(s, a->rd);
3399 TCGv_i64 val = tcg_temp_new_i64();
3400
3401 do_cntp(s, val, a->esz, a->pg, a->pg);
3402 do_sat_addsub_64(reg, val, a->u, a->d);
3403 }
3404 return true;
3405}
3406
3a7be554 3407static bool trans_SINCDECP_z(DisasContext *s, arg_incdec2_pred *a)
9ee3a611
RH
3408{
3409 if (a->esz == 0) {
3410 return false;
3411 }
3412 if (sve_access_check(s)) {
3413 TCGv_i64 val = tcg_temp_new_i64();
3414 do_cntp(s, val, a->esz, a->pg, a->pg);
3415 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, a->u, a->d);
3416 }
3417 return true;
3418}
3419
caf1cefc
RH
3420/*
3421 *** SVE Integer Compare Scalars Group
3422 */
3423
3a7be554 3424static bool trans_CTERM(DisasContext *s, arg_CTERM *a)
caf1cefc
RH
3425{
3426 if (!sve_access_check(s)) {
3427 return true;
3428 }
3429
3430 TCGCond cond = (a->ne ? TCG_COND_NE : TCG_COND_EQ);
3431 TCGv_i64 rn = read_cpu_reg(s, a->rn, a->sf);
3432 TCGv_i64 rm = read_cpu_reg(s, a->rm, a->sf);
3433 TCGv_i64 cmp = tcg_temp_new_i64();
3434
3435 tcg_gen_setcond_i64(cond, cmp, rn, rm);
3436 tcg_gen_extrl_i64_i32(cpu_NF, cmp);
3437 tcg_temp_free_i64(cmp);
3438
3439 /* VF = !NF & !CF. */
3440 tcg_gen_xori_i32(cpu_VF, cpu_NF, 1);
3441 tcg_gen_andc_i32(cpu_VF, cpu_VF, cpu_CF);
3442
3443 /* Both NF and VF actually look at bit 31. */
3444 tcg_gen_neg_i32(cpu_NF, cpu_NF);
3445 tcg_gen_neg_i32(cpu_VF, cpu_VF);
3446 return true;
3447}
3448
3a7be554 3449static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
caf1cefc 3450{
bbd0968c 3451 TCGv_i64 op0, op1, t0, t1, tmax;
caf1cefc
RH
3452 TCGv_i32 t2, t3;
3453 TCGv_ptr ptr;
e610906c
RH
3454 unsigned vsz = vec_full_reg_size(s);
3455 unsigned desc = 0;
caf1cefc 3456 TCGCond cond;
34688dbc
RH
3457 uint64_t maxval;
3458 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3459 bool eq = a->eq == a->lt;
caf1cefc 3460
34688dbc
RH
3461 /* The greater-than conditions are all SVE2. */
3462 if (!a->lt && !dc_isar_feature(aa64_sve2, s)) {
3463 return false;
3464 }
bbd0968c
RH
3465 if (!sve_access_check(s)) {
3466 return true;
3467 }
3468
3469 op0 = read_cpu_reg(s, a->rn, 1);
3470 op1 = read_cpu_reg(s, a->rm, 1);
3471
caf1cefc
RH
3472 if (!a->sf) {
3473 if (a->u) {
3474 tcg_gen_ext32u_i64(op0, op0);
3475 tcg_gen_ext32u_i64(op1, op1);
3476 } else {
3477 tcg_gen_ext32s_i64(op0, op0);
3478 tcg_gen_ext32s_i64(op1, op1);
3479 }
3480 }
3481
3482 /* For the helper, compress the different conditions into a computation
3483 * of how many iterations for which the condition is true.
caf1cefc 3484 */
bbd0968c
RH
3485 t0 = tcg_temp_new_i64();
3486 t1 = tcg_temp_new_i64();
34688dbc
RH
3487
3488 if (a->lt) {
3489 tcg_gen_sub_i64(t0, op1, op0);
3490 if (a->u) {
3491 maxval = a->sf ? UINT64_MAX : UINT32_MAX;
3492 cond = eq ? TCG_COND_LEU : TCG_COND_LTU;
3493 } else {
3494 maxval = a->sf ? INT64_MAX : INT32_MAX;
3495 cond = eq ? TCG_COND_LE : TCG_COND_LT;
3496 }
3497 } else {
3498 tcg_gen_sub_i64(t0, op0, op1);
3499 if (a->u) {
3500 maxval = 0;
3501 cond = eq ? TCG_COND_GEU : TCG_COND_GTU;
3502 } else {
3503 maxval = a->sf ? INT64_MIN : INT32_MIN;
3504 cond = eq ? TCG_COND_GE : TCG_COND_GT;
3505 }
3506 }
caf1cefc 3507
bbd0968c 3508 tmax = tcg_const_i64(vsz >> a->esz);
34688dbc 3509 if (eq) {
caf1cefc
RH
3510 /* Equality means one more iteration. */
3511 tcg_gen_addi_i64(t0, t0, 1);
bbd0968c 3512
34688dbc
RH
3513 /*
3514 * For the less-than while, if op1 is maxval (and the only time
3515 * the addition above could overflow), then we produce an all-true
3516 * predicate by setting the count to the vector length. This is
3517 * because the pseudocode is described as an increment + compare
3518 * loop, and the maximum integer would always compare true.
3519 * Similarly, the greater-than while has the same issue with the
3520 * minimum integer due to the decrement + compare loop.
bbd0968c 3521 */
34688dbc 3522 tcg_gen_movi_i64(t1, maxval);
bbd0968c 3523 tcg_gen_movcond_i64(TCG_COND_EQ, t0, op1, t1, tmax, t0);
caf1cefc
RH
3524 }
3525
bbd0968c
RH
3526 /* Bound to the maximum. */
3527 tcg_gen_umin_i64(t0, t0, tmax);
3528 tcg_temp_free_i64(tmax);
3529
3530 /* Set the count to zero if the condition is false. */
caf1cefc
RH
3531 tcg_gen_movi_i64(t1, 0);
3532 tcg_gen_movcond_i64(cond, t0, op0, op1, t0, t1);
bbd0968c 3533 tcg_temp_free_i64(t1);
caf1cefc 3534
bbd0968c 3535 /* Since we're bounded, pass as a 32-bit type. */
caf1cefc
RH
3536 t2 = tcg_temp_new_i32();
3537 tcg_gen_extrl_i64_i32(t2, t0);
3538 tcg_temp_free_i64(t0);
bbd0968c
RH
3539
3540 /* Scale elements to bits. */
3541 tcg_gen_shli_i32(t2, t2, a->esz);
caf1cefc 3542
e610906c
RH
3543 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8);
3544 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
caf1cefc
RH
3545 t3 = tcg_const_i32(desc);
3546
3547 ptr = tcg_temp_new_ptr();
3548 tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd));
3549
34688dbc
RH
3550 if (a->lt) {
3551 gen_helper_sve_whilel(t2, ptr, t2, t3);
3552 } else {
3553 gen_helper_sve_whileg(t2, ptr, t2, t3);
3554 }
caf1cefc
RH
3555 do_pred_flags(t2);
3556
3557 tcg_temp_free_ptr(ptr);
3558 tcg_temp_free_i32(t2);
3559 tcg_temp_free_i32(t3);
3560 return true;
3561}
3562
14f6dad1
RH
3563static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a)
3564{
3565 TCGv_i64 op0, op1, diff, t1, tmax;
3566 TCGv_i32 t2, t3;
3567 TCGv_ptr ptr;
3568 unsigned vsz = vec_full_reg_size(s);
3569 unsigned desc = 0;
3570
3571 if (!dc_isar_feature(aa64_sve2, s)) {
3572 return false;
3573 }
3574 if (!sve_access_check(s)) {
3575 return true;
3576 }
3577
3578 op0 = read_cpu_reg(s, a->rn, 1);
3579 op1 = read_cpu_reg(s, a->rm, 1);
3580
3581 tmax = tcg_const_i64(vsz);
3582 diff = tcg_temp_new_i64();
3583
3584 if (a->rw) {
3585 /* WHILERW */
3586 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3587 t1 = tcg_temp_new_i64();
3588 tcg_gen_sub_i64(diff, op0, op1);
3589 tcg_gen_sub_i64(t1, op1, op0);
3590 tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, diff, t1);
3591 tcg_temp_free_i64(t1);
3592 /* Round down to a multiple of ESIZE. */
3593 tcg_gen_andi_i64(diff, diff, -1 << a->esz);
3594 /* If op1 == op0, diff == 0, and the condition is always true. */
3595 tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff);
3596 } else {
3597 /* WHILEWR */
3598 tcg_gen_sub_i64(diff, op1, op0);
3599 /* Round down to a multiple of ESIZE. */
3600 tcg_gen_andi_i64(diff, diff, -1 << a->esz);
3601 /* If op0 >= op1, diff <= 0, the condition is always true. */
3602 tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff);
3603 }
3604
3605 /* Bound to the maximum. */
3606 tcg_gen_umin_i64(diff, diff, tmax);
3607 tcg_temp_free_i64(tmax);
3608
3609 /* Since we're bounded, pass as a 32-bit type. */
3610 t2 = tcg_temp_new_i32();
3611 tcg_gen_extrl_i64_i32(t2, diff);
3612 tcg_temp_free_i64(diff);
3613
3614 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8);
3615 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3616 t3 = tcg_const_i32(desc);
3617
3618 ptr = tcg_temp_new_ptr();
3619 tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd));
3620
3621 gen_helper_sve_whilel(t2, ptr, t2, t3);
3622 do_pred_flags(t2);
3623
3624 tcg_temp_free_ptr(ptr);
3625 tcg_temp_free_i32(t2);
3626 tcg_temp_free_i32(t3);
3627 return true;
3628}
3629
ed491961
RH
3630/*
3631 *** SVE Integer Wide Immediate - Unpredicated Group
3632 */
3633
3a7be554 3634static bool trans_FDUP(DisasContext *s, arg_FDUP *a)
ed491961
RH
3635{
3636 if (a->esz == 0) {
3637 return false;
3638 }
3639 if (sve_access_check(s)) {
3640 unsigned vsz = vec_full_reg_size(s);
3641 int dofs = vec_full_reg_offset(s, a->rd);
3642 uint64_t imm;
3643
3644 /* Decode the VFP immediate. */
3645 imm = vfp_expand_imm(a->esz, a->imm);
8711e71f 3646 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, imm);
ed491961
RH
3647 }
3648 return true;
3649}
3650
3a7be554 3651static bool trans_DUP_i(DisasContext *s, arg_DUP_i *a)
ed491961 3652{
3a7be554 3653 if (a->esz == 0 && extract32(s->insn, 13, 1)) {
ed491961
RH
3654 return false;
3655 }
3656 if (sve_access_check(s)) {
3657 unsigned vsz = vec_full_reg_size(s);
3658 int dofs = vec_full_reg_offset(s, a->rd);
3659
8711e71f 3660 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, a->imm);
ed491961
RH
3661 }
3662 return true;
3663}
3664
3a7be554 3665static bool trans_ADD_zzi(DisasContext *s, arg_rri_esz *a)
6e6a157d 3666{
3a7be554 3667 if (a->esz == 0 && extract32(s->insn, 13, 1)) {
6e6a157d
RH
3668 return false;
3669 }
3670 if (sve_access_check(s)) {
3671 unsigned vsz = vec_full_reg_size(s);
3672 tcg_gen_gvec_addi(a->esz, vec_full_reg_offset(s, a->rd),
3673 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
3674 }
3675 return true;
3676}
3677
3a7be554 3678static bool trans_SUB_zzi(DisasContext *s, arg_rri_esz *a)
6e6a157d
RH
3679{
3680 a->imm = -a->imm;
3a7be554 3681 return trans_ADD_zzi(s, a);
6e6a157d
RH
3682}
3683
3a7be554 3684static bool trans_SUBR_zzi(DisasContext *s, arg_rri_esz *a)
6e6a157d 3685{
53229a77 3686 static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 };
6e6a157d
RH
3687 static const GVecGen2s op[4] = {
3688 { .fni8 = tcg_gen_vec_sub8_i64,
3689 .fniv = tcg_gen_sub_vec,
3690 .fno = gen_helper_sve_subri_b,
53229a77 3691 .opt_opc = vecop_list,
6e6a157d
RH
3692 .vece = MO_8,
3693 .scalar_first = true },
3694 { .fni8 = tcg_gen_vec_sub16_i64,
3695 .fniv = tcg_gen_sub_vec,
3696 .fno = gen_helper_sve_subri_h,
53229a77 3697 .opt_opc = vecop_list,
6e6a157d
RH
3698 .vece = MO_16,
3699 .scalar_first = true },
3700 { .fni4 = tcg_gen_sub_i32,
3701 .fniv = tcg_gen_sub_vec,
3702 .fno = gen_helper_sve_subri_s,
53229a77 3703 .opt_opc = vecop_list,
6e6a157d
RH
3704 .vece = MO_32,
3705 .scalar_first = true },
3706 { .fni8 = tcg_gen_sub_i64,
3707 .fniv = tcg_gen_sub_vec,
3708 .fno = gen_helper_sve_subri_d,
53229a77 3709 .opt_opc = vecop_list,
6e6a157d
RH
3710 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3711 .vece = MO_64,
3712 .scalar_first = true }
3713 };
3714
3a7be554 3715 if (a->esz == 0 && extract32(s->insn, 13, 1)) {
6e6a157d
RH
3716 return false;
3717 }
3718 if (sve_access_check(s)) {
3719 unsigned vsz = vec_full_reg_size(s);
3720 TCGv_i64 c = tcg_const_i64(a->imm);
3721 tcg_gen_gvec_2s(vec_full_reg_offset(s, a->rd),
3722 vec_full_reg_offset(s, a->rn),
3723 vsz, vsz, c, &op[a->esz]);
3724 tcg_temp_free_i64(c);
3725 }
3726 return true;
3727}
3728
3a7be554 3729static bool trans_MUL_zzi(DisasContext *s, arg_rri_esz *a)
6e6a157d
RH
3730{
3731 if (sve_access_check(s)) {
3732 unsigned vsz = vec_full_reg_size(s);
3733 tcg_gen_gvec_muli(a->esz, vec_full_reg_offset(s, a->rd),
3734 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
3735 }
3736 return true;
3737}
3738
3a7be554 3739static bool do_zzi_sat(DisasContext *s, arg_rri_esz *a, bool u, bool d)
6e6a157d 3740{
3a7be554 3741 if (a->esz == 0 && extract32(s->insn, 13, 1)) {
6e6a157d
RH
3742 return false;
3743 }
3744 if (sve_access_check(s)) {
3745 TCGv_i64 val = tcg_const_i64(a->imm);
3746 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, u, d);
3747 tcg_temp_free_i64(val);
3748 }
3749 return true;
3750}
3751
3a7be554 3752static bool trans_SQADD_zzi(DisasContext *s, arg_rri_esz *a)
6e6a157d 3753{
3a7be554 3754 return do_zzi_sat(s, a, false, false);
6e6a157d
RH
3755}
3756
3a7be554 3757static bool trans_UQADD_zzi(DisasContext *s, arg_rri_esz *a)
6e6a157d 3758{
3a7be554 3759 return do_zzi_sat(s, a, true, false);
6e6a157d
RH
3760}
3761
3a7be554 3762static bool trans_SQSUB_zzi(DisasContext *s, arg_rri_esz *a)
6e6a157d 3763{
3a7be554 3764 return do_zzi_sat(s, a, false, true);
6e6a157d
RH
3765}
3766
3a7be554 3767static bool trans_UQSUB_zzi(DisasContext *s, arg_rri_esz *a)
6e6a157d 3768{
3a7be554 3769 return do_zzi_sat(s, a, true, true);
6e6a157d
RH
3770}
3771
3772static bool do_zzi_ool(DisasContext *s, arg_rri_esz *a, gen_helper_gvec_2i *fn)
3773{
3774 if (sve_access_check(s)) {
3775 unsigned vsz = vec_full_reg_size(s);
3776 TCGv_i64 c = tcg_const_i64(a->imm);
3777
3778 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd),
3779 vec_full_reg_offset(s, a->rn),
3780 c, vsz, vsz, 0, fn);
3781 tcg_temp_free_i64(c);
3782 }
3783 return true;
3784}
3785
3786#define DO_ZZI(NAME, name) \
3a7be554 3787static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
6e6a157d
RH
3788{ \
3789 static gen_helper_gvec_2i * const fns[4] = { \
3790 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3791 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3792 }; \
3793 return do_zzi_ool(s, a, fns[a->esz]); \
3794}
3795
3796DO_ZZI(SMAX, smax)
3797DO_ZZI(UMAX, umax)
3798DO_ZZI(SMIN, smin)
3799DO_ZZI(UMIN, umin)
3800
3801#undef DO_ZZI
3802
bc2bd697 3803static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a)
d730ecaa 3804{
bc2bd697 3805 static gen_helper_gvec_4 * const fns[2][2] = {
d730ecaa
RH
3806 { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
3807 { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
3808 };
3809
3810 if (sve_access_check(s)) {
bc2bd697 3811 gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0);
d730ecaa
RH
3812 }
3813 return true;
3814}
3815
814d4c52
RH
3816/*
3817 * SVE Multiply - Indexed
3818 */
3819
0a82d963
RH
3820static bool do_zzxz_ool(DisasContext *s, arg_rrxr_esz *a,
3821 gen_helper_gvec_4 *fn)
16fcfdc7 3822{
0a82d963
RH
3823 if (fn == NULL) {
3824 return false;
3825 }
16fcfdc7 3826 if (sve_access_check(s)) {
0a82d963 3827 gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index);
16fcfdc7
RH
3828 }
3829 return true;
3830}
3831
0a82d963
RH
3832#define DO_RRXR(NAME, FUNC) \
3833 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3834 { return do_zzxz_ool(s, a, FUNC); }
3835
3836DO_RRXR(trans_SDOT_zzxw_s, gen_helper_gvec_sdot_idx_b)
3837DO_RRXR(trans_SDOT_zzxw_d, gen_helper_gvec_sdot_idx_h)
3838DO_RRXR(trans_UDOT_zzxw_s, gen_helper_gvec_udot_idx_b)
3839DO_RRXR(trans_UDOT_zzxw_d, gen_helper_gvec_udot_idx_h)
3840
3841#undef DO_RRXR
16fcfdc7 3842
814d4c52
RH
3843static bool do_sve2_zzz_data(DisasContext *s, int rd, int rn, int rm, int data,
3844 gen_helper_gvec_3 *fn)
3845{
3846 if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
3847 return false;
3848 }
3849 if (sve_access_check(s)) {
3850 unsigned vsz = vec_full_reg_size(s);
3851 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
3852 vec_full_reg_offset(s, rn),
3853 vec_full_reg_offset(s, rm),
3854 vsz, vsz, data, fn);
3855 }
3856 return true;
3857}
3858
3859#define DO_SVE2_RRX(NAME, FUNC) \
3860 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
3861 { return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, a->index, FUNC); }
3862
3863DO_SVE2_RRX(trans_MUL_zzx_h, gen_helper_gvec_mul_idx_h)
3864DO_SVE2_RRX(trans_MUL_zzx_s, gen_helper_gvec_mul_idx_s)
3865DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d)
3866
3867#undef DO_SVE2_RRX
3868
8a02aac7
RH
3869static bool do_sve2_zzzz_data(DisasContext *s, int rd, int rn, int rm, int ra,
3870 int data, gen_helper_gvec_4 *fn)
3871{
3872 if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
3873 return false;
3874 }
3875 if (sve_access_check(s)) {
3876 unsigned vsz = vec_full_reg_size(s);
3877 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
3878 vec_full_reg_offset(s, rn),
3879 vec_full_reg_offset(s, rm),
3880 vec_full_reg_offset(s, ra),
3881 vsz, vsz, data, fn);
3882 }
3883 return true;
3884}
3885
3886#define DO_SVE2_RRXR(NAME, FUNC) \
3887 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3888 { return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, a->index, FUNC); }
3889
3890DO_SVE2_RRXR(trans_MLA_zzxz_h, gen_helper_gvec_mla_idx_h)
3891DO_SVE2_RRXR(trans_MLA_zzxz_s, gen_helper_gvec_mla_idx_s)
3892DO_SVE2_RRXR(trans_MLA_zzxz_d, gen_helper_gvec_mla_idx_d)
3893
3894DO_SVE2_RRXR(trans_MLS_zzxz_h, gen_helper_gvec_mls_idx_h)
3895DO_SVE2_RRXR(trans_MLS_zzxz_s, gen_helper_gvec_mls_idx_s)
3896DO_SVE2_RRXR(trans_MLS_zzxz_d, gen_helper_gvec_mls_idx_d)
3897
75d6d5fc
RH
3898DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_h, gen_helper_sve2_sqrdmlah_idx_h)
3899DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_s, gen_helper_sve2_sqrdmlah_idx_s)
3900DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_d, gen_helper_sve2_sqrdmlah_idx_d)
3901
3902DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_h, gen_helper_sve2_sqrdmlsh_idx_h)
3903DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_s, gen_helper_sve2_sqrdmlsh_idx_s)
3904DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d)
3905
8a02aac7
RH
3906#undef DO_SVE2_RRXR
3907
c5c455d7
RH
3908#define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3909 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3910 { \
3911 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->rd, \
3912 (a->index << 1) | TOP, FUNC); \
3913 }
3914
3915DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false)
3916DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false)
3917DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, true)
3918DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, true)
3919
3920DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, false)
3921DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false)
3922DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true)
3923DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true)
3924
3925#undef DO_SVE2_RRXR_TB
3926
ca40a6e6
RH
3927/*
3928 *** SVE Floating Point Multiply-Add Indexed Group
3929 */
3930
0a82d963 3931static bool do_FMLA_zzxz(DisasContext *s, arg_rrxr_esz *a, bool sub)
ca40a6e6
RH
3932{
3933 static gen_helper_gvec_4_ptr * const fns[3] = {
3934 gen_helper_gvec_fmla_idx_h,
3935 gen_helper_gvec_fmla_idx_s,
3936 gen_helper_gvec_fmla_idx_d,
3937 };
3938
3939 if (sve_access_check(s)) {
3940 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 3941 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
ca40a6e6
RH
3942 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
3943 vec_full_reg_offset(s, a->rn),
3944 vec_full_reg_offset(s, a->rm),
3945 vec_full_reg_offset(s, a->ra),
0a82d963 3946 status, vsz, vsz, (a->index << 1) | sub,
ca40a6e6
RH
3947 fns[a->esz - 1]);
3948 tcg_temp_free_ptr(status);
3949 }
3950 return true;
3951}
3952
0a82d963
RH
3953static bool trans_FMLA_zzxz(DisasContext *s, arg_FMLA_zzxz *a)
3954{
3955 return do_FMLA_zzxz(s, a, false);
3956}
3957
3958static bool trans_FMLS_zzxz(DisasContext *s, arg_FMLA_zzxz *a)
3959{
3960 return do_FMLA_zzxz(s, a, true);
3961}
3962
ca40a6e6
RH
3963/*
3964 *** SVE Floating Point Multiply Indexed Group
3965 */
3966
3a7be554 3967static bool trans_FMUL_zzx(DisasContext *s, arg_FMUL_zzx *a)
ca40a6e6
RH
3968{
3969 static gen_helper_gvec_3_ptr * const fns[3] = {
3970 gen_helper_gvec_fmul_idx_h,
3971 gen_helper_gvec_fmul_idx_s,
3972 gen_helper_gvec_fmul_idx_d,
3973 };
3974
3975 if (sve_access_check(s)) {
3976 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 3977 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
ca40a6e6
RH
3978 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
3979 vec_full_reg_offset(s, a->rn),
3980 vec_full_reg_offset(s, a->rm),
3981 status, vsz, vsz, a->index, fns[a->esz - 1]);
3982 tcg_temp_free_ptr(status);
3983 }
3984 return true;
3985}
3986
23fbe79f
RH
3987/*
3988 *** SVE Floating Point Fast Reduction Group
3989 */
3990
3991typedef void gen_helper_fp_reduce(TCGv_i64, TCGv_ptr, TCGv_ptr,
3992 TCGv_ptr, TCGv_i32);
3993
3994static void do_reduce(DisasContext *s, arg_rpr_esz *a,
3995 gen_helper_fp_reduce *fn)
3996{
3997 unsigned vsz = vec_full_reg_size(s);
3998 unsigned p2vsz = pow2ceil(vsz);
c648c9b7 3999 TCGv_i32 t_desc = tcg_const_i32(simd_desc(vsz, vsz, p2vsz));
23fbe79f
RH
4000 TCGv_ptr t_zn, t_pg, status;
4001 TCGv_i64 temp;
4002
4003 temp = tcg_temp_new_i64();
4004 t_zn = tcg_temp_new_ptr();
4005 t_pg = tcg_temp_new_ptr();
4006
4007 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn));
4008 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg));
cdfb22bb 4009 status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
23fbe79f
RH
4010
4011 fn(temp, t_zn, t_pg, status, t_desc);
4012 tcg_temp_free_ptr(t_zn);
4013 tcg_temp_free_ptr(t_pg);
4014 tcg_temp_free_ptr(status);
4015 tcg_temp_free_i32(t_desc);
4016
4017 write_fp_dreg(s, a->rd, temp);
4018 tcg_temp_free_i64(temp);
4019}
4020
4021#define DO_VPZ(NAME, name) \
3a7be554 4022static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
23fbe79f
RH
4023{ \
4024 static gen_helper_fp_reduce * const fns[3] = { \
4025 gen_helper_sve_##name##_h, \
4026 gen_helper_sve_##name##_s, \
4027 gen_helper_sve_##name##_d, \
4028 }; \
4029 if (a->esz == 0) { \
4030 return false; \
4031 } \
4032 if (sve_access_check(s)) { \
4033 do_reduce(s, a, fns[a->esz - 1]); \
4034 } \
4035 return true; \
4036}
4037
4038DO_VPZ(FADDV, faddv)
4039DO_VPZ(FMINNMV, fminnmv)
4040DO_VPZ(FMAXNMV, fmaxnmv)
4041DO_VPZ(FMINV, fminv)
4042DO_VPZ(FMAXV, fmaxv)
4043
3887c038
RH
4044/*
4045 *** SVE Floating Point Unary Operations - Unpredicated Group
4046 */
4047
4048static void do_zz_fp(DisasContext *s, arg_rr_esz *a, gen_helper_gvec_2_ptr *fn)
4049{
4050 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4051 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
3887c038
RH
4052
4053 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd),
4054 vec_full_reg_offset(s, a->rn),
4055 status, vsz, vsz, 0, fn);
4056 tcg_temp_free_ptr(status);
4057}
4058
3a7be554 4059static bool trans_FRECPE(DisasContext *s, arg_rr_esz *a)
3887c038
RH
4060{
4061 static gen_helper_gvec_2_ptr * const fns[3] = {
4062 gen_helper_gvec_frecpe_h,
4063 gen_helper_gvec_frecpe_s,
4064 gen_helper_gvec_frecpe_d,
4065 };
4066 if (a->esz == 0) {
4067 return false;
4068 }
4069 if (sve_access_check(s)) {
4070 do_zz_fp(s, a, fns[a->esz - 1]);
4071 }
4072 return true;
4073}
4074
3a7be554 4075static bool trans_FRSQRTE(DisasContext *s, arg_rr_esz *a)
3887c038
RH
4076{
4077 static gen_helper_gvec_2_ptr * const fns[3] = {
4078 gen_helper_gvec_frsqrte_h,
4079 gen_helper_gvec_frsqrte_s,
4080 gen_helper_gvec_frsqrte_d,
4081 };
4082 if (a->esz == 0) {
4083 return false;
4084 }
4085 if (sve_access_check(s)) {
4086 do_zz_fp(s, a, fns[a->esz - 1]);
4087 }
4088 return true;
4089}
4090
4d2e2a03
RH
4091/*
4092 *** SVE Floating Point Compare with Zero Group
4093 */
4094
4095static void do_ppz_fp(DisasContext *s, arg_rpr_esz *a,
4096 gen_helper_gvec_3_ptr *fn)
4097{
4098 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4099 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
4d2e2a03
RH
4100
4101 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd),
4102 vec_full_reg_offset(s, a->rn),
4103 pred_full_reg_offset(s, a->pg),
4104 status, vsz, vsz, 0, fn);
4105 tcg_temp_free_ptr(status);
4106}
4107
4108#define DO_PPZ(NAME, name) \
3a7be554 4109static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4d2e2a03
RH
4110{ \
4111 static gen_helper_gvec_3_ptr * const fns[3] = { \
4112 gen_helper_sve_##name##_h, \
4113 gen_helper_sve_##name##_s, \
4114 gen_helper_sve_##name##_d, \
4115 }; \
4116 if (a->esz == 0) { \
4117 return false; \
4118 } \
4119 if (sve_access_check(s)) { \
4120 do_ppz_fp(s, a, fns[a->esz - 1]); \
4121 } \
4122 return true; \
4123}
4124
4125DO_PPZ(FCMGE_ppz0, fcmge0)
4126DO_PPZ(FCMGT_ppz0, fcmgt0)
4127DO_PPZ(FCMLE_ppz0, fcmle0)
4128DO_PPZ(FCMLT_ppz0, fcmlt0)
4129DO_PPZ(FCMEQ_ppz0, fcmeq0)
4130DO_PPZ(FCMNE_ppz0, fcmne0)
4131
4132#undef DO_PPZ
4133
67fcd9ad
RH
4134/*
4135 *** SVE floating-point trig multiply-add coefficient
4136 */
4137
3a7be554 4138static bool trans_FTMAD(DisasContext *s, arg_FTMAD *a)
67fcd9ad
RH
4139{
4140 static gen_helper_gvec_3_ptr * const fns[3] = {
4141 gen_helper_sve_ftmad_h,
4142 gen_helper_sve_ftmad_s,
4143 gen_helper_sve_ftmad_d,
4144 };
4145
4146 if (a->esz == 0) {
4147 return false;
4148 }
4149 if (sve_access_check(s)) {
4150 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4151 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
67fcd9ad
RH
4152 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
4153 vec_full_reg_offset(s, a->rn),
4154 vec_full_reg_offset(s, a->rm),
4155 status, vsz, vsz, a->imm, fns[a->esz - 1]);
4156 tcg_temp_free_ptr(status);
4157 }
4158 return true;
4159}
4160
7f9ddf64
RH
4161/*
4162 *** SVE Floating Point Accumulating Reduction Group
4163 */
4164
3a7be554 4165static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a)
7f9ddf64
RH
4166{
4167 typedef void fadda_fn(TCGv_i64, TCGv_i64, TCGv_ptr,
4168 TCGv_ptr, TCGv_ptr, TCGv_i32);
4169 static fadda_fn * const fns[3] = {
4170 gen_helper_sve_fadda_h,
4171 gen_helper_sve_fadda_s,
4172 gen_helper_sve_fadda_d,
4173 };
4174 unsigned vsz = vec_full_reg_size(s);
4175 TCGv_ptr t_rm, t_pg, t_fpst;
4176 TCGv_i64 t_val;
4177 TCGv_i32 t_desc;
4178
4179 if (a->esz == 0) {
4180 return false;
4181 }
4182 if (!sve_access_check(s)) {
4183 return true;
4184 }
4185
4186 t_val = load_esz(cpu_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz);
4187 t_rm = tcg_temp_new_ptr();
4188 t_pg = tcg_temp_new_ptr();
4189 tcg_gen_addi_ptr(t_rm, cpu_env, vec_full_reg_offset(s, a->rm));
4190 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg));
cdfb22bb 4191 t_fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
7f9ddf64
RH
4192 t_desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
4193
4194 fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc);
4195
4196 tcg_temp_free_i32(t_desc);
4197 tcg_temp_free_ptr(t_fpst);
4198 tcg_temp_free_ptr(t_pg);
4199 tcg_temp_free_ptr(t_rm);
4200
4201 write_fp_dreg(s, a->rd, t_val);
4202 tcg_temp_free_i64(t_val);
4203 return true;
4204}
4205
29b80469
RH
4206/*
4207 *** SVE Floating Point Arithmetic - Unpredicated Group
4208 */
4209
4210static bool do_zzz_fp(DisasContext *s, arg_rrr_esz *a,
4211 gen_helper_gvec_3_ptr *fn)
4212{
4213 if (fn == NULL) {
4214 return false;
4215 }
4216 if (sve_access_check(s)) {
4217 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4218 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
29b80469
RH
4219 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
4220 vec_full_reg_offset(s, a->rn),
4221 vec_full_reg_offset(s, a->rm),
4222 status, vsz, vsz, 0, fn);
4223 tcg_temp_free_ptr(status);
4224 }
4225 return true;
4226}
4227
4228
4229#define DO_FP3(NAME, name) \
3a7be554 4230static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
29b80469
RH
4231{ \
4232 static gen_helper_gvec_3_ptr * const fns[4] = { \
4233 NULL, gen_helper_gvec_##name##_h, \
4234 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4235 }; \
4236 return do_zzz_fp(s, a, fns[a->esz]); \
4237}
4238
4239DO_FP3(FADD_zzz, fadd)
4240DO_FP3(FSUB_zzz, fsub)
4241DO_FP3(FMUL_zzz, fmul)
4242DO_FP3(FTSMUL, ftsmul)
4243DO_FP3(FRECPS, recps)
4244DO_FP3(FRSQRTS, rsqrts)
4245
4246#undef DO_FP3
4247
ec3b87c2
RH
4248/*
4249 *** SVE Floating Point Arithmetic - Predicated Group
4250 */
4251
4252static bool do_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
4253 gen_helper_gvec_4_ptr *fn)
4254{
4255 if (fn == NULL) {
4256 return false;
4257 }
4258 if (sve_access_check(s)) {
4259 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4260 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
ec3b87c2
RH
4261 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
4262 vec_full_reg_offset(s, a->rn),
4263 vec_full_reg_offset(s, a->rm),
4264 pred_full_reg_offset(s, a->pg),
4265 status, vsz, vsz, 0, fn);
4266 tcg_temp_free_ptr(status);
4267 }
4268 return true;
4269}
4270
4271#define DO_FP3(NAME, name) \
3a7be554 4272static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
ec3b87c2
RH
4273{ \
4274 static gen_helper_gvec_4_ptr * const fns[4] = { \
4275 NULL, gen_helper_sve_##name##_h, \
4276 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4277 }; \
4278 return do_zpzz_fp(s, a, fns[a->esz]); \
4279}
4280
4281DO_FP3(FADD_zpzz, fadd)
4282DO_FP3(FSUB_zpzz, fsub)
4283DO_FP3(FMUL_zpzz, fmul)
4284DO_FP3(FMIN_zpzz, fmin)
4285DO_FP3(FMAX_zpzz, fmax)
4286DO_FP3(FMINNM_zpzz, fminnum)
4287DO_FP3(FMAXNM_zpzz, fmaxnum)
4288DO_FP3(FABD, fabd)
4289DO_FP3(FSCALE, fscalbn)
4290DO_FP3(FDIV, fdiv)
4291DO_FP3(FMULX, fmulx)
4292
4293#undef DO_FP3
8092c6a3 4294
cc48affe
RH
4295typedef void gen_helper_sve_fp2scalar(TCGv_ptr, TCGv_ptr, TCGv_ptr,
4296 TCGv_i64, TCGv_ptr, TCGv_i32);
4297
4298static void do_fp_scalar(DisasContext *s, int zd, int zn, int pg, bool is_fp16,
4299 TCGv_i64 scalar, gen_helper_sve_fp2scalar *fn)
4300{
4301 unsigned vsz = vec_full_reg_size(s);
4302 TCGv_ptr t_zd, t_zn, t_pg, status;
4303 TCGv_i32 desc;
4304
4305 t_zd = tcg_temp_new_ptr();
4306 t_zn = tcg_temp_new_ptr();
4307 t_pg = tcg_temp_new_ptr();
4308 tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, zd));
4309 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, zn));
4310 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
4311
cdfb22bb 4312 status = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
cc48affe
RH
4313 desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
4314 fn(t_zd, t_zn, t_pg, scalar, status, desc);
4315
4316 tcg_temp_free_i32(desc);
4317 tcg_temp_free_ptr(status);
4318 tcg_temp_free_ptr(t_pg);
4319 tcg_temp_free_ptr(t_zn);
4320 tcg_temp_free_ptr(t_zd);
4321}
4322
4323static void do_fp_imm(DisasContext *s, arg_rpri_esz *a, uint64_t imm,
4324 gen_helper_sve_fp2scalar *fn)
4325{
4326 TCGv_i64 temp = tcg_const_i64(imm);
4327 do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16, temp, fn);
4328 tcg_temp_free_i64(temp);
4329}
4330
4331#define DO_FP_IMM(NAME, name, const0, const1) \
3a7be554 4332static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
cc48affe
RH
4333{ \
4334 static gen_helper_sve_fp2scalar * const fns[3] = { \
4335 gen_helper_sve_##name##_h, \
4336 gen_helper_sve_##name##_s, \
4337 gen_helper_sve_##name##_d \
4338 }; \
4339 static uint64_t const val[3][2] = { \
4340 { float16_##const0, float16_##const1 }, \
4341 { float32_##const0, float32_##const1 }, \
4342 { float64_##const0, float64_##const1 }, \
4343 }; \
4344 if (a->esz == 0) { \
4345 return false; \
4346 } \
4347 if (sve_access_check(s)) { \
4348 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4349 } \
4350 return true; \
4351}
4352
cc48affe
RH
4353DO_FP_IMM(FADD, fadds, half, one)
4354DO_FP_IMM(FSUB, fsubs, half, one)
4355DO_FP_IMM(FMUL, fmuls, half, two)
4356DO_FP_IMM(FSUBR, fsubrs, half, one)
4357DO_FP_IMM(FMAXNM, fmaxnms, zero, one)
4358DO_FP_IMM(FMINNM, fminnms, zero, one)
4359DO_FP_IMM(FMAX, fmaxs, zero, one)
4360DO_FP_IMM(FMIN, fmins, zero, one)
4361
4362#undef DO_FP_IMM
4363
abfdefd5
RH
4364static bool do_fp_cmp(DisasContext *s, arg_rprr_esz *a,
4365 gen_helper_gvec_4_ptr *fn)
4366{
4367 if (fn == NULL) {
4368 return false;
4369 }
4370 if (sve_access_check(s)) {
4371 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4372 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
abfdefd5
RH
4373 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd),
4374 vec_full_reg_offset(s, a->rn),
4375 vec_full_reg_offset(s, a->rm),
4376 pred_full_reg_offset(s, a->pg),
4377 status, vsz, vsz, 0, fn);
4378 tcg_temp_free_ptr(status);
4379 }
4380 return true;
4381}
4382
4383#define DO_FPCMP(NAME, name) \
3a7be554 4384static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
abfdefd5
RH
4385{ \
4386 static gen_helper_gvec_4_ptr * const fns[4] = { \
4387 NULL, gen_helper_sve_##name##_h, \
4388 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4389 }; \
4390 return do_fp_cmp(s, a, fns[a->esz]); \
4391}
4392
4393DO_FPCMP(FCMGE, fcmge)
4394DO_FPCMP(FCMGT, fcmgt)
4395DO_FPCMP(FCMEQ, fcmeq)
4396DO_FPCMP(FCMNE, fcmne)
4397DO_FPCMP(FCMUO, fcmuo)
4398DO_FPCMP(FACGE, facge)
4399DO_FPCMP(FACGT, facgt)
4400
4401#undef DO_FPCMP
4402
3a7be554 4403static bool trans_FCADD(DisasContext *s, arg_FCADD *a)
76a9d9cd
RH
4404{
4405 static gen_helper_gvec_4_ptr * const fns[3] = {
4406 gen_helper_sve_fcadd_h,
4407 gen_helper_sve_fcadd_s,
4408 gen_helper_sve_fcadd_d
4409 };
4410
4411 if (a->esz == 0) {
4412 return false;
4413 }
4414 if (sve_access_check(s)) {
4415 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4416 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
76a9d9cd
RH
4417 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
4418 vec_full_reg_offset(s, a->rn),
4419 vec_full_reg_offset(s, a->rm),
4420 pred_full_reg_offset(s, a->pg),
4421 status, vsz, vsz, a->rot, fns[a->esz - 1]);
4422 tcg_temp_free_ptr(status);
4423 }
4424 return true;
4425}
4426
08975da9
RH
4427static bool do_fmla(DisasContext *s, arg_rprrr_esz *a,
4428 gen_helper_gvec_5_ptr *fn)
6ceabaad 4429{
08975da9 4430 if (a->esz == 0) {
6ceabaad
RH
4431 return false;
4432 }
08975da9
RH
4433 if (sve_access_check(s)) {
4434 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4435 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
08975da9
RH
4436 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s, a->rd),
4437 vec_full_reg_offset(s, a->rn),
4438 vec_full_reg_offset(s, a->rm),
4439 vec_full_reg_offset(s, a->ra),
4440 pred_full_reg_offset(s, a->pg),
4441 status, vsz, vsz, 0, fn);
4442 tcg_temp_free_ptr(status);
6ceabaad 4443 }
6ceabaad
RH
4444 return true;
4445}
4446
4447#define DO_FMLA(NAME, name) \
3a7be554 4448static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
6ceabaad 4449{ \
08975da9 4450 static gen_helper_gvec_5_ptr * const fns[4] = { \
6ceabaad
RH
4451 NULL, gen_helper_sve_##name##_h, \
4452 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4453 }; \
4454 return do_fmla(s, a, fns[a->esz]); \
4455}
4456
4457DO_FMLA(FMLA_zpzzz, fmla_zpzzz)
4458DO_FMLA(FMLS_zpzzz, fmls_zpzzz)
4459DO_FMLA(FNMLA_zpzzz, fnmla_zpzzz)
4460DO_FMLA(FNMLS_zpzzz, fnmls_zpzzz)
4461
4462#undef DO_FMLA
4463
3a7be554 4464static bool trans_FCMLA_zpzzz(DisasContext *s, arg_FCMLA_zpzzz *a)
05f48bab 4465{
08975da9
RH
4466 static gen_helper_gvec_5_ptr * const fns[4] = {
4467 NULL,
05f48bab
RH
4468 gen_helper_sve_fcmla_zpzzz_h,
4469 gen_helper_sve_fcmla_zpzzz_s,
4470 gen_helper_sve_fcmla_zpzzz_d,
4471 };
4472
4473 if (a->esz == 0) {
4474 return false;
4475 }
4476 if (sve_access_check(s)) {
4477 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4478 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
08975da9
RH
4479 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s, a->rd),
4480 vec_full_reg_offset(s, a->rn),
4481 vec_full_reg_offset(s, a->rm),
4482 vec_full_reg_offset(s, a->ra),
4483 pred_full_reg_offset(s, a->pg),
4484 status, vsz, vsz, a->rot, fns[a->esz]);
4485 tcg_temp_free_ptr(status);
05f48bab
RH
4486 }
4487 return true;
4488}
4489
3a7be554 4490static bool trans_FCMLA_zzxz(DisasContext *s, arg_FCMLA_zzxz *a)
18fc2405 4491{
636ddeb1 4492 static gen_helper_gvec_4_ptr * const fns[2] = {
18fc2405
RH
4493 gen_helper_gvec_fcmlah_idx,
4494 gen_helper_gvec_fcmlas_idx,
4495 };
4496
4497 tcg_debug_assert(a->esz == 1 || a->esz == 2);
4498 tcg_debug_assert(a->rd == a->ra);
4499 if (sve_access_check(s)) {
4500 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4501 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
636ddeb1 4502 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
18fc2405
RH
4503 vec_full_reg_offset(s, a->rn),
4504 vec_full_reg_offset(s, a->rm),
636ddeb1 4505 vec_full_reg_offset(s, a->ra),
18fc2405
RH
4506 status, vsz, vsz,
4507 a->index * 4 + a->rot,
4508 fns[a->esz - 1]);
4509 tcg_temp_free_ptr(status);
4510 }
4511 return true;
4512}
4513
8092c6a3
RH
4514/*
4515 *** SVE Floating Point Unary Operations Predicated Group
4516 */
4517
4518static bool do_zpz_ptr(DisasContext *s, int rd, int rn, int pg,
4519 bool is_fp16, gen_helper_gvec_3_ptr *fn)
4520{
4521 if (sve_access_check(s)) {
4522 unsigned vsz = vec_full_reg_size(s);
cdfb22bb 4523 TCGv_ptr status = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
8092c6a3
RH
4524 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
4525 vec_full_reg_offset(s, rn),
4526 pred_full_reg_offset(s, pg),
4527 status, vsz, vsz, 0, fn);
4528 tcg_temp_free_ptr(status);
4529 }
4530 return true;
4531}
4532
3a7be554 4533static bool trans_FCVT_sh(DisasContext *s, arg_rpr_esz *a)
46d33d1e 4534{
e4ab5124 4535 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_sh);
46d33d1e
RH
4536}
4537
3a7be554 4538static bool trans_FCVT_hs(DisasContext *s, arg_rpr_esz *a)
46d33d1e
RH
4539{
4540 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_hs);
4541}
4542
3a7be554 4543static bool trans_FCVT_dh(DisasContext *s, arg_rpr_esz *a)
46d33d1e 4544{
e4ab5124 4545 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_dh);
46d33d1e
RH
4546}
4547
3a7be554 4548static bool trans_FCVT_hd(DisasContext *s, arg_rpr_esz *a)
46d33d1e
RH
4549{
4550 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_hd);
4551}
4552
3a7be554 4553static bool trans_FCVT_ds(DisasContext *s, arg_rpr_esz *a)
46d33d1e
RH
4554{
4555 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_ds);
4556}
4557
3a7be554 4558static bool trans_FCVT_sd(DisasContext *s, arg_rpr_esz *a)
46d33d1e
RH
4559{
4560 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_sd);
4561}
4562
3a7be554 4563static bool trans_FCVTZS_hh(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4564{
4565 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzs_hh);
4566}
4567
3a7be554 4568static bool trans_FCVTZU_hh(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4569{
4570 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzu_hh);
4571}
4572
3a7be554 4573static bool trans_FCVTZS_hs(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4574{
4575 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzs_hs);
4576}
4577
3a7be554 4578static bool trans_FCVTZU_hs(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4579{
4580 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzu_hs);
4581}
4582
3a7be554 4583static bool trans_FCVTZS_hd(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4584{
4585 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzs_hd);
4586}
4587
3a7be554 4588static bool trans_FCVTZU_hd(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4589{
4590 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzu_hd);
4591}
4592
3a7be554 4593static bool trans_FCVTZS_ss(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4594{
4595 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzs_ss);
4596}
4597
3a7be554 4598static bool trans_FCVTZU_ss(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4599{
4600 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzu_ss);
4601}
4602
3a7be554 4603static bool trans_FCVTZS_sd(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4604{
4605 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzs_sd);
4606}
4607
3a7be554 4608static bool trans_FCVTZU_sd(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4609{
4610 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzu_sd);
4611}
4612
3a7be554 4613static bool trans_FCVTZS_ds(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4614{
4615 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzs_ds);
4616}
4617
3a7be554 4618static bool trans_FCVTZU_ds(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4619{
4620 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzu_ds);
4621}
4622
3a7be554 4623static bool trans_FCVTZS_dd(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4624{
4625 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzs_dd);
4626}
4627
3a7be554 4628static bool trans_FCVTZU_dd(DisasContext *s, arg_rpr_esz *a)
df4de1af
RH
4629{
4630 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzu_dd);
4631}
4632
cda3c753
RH
4633static gen_helper_gvec_3_ptr * const frint_fns[3] = {
4634 gen_helper_sve_frint_h,
4635 gen_helper_sve_frint_s,
4636 gen_helper_sve_frint_d
4637};
4638
3a7be554 4639static bool trans_FRINTI(DisasContext *s, arg_rpr_esz *a)
cda3c753
RH
4640{
4641 if (a->esz == 0) {
4642 return false;
4643 }
4644 return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16,
4645 frint_fns[a->esz - 1]);
4646}
4647
3a7be554 4648static bool trans_FRINTX(DisasContext *s, arg_rpr_esz *a)
cda3c753
RH
4649{
4650 static gen_helper_gvec_3_ptr * const fns[3] = {
4651 gen_helper_sve_frintx_h,
4652 gen_helper_sve_frintx_s,
4653 gen_helper_sve_frintx_d
4654 };
4655 if (a->esz == 0) {
4656 return false;
4657 }
4658 return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16, fns[a->esz - 1]);
4659}
4660
4661static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, int mode)
4662{
4663 if (a->esz == 0) {
4664 return false;
4665 }
4666 if (sve_access_check(s)) {
4667 unsigned vsz = vec_full_reg_size(s);
4668 TCGv_i32 tmode = tcg_const_i32(mode);
cdfb22bb 4669 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
cda3c753
RH
4670
4671 gen_helper_set_rmode(tmode, tmode, status);
4672
4673 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
4674 vec_full_reg_offset(s, a->rn),
4675 pred_full_reg_offset(s, a->pg),
4676 status, vsz, vsz, 0, frint_fns[a->esz - 1]);
4677
4678 gen_helper_set_rmode(tmode, tmode, status);
4679 tcg_temp_free_i32(tmode);
4680 tcg_temp_free_ptr(status);
4681 }
4682 return true;
4683}
4684
3a7be554 4685static bool trans_FRINTN(DisasContext *s, arg_rpr_esz *a)
cda3c753
RH
4686{
4687 return do_frint_mode(s, a, float_round_nearest_even);
4688}
4689
3a7be554 4690static bool trans_FRINTP(DisasContext *s, arg_rpr_esz *a)
cda3c753
RH
4691{
4692 return do_frint_mode(s, a, float_round_up);
4693}
4694
3a7be554 4695static bool trans_FRINTM(DisasContext *s, arg_rpr_esz *a)
cda3c753
RH
4696{
4697 return do_frint_mode(s, a, float_round_down);
4698}
4699
3a7be554 4700static bool trans_FRINTZ(DisasContext *s, arg_rpr_esz *a)
cda3c753
RH
4701{
4702 return do_frint_mode(s, a, float_round_to_zero);
4703}
4704
3a7be554 4705static bool trans_FRINTA(DisasContext *s, arg_rpr_esz *a)
cda3c753
RH
4706{
4707 return do_frint_mode(s, a, float_round_ties_away);
4708}
4709
3a7be554 4710static bool trans_FRECPX(DisasContext *s, arg_rpr_esz *a)
ec5b375b
RH
4711{
4712 static gen_helper_gvec_3_ptr * const fns[3] = {
4713 gen_helper_sve_frecpx_h,
4714 gen_helper_sve_frecpx_s,
4715 gen_helper_sve_frecpx_d
4716 };
4717 if (a->esz == 0) {
4718 return false;
4719 }
4720 return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16, fns[a->esz - 1]);
4721}
4722
3a7be554 4723static bool trans_FSQRT(DisasContext *s, arg_rpr_esz *a)
ec5b375b
RH
4724{
4725 static gen_helper_gvec_3_ptr * const fns[3] = {
4726 gen_helper_sve_fsqrt_h,
4727 gen_helper_sve_fsqrt_s,
4728 gen_helper_sve_fsqrt_d
4729 };
4730 if (a->esz == 0) {
4731 return false;
4732 }
4733 return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16, fns[a->esz - 1]);
4734}
4735
3a7be554 4736static bool trans_SCVTF_hh(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4737{
4738 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_scvt_hh);
4739}
4740
3a7be554 4741static bool trans_SCVTF_sh(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4742{
4743 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_scvt_sh);
4744}
4745
3a7be554 4746static bool trans_SCVTF_dh(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4747{
4748 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_scvt_dh);
4749}
4750
3a7be554 4751static bool trans_SCVTF_ss(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4752{
4753 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_scvt_ss);
4754}
4755
3a7be554 4756static bool trans_SCVTF_ds(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4757{
4758 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_scvt_ds);
4759}
4760
3a7be554 4761static bool trans_SCVTF_sd(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4762{
4763 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_scvt_sd);
4764}
4765
3a7be554 4766static bool trans_SCVTF_dd(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4767{
4768 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_scvt_dd);
4769}
4770
3a7be554 4771static bool trans_UCVTF_hh(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4772{
4773 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_ucvt_hh);
4774}
4775
3a7be554 4776static bool trans_UCVTF_sh(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4777{
4778 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_ucvt_sh);
4779}
4780
3a7be554 4781static bool trans_UCVTF_dh(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4782{
4783 return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_ucvt_dh);
4784}
4785
3a7be554 4786static bool trans_UCVTF_ss(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4787{
4788 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_ucvt_ss);
4789}
4790
3a7be554 4791static bool trans_UCVTF_ds(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4792{
4793 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_ucvt_ds);
4794}
4795
3a7be554 4796static bool trans_UCVTF_sd(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4797{
4798 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_ucvt_sd);
4799}
4800
3a7be554 4801static bool trans_UCVTF_dd(DisasContext *s, arg_rpr_esz *a)
8092c6a3
RH
4802{
4803 return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_ucvt_dd);
4804}
4805
d1822297
RH
4806/*
4807 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4808 */
4809
4810/* Subroutine loading a vector register at VOFS of LEN bytes.
4811 * The load should begin at the address Rn + IMM.
4812 */
4813
19f2acc9 4814static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
d1822297 4815{
19f2acc9
RH
4816 int len_align = QEMU_ALIGN_DOWN(len, 8);
4817 int len_remain = len % 8;
4818 int nparts = len / 8 + ctpop8(len_remain);
d1822297 4819 int midx = get_mem_index(s);
b2aa8879 4820 TCGv_i64 dirty_addr, clean_addr, t0, t1;
d1822297 4821
b2aa8879
RH
4822 dirty_addr = tcg_temp_new_i64();
4823 tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm);
33e74c31 4824 clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len);
b2aa8879 4825 tcg_temp_free_i64(dirty_addr);
d1822297 4826
b2aa8879
RH
4827 /*
4828 * Note that unpredicated load/store of vector/predicate registers
d1822297 4829 * are defined as a stream of bytes, which equates to little-endian
b2aa8879 4830 * operations on larger quantities.
d1822297
RH
4831 * Attempt to keep code expansion to a minimum by limiting the
4832 * amount of unrolling done.
4833 */
4834 if (nparts <= 4) {
4835 int i;
4836
b2aa8879 4837 t0 = tcg_temp_new_i64();
d1822297 4838 for (i = 0; i < len_align; i += 8) {
b2aa8879 4839 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEQ);
d1822297 4840 tcg_gen_st_i64(t0, cpu_env, vofs + i);
d8227b09 4841 tcg_gen_addi_i64(clean_addr, clean_addr, 8);
d1822297 4842 }
b2aa8879 4843 tcg_temp_free_i64(t0);
d1822297
RH
4844 } else {
4845 TCGLabel *loop = gen_new_label();
4846 TCGv_ptr tp, i = tcg_const_local_ptr(0);
4847
b2aa8879
RH
4848 /* Copy the clean address into a local temp, live across the loop. */
4849 t0 = clean_addr;
4b4dc975 4850 clean_addr = new_tmp_a64_local(s);
b2aa8879 4851 tcg_gen_mov_i64(clean_addr, t0);
d1822297 4852
b2aa8879 4853 gen_set_label(loop);
d1822297 4854
b2aa8879
RH
4855 t0 = tcg_temp_new_i64();
4856 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEQ);
4857 tcg_gen_addi_i64(clean_addr, clean_addr, 8);
d1822297 4858
b2aa8879 4859 tp = tcg_temp_new_ptr();
d1822297
RH
4860 tcg_gen_add_ptr(tp, cpu_env, i);
4861 tcg_gen_addi_ptr(i, i, 8);
4862 tcg_gen_st_i64(t0, tp, vofs);
4863 tcg_temp_free_ptr(tp);
b2aa8879 4864 tcg_temp_free_i64(t0);
d1822297
RH
4865
4866 tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
4867 tcg_temp_free_ptr(i);
4868 }
4869
b2aa8879
RH
4870 /*
4871 * Predicate register loads can be any multiple of 2.
d1822297
RH
4872 * Note that we still store the entire 64-bit unit into cpu_env.
4873 */
4874 if (len_remain) {
b2aa8879 4875 t0 = tcg_temp_new_i64();
d1822297
RH
4876 switch (len_remain) {
4877 case 2:
4878 case 4:
4879 case 8:
b2aa8879
RH
4880 tcg_gen_qemu_ld_i64(t0, clean_addr, midx,
4881 MO_LE | ctz32(len_remain));
d1822297
RH
4882 break;
4883
4884 case 6:
4885 t1 = tcg_temp_new_i64();
b2aa8879
RH
4886 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUL);
4887 tcg_gen_addi_i64(clean_addr, clean_addr, 4);
4888 tcg_gen_qemu_ld_i64(t1, clean_addr, midx, MO_LEUW);
d1822297
RH
4889 tcg_gen_deposit_i64(t0, t0, t1, 32, 32);
4890 tcg_temp_free_i64(t1);
4891 break;
4892
4893 default:
4894 g_assert_not_reached();
4895 }
4896 tcg_gen_st_i64(t0, cpu_env, vofs + len_align);
b2aa8879 4897 tcg_temp_free_i64(t0);
d1822297 4898 }
d1822297
RH
4899}
4900
5047c204 4901/* Similarly for stores. */
19f2acc9 4902static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
5047c204 4903{
19f2acc9
RH
4904 int len_align = QEMU_ALIGN_DOWN(len, 8);
4905 int len_remain = len % 8;
4906 int nparts = len / 8 + ctpop8(len_remain);
5047c204 4907 int midx = get_mem_index(s);
bba87d0a 4908 TCGv_i64 dirty_addr, clean_addr, t0;
5047c204 4909
bba87d0a
RH
4910 dirty_addr = tcg_temp_new_i64();
4911 tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm);
33e74c31 4912 clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len);
bba87d0a 4913 tcg_temp_free_i64(dirty_addr);
5047c204
RH
4914
4915 /* Note that unpredicated load/store of vector/predicate registers
4916 * are defined as a stream of bytes, which equates to little-endian
4917 * operations on larger quantities. There is no nice way to force
4918 * a little-endian store for aarch64_be-linux-user out of line.
4919 *
4920 * Attempt to keep code expansion to a minimum by limiting the
4921 * amount of unrolling done.
4922 */
4923 if (nparts <= 4) {
4924 int i;
4925
bba87d0a 4926 t0 = tcg_temp_new_i64();
5047c204
RH
4927 for (i = 0; i < len_align; i += 8) {
4928 tcg_gen_ld_i64(t0, cpu_env, vofs + i);
bba87d0a 4929 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEQ);
d8227b09 4930 tcg_gen_addi_i64(clean_addr, clean_addr, 8);
5047c204 4931 }
bba87d0a 4932 tcg_temp_free_i64(t0);
5047c204
RH
4933 } else {
4934 TCGLabel *loop = gen_new_label();
bba87d0a 4935 TCGv_ptr tp, i = tcg_const_local_ptr(0);
5047c204 4936
bba87d0a
RH
4937 /* Copy the clean address into a local temp, live across the loop. */
4938 t0 = clean_addr;
4b4dc975 4939 clean_addr = new_tmp_a64_local(s);
bba87d0a 4940 tcg_gen_mov_i64(clean_addr, t0);
5047c204 4941
bba87d0a 4942 gen_set_label(loop);
5047c204 4943
bba87d0a
RH
4944 t0 = tcg_temp_new_i64();
4945 tp = tcg_temp_new_ptr();
4946 tcg_gen_add_ptr(tp, cpu_env, i);
4947 tcg_gen_ld_i64(t0, tp, vofs);
5047c204 4948 tcg_gen_addi_ptr(i, i, 8);
bba87d0a
RH
4949 tcg_temp_free_ptr(tp);
4950
4951 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEQ);
4952 tcg_gen_addi_i64(clean_addr, clean_addr, 8);
4953 tcg_temp_free_i64(t0);
5047c204
RH
4954
4955 tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
4956 tcg_temp_free_ptr(i);
4957 }
4958
4959 /* Predicate register stores can be any multiple of 2. */
4960 if (len_remain) {
bba87d0a 4961 t0 = tcg_temp_new_i64();
5047c204 4962 tcg_gen_ld_i64(t0, cpu_env, vofs + len_align);
5047c204
RH
4963
4964 switch (len_remain) {
4965 case 2:
4966 case 4:
4967 case 8:
bba87d0a
RH
4968 tcg_gen_qemu_st_i64(t0, clean_addr, midx,
4969 MO_LE | ctz32(len_remain));
5047c204
RH
4970 break;
4971
4972 case 6:
bba87d0a
RH
4973 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUL);
4974 tcg_gen_addi_i64(clean_addr, clean_addr, 4);
5047c204 4975 tcg_gen_shri_i64(t0, t0, 32);
bba87d0a 4976 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUW);
5047c204
RH
4977 break;
4978
4979 default:
4980 g_assert_not_reached();
4981 }
bba87d0a 4982 tcg_temp_free_i64(t0);
5047c204 4983 }
5047c204
RH
4984}
4985
3a7be554 4986static bool trans_LDR_zri(DisasContext *s, arg_rri *a)
d1822297
RH
4987{
4988 if (sve_access_check(s)) {
4989 int size = vec_full_reg_size(s);
4990 int off = vec_full_reg_offset(s, a->rd);
4991 do_ldr(s, off, size, a->rn, a->imm * size);
4992 }
4993 return true;
4994}
4995
3a7be554 4996static bool trans_LDR_pri(DisasContext *s, arg_rri *a)
d1822297
RH
4997{
4998 if (sve_access_check(s)) {
4999 int size = pred_full_reg_size(s);
5000 int off = pred_full_reg_offset(s, a->rd);
5001 do_ldr(s, off, size, a->rn, a->imm * size);
5002 }
5003 return true;
5004}
c4e7c493 5005
3a7be554 5006static bool trans_STR_zri(DisasContext *s, arg_rri *a)
5047c204
RH
5007{
5008 if (sve_access_check(s)) {
5009 int size = vec_full_reg_size(s);
5010 int off = vec_full_reg_offset(s, a->rd);
5011 do_str(s, off, size, a->rn, a->imm * size);
5012 }
5013 return true;
5014}
5015
3a7be554 5016static bool trans_STR_pri(DisasContext *s, arg_rri *a)
5047c204
RH
5017{
5018 if (sve_access_check(s)) {
5019 int size = pred_full_reg_size(s);
5020 int off = pred_full_reg_offset(s, a->rd);
5021 do_str(s, off, size, a->rn, a->imm * size);
5022 }
5023 return true;
5024}
5025
c4e7c493
RH
5026/*
5027 *** SVE Memory - Contiguous Load Group
5028 */
5029
5030/* The memory mode of the dtype. */
14776ab5 5031static const MemOp dtype_mop[16] = {
c4e7c493
RH
5032 MO_UB, MO_UB, MO_UB, MO_UB,
5033 MO_SL, MO_UW, MO_UW, MO_UW,
5034 MO_SW, MO_SW, MO_UL, MO_UL,
5035 MO_SB, MO_SB, MO_SB, MO_Q
5036};
5037
5038#define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
5039
5040/* The vector element size of dtype. */
5041static const uint8_t dtype_esz[16] = {
5042 0, 1, 2, 3,
5043 3, 1, 2, 3,
5044 3, 2, 2, 3,
5045 3, 2, 1, 3
5046};
5047
5048static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
206adacf
RH
5049 int dtype, uint32_t mte_n, bool is_write,
5050 gen_helper_gvec_mem *fn)
c4e7c493
RH
5051{
5052 unsigned vsz = vec_full_reg_size(s);
5053 TCGv_ptr t_pg;
500d0484 5054 TCGv_i32 t_desc;
206adacf 5055 int desc = 0;
c4e7c493 5056
206adacf
RH
5057 /*
5058 * For e.g. LD4, there are not enough arguments to pass all 4
c4e7c493
RH
5059 * registers as pointers, so encode the regno into the data field.
5060 * For consistency, do this even for LD1.
5061 */
9473d0ec 5062 if (s->mte_active[0]) {
206adacf
RH
5063 int msz = dtype_msz(dtype);
5064
5065 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
5066 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
5067 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
5068 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
28f32503 5069 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (mte_n << msz) - 1);
206adacf 5070 desc <<= SVE_MTEDESC_SHIFT;
9473d0ec
RH
5071 } else {
5072 addr = clean_data_tbi(s, addr);
206adacf 5073 }
9473d0ec 5074
206adacf 5075 desc = simd_desc(vsz, vsz, zt | desc);
500d0484 5076 t_desc = tcg_const_i32(desc);
c4e7c493
RH
5077 t_pg = tcg_temp_new_ptr();
5078
5079 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
500d0484 5080 fn(cpu_env, t_pg, addr, t_desc);
c4e7c493
RH
5081
5082 tcg_temp_free_ptr(t_pg);
500d0484 5083 tcg_temp_free_i32(t_desc);
c4e7c493
RH
5084}
5085
5086static void do_ld_zpa(DisasContext *s, int zt, int pg,
5087 TCGv_i64 addr, int dtype, int nreg)
5088{
206adacf
RH
5089 static gen_helper_gvec_mem * const fns[2][2][16][4] = {
5090 { /* mte inactive, little-endian */
5091 { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
7d0a57a2 5092 gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
206adacf
RH
5093 { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
5094 { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
5095 { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
5096
5097 { gen_helper_sve_ld1sds_le_r, NULL, NULL, NULL },
5098 { gen_helper_sve_ld1hh_le_r, gen_helper_sve_ld2hh_le_r,
5099 gen_helper_sve_ld3hh_le_r, gen_helper_sve_ld4hh_le_r },
5100 { gen_helper_sve_ld1hsu_le_r, NULL, NULL, NULL },
5101 { gen_helper_sve_ld1hdu_le_r, NULL, NULL, NULL },
5102
5103 { gen_helper_sve_ld1hds_le_r, NULL, NULL, NULL },
5104 { gen_helper_sve_ld1hss_le_r, NULL, NULL, NULL },
5105 { gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld2ss_le_r,
5106 gen_helper_sve_ld3ss_le_r, gen_helper_sve_ld4ss_le_r },
5107 { gen_helper_sve_ld1sdu_le_r, NULL, NULL, NULL },
5108
5109 { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
5110 { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
5111 { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
5112 { gen_helper_sve_ld1dd_le_r, gen_helper_sve_ld2dd_le_r,
5113 gen_helper_sve_ld3dd_le_r, gen_helper_sve_ld4dd_le_r } },
5114
5115 /* mte inactive, big-endian */
5116 { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
5117 gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
5118 { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
5119 { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
5120 { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
5121
5122 { gen_helper_sve_ld1sds_be_r, NULL, NULL, NULL },
5123 { gen_helper_sve_ld1hh_be_r, gen_helper_sve_ld2hh_be_r,
5124 gen_helper_sve_ld3hh_be_r, gen_helper_sve_ld4hh_be_r },
5125 { gen_helper_sve_ld1hsu_be_r, NULL, NULL, NULL },
5126 { gen_helper_sve_ld1hdu_be_r, NULL, NULL, NULL },
5127
5128 { gen_helper_sve_ld1hds_be_r, NULL, NULL, NULL },
5129 { gen_helper_sve_ld1hss_be_r, NULL, NULL, NULL },
5130 { gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld2ss_be_r,
5131 gen_helper_sve_ld3ss_be_r, gen_helper_sve_ld4ss_be_r },
5132 { gen_helper_sve_ld1sdu_be_r, NULL, NULL, NULL },
5133
5134 { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
5135 { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
5136 { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
5137 { gen_helper_sve_ld1dd_be_r, gen_helper_sve_ld2dd_be_r,
5138 gen_helper_sve_ld3dd_be_r, gen_helper_sve_ld4dd_be_r } } },
5139
5140 { /* mte active, little-endian */
5141 { { gen_helper_sve_ld1bb_r_mte,
5142 gen_helper_sve_ld2bb_r_mte,
5143 gen_helper_sve_ld3bb_r_mte,
5144 gen_helper_sve_ld4bb_r_mte },
5145 { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
5146 { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
5147 { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
5148
5149 { gen_helper_sve_ld1sds_le_r_mte, NULL, NULL, NULL },
5150 { gen_helper_sve_ld1hh_le_r_mte,
5151 gen_helper_sve_ld2hh_le_r_mte,
5152 gen_helper_sve_ld3hh_le_r_mte,
5153 gen_helper_sve_ld4hh_le_r_mte },
5154 { gen_helper_sve_ld1hsu_le_r_mte, NULL, NULL, NULL },
5155 { gen_helper_sve_ld1hdu_le_r_mte, NULL, NULL, NULL },
5156
5157 { gen_helper_sve_ld1hds_le_r_mte, NULL, NULL, NULL },
5158 { gen_helper_sve_ld1hss_le_r_mte, NULL, NULL, NULL },
5159 { gen_helper_sve_ld1ss_le_r_mte,
5160 gen_helper_sve_ld2ss_le_r_mte,
5161 gen_helper_sve_ld3ss_le_r_mte,
5162 gen_helper_sve_ld4ss_le_r_mte },
5163 { gen_helper_sve_ld1sdu_le_r_mte, NULL, NULL, NULL },
5164
5165 { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
5166 { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
5167 { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
5168 { gen_helper_sve_ld1dd_le_r_mte,
5169 gen_helper_sve_ld2dd_le_r_mte,
5170 gen_helper_sve_ld3dd_le_r_mte,
5171 gen_helper_sve_ld4dd_le_r_mte } },
5172
5173 /* mte active, big-endian */
5174 { { gen_helper_sve_ld1bb_r_mte,
5175 gen_helper_sve_ld2bb_r_mte,
5176 gen_helper_sve_ld3bb_r_mte,
5177 gen_helper_sve_ld4bb_r_mte },
5178 { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
5179 { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
5180 { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
5181
5182 { gen_helper_sve_ld1sds_be_r_mte, NULL, NULL, NULL },
5183 { gen_helper_sve_ld1hh_be_r_mte,
5184 gen_helper_sve_ld2hh_be_r_mte,
5185 gen_helper_sve_ld3hh_be_r_mte,
5186 gen_helper_sve_ld4hh_be_r_mte },
5187 { gen_helper_sve_ld1hsu_be_r_mte, NULL, NULL, NULL },
5188 { gen_helper_sve_ld1hdu_be_r_mte, NULL, NULL, NULL },
5189
5190 { gen_helper_sve_ld1hds_be_r_mte, NULL, NULL, NULL },
5191 { gen_helper_sve_ld1hss_be_r_mte, NULL, NULL, NULL },
5192 { gen_helper_sve_ld1ss_be_r_mte,
5193 gen_helper_sve_ld2ss_be_r_mte,
5194 gen_helper_sve_ld3ss_be_r_mte,
5195 gen_helper_sve_ld4ss_be_r_mte },
5196 { gen_helper_sve_ld1sdu_be_r_mte, NULL, NULL, NULL },
5197
5198 { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
5199 { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
5200 { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
5201 { gen_helper_sve_ld1dd_be_r_mte,
5202 gen_helper_sve_ld2dd_be_r_mte,
5203 gen_helper_sve_ld3dd_be_r_mte,
5204 gen_helper_sve_ld4dd_be_r_mte } } },
c4e7c493 5205 };
206adacf
RH
5206 gen_helper_gvec_mem *fn
5207 = fns[s->mte_active[0]][s->be_data == MO_BE][dtype][nreg];
c4e7c493 5208
206adacf
RH
5209 /*
5210 * While there are holes in the table, they are not
c4e7c493
RH
5211 * accessible via the instruction encoding.
5212 */
5213 assert(fn != NULL);
206adacf 5214 do_mem_zpa(s, zt, pg, addr, dtype, nreg, false, fn);
c4e7c493
RH
5215}
5216
3a7be554 5217static bool trans_LD_zprr(DisasContext *s, arg_rprr_load *a)
c4e7c493
RH
5218{
5219 if (a->rm == 31) {
5220 return false;
5221 }
5222 if (sve_access_check(s)) {
5223 TCGv_i64 addr = new_tmp_a64(s);
50ef1cbf 5224 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
c4e7c493
RH
5225 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
5226 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg);
5227 }
5228 return true;
5229}
5230
3a7be554 5231static bool trans_LD_zpri(DisasContext *s, arg_rpri_load *a)
c4e7c493
RH
5232{
5233 if (sve_access_check(s)) {
5234 int vsz = vec_full_reg_size(s);
5235 int elements = vsz >> dtype_esz[a->dtype];
5236 TCGv_i64 addr = new_tmp_a64(s);
5237
5238 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn),
5239 (a->imm * elements * (a->nreg + 1))
5240 << dtype_msz(a->dtype));
5241 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg);
5242 }
5243 return true;
5244}
e2654d75 5245
3a7be554 5246static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a)
e2654d75 5247{
aa13f7c3
RH
5248 static gen_helper_gvec_mem * const fns[2][2][16] = {
5249 { /* mte inactive, little-endian */
5250 { gen_helper_sve_ldff1bb_r,
5251 gen_helper_sve_ldff1bhu_r,
5252 gen_helper_sve_ldff1bsu_r,
5253 gen_helper_sve_ldff1bdu_r,
5254
5255 gen_helper_sve_ldff1sds_le_r,
5256 gen_helper_sve_ldff1hh_le_r,
5257 gen_helper_sve_ldff1hsu_le_r,
5258 gen_helper_sve_ldff1hdu_le_r,
5259
5260 gen_helper_sve_ldff1hds_le_r,
5261 gen_helper_sve_ldff1hss_le_r,
5262 gen_helper_sve_ldff1ss_le_r,
5263 gen_helper_sve_ldff1sdu_le_r,
5264
5265 gen_helper_sve_ldff1bds_r,
5266 gen_helper_sve_ldff1bss_r,
5267 gen_helper_sve_ldff1bhs_r,
5268 gen_helper_sve_ldff1dd_le_r },
5269
5270 /* mte inactive, big-endian */
5271 { gen_helper_sve_ldff1bb_r,
5272 gen_helper_sve_ldff1bhu_r,
5273 gen_helper_sve_ldff1bsu_r,
5274 gen_helper_sve_ldff1bdu_r,
5275
5276 gen_helper_sve_ldff1sds_be_r,
5277 gen_helper_sve_ldff1hh_be_r,
5278 gen_helper_sve_ldff1hsu_be_r,
5279 gen_helper_sve_ldff1hdu_be_r,
5280
5281 gen_helper_sve_ldff1hds_be_r,
5282 gen_helper_sve_ldff1hss_be_r,
5283 gen_helper_sve_ldff1ss_be_r,
5284 gen_helper_sve_ldff1sdu_be_r,
5285
5286 gen_helper_sve_ldff1bds_r,
5287 gen_helper_sve_ldff1bss_r,
5288 gen_helper_sve_ldff1bhs_r,
5289 gen_helper_sve_ldff1dd_be_r } },
5290
5291 { /* mte active, little-endian */
5292 { gen_helper_sve_ldff1bb_r_mte,
5293 gen_helper_sve_ldff1bhu_r_mte,
5294 gen_helper_sve_ldff1bsu_r_mte,
5295 gen_helper_sve_ldff1bdu_r_mte,
5296
5297 gen_helper_sve_ldff1sds_le_r_mte,
5298 gen_helper_sve_ldff1hh_le_r_mte,
5299 gen_helper_sve_ldff1hsu_le_r_mte,
5300 gen_helper_sve_ldff1hdu_le_r_mte,
5301
5302 gen_helper_sve_ldff1hds_le_r_mte,
5303 gen_helper_sve_ldff1hss_le_r_mte,
5304 gen_helper_sve_ldff1ss_le_r_mte,
5305 gen_helper_sve_ldff1sdu_le_r_mte,
5306
5307 gen_helper_sve_ldff1bds_r_mte,
5308 gen_helper_sve_ldff1bss_r_mte,
5309 gen_helper_sve_ldff1bhs_r_mte,
5310 gen_helper_sve_ldff1dd_le_r_mte },
5311
5312 /* mte active, big-endian */
5313 { gen_helper_sve_ldff1bb_r_mte,
5314 gen_helper_sve_ldff1bhu_r_mte,
5315 gen_helper_sve_ldff1bsu_r_mte,
5316 gen_helper_sve_ldff1bdu_r_mte,
5317
5318 gen_helper_sve_ldff1sds_be_r_mte,
5319 gen_helper_sve_ldff1hh_be_r_mte,
5320 gen_helper_sve_ldff1hsu_be_r_mte,
5321 gen_helper_sve_ldff1hdu_be_r_mte,
5322
5323 gen_helper_sve_ldff1hds_be_r_mte,
5324 gen_helper_sve_ldff1hss_be_r_mte,
5325 gen_helper_sve_ldff1ss_be_r_mte,
5326 gen_helper_sve_ldff1sdu_be_r_mte,
5327
5328 gen_helper_sve_ldff1bds_r_mte,
5329 gen_helper_sve_ldff1bss_r_mte,
5330 gen_helper_sve_ldff1bhs_r_mte,
5331 gen_helper_sve_ldff1dd_be_r_mte } },
e2654d75
RH
5332 };
5333
5334 if (sve_access_check(s)) {
5335 TCGv_i64 addr = new_tmp_a64(s);
5336 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
5337 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
aa13f7c3
RH
5338 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false,
5339 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]);
e2654d75
RH
5340 }
5341 return true;
5342}
5343
3a7be554 5344static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a)
e2654d75 5345{
aa13f7c3
RH
5346 static gen_helper_gvec_mem * const fns[2][2][16] = {
5347 { /* mte inactive, little-endian */
5348 { gen_helper_sve_ldnf1bb_r,
5349 gen_helper_sve_ldnf1bhu_r,
5350 gen_helper_sve_ldnf1bsu_r,
5351 gen_helper_sve_ldnf1bdu_r,
5352
5353 gen_helper_sve_ldnf1sds_le_r,
5354 gen_helper_sve_ldnf1hh_le_r,
5355 gen_helper_sve_ldnf1hsu_le_r,
5356 gen_helper_sve_ldnf1hdu_le_r,
5357
5358 gen_helper_sve_ldnf1hds_le_r,
5359 gen_helper_sve_ldnf1hss_le_r,
5360 gen_helper_sve_ldnf1ss_le_r,
5361 gen_helper_sve_ldnf1sdu_le_r,
5362
5363 gen_helper_sve_ldnf1bds_r,
5364 gen_helper_sve_ldnf1bss_r,
5365 gen_helper_sve_ldnf1bhs_r,
5366 gen_helper_sve_ldnf1dd_le_r },
5367
5368 /* mte inactive, big-endian */
5369 { gen_helper_sve_ldnf1bb_r,
5370 gen_helper_sve_ldnf1bhu_r,
5371 gen_helper_sve_ldnf1bsu_r,
5372 gen_helper_sve_ldnf1bdu_r,
5373
5374 gen_helper_sve_ldnf1sds_be_r,
5375 gen_helper_sve_ldnf1hh_be_r,
5376 gen_helper_sve_ldnf1hsu_be_r,
5377 gen_helper_sve_ldnf1hdu_be_r,
5378
5379 gen_helper_sve_ldnf1hds_be_r,
5380 gen_helper_sve_ldnf1hss_be_r,
5381 gen_helper_sve_ldnf1ss_be_r,
5382 gen_helper_sve_ldnf1sdu_be_r,
5383
5384 gen_helper_sve_ldnf1bds_r,
5385 gen_helper_sve_ldnf1bss_r,
5386 gen_helper_sve_ldnf1bhs_r,
5387 gen_helper_sve_ldnf1dd_be_r } },
5388
5389 { /* mte inactive, little-endian */
5390 { gen_helper_sve_ldnf1bb_r_mte,
5391 gen_helper_sve_ldnf1bhu_r_mte,
5392 gen_helper_sve_ldnf1bsu_r_mte,
5393 gen_helper_sve_ldnf1bdu_r_mte,
5394
5395 gen_helper_sve_ldnf1sds_le_r_mte,
5396 gen_helper_sve_ldnf1hh_le_r_mte,
5397 gen_helper_sve_ldnf1hsu_le_r_mte,
5398 gen_helper_sve_ldnf1hdu_le_r_mte,
5399
5400 gen_helper_sve_ldnf1hds_le_r_mte,
5401 gen_helper_sve_ldnf1hss_le_r_mte,
5402 gen_helper_sve_ldnf1ss_le_r_mte,
5403 gen_helper_sve_ldnf1sdu_le_r_mte,
5404
5405 gen_helper_sve_ldnf1bds_r_mte,
5406 gen_helper_sve_ldnf1bss_r_mte,
5407 gen_helper_sve_ldnf1bhs_r_mte,
5408 gen_helper_sve_ldnf1dd_le_r_mte },
5409
5410 /* mte inactive, big-endian */
5411 { gen_helper_sve_ldnf1bb_r_mte,
5412 gen_helper_sve_ldnf1bhu_r_mte,
5413 gen_helper_sve_ldnf1bsu_r_mte,
5414 gen_helper_sve_ldnf1bdu_r_mte,
5415
5416 gen_helper_sve_ldnf1sds_be_r_mte,
5417 gen_helper_sve_ldnf1hh_be_r_mte,
5418 gen_helper_sve_ldnf1hsu_be_r_mte,
5419 gen_helper_sve_ldnf1hdu_be_r_mte,
5420
5421 gen_helper_sve_ldnf1hds_be_r_mte,
5422 gen_helper_sve_ldnf1hss_be_r_mte,
5423 gen_helper_sve_ldnf1ss_be_r_mte,
5424 gen_helper_sve_ldnf1sdu_be_r_mte,
5425
5426 gen_helper_sve_ldnf1bds_r_mte,
5427 gen_helper_sve_ldnf1bss_r_mte,
5428 gen_helper_sve_ldnf1bhs_r_mte,
5429 gen_helper_sve_ldnf1dd_be_r_mte } },
e2654d75
RH
5430 };
5431
5432 if (sve_access_check(s)) {
5433 int vsz = vec_full_reg_size(s);
5434 int elements = vsz >> dtype_esz[a->dtype];
5435 int off = (a->imm * elements) << dtype_msz(a->dtype);
5436 TCGv_i64 addr = new_tmp_a64(s);
5437
5438 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off);
aa13f7c3
RH
5439 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false,
5440 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]);
e2654d75
RH
5441 }
5442 return true;
5443}
1a039c7e 5444
05abe304
RH
5445static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int msz)
5446{
7d0a57a2
RH
5447 static gen_helper_gvec_mem * const fns[2][4] = {
5448 { gen_helper_sve_ld1bb_r, gen_helper_sve_ld1hh_le_r,
5449 gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld1dd_le_r },
5450 { gen_helper_sve_ld1bb_r, gen_helper_sve_ld1hh_be_r,
5451 gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld1dd_be_r },
05abe304
RH
5452 };
5453 unsigned vsz = vec_full_reg_size(s);
5454 TCGv_ptr t_pg;
500d0484
RH
5455 TCGv_i32 t_desc;
5456 int desc, poff;
05abe304
RH
5457
5458 /* Load the first quadword using the normal predicated load helpers. */
ba080b86 5459 desc = simd_desc(16, 16, zt);
500d0484 5460 t_desc = tcg_const_i32(desc);
2a99ab2b
RH
5461
5462 poff = pred_full_reg_offset(s, pg);
5463 if (vsz > 16) {
5464 /*
5465 * Zero-extend the first 16 bits of the predicate into a temporary.
5466 * This avoids triggering an assert making sure we don't have bits
5467 * set within a predicate beyond VQ, but we have lowered VQ to 1
5468 * for this load operation.
5469 */
5470 TCGv_i64 tmp = tcg_temp_new_i64();
5471#ifdef HOST_WORDS_BIGENDIAN
5472 poff += 6;
5473#endif
5474 tcg_gen_ld16u_i64(tmp, cpu_env, poff);
5475
5476 poff = offsetof(CPUARMState, vfp.preg_tmp);
5477 tcg_gen_st_i64(tmp, cpu_env, poff);
5478 tcg_temp_free_i64(tmp);
5479 }
5480
05abe304 5481 t_pg = tcg_temp_new_ptr();
2a99ab2b 5482 tcg_gen_addi_ptr(t_pg, cpu_env, poff);
05abe304 5483
500d0484 5484 fns[s->be_data == MO_BE][msz](cpu_env, t_pg, addr, t_desc);
05abe304
RH
5485
5486 tcg_temp_free_ptr(t_pg);
500d0484 5487 tcg_temp_free_i32(t_desc);
05abe304
RH
5488
5489 /* Replicate that first quadword. */
5490 if (vsz > 16) {
5491 unsigned dofs = vec_full_reg_offset(s, zt);
5492 tcg_gen_gvec_dup_mem(4, dofs + 16, dofs, vsz - 16, vsz - 16);
5493 }
5494}
5495
3a7be554 5496static bool trans_LD1RQ_zprr(DisasContext *s, arg_rprr_load *a)
05abe304
RH
5497{
5498 if (a->rm == 31) {
5499 return false;
5500 }
5501 if (sve_access_check(s)) {
5502 int msz = dtype_msz(a->dtype);
5503 TCGv_i64 addr = new_tmp_a64(s);
5504 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz);
5505 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
5506 do_ldrq(s, a->rd, a->pg, addr, msz);
5507 }
5508 return true;
5509}
5510
3a7be554 5511static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a)
05abe304
RH
5512{
5513 if (sve_access_check(s)) {
5514 TCGv_i64 addr = new_tmp_a64(s);
5515 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16);
5516 do_ldrq(s, a->rd, a->pg, addr, dtype_msz(a->dtype));
5517 }
5518 return true;
5519}
5520
68459864 5521/* Load and broadcast element. */
3a7be554 5522static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a)
68459864 5523{
68459864
RH
5524 unsigned vsz = vec_full_reg_size(s);
5525 unsigned psz = pred_full_reg_size(s);
5526 unsigned esz = dtype_esz[a->dtype];
d0e372b0 5527 unsigned msz = dtype_msz(a->dtype);
c0ed9166 5528 TCGLabel *over;
4ac430e1 5529 TCGv_i64 temp, clean_addr;
68459864 5530
c0ed9166
RH
5531 if (!sve_access_check(s)) {
5532 return true;
5533 }
5534
5535 over = gen_new_label();
5536
68459864
RH
5537 /* If the guarding predicate has no bits set, no load occurs. */
5538 if (psz <= 8) {
5539 /* Reduce the pred_esz_masks value simply to reduce the
5540 * size of the code generated here.
5541 */
5542 uint64_t psz_mask = MAKE_64BIT_MASK(0, psz * 8);
5543 temp = tcg_temp_new_i64();
5544 tcg_gen_ld_i64(temp, cpu_env, pred_full_reg_offset(s, a->pg));
5545 tcg_gen_andi_i64(temp, temp, pred_esz_masks[esz] & psz_mask);
5546 tcg_gen_brcondi_i64(TCG_COND_EQ, temp, 0, over);
5547 tcg_temp_free_i64(temp);
5548 } else {
5549 TCGv_i32 t32 = tcg_temp_new_i32();
5550 find_last_active(s, t32, esz, a->pg);
5551 tcg_gen_brcondi_i32(TCG_COND_LT, t32, 0, over);
5552 tcg_temp_free_i32(t32);
5553 }
5554
5555 /* Load the data. */
5556 temp = tcg_temp_new_i64();
d0e372b0 5557 tcg_gen_addi_i64(temp, cpu_reg_sp(s, a->rn), a->imm << msz);
4ac430e1
RH
5558 clean_addr = gen_mte_check1(s, temp, false, true, msz);
5559
5560 tcg_gen_qemu_ld_i64(temp, clean_addr, get_mem_index(s),
0ca0f872 5561 finalize_memop(s, dtype_mop[a->dtype]));
68459864
RH
5562
5563 /* Broadcast to *all* elements. */
5564 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd),
5565 vsz, vsz, temp);
5566 tcg_temp_free_i64(temp);
5567
5568 /* Zero the inactive elements. */
5569 gen_set_label(over);
60245996 5570 return do_movz_zpz(s, a->rd, a->rd, a->pg, esz, false);
68459864
RH
5571}
5572
1a039c7e
RH
5573static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
5574 int msz, int esz, int nreg)
5575{
71b9f394
RH
5576 static gen_helper_gvec_mem * const fn_single[2][2][4][4] = {
5577 { { { gen_helper_sve_st1bb_r,
5578 gen_helper_sve_st1bh_r,
5579 gen_helper_sve_st1bs_r,
5580 gen_helper_sve_st1bd_r },
5581 { NULL,
5582 gen_helper_sve_st1hh_le_r,
5583 gen_helper_sve_st1hs_le_r,
5584 gen_helper_sve_st1hd_le_r },
5585 { NULL, NULL,
5586 gen_helper_sve_st1ss_le_r,
5587 gen_helper_sve_st1sd_le_r },
5588 { NULL, NULL, NULL,
5589 gen_helper_sve_st1dd_le_r } },
5590 { { gen_helper_sve_st1bb_r,
5591 gen_helper_sve_st1bh_r,
5592 gen_helper_sve_st1bs_r,
5593 gen_helper_sve_st1bd_r },
5594 { NULL,
5595 gen_helper_sve_st1hh_be_r,
5596 gen_helper_sve_st1hs_be_r,
5597 gen_helper_sve_st1hd_be_r },
5598 { NULL, NULL,
5599 gen_helper_sve_st1ss_be_r,
5600 gen_helper_sve_st1sd_be_r },
5601 { NULL, NULL, NULL,
5602 gen_helper_sve_st1dd_be_r } } },
5603
5604 { { { gen_helper_sve_st1bb_r_mte,
5605 gen_helper_sve_st1bh_r_mte,
5606 gen_helper_sve_st1bs_r_mte,
5607 gen_helper_sve_st1bd_r_mte },
5608 { NULL,
5609 gen_helper_sve_st1hh_le_r_mte,
5610 gen_helper_sve_st1hs_le_r_mte,
5611 gen_helper_sve_st1hd_le_r_mte },
5612 { NULL, NULL,
5613 gen_helper_sve_st1ss_le_r_mte,
5614 gen_helper_sve_st1sd_le_r_mte },
5615 { NULL, NULL, NULL,
5616 gen_helper_sve_st1dd_le_r_mte } },
5617 { { gen_helper_sve_st1bb_r_mte,
5618 gen_helper_sve_st1bh_r_mte,
5619 gen_helper_sve_st1bs_r_mte,
5620 gen_helper_sve_st1bd_r_mte },
5621 { NULL,
5622 gen_helper_sve_st1hh_be_r_mte,
5623 gen_helper_sve_st1hs_be_r_mte,
5624 gen_helper_sve_st1hd_be_r_mte },
5625 { NULL, NULL,
5626 gen_helper_sve_st1ss_be_r_mte,
5627 gen_helper_sve_st1sd_be_r_mte },
5628 { NULL, NULL, NULL,
5629 gen_helper_sve_st1dd_be_r_mte } } },
1a039c7e 5630 };
71b9f394
RH
5631 static gen_helper_gvec_mem * const fn_multiple[2][2][3][4] = {
5632 { { { gen_helper_sve_st2bb_r,
5633 gen_helper_sve_st2hh_le_r,
5634 gen_helper_sve_st2ss_le_r,
5635 gen_helper_sve_st2dd_le_r },
5636 { gen_helper_sve_st3bb_r,
5637 gen_helper_sve_st3hh_le_r,
5638 gen_helper_sve_st3ss_le_r,
5639 gen_helper_sve_st3dd_le_r },
5640 { gen_helper_sve_st4bb_r,
5641 gen_helper_sve_st4hh_le_r,
5642 gen_helper_sve_st4ss_le_r,
5643 gen_helper_sve_st4dd_le_r } },
5644 { { gen_helper_sve_st2bb_r,
5645 gen_helper_sve_st2hh_be_r,
5646 gen_helper_sve_st2ss_be_r,
5647 gen_helper_sve_st2dd_be_r },
5648 { gen_helper_sve_st3bb_r,
5649 gen_helper_sve_st3hh_be_r,
5650 gen_helper_sve_st3ss_be_r,
5651 gen_helper_sve_st3dd_be_r },
5652 { gen_helper_sve_st4bb_r,
5653 gen_helper_sve_st4hh_be_r,
5654 gen_helper_sve_st4ss_be_r,
5655 gen_helper_sve_st4dd_be_r } } },
5656 { { { gen_helper_sve_st2bb_r_mte,
5657 gen_helper_sve_st2hh_le_r_mte,
5658 gen_helper_sve_st2ss_le_r_mte,
5659 gen_helper_sve_st2dd_le_r_mte },
5660 { gen_helper_sve_st3bb_r_mte,
5661 gen_helper_sve_st3hh_le_r_mte,
5662 gen_helper_sve_st3ss_le_r_mte,
5663 gen_helper_sve_st3dd_le_r_mte },
5664 { gen_helper_sve_st4bb_r_mte,
5665 gen_helper_sve_st4hh_le_r_mte,
5666 gen_helper_sve_st4ss_le_r_mte,
5667 gen_helper_sve_st4dd_le_r_mte } },
5668 { { gen_helper_sve_st2bb_r_mte,
5669 gen_helper_sve_st2hh_be_r_mte,
5670 gen_helper_sve_st2ss_be_r_mte,
5671 gen_helper_sve_st2dd_be_r_mte },
5672 { gen_helper_sve_st3bb_r_mte,
5673 gen_helper_sve_st3hh_be_r_mte,
5674 gen_helper_sve_st3ss_be_r_mte,
5675 gen_helper_sve_st3dd_be_r_mte },
5676 { gen_helper_sve_st4bb_r_mte,
5677 gen_helper_sve_st4hh_be_r_mte,
5678 gen_helper_sve_st4ss_be_r_mte,
5679 gen_helper_sve_st4dd_be_r_mte } } },
1a039c7e
RH
5680 };
5681 gen_helper_gvec_mem *fn;
28d57f2d 5682 int be = s->be_data == MO_BE;
1a039c7e
RH
5683
5684 if (nreg == 0) {
5685 /* ST1 */
71b9f394
RH
5686 fn = fn_single[s->mte_active[0]][be][msz][esz];
5687 nreg = 1;
1a039c7e
RH
5688 } else {
5689 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5690 assert(msz == esz);
71b9f394 5691 fn = fn_multiple[s->mte_active[0]][be][nreg - 1][msz];
1a039c7e
RH
5692 }
5693 assert(fn != NULL);
71b9f394 5694 do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), nreg, true, fn);
1a039c7e
RH
5695}
5696
3a7be554 5697static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a)
1a039c7e
RH
5698{
5699 if (a->rm == 31 || a->msz > a->esz) {
5700 return false;
5701 }
5702 if (sve_access_check(s)) {
5703 TCGv_i64 addr = new_tmp_a64(s);
50ef1cbf 5704 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->msz);
1a039c7e
RH
5705 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
5706 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg);
5707 }
5708 return true;
5709}
5710
3a7be554 5711static bool trans_ST_zpri(DisasContext *s, arg_rpri_store *a)
1a039c7e
RH
5712{
5713 if (a->msz > a->esz) {
5714 return false;
5715 }
5716 if (sve_access_check(s)) {
5717 int vsz = vec_full_reg_size(s);
5718 int elements = vsz >> a->esz;
5719 TCGv_i64 addr = new_tmp_a64(s);
5720
5721 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn),
5722 (a->imm * elements * (a->nreg + 1)) << a->msz);
5723 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg);
5724 }
5725 return true;
5726}
f6dbf62a
RH
5727
5728/*
5729 *** SVE gather loads / scatter stores
5730 */
5731
500d0484 5732static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm,
d28d12f0 5733 int scale, TCGv_i64 scalar, int msz, bool is_write,
500d0484 5734 gen_helper_gvec_mem_scatter *fn)
f6dbf62a
RH
5735{
5736 unsigned vsz = vec_full_reg_size(s);
f6dbf62a
RH
5737 TCGv_ptr t_zm = tcg_temp_new_ptr();
5738 TCGv_ptr t_pg = tcg_temp_new_ptr();
5739 TCGv_ptr t_zt = tcg_temp_new_ptr();
500d0484 5740 TCGv_i32 t_desc;
d28d12f0 5741 int desc = 0;
500d0484 5742
d28d12f0
RH
5743 if (s->mte_active[0]) {
5744 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
5745 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
5746 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
5747 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
28f32503 5748 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << msz) - 1);
d28d12f0
RH
5749 desc <<= SVE_MTEDESC_SHIFT;
5750 }
cdecb3fc 5751 desc = simd_desc(vsz, vsz, desc | scale);
500d0484 5752 t_desc = tcg_const_i32(desc);
f6dbf62a
RH
5753
5754 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
5755 tcg_gen_addi_ptr(t_zm, cpu_env, vec_full_reg_offset(s, zm));
5756 tcg_gen_addi_ptr(t_zt, cpu_env, vec_full_reg_offset(s, zt));
500d0484 5757 fn(cpu_env, t_zt, t_pg, t_zm, scalar, t_desc);
f6dbf62a
RH
5758
5759 tcg_temp_free_ptr(t_zt);
5760 tcg_temp_free_ptr(t_zm);
5761 tcg_temp_free_ptr(t_pg);
500d0484 5762 tcg_temp_free_i32(t_desc);
f6dbf62a
RH
5763}
5764
d28d12f0
RH
5765/* Indexed by [mte][be][ff][xs][u][msz]. */
5766static gen_helper_gvec_mem_scatter * const
5767gather_load_fn32[2][2][2][2][2][3] = {
5768 { /* MTE Inactive */
5769 { /* Little-endian */
5770 { { { gen_helper_sve_ldbss_zsu,
5771 gen_helper_sve_ldhss_le_zsu,
5772 NULL, },
5773 { gen_helper_sve_ldbsu_zsu,
5774 gen_helper_sve_ldhsu_le_zsu,
5775 gen_helper_sve_ldss_le_zsu, } },
5776 { { gen_helper_sve_ldbss_zss,
5777 gen_helper_sve_ldhss_le_zss,
5778 NULL, },
5779 { gen_helper_sve_ldbsu_zss,
5780 gen_helper_sve_ldhsu_le_zss,
5781 gen_helper_sve_ldss_le_zss, } } },
5782
5783 /* First-fault */
5784 { { { gen_helper_sve_ldffbss_zsu,
5785 gen_helper_sve_ldffhss_le_zsu,
5786 NULL, },
5787 { gen_helper_sve_ldffbsu_zsu,
5788 gen_helper_sve_ldffhsu_le_zsu,
5789 gen_helper_sve_ldffss_le_zsu, } },
5790 { { gen_helper_sve_ldffbss_zss,
5791 gen_helper_sve_ldffhss_le_zss,
5792 NULL, },
5793 { gen_helper_sve_ldffbsu_zss,
5794 gen_helper_sve_ldffhsu_le_zss,
5795 gen_helper_sve_ldffss_le_zss, } } } },
5796
5797 { /* Big-endian */
5798 { { { gen_helper_sve_ldbss_zsu,
5799 gen_helper_sve_ldhss_be_zsu,
5800 NULL, },
5801 { gen_helper_sve_ldbsu_zsu,
5802 gen_helper_sve_ldhsu_be_zsu,
5803 gen_helper_sve_ldss_be_zsu, } },
5804 { { gen_helper_sve_ldbss_zss,
5805 gen_helper_sve_ldhss_be_zss,
5806 NULL, },
5807 { gen_helper_sve_ldbsu_zss,
5808 gen_helper_sve_ldhsu_be_zss,
5809 gen_helper_sve_ldss_be_zss, } } },
5810
5811 /* First-fault */
5812 { { { gen_helper_sve_ldffbss_zsu,
5813 gen_helper_sve_ldffhss_be_zsu,
5814 NULL, },
5815 { gen_helper_sve_ldffbsu_zsu,
5816 gen_helper_sve_ldffhsu_be_zsu,
5817 gen_helper_sve_ldffss_be_zsu, } },
5818 { { gen_helper_sve_ldffbss_zss,
5819 gen_helper_sve_ldffhss_be_zss,
5820 NULL, },
5821 { gen_helper_sve_ldffbsu_zss,
5822 gen_helper_sve_ldffhsu_be_zss,
5823 gen_helper_sve_ldffss_be_zss, } } } } },
5824 { /* MTE Active */
5825 { /* Little-endian */
5826 { { { gen_helper_sve_ldbss_zsu_mte,
5827 gen_helper_sve_ldhss_le_zsu_mte,
5828 NULL, },
5829 { gen_helper_sve_ldbsu_zsu_mte,
5830 gen_helper_sve_ldhsu_le_zsu_mte,
5831 gen_helper_sve_ldss_le_zsu_mte, } },
5832 { { gen_helper_sve_ldbss_zss_mte,
5833 gen_helper_sve_ldhss_le_zss_mte,
5834 NULL, },
5835 { gen_helper_sve_ldbsu_zss_mte,
5836 gen_helper_sve_ldhsu_le_zss_mte,
5837 gen_helper_sve_ldss_le_zss_mte, } } },
5838
5839 /* First-fault */
5840 { { { gen_helper_sve_ldffbss_zsu_mte,
5841 gen_helper_sve_ldffhss_le_zsu_mte,
5842 NULL, },
5843 { gen_helper_sve_ldffbsu_zsu_mte,
5844 gen_helper_sve_ldffhsu_le_zsu_mte,
5845 gen_helper_sve_ldffss_le_zsu_mte, } },
5846 { { gen_helper_sve_ldffbss_zss_mte,
5847 gen_helper_sve_ldffhss_le_zss_mte,
5848 NULL, },
5849 { gen_helper_sve_ldffbsu_zss_mte,
5850 gen_helper_sve_ldffhsu_le_zss_mte,
5851 gen_helper_sve_ldffss_le_zss_mte, } } } },
5852
5853 { /* Big-endian */
5854 { { { gen_helper_sve_ldbss_zsu_mte,
5855 gen_helper_sve_ldhss_be_zsu_mte,
5856 NULL, },
5857 { gen_helper_sve_ldbsu_zsu_mte,
5858 gen_helper_sve_ldhsu_be_zsu_mte,
5859 gen_helper_sve_ldss_be_zsu_mte, } },
5860 { { gen_helper_sve_ldbss_zss_mte,
5861 gen_helper_sve_ldhss_be_zss_mte,
5862 NULL, },
5863 { gen_helper_sve_ldbsu_zss_mte,
5864 gen_helper_sve_ldhsu_be_zss_mte,
5865 gen_helper_sve_ldss_be_zss_mte, } } },
5866
5867 /* First-fault */
5868 { { { gen_helper_sve_ldffbss_zsu_mte,
5869 gen_helper_sve_ldffhss_be_zsu_mte,
5870 NULL, },
5871 { gen_helper_sve_ldffbsu_zsu_mte,
5872 gen_helper_sve_ldffhsu_be_zsu_mte,
5873 gen_helper_sve_ldffss_be_zsu_mte, } },
5874 { { gen_helper_sve_ldffbss_zss_mte,
5875 gen_helper_sve_ldffhss_be_zss_mte,
5876 NULL, },
5877 { gen_helper_sve_ldffbsu_zss_mte,
5878 gen_helper_sve_ldffhsu_be_zss_mte,
5879 gen_helper_sve_ldffss_be_zss_mte, } } } } },
673e9fa6
RH
5880};
5881
5882/* Note that we overload xs=2 to indicate 64-bit offset. */
d28d12f0
RH
5883static gen_helper_gvec_mem_scatter * const
5884gather_load_fn64[2][2][2][3][2][4] = {
5885 { /* MTE Inactive */
5886 { /* Little-endian */
5887 { { { gen_helper_sve_ldbds_zsu,
5888 gen_helper_sve_ldhds_le_zsu,
5889 gen_helper_sve_ldsds_le_zsu,
5890 NULL, },
5891 { gen_helper_sve_ldbdu_zsu,
5892 gen_helper_sve_ldhdu_le_zsu,
5893 gen_helper_sve_ldsdu_le_zsu,
5894 gen_helper_sve_lddd_le_zsu, } },
5895 { { gen_helper_sve_ldbds_zss,
5896 gen_helper_sve_ldhds_le_zss,
5897 gen_helper_sve_ldsds_le_zss,
5898 NULL, },
5899 { gen_helper_sve_ldbdu_zss,
5900 gen_helper_sve_ldhdu_le_zss,
5901 gen_helper_sve_ldsdu_le_zss,
5902 gen_helper_sve_lddd_le_zss, } },
5903 { { gen_helper_sve_ldbds_zd,
5904 gen_helper_sve_ldhds_le_zd,
5905 gen_helper_sve_ldsds_le_zd,
5906 NULL, },
5907 { gen_helper_sve_ldbdu_zd,
5908 gen_helper_sve_ldhdu_le_zd,
5909 gen_helper_sve_ldsdu_le_zd,
5910 gen_helper_sve_lddd_le_zd, } } },
5911
5912 /* First-fault */
5913 { { { gen_helper_sve_ldffbds_zsu,
5914 gen_helper_sve_ldffhds_le_zsu,
5915 gen_helper_sve_ldffsds_le_zsu,
5916 NULL, },
5917 { gen_helper_sve_ldffbdu_zsu,
5918 gen_helper_sve_ldffhdu_le_zsu,
5919 gen_helper_sve_ldffsdu_le_zsu,
5920 gen_helper_sve_ldffdd_le_zsu, } },
5921 { { gen_helper_sve_ldffbds_zss,
5922 gen_helper_sve_ldffhds_le_zss,
5923 gen_helper_sve_ldffsds_le_zss,
5924 NULL, },
5925 { gen_helper_sve_ldffbdu_zss,
5926 gen_helper_sve_ldffhdu_le_zss,
5927 gen_helper_sve_ldffsdu_le_zss,
5928 gen_helper_sve_ldffdd_le_zss, } },
5929 { { gen_helper_sve_ldffbds_zd,
5930 gen_helper_sve_ldffhds_le_zd,
5931 gen_helper_sve_ldffsds_le_zd,
5932 NULL, },
5933 { gen_helper_sve_ldffbdu_zd,
5934 gen_helper_sve_ldffhdu_le_zd,
5935 gen_helper_sve_ldffsdu_le_zd,
5936 gen_helper_sve_ldffdd_le_zd, } } } },
5937 { /* Big-endian */
5938 { { { gen_helper_sve_ldbds_zsu,
5939 gen_helper_sve_ldhds_be_zsu,
5940 gen_helper_sve_ldsds_be_zsu,
5941 NULL, },
5942 { gen_helper_sve_ldbdu_zsu,
5943 gen_helper_sve_ldhdu_be_zsu,
5944 gen_helper_sve_ldsdu_be_zsu,
5945 gen_helper_sve_lddd_be_zsu, } },
5946 { { gen_helper_sve_ldbds_zss,
5947 gen_helper_sve_ldhds_be_zss,
5948 gen_helper_sve_ldsds_be_zss,
5949 NULL, },
5950 { gen_helper_sve_ldbdu_zss,
5951 gen_helper_sve_ldhdu_be_zss,
5952 gen_helper_sve_ldsdu_be_zss,
5953 gen_helper_sve_lddd_be_zss, } },
5954 { { gen_helper_sve_ldbds_zd,
5955 gen_helper_sve_ldhds_be_zd,
5956 gen_helper_sve_ldsds_be_zd,
5957 NULL, },
5958 { gen_helper_sve_ldbdu_zd,
5959 gen_helper_sve_ldhdu_be_zd,
5960 gen_helper_sve_ldsdu_be_zd,
5961 gen_helper_sve_lddd_be_zd, } } },
5962
5963 /* First-fault */
5964 { { { gen_helper_sve_ldffbds_zsu,
5965 gen_helper_sve_ldffhds_be_zsu,
5966 gen_helper_sve_ldffsds_be_zsu,
5967 NULL, },
5968 { gen_helper_sve_ldffbdu_zsu,
5969 gen_helper_sve_ldffhdu_be_zsu,
5970 gen_helper_sve_ldffsdu_be_zsu,
5971 gen_helper_sve_ldffdd_be_zsu, } },
5972 { { gen_helper_sve_ldffbds_zss,
5973 gen_helper_sve_ldffhds_be_zss,
5974 gen_helper_sve_ldffsds_be_zss,
5975 NULL, },
5976 { gen_helper_sve_ldffbdu_zss,
5977 gen_helper_sve_ldffhdu_be_zss,
5978 gen_helper_sve_ldffsdu_be_zss,
5979 gen_helper_sve_ldffdd_be_zss, } },
5980 { { gen_helper_sve_ldffbds_zd,
5981 gen_helper_sve_ldffhds_be_zd,
5982 gen_helper_sve_ldffsds_be_zd,
5983 NULL, },
5984 { gen_helper_sve_ldffbdu_zd,
5985 gen_helper_sve_ldffhdu_be_zd,
5986 gen_helper_sve_ldffsdu_be_zd,
5987 gen_helper_sve_ldffdd_be_zd, } } } } },
5988 { /* MTE Active */
5989 { /* Little-endian */
5990 { { { gen_helper_sve_ldbds_zsu_mte,
5991 gen_helper_sve_ldhds_le_zsu_mte,
5992 gen_helper_sve_ldsds_le_zsu_mte,
5993 NULL, },
5994 { gen_helper_sve_ldbdu_zsu_mte,
5995 gen_helper_sve_ldhdu_le_zsu_mte,
5996 gen_helper_sve_ldsdu_le_zsu_mte,
5997 gen_helper_sve_lddd_le_zsu_mte, } },
5998 { { gen_helper_sve_ldbds_zss_mte,
5999 gen_helper_sve_ldhds_le_zss_mte,
6000 gen_helper_sve_ldsds_le_zss_mte,
6001 NULL, },
6002 { gen_helper_sve_ldbdu_zss_mte,
6003 gen_helper_sve_ldhdu_le_zss_mte,
6004 gen_helper_sve_ldsdu_le_zss_mte,
6005 gen_helper_sve_lddd_le_zss_mte, } },
6006 { { gen_helper_sve_ldbds_zd_mte,
6007 gen_helper_sve_ldhds_le_zd_mte,
6008 gen_helper_sve_ldsds_le_zd_mte,
6009 NULL, },
6010 { gen_helper_sve_ldbdu_zd_mte,
6011 gen_helper_sve_ldhdu_le_zd_mte,
6012 gen_helper_sve_ldsdu_le_zd_mte,
6013 gen_helper_sve_lddd_le_zd_mte, } } },
6014
6015 /* First-fault */
6016 { { { gen_helper_sve_ldffbds_zsu_mte,
6017 gen_helper_sve_ldffhds_le_zsu_mte,
6018 gen_helper_sve_ldffsds_le_zsu_mte,
6019 NULL, },
6020 { gen_helper_sve_ldffbdu_zsu_mte,
6021 gen_helper_sve_ldffhdu_le_zsu_mte,
6022 gen_helper_sve_ldffsdu_le_zsu_mte,
6023 gen_helper_sve_ldffdd_le_zsu_mte, } },
6024 { { gen_helper_sve_ldffbds_zss_mte,
6025 gen_helper_sve_ldffhds_le_zss_mte,
6026 gen_helper_sve_ldffsds_le_zss_mte,
6027 NULL, },
6028 { gen_helper_sve_ldffbdu_zss_mte,
6029 gen_helper_sve_ldffhdu_le_zss_mte,
6030 gen_helper_sve_ldffsdu_le_zss_mte,
6031 gen_helper_sve_ldffdd_le_zss_mte, } },
6032 { { gen_helper_sve_ldffbds_zd_mte,
6033 gen_helper_sve_ldffhds_le_zd_mte,
6034 gen_helper_sve_ldffsds_le_zd_mte,
6035 NULL, },
6036 { gen_helper_sve_ldffbdu_zd_mte,
6037 gen_helper_sve_ldffhdu_le_zd_mte,
6038 gen_helper_sve_ldffsdu_le_zd_mte,
6039 gen_helper_sve_ldffdd_le_zd_mte, } } } },
6040 { /* Big-endian */
6041 { { { gen_helper_sve_ldbds_zsu_mte,
6042 gen_helper_sve_ldhds_be_zsu_mte,
6043 gen_helper_sve_ldsds_be_zsu_mte,
6044 NULL, },
6045 { gen_helper_sve_ldbdu_zsu_mte,
6046 gen_helper_sve_ldhdu_be_zsu_mte,
6047 gen_helper_sve_ldsdu_be_zsu_mte,
6048 gen_helper_sve_lddd_be_zsu_mte, } },
6049 { { gen_helper_sve_ldbds_zss_mte,
6050 gen_helper_sve_ldhds_be_zss_mte,
6051 gen_helper_sve_ldsds_be_zss_mte,
6052 NULL, },
6053 { gen_helper_sve_ldbdu_zss_mte,
6054 gen_helper_sve_ldhdu_be_zss_mte,
6055 gen_helper_sve_ldsdu_be_zss_mte,
6056 gen_helper_sve_lddd_be_zss_mte, } },
6057 { { gen_helper_sve_ldbds_zd_mte,
6058 gen_helper_sve_ldhds_be_zd_mte,
6059 gen_helper_sve_ldsds_be_zd_mte,
6060 NULL, },
6061 { gen_helper_sve_ldbdu_zd_mte,
6062 gen_helper_sve_ldhdu_be_zd_mte,
6063 gen_helper_sve_ldsdu_be_zd_mte,
6064 gen_helper_sve_lddd_be_zd_mte, } } },
6065
6066 /* First-fault */
6067 { { { gen_helper_sve_ldffbds_zsu_mte,
6068 gen_helper_sve_ldffhds_be_zsu_mte,
6069 gen_helper_sve_ldffsds_be_zsu_mte,
6070 NULL, },
6071 { gen_helper_sve_ldffbdu_zsu_mte,
6072 gen_helper_sve_ldffhdu_be_zsu_mte,
6073 gen_helper_sve_ldffsdu_be_zsu_mte,
6074 gen_helper_sve_ldffdd_be_zsu_mte, } },
6075 { { gen_helper_sve_ldffbds_zss_mte,
6076 gen_helper_sve_ldffhds_be_zss_mte,
6077 gen_helper_sve_ldffsds_be_zss_mte,
6078 NULL, },
6079 { gen_helper_sve_ldffbdu_zss_mte,
6080 gen_helper_sve_ldffhdu_be_zss_mte,
6081 gen_helper_sve_ldffsdu_be_zss_mte,
6082 gen_helper_sve_ldffdd_be_zss_mte, } },
6083 { { gen_helper_sve_ldffbds_zd_mte,
6084 gen_helper_sve_ldffhds_be_zd_mte,
6085 gen_helper_sve_ldffsds_be_zd_mte,
6086 NULL, },
6087 { gen_helper_sve_ldffbdu_zd_mte,
6088 gen_helper_sve_ldffhdu_be_zd_mte,
6089 gen_helper_sve_ldffsdu_be_zd_mte,
6090 gen_helper_sve_ldffdd_be_zd_mte, } } } } },
673e9fa6
RH
6091};
6092
3a7be554 6093static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
673e9fa6
RH
6094{
6095 gen_helper_gvec_mem_scatter *fn = NULL;
d28d12f0
RH
6096 bool be = s->be_data == MO_BE;
6097 bool mte = s->mte_active[0];
673e9fa6
RH
6098
6099 if (!sve_access_check(s)) {
6100 return true;
6101 }
6102
6103 switch (a->esz) {
6104 case MO_32:
d28d12f0 6105 fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz];
673e9fa6
RH
6106 break;
6107 case MO_64:
d28d12f0 6108 fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz];
673e9fa6
RH
6109 break;
6110 }
6111 assert(fn != NULL);
6112
6113 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
d28d12f0 6114 cpu_reg_sp(s, a->rn), a->msz, false, fn);
673e9fa6
RH
6115 return true;
6116}
6117
3a7be554 6118static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
673e9fa6
RH
6119{
6120 gen_helper_gvec_mem_scatter *fn = NULL;
d28d12f0
RH
6121 bool be = s->be_data == MO_BE;
6122 bool mte = s->mte_active[0];
673e9fa6
RH
6123 TCGv_i64 imm;
6124
6125 if (a->esz < a->msz || (a->esz == a->msz && !a->u)) {
6126 return false;
6127 }
6128 if (!sve_access_check(s)) {
6129 return true;
6130 }
6131
6132 switch (a->esz) {
6133 case MO_32:
d28d12f0 6134 fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz];
673e9fa6
RH
6135 break;
6136 case MO_64:
d28d12f0 6137 fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz];
673e9fa6
RH
6138 break;
6139 }
6140 assert(fn != NULL);
6141
6142 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6143 * by loading the immediate into the scalar parameter.
6144 */
6145 imm = tcg_const_i64(a->imm << a->msz);
d28d12f0 6146 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, false, fn);
673e9fa6
RH
6147 tcg_temp_free_i64(imm);
6148 return true;
6149}
6150
cf327449
SL
6151static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a)
6152{
6153 if (!dc_isar_feature(aa64_sve2, s)) {
6154 return false;
6155 }
6156 return trans_LD1_zprz(s, a);
6157}
6158
d28d12f0
RH
6159/* Indexed by [mte][be][xs][msz]. */
6160static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = {
6161 { /* MTE Inactive */
6162 { /* Little-endian */
6163 { gen_helper_sve_stbs_zsu,
6164 gen_helper_sve_sths_le_zsu,
6165 gen_helper_sve_stss_le_zsu, },
6166 { gen_helper_sve_stbs_zss,
6167 gen_helper_sve_sths_le_zss,
6168 gen_helper_sve_stss_le_zss, } },
6169 { /* Big-endian */
6170 { gen_helper_sve_stbs_zsu,
6171 gen_helper_sve_sths_be_zsu,
6172 gen_helper_sve_stss_be_zsu, },
6173 { gen_helper_sve_stbs_zss,
6174 gen_helper_sve_sths_be_zss,
6175 gen_helper_sve_stss_be_zss, } } },
6176 { /* MTE Active */
6177 { /* Little-endian */
6178 { gen_helper_sve_stbs_zsu_mte,
6179 gen_helper_sve_sths_le_zsu_mte,
6180 gen_helper_sve_stss_le_zsu_mte, },
6181 { gen_helper_sve_stbs_zss_mte,
6182 gen_helper_sve_sths_le_zss_mte,
6183 gen_helper_sve_stss_le_zss_mte, } },
6184 { /* Big-endian */
6185 { gen_helper_sve_stbs_zsu_mte,
6186 gen_helper_sve_sths_be_zsu_mte,
6187 gen_helper_sve_stss_be_zsu_mte, },
6188 { gen_helper_sve_stbs_zss_mte,
6189 gen_helper_sve_sths_be_zss_mte,
6190 gen_helper_sve_stss_be_zss_mte, } } },
408ecde9
RH
6191};
6192
6193/* Note that we overload xs=2 to indicate 64-bit offset. */
d28d12f0
RH
6194static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][2][3][4] = {
6195 { /* MTE Inactive */
6196 { /* Little-endian */
6197 { gen_helper_sve_stbd_zsu,
6198 gen_helper_sve_sthd_le_zsu,
6199 gen_helper_sve_stsd_le_zsu,
6200 gen_helper_sve_stdd_le_zsu, },
6201 { gen_helper_sve_stbd_zss,
6202 gen_helper_sve_sthd_le_zss,
6203 gen_helper_sve_stsd_le_zss,
6204 gen_helper_sve_stdd_le_zss, },
6205 { gen_helper_sve_stbd_zd,
6206 gen_helper_sve_sthd_le_zd,
6207 gen_helper_sve_stsd_le_zd,
6208 gen_helper_sve_stdd_le_zd, } },
6209 { /* Big-endian */
6210 { gen_helper_sve_stbd_zsu,
6211 gen_helper_sve_sthd_be_zsu,
6212 gen_helper_sve_stsd_be_zsu,
6213 gen_helper_sve_stdd_be_zsu, },
6214 { gen_helper_sve_stbd_zss,
6215 gen_helper_sve_sthd_be_zss,
6216 gen_helper_sve_stsd_be_zss,
6217 gen_helper_sve_stdd_be_zss, },
6218 { gen_helper_sve_stbd_zd,
6219 gen_helper_sve_sthd_be_zd,
6220 gen_helper_sve_stsd_be_zd,
6221 gen_helper_sve_stdd_be_zd, } } },
6222 { /* MTE Inactive */
6223 { /* Little-endian */
6224 { gen_helper_sve_stbd_zsu_mte,
6225 gen_helper_sve_sthd_le_zsu_mte,
6226 gen_helper_sve_stsd_le_zsu_mte,
6227 gen_helper_sve_stdd_le_zsu_mte, },
6228 { gen_helper_sve_stbd_zss_mte,
6229 gen_helper_sve_sthd_le_zss_mte,
6230 gen_helper_sve_stsd_le_zss_mte,
6231 gen_helper_sve_stdd_le_zss_mte, },
6232 { gen_helper_sve_stbd_zd_mte,
6233 gen_helper_sve_sthd_le_zd_mte,
6234 gen_helper_sve_stsd_le_zd_mte,
6235 gen_helper_sve_stdd_le_zd_mte, } },
6236 { /* Big-endian */
6237 { gen_helper_sve_stbd_zsu_mte,
6238 gen_helper_sve_sthd_be_zsu_mte,
6239 gen_helper_sve_stsd_be_zsu_mte,
6240 gen_helper_sve_stdd_be_zsu_mte, },
6241 { gen_helper_sve_stbd_zss_mte,
6242 gen_helper_sve_sthd_be_zss_mte,
6243 gen_helper_sve_stsd_be_zss_mte,
6244 gen_helper_sve_stdd_be_zss_mte, },
6245 { gen_helper_sve_stbd_zd_mte,
6246 gen_helper_sve_sthd_be_zd_mte,
6247 gen_helper_sve_stsd_be_zd_mte,
6248 gen_helper_sve_stdd_be_zd_mte, } } },
408ecde9
RH
6249};
6250
3a7be554 6251static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
f6dbf62a 6252{
f6dbf62a 6253 gen_helper_gvec_mem_scatter *fn;
d28d12f0
RH
6254 bool be = s->be_data == MO_BE;
6255 bool mte = s->mte_active[0];
f6dbf62a
RH
6256
6257 if (a->esz < a->msz || (a->msz == 0 && a->scale)) {
6258 return false;
6259 }
6260 if (!sve_access_check(s)) {
6261 return true;
6262 }
6263 switch (a->esz) {
6264 case MO_32:
d28d12f0 6265 fn = scatter_store_fn32[mte][be][a->xs][a->msz];
f6dbf62a
RH
6266 break;
6267 case MO_64:
d28d12f0 6268 fn = scatter_store_fn64[mte][be][a->xs][a->msz];
f6dbf62a
RH
6269 break;
6270 default:
6271 g_assert_not_reached();
6272 }
6273 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
d28d12f0 6274 cpu_reg_sp(s, a->rn), a->msz, true, fn);
f6dbf62a
RH
6275 return true;
6276}
dec6cf6b 6277
3a7be554 6278static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
408ecde9
RH
6279{
6280 gen_helper_gvec_mem_scatter *fn = NULL;
d28d12f0
RH
6281 bool be = s->be_data == MO_BE;
6282 bool mte = s->mte_active[0];
408ecde9
RH
6283 TCGv_i64 imm;
6284
6285 if (a->esz < a->msz) {
6286 return false;
6287 }
6288 if (!sve_access_check(s)) {
6289 return true;
6290 }
6291
6292 switch (a->esz) {
6293 case MO_32:
d28d12f0 6294 fn = scatter_store_fn32[mte][be][0][a->msz];
408ecde9
RH
6295 break;
6296 case MO_64:
d28d12f0 6297 fn = scatter_store_fn64[mte][be][2][a->msz];
408ecde9
RH
6298 break;
6299 }
6300 assert(fn != NULL);
6301
6302 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6303 * by loading the immediate into the scalar parameter.
6304 */
6305 imm = tcg_const_i64(a->imm << a->msz);
d28d12f0 6306 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, true, fn);
408ecde9
RH
6307 tcg_temp_free_i64(imm);
6308 return true;
6309}
6310
6ebca45f
SL
6311static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a)
6312{
6313 if (!dc_isar_feature(aa64_sve2, s)) {
6314 return false;
6315 }
6316 return trans_ST1_zprz(s, a);
6317}
6318
dec6cf6b
RH
6319/*
6320 * Prefetches
6321 */
6322
3a7be554 6323static bool trans_PRF(DisasContext *s, arg_PRF *a)
dec6cf6b
RH
6324{
6325 /* Prefetch is a nop within QEMU. */
2f95a3b0 6326 (void)sve_access_check(s);
dec6cf6b
RH
6327 return true;
6328}
6329
3a7be554 6330static bool trans_PRF_rr(DisasContext *s, arg_PRF_rr *a)
dec6cf6b
RH
6331{
6332 if (a->rm == 31) {
6333 return false;
6334 }
6335 /* Prefetch is a nop within QEMU. */
2f95a3b0 6336 (void)sve_access_check(s);
dec6cf6b
RH
6337 return true;
6338}
a2103582
RH
6339
6340/*
6341 * Move Prefix
6342 *
6343 * TODO: The implementation so far could handle predicated merging movprfx.
6344 * The helper functions as written take an extra source register to
6345 * use in the operation, but the result is only written when predication
6346 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6347 * to allow the final write back to the destination to be unconditional.
6348 * For predicated zeroing movprfx, we need to rearrange the helpers to
6349 * allow the final write back to zero inactives.
6350 *
6351 * In the meantime, just emit the moves.
6352 */
6353
3a7be554 6354static bool trans_MOVPRFX(DisasContext *s, arg_MOVPRFX *a)
a2103582
RH
6355{
6356 return do_mov_z(s, a->rd, a->rn);
6357}
6358
3a7be554 6359static bool trans_MOVPRFX_m(DisasContext *s, arg_rpr_esz *a)
a2103582
RH
6360{
6361 if (sve_access_check(s)) {
6362 do_sel_z(s, a->rd, a->rn, a->rd, a->pg, a->esz);
6363 }
6364 return true;
6365}
6366
3a7be554 6367static bool trans_MOVPRFX_z(DisasContext *s, arg_rpr_esz *a)
a2103582 6368{
60245996 6369 return do_movz_zpz(s, a->rd, a->rn, a->pg, a->esz, false);
a2103582 6370}
5dad1ba5
RH
6371
6372/*
6373 * SVE2 Integer Multiply - Unpredicated
6374 */
6375
6376static bool trans_MUL_zzz(DisasContext *s, arg_rrr_esz *a)
6377{
6378 if (!dc_isar_feature(aa64_sve2, s)) {
6379 return false;
6380 }
6381 if (sve_access_check(s)) {
6382 gen_gvec_fn_zzz(s, tcg_gen_gvec_mul, a->esz, a->rd, a->rn, a->rm);
6383 }
6384 return true;
6385}
6386
6387static bool do_sve2_zzz_ool(DisasContext *s, arg_rrr_esz *a,
6388 gen_helper_gvec_3 *fn)
6389{
6390 if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
6391 return false;
6392 }
6393 if (sve_access_check(s)) {
6394 gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, 0);
6395 }
6396 return true;
6397}
6398
6399static bool trans_SMULH_zzz(DisasContext *s, arg_rrr_esz *a)
6400{
6401 static gen_helper_gvec_3 * const fns[4] = {
6402 gen_helper_gvec_smulh_b, gen_helper_gvec_smulh_h,
6403 gen_helper_gvec_smulh_s, gen_helper_gvec_smulh_d,
6404 };
6405 return do_sve2_zzz_ool(s, a, fns[a->esz]);
6406}
6407
6408static bool trans_UMULH_zzz(DisasContext *s, arg_rrr_esz *a)
6409{
6410 static gen_helper_gvec_3 * const fns[4] = {
6411 gen_helper_gvec_umulh_b, gen_helper_gvec_umulh_h,
6412 gen_helper_gvec_umulh_s, gen_helper_gvec_umulh_d,
6413 };
6414 return do_sve2_zzz_ool(s, a, fns[a->esz]);
6415}
6416
6417static bool trans_PMUL_zzz(DisasContext *s, arg_rrr_esz *a)
6418{
6419 return do_sve2_zzz_ool(s, a, gen_helper_gvec_pmul_b);
6420}
d4b1e59d
RH
6421
6422/*
6423 * SVE2 Integer - Predicated
6424 */
6425
6426static bool do_sve2_zpzz_ool(DisasContext *s, arg_rprr_esz *a,
6427 gen_helper_gvec_4 *fn)
6428{
6429 if (!dc_isar_feature(aa64_sve2, s)) {
6430 return false;
6431 }
6432 return do_zpzz_ool(s, a, fn);
6433}
6434
6435static bool trans_SADALP_zpzz(DisasContext *s, arg_rprr_esz *a)
6436{
6437 static gen_helper_gvec_4 * const fns[3] = {
6438 gen_helper_sve2_sadalp_zpzz_h,
6439 gen_helper_sve2_sadalp_zpzz_s,
6440 gen_helper_sve2_sadalp_zpzz_d,
6441 };
6442 if (a->esz == 0) {
6443 return false;
6444 }
6445 return do_sve2_zpzz_ool(s, a, fns[a->esz - 1]);
6446}
6447
6448static bool trans_UADALP_zpzz(DisasContext *s, arg_rprr_esz *a)
6449{
6450 static gen_helper_gvec_4 * const fns[3] = {
6451 gen_helper_sve2_uadalp_zpzz_h,
6452 gen_helper_sve2_uadalp_zpzz_s,
6453 gen_helper_sve2_uadalp_zpzz_d,
6454 };
6455 if (a->esz == 0) {
6456 return false;
6457 }
6458 return do_sve2_zpzz_ool(s, a, fns[a->esz - 1]);
6459}
db366da8
RH
6460
6461/*
6462 * SVE2 integer unary operations (predicated)
6463 */
6464
6465static bool do_sve2_zpz_ool(DisasContext *s, arg_rpr_esz *a,
6466 gen_helper_gvec_3 *fn)
6467{
6468 if (!dc_isar_feature(aa64_sve2, s)) {
6469 return false;
6470 }
6471 return do_zpz_ool(s, a, fn);
6472}
6473
6474static bool trans_URECPE(DisasContext *s, arg_rpr_esz *a)
6475{
6476 if (a->esz != 2) {
6477 return false;
6478 }
6479 return do_sve2_zpz_ool(s, a, gen_helper_sve2_urecpe_s);
6480}
6481
6482static bool trans_URSQRTE(DisasContext *s, arg_rpr_esz *a)
6483{
6484 if (a->esz != 2) {
6485 return false;
6486 }
6487 return do_sve2_zpz_ool(s, a, gen_helper_sve2_ursqrte_s);
6488}
6489
6490static bool trans_SQABS(DisasContext *s, arg_rpr_esz *a)
6491{
6492 static gen_helper_gvec_3 * const fns[4] = {
6493 gen_helper_sve2_sqabs_b, gen_helper_sve2_sqabs_h,
6494 gen_helper_sve2_sqabs_s, gen_helper_sve2_sqabs_d,
6495 };
6496 return do_sve2_zpz_ool(s, a, fns[a->esz]);
6497}
6498
6499static bool trans_SQNEG(DisasContext *s, arg_rpr_esz *a)
6500{
6501 static gen_helper_gvec_3 * const fns[4] = {
6502 gen_helper_sve2_sqneg_b, gen_helper_sve2_sqneg_h,
6503 gen_helper_sve2_sqneg_s, gen_helper_sve2_sqneg_d,
6504 };
6505 return do_sve2_zpz_ool(s, a, fns[a->esz]);
6506}
45d9503d
RH
6507
6508#define DO_SVE2_ZPZZ(NAME, name) \
6509static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
6510{ \
6511 static gen_helper_gvec_4 * const fns[4] = { \
6512 gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
6513 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
6514 }; \
6515 return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
6516}
6517
6518DO_SVE2_ZPZZ(SQSHL, sqshl)
6519DO_SVE2_ZPZZ(SQRSHL, sqrshl)
6520DO_SVE2_ZPZZ(SRSHL, srshl)
6521
6522DO_SVE2_ZPZZ(UQSHL, uqshl)
6523DO_SVE2_ZPZZ(UQRSHL, uqrshl)
6524DO_SVE2_ZPZZ(URSHL, urshl)
a47dc220
RH
6525
6526DO_SVE2_ZPZZ(SHADD, shadd)
6527DO_SVE2_ZPZZ(SRHADD, srhadd)
6528DO_SVE2_ZPZZ(SHSUB, shsub)
6529
6530DO_SVE2_ZPZZ(UHADD, uhadd)
6531DO_SVE2_ZPZZ(URHADD, urhadd)
6532DO_SVE2_ZPZZ(UHSUB, uhsub)
8597dc8b
RH
6533
6534DO_SVE2_ZPZZ(ADDP, addp)
6535DO_SVE2_ZPZZ(SMAXP, smaxp)
6536DO_SVE2_ZPZZ(UMAXP, umaxp)
6537DO_SVE2_ZPZZ(SMINP, sminp)
6538DO_SVE2_ZPZZ(UMINP, uminp)
4f07fbeb
RH
6539
6540DO_SVE2_ZPZZ(SQADD_zpzz, sqadd)
6541DO_SVE2_ZPZZ(UQADD_zpzz, uqadd)
6542DO_SVE2_ZPZZ(SQSUB_zpzz, sqsub)
6543DO_SVE2_ZPZZ(UQSUB_zpzz, uqsub)
6544DO_SVE2_ZPZZ(SUQADD, suqadd)
6545DO_SVE2_ZPZZ(USQADD, usqadd)
0ce1dda8
RH
6546
6547/*
6548 * SVE2 Widening Integer Arithmetic
6549 */
6550
6551static bool do_sve2_zzw_ool(DisasContext *s, arg_rrr_esz *a,
6552 gen_helper_gvec_3 *fn, int data)
6553{
6554 if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
6555 return false;
6556 }
6557 if (sve_access_check(s)) {
6558 unsigned vsz = vec_full_reg_size(s);
6559 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
6560 vec_full_reg_offset(s, a->rn),
6561 vec_full_reg_offset(s, a->rm),
6562 vsz, vsz, data, fn);
6563 }
6564 return true;
6565}
6566
6567#define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \
6568static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6569{ \
6570 static gen_helper_gvec_3 * const fns[4] = { \
6571 NULL, gen_helper_sve2_##name##_h, \
6572 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6573 }; \
6574 return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \
6575}
6576
6577DO_SVE2_ZZZ_TB(SADDLB, saddl, false, false)
6578DO_SVE2_ZZZ_TB(SSUBLB, ssubl, false, false)
6579DO_SVE2_ZZZ_TB(SABDLB, sabdl, false, false)
6580
6581DO_SVE2_ZZZ_TB(UADDLB, uaddl, false, false)
6582DO_SVE2_ZZZ_TB(USUBLB, usubl, false, false)
6583DO_SVE2_ZZZ_TB(UABDLB, uabdl, false, false)
6584
6585DO_SVE2_ZZZ_TB(SADDLT, saddl, true, true)
6586DO_SVE2_ZZZ_TB(SSUBLT, ssubl, true, true)
6587DO_SVE2_ZZZ_TB(SABDLT, sabdl, true, true)
6588
6589DO_SVE2_ZZZ_TB(UADDLT, uaddl, true, true)
6590DO_SVE2_ZZZ_TB(USUBLT, usubl, true, true)
6591DO_SVE2_ZZZ_TB(UABDLT, uabdl, true, true)
daec426b
RH
6592
6593DO_SVE2_ZZZ_TB(SADDLBT, saddl, false, true)
6594DO_SVE2_ZZZ_TB(SSUBLBT, ssubl, false, true)
6595DO_SVE2_ZZZ_TB(SSUBLTB, ssubl, true, false)
81fccf09 6596
69ccc099
RH
6597DO_SVE2_ZZZ_TB(SQDMULLB_zzz, sqdmull_zzz, false, false)
6598DO_SVE2_ZZZ_TB(SQDMULLT_zzz, sqdmull_zzz, true, true)
6599
6600DO_SVE2_ZZZ_TB(SMULLB_zzz, smull_zzz, false, false)
6601DO_SVE2_ZZZ_TB(SMULLT_zzz, smull_zzz, true, true)
6602
6603DO_SVE2_ZZZ_TB(UMULLB_zzz, umull_zzz, false, false)
6604DO_SVE2_ZZZ_TB(UMULLT_zzz, umull_zzz, true, true)
6605
2df3ca55
RH
6606static bool do_eor_tb(DisasContext *s, arg_rrr_esz *a, bool sel1)
6607{
6608 static gen_helper_gvec_3 * const fns[4] = {
6609 gen_helper_sve2_eoril_b, gen_helper_sve2_eoril_h,
6610 gen_helper_sve2_eoril_s, gen_helper_sve2_eoril_d,
6611 };
6612 return do_sve2_zzw_ool(s, a, fns[a->esz], (!sel1 << 1) | sel1);
6613}
6614
6615static bool trans_EORBT(DisasContext *s, arg_rrr_esz *a)
6616{
6617 return do_eor_tb(s, a, false);
6618}
6619
6620static bool trans_EORTB(DisasContext *s, arg_rrr_esz *a)
6621{
6622 return do_eor_tb(s, a, true);
6623}
6624
e3a56131
RH
6625static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel)
6626{
6627 static gen_helper_gvec_3 * const fns[4] = {
6628 gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h,
6629 NULL, gen_helper_sve2_pmull_d,
6630 };
6631 if (a->esz == 0 && !dc_isar_feature(aa64_sve2_pmull128, s)) {
6632 return false;
6633 }
6634 return do_sve2_zzw_ool(s, a, fns[a->esz], sel);
6635}
6636
6637static bool trans_PMULLB(DisasContext *s, arg_rrr_esz *a)
6638{
6639 return do_trans_pmull(s, a, false);
6640}
6641
6642static bool trans_PMULLT(DisasContext *s, arg_rrr_esz *a)
6643{
6644 return do_trans_pmull(s, a, true);
6645}
6646
81fccf09
RH
6647#define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
6648static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6649{ \
6650 static gen_helper_gvec_3 * const fns[4] = { \
6651 NULL, gen_helper_sve2_##name##_h, \
6652 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6653 }; \
6654 return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \
6655}
6656
6657DO_SVE2_ZZZ_WTB(SADDWB, saddw, false)
6658DO_SVE2_ZZZ_WTB(SADDWT, saddw, true)
6659DO_SVE2_ZZZ_WTB(SSUBWB, ssubw, false)
6660DO_SVE2_ZZZ_WTB(SSUBWT, ssubw, true)
6661
6662DO_SVE2_ZZZ_WTB(UADDWB, uaddw, false)
6663DO_SVE2_ZZZ_WTB(UADDWT, uaddw, true)
6664DO_SVE2_ZZZ_WTB(USUBWB, usubw, false)
6665DO_SVE2_ZZZ_WTB(USUBWT, usubw, true)
4269fef1
RH
6666
6667static void gen_sshll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
6668{
6669 int top = imm & 1;
6670 int shl = imm >> 1;
6671 int halfbits = 4 << vece;
6672
6673 if (top) {
6674 if (shl == halfbits) {
6675 TCGv_vec t = tcg_temp_new_vec_matching(d);
6676 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits));
6677 tcg_gen_and_vec(vece, d, n, t);
6678 tcg_temp_free_vec(t);
6679 } else {
6680 tcg_gen_sari_vec(vece, d, n, halfbits);
6681 tcg_gen_shli_vec(vece, d, d, shl);
6682 }
6683 } else {
6684 tcg_gen_shli_vec(vece, d, n, halfbits);
6685 tcg_gen_sari_vec(vece, d, d, halfbits - shl);
6686 }
6687}
6688
6689static void gen_ushll_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int imm)
6690{
6691 int halfbits = 4 << vece;
6692 int top = imm & 1;
6693 int shl = (imm >> 1);
6694 int shift;
6695 uint64_t mask;
6696
6697 mask = MAKE_64BIT_MASK(0, halfbits);
6698 mask <<= shl;
6699 mask = dup_const(vece, mask);
6700
6701 shift = shl - top * halfbits;
6702 if (shift < 0) {
6703 tcg_gen_shri_i64(d, n, -shift);
6704 } else {
6705 tcg_gen_shli_i64(d, n, shift);
6706 }
6707 tcg_gen_andi_i64(d, d, mask);
6708}
6709
6710static void gen_ushll16_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6711{
6712 gen_ushll_i64(MO_16, d, n, imm);
6713}
6714
6715static void gen_ushll32_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6716{
6717 gen_ushll_i64(MO_32, d, n, imm);
6718}
6719
6720static void gen_ushll64_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6721{
6722 gen_ushll_i64(MO_64, d, n, imm);
6723}
6724
6725static void gen_ushll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
6726{
6727 int halfbits = 4 << vece;
6728 int top = imm & 1;
6729 int shl = imm >> 1;
6730
6731 if (top) {
6732 if (shl == halfbits) {
6733 TCGv_vec t = tcg_temp_new_vec_matching(d);
6734 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits));
6735 tcg_gen_and_vec(vece, d, n, t);
6736 tcg_temp_free_vec(t);
6737 } else {
6738 tcg_gen_shri_vec(vece, d, n, halfbits);
6739 tcg_gen_shli_vec(vece, d, d, shl);
6740 }
6741 } else {
6742 if (shl == 0) {
6743 TCGv_vec t = tcg_temp_new_vec_matching(d);
6744 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
6745 tcg_gen_and_vec(vece, d, n, t);
6746 tcg_temp_free_vec(t);
6747 } else {
6748 tcg_gen_shli_vec(vece, d, n, halfbits);
6749 tcg_gen_shri_vec(vece, d, d, halfbits - shl);
6750 }
6751 }
6752}
6753
6754static bool do_sve2_shll_tb(DisasContext *s, arg_rri_esz *a,
6755 bool sel, bool uns)
6756{
6757 static const TCGOpcode sshll_list[] = {
6758 INDEX_op_shli_vec, INDEX_op_sari_vec, 0
6759 };
6760 static const TCGOpcode ushll_list[] = {
6761 INDEX_op_shli_vec, INDEX_op_shri_vec, 0
6762 };
6763 static const GVecGen2i ops[2][3] = {
6764 { { .fniv = gen_sshll_vec,
6765 .opt_opc = sshll_list,
6766 .fno = gen_helper_sve2_sshll_h,
6767 .vece = MO_16 },
6768 { .fniv = gen_sshll_vec,
6769 .opt_opc = sshll_list,
6770 .fno = gen_helper_sve2_sshll_s,
6771 .vece = MO_32 },
6772 { .fniv = gen_sshll_vec,
6773 .opt_opc = sshll_list,
6774 .fno = gen_helper_sve2_sshll_d,
6775 .vece = MO_64 } },
6776 { { .fni8 = gen_ushll16_i64,
6777 .fniv = gen_ushll_vec,
6778 .opt_opc = ushll_list,
6779 .fno = gen_helper_sve2_ushll_h,
6780 .vece = MO_16 },
6781 { .fni8 = gen_ushll32_i64,
6782 .fniv = gen_ushll_vec,
6783 .opt_opc = ushll_list,
6784 .fno = gen_helper_sve2_ushll_s,
6785 .vece = MO_32 },
6786 { .fni8 = gen_ushll64_i64,
6787 .fniv = gen_ushll_vec,
6788 .opt_opc = ushll_list,
6789 .fno = gen_helper_sve2_ushll_d,
6790 .vece = MO_64 } },
6791 };
6792
6793 if (a->esz < 0 || a->esz > 2 || !dc_isar_feature(aa64_sve2, s)) {
6794 return false;
6795 }
6796 if (sve_access_check(s)) {
6797 unsigned vsz = vec_full_reg_size(s);
6798 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd),
6799 vec_full_reg_offset(s, a->rn),
6800 vsz, vsz, (a->imm << 1) | sel,
6801 &ops[uns][a->esz]);
6802 }
6803 return true;
6804}
6805
6806static bool trans_SSHLLB(DisasContext *s, arg_rri_esz *a)
6807{
6808 return do_sve2_shll_tb(s, a, false, false);
6809}
6810
6811static bool trans_SSHLLT(DisasContext *s, arg_rri_esz *a)
6812{
6813 return do_sve2_shll_tb(s, a, true, false);
6814}
6815
6816static bool trans_USHLLB(DisasContext *s, arg_rri_esz *a)
6817{
6818 return do_sve2_shll_tb(s, a, false, true);
6819}
6820
6821static bool trans_USHLLT(DisasContext *s, arg_rri_esz *a)
6822{
6823 return do_sve2_shll_tb(s, a, true, true);
6824}
cb9c33b8
RH
6825
6826static bool trans_BEXT(DisasContext *s, arg_rrr_esz *a)
6827{
6828 static gen_helper_gvec_3 * const fns[4] = {
6829 gen_helper_sve2_bext_b, gen_helper_sve2_bext_h,
6830 gen_helper_sve2_bext_s, gen_helper_sve2_bext_d,
6831 };
6832 if (!dc_isar_feature(aa64_sve2_bitperm, s)) {
6833 return false;
6834 }
6835 return do_sve2_zzw_ool(s, a, fns[a->esz], 0);
6836}
6837
6838static bool trans_BDEP(DisasContext *s, arg_rrr_esz *a)
6839{
6840 static gen_helper_gvec_3 * const fns[4] = {
6841 gen_helper_sve2_bdep_b, gen_helper_sve2_bdep_h,
6842 gen_helper_sve2_bdep_s, gen_helper_sve2_bdep_d,
6843 };
6844 if (!dc_isar_feature(aa64_sve2_bitperm, s)) {
6845 return false;
6846 }
6847 return do_sve2_zzw_ool(s, a, fns[a->esz], 0);
6848}
6849
6850static bool trans_BGRP(DisasContext *s, arg_rrr_esz *a)
6851{
6852 static gen_helper_gvec_3 * const fns[4] = {
6853 gen_helper_sve2_bgrp_b, gen_helper_sve2_bgrp_h,
6854 gen_helper_sve2_bgrp_s, gen_helper_sve2_bgrp_d,
6855 };
6856 if (!dc_isar_feature(aa64_sve2_bitperm, s)) {
6857 return false;
6858 }
6859 return do_sve2_zzw_ool(s, a, fns[a->esz], 0);
6860}
ed4a6387
RH
6861
6862static bool do_cadd(DisasContext *s, arg_rrr_esz *a, bool sq, bool rot)
6863{
6864 static gen_helper_gvec_3 * const fns[2][4] = {
6865 { gen_helper_sve2_cadd_b, gen_helper_sve2_cadd_h,
6866 gen_helper_sve2_cadd_s, gen_helper_sve2_cadd_d },
6867 { gen_helper_sve2_sqcadd_b, gen_helper_sve2_sqcadd_h,
6868 gen_helper_sve2_sqcadd_s, gen_helper_sve2_sqcadd_d },
6869 };
6870 return do_sve2_zzw_ool(s, a, fns[sq][a->esz], rot);
6871}
6872
6873static bool trans_CADD_rot90(DisasContext *s, arg_rrr_esz *a)
6874{
6875 return do_cadd(s, a, false, false);
6876}
6877
6878static bool trans_CADD_rot270(DisasContext *s, arg_rrr_esz *a)
6879{
6880 return do_cadd(s, a, false, true);
6881}
6882
6883static bool trans_SQCADD_rot90(DisasContext *s, arg_rrr_esz *a)
6884{
6885 return do_cadd(s, a, true, false);
6886}
6887
6888static bool trans_SQCADD_rot270(DisasContext *s, arg_rrr_esz *a)
6889{
6890 return do_cadd(s, a, true, true);
6891}
38650638
RH
6892
6893static bool do_sve2_zzzz_ool(DisasContext *s, arg_rrrr_esz *a,
6894 gen_helper_gvec_4 *fn, int data)
6895{
6896 if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
6897 return false;
6898 }
6899 if (sve_access_check(s)) {
6900 gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
6901 }
6902 return true;
6903}
6904
6905static bool do_abal(DisasContext *s, arg_rrrr_esz *a, bool uns, bool sel)
6906{
6907 static gen_helper_gvec_4 * const fns[2][4] = {
6908 { NULL, gen_helper_sve2_sabal_h,
6909 gen_helper_sve2_sabal_s, gen_helper_sve2_sabal_d },
6910 { NULL, gen_helper_sve2_uabal_h,
6911 gen_helper_sve2_uabal_s, gen_helper_sve2_uabal_d },
6912 };
6913 return do_sve2_zzzz_ool(s, a, fns[uns][a->esz], sel);
6914}
6915
6916static bool trans_SABALB(DisasContext *s, arg_rrrr_esz *a)
6917{
6918 return do_abal(s, a, false, false);
6919}
6920
6921static bool trans_SABALT(DisasContext *s, arg_rrrr_esz *a)
6922{
6923 return do_abal(s, a, false, true);
6924}
6925
6926static bool trans_UABALB(DisasContext *s, arg_rrrr_esz *a)
6927{
6928 return do_abal(s, a, true, false);
6929}
6930
6931static bool trans_UABALT(DisasContext *s, arg_rrrr_esz *a)
6932{
6933 return do_abal(s, a, true, true);
6934}
b8295dfb
RH
6935
6936static bool do_adcl(DisasContext *s, arg_rrrr_esz *a, bool sel)
6937{
6938 static gen_helper_gvec_4 * const fns[2] = {
6939 gen_helper_sve2_adcl_s,
6940 gen_helper_sve2_adcl_d,
6941 };
6942 /*
6943 * Note that in this case the ESZ field encodes both size and sign.
6944 * Split out 'subtract' into bit 1 of the data field for the helper.
6945 */
6946 return do_sve2_zzzz_ool(s, a, fns[a->esz & 1], (a->esz & 2) | sel);
6947}
6948
6949static bool trans_ADCLB(DisasContext *s, arg_rrrr_esz *a)
6950{
6951 return do_adcl(s, a, false);
6952}
6953
6954static bool trans_ADCLT(DisasContext *s, arg_rrrr_esz *a)
6955{
6956 return do_adcl(s, a, true);
6957}
a7e3a90e
RH
6958
6959static bool do_sve2_fn2i(DisasContext *s, arg_rri_esz *a, GVecGen2iFn *fn)
6960{
6961 if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
6962 return false;
6963 }
6964 if (sve_access_check(s)) {
6965 unsigned vsz = vec_full_reg_size(s);
6966 unsigned rd_ofs = vec_full_reg_offset(s, a->rd);
6967 unsigned rn_ofs = vec_full_reg_offset(s, a->rn);
6968 fn(a->esz, rd_ofs, rn_ofs, a->imm, vsz, vsz);
6969 }
6970 return true;
6971}
6972
6973static bool trans_SSRA(DisasContext *s, arg_rri_esz *a)
6974{
6975 return do_sve2_fn2i(s, a, gen_gvec_ssra);
6976}
6977
6978static bool trans_USRA(DisasContext *s, arg_rri_esz *a)
6979{
6980 return do_sve2_fn2i(s, a, gen_gvec_usra);
6981}
6982
6983static bool trans_SRSRA(DisasContext *s, arg_rri_esz *a)
6984{
6985 return do_sve2_fn2i(s, a, gen_gvec_srsra);
6986}
6987
6988static bool trans_URSRA(DisasContext *s, arg_rri_esz *a)
6989{
6990 return do_sve2_fn2i(s, a, gen_gvec_ursra);
6991}
fc12b46a
RH
6992
6993static bool trans_SRI(DisasContext *s, arg_rri_esz *a)
6994{
6995 return do_sve2_fn2i(s, a, gen_gvec_sri);
6996}
6997
6998static bool trans_SLI(DisasContext *s, arg_rri_esz *a)
6999{
7000 return do_sve2_fn2i(s, a, gen_gvec_sli);
7001}
289a1797
RH
7002
7003static bool do_sve2_fn_zzz(DisasContext *s, arg_rrr_esz *a, GVecGen3Fn *fn)
7004{
7005 if (!dc_isar_feature(aa64_sve2, s)) {
7006 return false;
7007 }
7008 if (sve_access_check(s)) {
7009 gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm);
7010 }
7011 return true;
7012}
7013
7014static bool trans_SABA(DisasContext *s, arg_rrr_esz *a)
7015{
7016 return do_sve2_fn_zzz(s, a, gen_gvec_saba);
7017}
7018
7019static bool trans_UABA(DisasContext *s, arg_rrr_esz *a)
7020{
7021 return do_sve2_fn_zzz(s, a, gen_gvec_uaba);
7022}
5ff2838d
RH
7023
7024static bool do_sve2_narrow_extract(DisasContext *s, arg_rri_esz *a,
7025 const GVecGen2 ops[3])
7026{
7027 if (a->esz < 0 || a->esz > MO_32 || a->imm != 0 ||
7028 !dc_isar_feature(aa64_sve2, s)) {
7029 return false;
7030 }
7031 if (sve_access_check(s)) {
7032 unsigned vsz = vec_full_reg_size(s);
7033 tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd),
7034 vec_full_reg_offset(s, a->rn),
7035 vsz, vsz, &ops[a->esz]);
7036 }
7037 return true;
7038}
7039
7040static const TCGOpcode sqxtn_list[] = {
7041 INDEX_op_shli_vec, INDEX_op_smin_vec, INDEX_op_smax_vec, 0
7042};
7043
7044static void gen_sqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
7045{
7046 TCGv_vec t = tcg_temp_new_vec_matching(d);
7047 int halfbits = 4 << vece;
7048 int64_t mask = (1ull << halfbits) - 1;
7049 int64_t min = -1ull << (halfbits - 1);
7050 int64_t max = -min - 1;
7051
7052 tcg_gen_dupi_vec(vece, t, min);
7053 tcg_gen_smax_vec(vece, d, n, t);
7054 tcg_gen_dupi_vec(vece, t, max);
7055 tcg_gen_smin_vec(vece, d, d, t);
7056 tcg_gen_dupi_vec(vece, t, mask);
7057 tcg_gen_and_vec(vece, d, d, t);
7058 tcg_temp_free_vec(t);
7059}
7060
7061static bool trans_SQXTNB(DisasContext *s, arg_rri_esz *a)
7062{
7063 static const GVecGen2 ops[3] = {
7064 { .fniv = gen_sqxtnb_vec,
7065 .opt_opc = sqxtn_list,
7066 .fno = gen_helper_sve2_sqxtnb_h,
7067 .vece = MO_16 },
7068 { .fniv = gen_sqxtnb_vec,
7069 .opt_opc = sqxtn_list,
7070 .fno = gen_helper_sve2_sqxtnb_s,
7071 .vece = MO_32 },
7072 { .fniv = gen_sqxtnb_vec,
7073 .opt_opc = sqxtn_list,
7074 .fno = gen_helper_sve2_sqxtnb_d,
7075 .vece = MO_64 },
7076 };
7077 return do_sve2_narrow_extract(s, a, ops);
7078}
7079
7080static void gen_sqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
7081{
7082 TCGv_vec t = tcg_temp_new_vec_matching(d);
7083 int halfbits = 4 << vece;
7084 int64_t mask = (1ull << halfbits) - 1;
7085 int64_t min = -1ull << (halfbits - 1);
7086 int64_t max = -min - 1;
7087
7088 tcg_gen_dupi_vec(vece, t, min);
7089 tcg_gen_smax_vec(vece, n, n, t);
7090 tcg_gen_dupi_vec(vece, t, max);
7091 tcg_gen_smin_vec(vece, n, n, t);
7092 tcg_gen_shli_vec(vece, n, n, halfbits);
7093 tcg_gen_dupi_vec(vece, t, mask);
7094 tcg_gen_bitsel_vec(vece, d, t, d, n);
7095 tcg_temp_free_vec(t);
7096}
7097
7098static bool trans_SQXTNT(DisasContext *s, arg_rri_esz *a)
7099{
7100 static const GVecGen2 ops[3] = {
7101 { .fniv = gen_sqxtnt_vec,
7102 .opt_opc = sqxtn_list,
7103 .load_dest = true,
7104 .fno = gen_helper_sve2_sqxtnt_h,
7105 .vece = MO_16 },
7106 { .fniv = gen_sqxtnt_vec,
7107 .opt_opc = sqxtn_list,
7108 .load_dest = true,
7109 .fno = gen_helper_sve2_sqxtnt_s,
7110 .vece = MO_32 },
7111 { .fniv = gen_sqxtnt_vec,
7112 .opt_opc = sqxtn_list,
7113 .load_dest = true,
7114 .fno = gen_helper_sve2_sqxtnt_d,
7115 .vece = MO_64 },
7116 };
7117 return do_sve2_narrow_extract(s, a, ops);
7118}
7119
7120static const TCGOpcode uqxtn_list[] = {
7121 INDEX_op_shli_vec, INDEX_op_umin_vec, 0
7122};
7123
7124static void gen_uqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
7125{
7126 TCGv_vec t = tcg_temp_new_vec_matching(d);
7127 int halfbits = 4 << vece;
7128 int64_t max = (1ull << halfbits) - 1;
7129
7130 tcg_gen_dupi_vec(vece, t, max);
7131 tcg_gen_umin_vec(vece, d, n, t);
7132 tcg_temp_free_vec(t);
7133}
7134
7135static bool trans_UQXTNB(DisasContext *s, arg_rri_esz *a)
7136{
7137 static const GVecGen2 ops[3] = {
7138 { .fniv = gen_uqxtnb_vec,
7139 .opt_opc = uqxtn_list,
7140 .fno = gen_helper_sve2_uqxtnb_h,
7141 .vece = MO_16 },
7142 { .fniv = gen_uqxtnb_vec,
7143 .opt_opc = uqxtn_list,
7144 .fno = gen_helper_sve2_uqxtnb_s,
7145 .vece = MO_32 },
7146 { .fniv = gen_uqxtnb_vec,
7147 .opt_opc = uqxtn_list,
7148 .fno = gen_helper_sve2_uqxtnb_d,
7149 .vece = MO_64 },
7150 };
7151 return do_sve2_narrow_extract(s, a, ops);
7152}
7153
7154static void gen_uqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
7155{
7156 TCGv_vec t = tcg_temp_new_vec_matching(d);
7157 int halfbits = 4 << vece;
7158 int64_t max = (1ull << halfbits) - 1;
7159
7160 tcg_gen_dupi_vec(vece, t, max);
7161 tcg_gen_umin_vec(vece, n, n, t);
7162 tcg_gen_shli_vec(vece, n, n, halfbits);
7163 tcg_gen_bitsel_vec(vece, d, t, d, n);
7164 tcg_temp_free_vec(t);
7165}
7166
7167static bool trans_UQXTNT(DisasContext *s, arg_rri_esz *a)
7168{
7169 static const GVecGen2 ops[3] = {
7170 { .fniv = gen_uqxtnt_vec,
7171 .opt_opc = uqxtn_list,
7172 .load_dest = true,
7173 .fno = gen_helper_sve2_uqxtnt_h,
7174 .vece = MO_16 },
7175 { .fniv = gen_uqxtnt_vec,
7176 .opt_opc = uqxtn_list,
7177 .load_dest = true,
7178 .fno = gen_helper_sve2_uqxtnt_s,
7179 .vece = MO_32 },
7180 { .fniv = gen_uqxtnt_vec,
7181 .opt_opc = uqxtn_list,
7182 .load_dest = true,
7183 .fno = gen_helper_sve2_uqxtnt_d,
7184 .vece = MO_64 },
7185 };
7186 return do_sve2_narrow_extract(s, a, ops);
7187}
7188
7189static const TCGOpcode sqxtun_list[] = {
7190 INDEX_op_shli_vec, INDEX_op_umin_vec, INDEX_op_smax_vec, 0
7191};
7192
7193static void gen_sqxtunb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
7194{
7195 TCGv_vec t = tcg_temp_new_vec_matching(d);
7196 int halfbits = 4 << vece;
7197 int64_t max = (1ull << halfbits) - 1;
7198
7199 tcg_gen_dupi_vec(vece, t, 0);
7200 tcg_gen_smax_vec(vece, d, n, t);
7201 tcg_gen_dupi_vec(vece, t, max);
7202 tcg_gen_umin_vec(vece, d, d, t);
7203 tcg_temp_free_vec(t);
7204}
7205
7206static bool trans_SQXTUNB(DisasContext *s, arg_rri_esz *a)
7207{
7208 static const GVecGen2 ops[3] = {
7209 { .fniv = gen_sqxtunb_vec,
7210 .opt_opc = sqxtun_list,
7211 .fno = gen_helper_sve2_sqxtunb_h,
7212 .vece = MO_16 },
7213 { .fniv = gen_sqxtunb_vec,
7214 .opt_opc = sqxtun_list,
7215 .fno = gen_helper_sve2_sqxtunb_s,
7216 .vece = MO_32 },
7217 { .fniv = gen_sqxtunb_vec,
7218 .opt_opc = sqxtun_list,
7219 .fno = gen_helper_sve2_sqxtunb_d,
7220 .vece = MO_64 },
7221 };
7222 return do_sve2_narrow_extract(s, a, ops);
7223}
7224
7225static void gen_sqxtunt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
7226{
7227 TCGv_vec t = tcg_temp_new_vec_matching(d);
7228 int halfbits = 4 << vece;
7229 int64_t max = (1ull << halfbits) - 1;
7230
7231 tcg_gen_dupi_vec(vece, t, 0);
7232 tcg_gen_smax_vec(vece, n, n, t);
7233 tcg_gen_dupi_vec(vece, t, max);
7234 tcg_gen_umin_vec(vece, n, n, t);
7235 tcg_gen_shli_vec(vece, n, n, halfbits);
7236 tcg_gen_bitsel_vec(vece, d, t, d, n);
7237 tcg_temp_free_vec(t);
7238}
7239
7240static bool trans_SQXTUNT(DisasContext *s, arg_rri_esz *a)
7241{
7242 static const GVecGen2 ops[3] = {
7243 { .fniv = gen_sqxtunt_vec,
7244 .opt_opc = sqxtun_list,
7245 .load_dest = true,
7246 .fno = gen_helper_sve2_sqxtunt_h,
7247 .vece = MO_16 },
7248 { .fniv = gen_sqxtunt_vec,
7249 .opt_opc = sqxtun_list,
7250 .load_dest = true,
7251 .fno = gen_helper_sve2_sqxtunt_s,
7252 .vece = MO_32 },
7253 { .fniv = gen_sqxtunt_vec,
7254 .opt_opc = sqxtun_list,
7255 .load_dest = true,
7256 .fno = gen_helper_sve2_sqxtunt_d,
7257 .vece = MO_64 },
7258 };
7259 return do_sve2_narrow_extract(s, a, ops);
46d111b2
RH
7260}
7261
7262static bool do_sve2_shr_narrow(DisasContext *s, arg_rri_esz *a,
7263 const GVecGen2i ops[3])
7264{
7265 if (a->esz < 0 || a->esz > MO_32 || !dc_isar_feature(aa64_sve2, s)) {
7266 return false;
7267 }
7268 assert(a->imm > 0 && a->imm <= (8 << a->esz));
7269 if (sve_access_check(s)) {
7270 unsigned vsz = vec_full_reg_size(s);
7271 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd),
7272 vec_full_reg_offset(s, a->rn),
7273 vsz, vsz, a->imm, &ops[a->esz]);
7274 }
7275 return true;
7276}
7277
7278static void gen_shrnb_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr)
7279{
7280 int halfbits = 4 << vece;
7281 uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits));
7282
7283 tcg_gen_shri_i64(d, n, shr);
7284 tcg_gen_andi_i64(d, d, mask);
7285}
7286
7287static void gen_shrnb16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
7288{
7289 gen_shrnb_i64(MO_16, d, n, shr);
7290}
7291
7292static void gen_shrnb32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
7293{
7294 gen_shrnb_i64(MO_32, d, n, shr);
7295}
7296
7297static void gen_shrnb64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
7298{
7299 gen_shrnb_i64(MO_64, d, n, shr);
7300}
7301
7302static void gen_shrnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr)
7303{
7304 TCGv_vec t = tcg_temp_new_vec_matching(d);
7305 int halfbits = 4 << vece;
7306 uint64_t mask = MAKE_64BIT_MASK(0, halfbits);
7307
7308 tcg_gen_shri_vec(vece, n, n, shr);
7309 tcg_gen_dupi_vec(vece, t, mask);
7310 tcg_gen_and_vec(vece, d, n, t);
7311 tcg_temp_free_vec(t);
7312}
7313
7314static bool trans_SHRNB(DisasContext *s, arg_rri_esz *a)
7315{
7316 static const TCGOpcode vec_list[] = { INDEX_op_shri_vec, 0 };
7317 static const GVecGen2i ops[3] = {
7318 { .fni8 = gen_shrnb16_i64,
7319 .fniv = gen_shrnb_vec,
7320 .opt_opc = vec_list,
7321 .fno = gen_helper_sve2_shrnb_h,
7322 .vece = MO_16 },
7323 { .fni8 = gen_shrnb32_i64,
7324 .fniv = gen_shrnb_vec,
7325 .opt_opc = vec_list,
7326 .fno = gen_helper_sve2_shrnb_s,
7327 .vece = MO_32 },
7328 { .fni8 = gen_shrnb64_i64,
7329 .fniv = gen_shrnb_vec,
7330 .opt_opc = vec_list,
7331 .fno = gen_helper_sve2_shrnb_d,
7332 .vece = MO_64 },
7333 };
7334 return do_sve2_shr_narrow(s, a, ops);
7335}
7336
7337static void gen_shrnt_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr)
7338{
7339 int halfbits = 4 << vece;
7340 uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits));
7341
7342 tcg_gen_shli_i64(n, n, halfbits - shr);
7343 tcg_gen_andi_i64(n, n, ~mask);
7344 tcg_gen_andi_i64(d, d, mask);
7345 tcg_gen_or_i64(d, d, n);
7346}
7347
7348static void gen_shrnt16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
7349{
7350 gen_shrnt_i64(MO_16, d, n, shr);
7351}
7352
7353static void gen_shrnt32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
7354{
7355 gen_shrnt_i64(MO_32, d, n, shr);
7356}
7357
7358static void gen_shrnt64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
7359{
7360 tcg_gen_shri_i64(n, n, shr);
7361 tcg_gen_deposit_i64(d, d, n, 32, 32);
7362}
7363
7364static void gen_shrnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr)
7365{
7366 TCGv_vec t = tcg_temp_new_vec_matching(d);
7367 int halfbits = 4 << vece;
7368 uint64_t mask = MAKE_64BIT_MASK(0, halfbits);
7369
7370 tcg_gen_shli_vec(vece, n, n, halfbits - shr);
7371 tcg_gen_dupi_vec(vece, t, mask);
7372 tcg_gen_bitsel_vec(vece, d, t, d, n);
7373 tcg_temp_free_vec(t);
7374}
7375
7376static bool trans_SHRNT(DisasContext *s, arg_rri_esz *a)
7377{
7378 static const TCGOpcode vec_list[] = { INDEX_op_shli_vec, 0 };
7379 static const GVecGen2i ops[3] = {
7380 { .fni8 = gen_shrnt16_i64,
7381 .fniv = gen_shrnt_vec,
7382 .opt_opc = vec_list,
7383 .load_dest = true,
7384 .fno = gen_helper_sve2_shrnt_h,
7385 .vece = MO_16 },
7386 { .fni8 = gen_shrnt32_i64,
7387 .fniv = gen_shrnt_vec,
7388 .opt_opc = vec_list,
7389 .load_dest = true,
7390 .fno = gen_helper_sve2_shrnt_s,
7391 .vece = MO_32 },
7392 { .fni8 = gen_shrnt64_i64,
7393 .fniv = gen_shrnt_vec,
7394 .opt_opc = vec_list,
7395 .load_dest = true,
7396 .fno = gen_helper_sve2_shrnt_d,
7397 .vece = MO_64 },
7398 };
7399 return do_sve2_shr_narrow(s, a, ops);
7400}
7401
7402static bool trans_RSHRNB(DisasContext *s, arg_rri_esz *a)
7403{
7404 static const GVecGen2i ops[3] = {
7405 { .fno = gen_helper_sve2_rshrnb_h },
7406 { .fno = gen_helper_sve2_rshrnb_s },
7407 { .fno = gen_helper_sve2_rshrnb_d },
7408 };
7409 return do_sve2_shr_narrow(s, a, ops);
7410}
7411
7412static bool trans_RSHRNT(DisasContext *s, arg_rri_esz *a)
7413{
7414 static const GVecGen2i ops[3] = {
7415 { .fno = gen_helper_sve2_rshrnt_h },
7416 { .fno = gen_helper_sve2_rshrnt_s },
7417 { .fno = gen_helper_sve2_rshrnt_d },
7418 };
7419 return do_sve2_shr_narrow(s, a, ops);
81fd3e6e
RH
7420}
7421
7422static void gen_sqshrunb_vec(unsigned vece, TCGv_vec d,
7423 TCGv_vec n, int64_t shr)
7424{
7425 TCGv_vec t = tcg_temp_new_vec_matching(d);
7426 int halfbits = 4 << vece;
7427
7428 tcg_gen_sari_vec(vece, n, n, shr);
7429 tcg_gen_dupi_vec(vece, t, 0);
7430 tcg_gen_smax_vec(vece, n, n, t);
7431 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
7432 tcg_gen_umin_vec(vece, d, n, t);
7433 tcg_temp_free_vec(t);
7434}
7435
7436static bool trans_SQSHRUNB(DisasContext *s, arg_rri_esz *a)
7437{
7438 static const TCGOpcode vec_list[] = {
7439 INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_umin_vec, 0
7440 };
7441 static const GVecGen2i ops[3] = {
7442 { .fniv = gen_sqshrunb_vec,
7443 .opt_opc = vec_list,
7444 .fno = gen_helper_sve2_sqshrunb_h,
7445 .vece = MO_16 },
7446 { .fniv = gen_sqshrunb_vec,
7447 .opt_opc = vec_list,
7448 .fno = gen_helper_sve2_sqshrunb_s,
7449 .vece = MO_32 },
7450 { .fniv = gen_sqshrunb_vec,
7451 .opt_opc = vec_list,
7452 .fno = gen_helper_sve2_sqshrunb_d,
7453 .vece = MO_64 },
7454 };
7455 return do_sve2_shr_narrow(s, a, ops);
7456}
7457
7458static void gen_sqshrunt_vec(unsigned vece, TCGv_vec d,
7459 TCGv_vec n, int64_t shr)
7460{
7461 TCGv_vec t = tcg_temp_new_vec_matching(d);
7462 int halfbits = 4 << vece;
7463
7464 tcg_gen_sari_vec(vece, n, n, shr);
7465 tcg_gen_dupi_vec(vece, t, 0);
7466 tcg_gen_smax_vec(vece, n, n, t);
7467 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
7468 tcg_gen_umin_vec(vece, n, n, t);
7469 tcg_gen_shli_vec(vece, n, n, halfbits);
7470 tcg_gen_bitsel_vec(vece, d, t, d, n);
7471 tcg_temp_free_vec(t);
7472}
7473
7474static bool trans_SQSHRUNT(DisasContext *s, arg_rri_esz *a)
7475{
7476 static const TCGOpcode vec_list[] = {
7477 INDEX_op_shli_vec, INDEX_op_sari_vec,
7478 INDEX_op_smax_vec, INDEX_op_umin_vec, 0
7479 };
7480 static const GVecGen2i ops[3] = {
7481 { .fniv = gen_sqshrunt_vec,
7482 .opt_opc = vec_list,
7483 .load_dest = true,
7484 .fno = gen_helper_sve2_sqshrunt_h,
7485 .vece = MO_16 },
7486 { .fniv = gen_sqshrunt_vec,
7487 .opt_opc = vec_list,
7488 .load_dest = true,
7489 .fno = gen_helper_sve2_sqshrunt_s,
7490 .vece = MO_32 },
7491 { .fniv = gen_sqshrunt_vec,
7492 .opt_opc = vec_list,
7493 .load_dest = true,
7494 .fno = gen_helper_sve2_sqshrunt_d,
7495 .vece = MO_64 },
7496 };
7497 return do_sve2_shr_narrow(s, a, ops);
7498}
7499
7500static bool trans_SQRSHRUNB(DisasContext *s, arg_rri_esz *a)
7501{
7502 static const GVecGen2i ops[3] = {
7503 { .fno = gen_helper_sve2_sqrshrunb_h },
7504 { .fno = gen_helper_sve2_sqrshrunb_s },
7505 { .fno = gen_helper_sve2_sqrshrunb_d },
7506 };
7507 return do_sve2_shr_narrow(s, a, ops);
7508}
7509
7510static bool trans_SQRSHRUNT(DisasContext *s, arg_rri_esz *a)
7511{
7512 static const GVecGen2i ops[3] = {
7513 { .fno = gen_helper_sve2_sqrshrunt_h },
7514 { .fno = gen_helper_sve2_sqrshrunt_s },
7515 { .fno = gen_helper_sve2_sqrshrunt_d },
7516 };
7517 return do_sve2_shr_narrow(s, a, ops);
c13418da
RH
7518}
7519
743bb147
RH
7520static void gen_sqshrnb_vec(unsigned vece, TCGv_vec d,
7521 TCGv_vec n, int64_t shr)
7522{
7523 TCGv_vec t = tcg_temp_new_vec_matching(d);
7524 int halfbits = 4 << vece;
7525 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1);
7526 int64_t min = -max - 1;
7527
7528 tcg_gen_sari_vec(vece, n, n, shr);
7529 tcg_gen_dupi_vec(vece, t, min);
7530 tcg_gen_smax_vec(vece, n, n, t);
7531 tcg_gen_dupi_vec(vece, t, max);
7532 tcg_gen_smin_vec(vece, n, n, t);
7533 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
7534 tcg_gen_and_vec(vece, d, n, t);
7535 tcg_temp_free_vec(t);
7536}
7537
7538static bool trans_SQSHRNB(DisasContext *s, arg_rri_esz *a)
7539{
7540 static const TCGOpcode vec_list[] = {
7541 INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_smin_vec, 0
7542 };
7543 static const GVecGen2i ops[3] = {
7544 { .fniv = gen_sqshrnb_vec,
7545 .opt_opc = vec_list,
7546 .fno = gen_helper_sve2_sqshrnb_h,
7547 .vece = MO_16 },
7548 { .fniv = gen_sqshrnb_vec,
7549 .opt_opc = vec_list,
7550 .fno = gen_helper_sve2_sqshrnb_s,
7551 .vece = MO_32 },
7552 { .fniv = gen_sqshrnb_vec,
7553 .opt_opc = vec_list,
7554 .fno = gen_helper_sve2_sqshrnb_d,
7555 .vece = MO_64 },
7556 };
7557 return do_sve2_shr_narrow(s, a, ops);
7558}
7559
7560static void gen_sqshrnt_vec(unsigned vece, TCGv_vec d,
7561 TCGv_vec n, int64_t shr)
7562{
7563 TCGv_vec t = tcg_temp_new_vec_matching(d);
7564 int halfbits = 4 << vece;
7565 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1);
7566 int64_t min = -max - 1;
7567
7568 tcg_gen_sari_vec(vece, n, n, shr);
7569 tcg_gen_dupi_vec(vece, t, min);
7570 tcg_gen_smax_vec(vece, n, n, t);
7571 tcg_gen_dupi_vec(vece, t, max);
7572 tcg_gen_smin_vec(vece, n, n, t);
7573 tcg_gen_shli_vec(vece, n, n, halfbits);
7574 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
7575 tcg_gen_bitsel_vec(vece, d, t, d, n);
7576 tcg_temp_free_vec(t);
7577}
7578
7579static bool trans_SQSHRNT(DisasContext *s, arg_rri_esz *a)
7580{
7581 static const TCGOpcode vec_list[] = {
7582 INDEX_op_shli_vec, INDEX_op_sari_vec,
7583 INDEX_op_smax_vec, INDEX_op_smin_vec, 0
7584 };
7585 static const GVecGen2i ops[3] = {
7586 { .fniv = gen_sqshrnt_vec,
7587 .opt_opc = vec_list,
7588 .load_dest = true,
7589 .fno = gen_helper_sve2_sqshrnt_h,
7590 .vece = MO_16 },
7591 { .fniv = gen_sqshrnt_vec,
7592 .opt_opc = vec_list,
7593 .load_dest = true,
7594 .fno = gen_helper_sve2_sqshrnt_s,
7595 .vece = MO_32 },
7596 { .fniv = gen_sqshrnt_vec,
7597 .opt_opc = vec_list,
7598 .load_dest = true,
7599 .fno = gen_helper_sve2_sqshrnt_d,
7600 .vece = MO_64 },
7601 };
7602 return do_sve2_shr_narrow(s, a, ops);
7603}
7604
7605static bool trans_SQRSHRNB(DisasContext *s, arg_rri_esz *a)
7606{
7607 static const GVecGen2i ops[3] = {
7608 { .fno = gen_helper_sve2_sqrshrnb_h },
7609 { .fno = gen_helper_sve2_sqrshrnb_s },
7610 { .fno = gen_helper_sve2_sqrshrnb_d },
7611 };
7612 return do_sve2_shr_narrow(s, a, ops);
7613}
7614
7615static bool trans_SQRSHRNT(DisasContext *s, arg_rri_esz *a)
7616{
7617 static const GVecGen2i ops[3] = {
7618 { .fno = gen_helper_sve2_sqrshrnt_h },
7619 { .fno = gen_helper_sve2_sqrshrnt_s },
7620 { .fno = gen_helper_sve2_sqrshrnt_d },
7621 };
7622 return do_sve2_shr_narrow(s, a, ops);
7623}
7624
c13418da
RH
7625static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d,
7626 TCGv_vec n, int64_t shr)
7627{
7628 TCGv_vec t = tcg_temp_new_vec_matching(d);
7629 int halfbits = 4 << vece;
7630
7631 tcg_gen_shri_vec(vece, n, n, shr);
7632 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
7633 tcg_gen_umin_vec(vece, d, n, t);
7634 tcg_temp_free_vec(t);
7635}
7636
7637static bool trans_UQSHRNB(DisasContext *s, arg_rri_esz *a)
7638{
7639 static const TCGOpcode vec_list[] = {
7640 INDEX_op_shri_vec, INDEX_op_umin_vec, 0
7641 };
7642 static const GVecGen2i ops[3] = {
7643 { .fniv = gen_uqshrnb_vec,
7644 .opt_opc = vec_list,
7645 .fno = gen_helper_sve2_uqshrnb_h,
7646 .vece = MO_16 },
7647 { .fniv = gen_uqshrnb_vec,
7648 .opt_opc = vec_list,
7649 .fno = gen_helper_sve2_uqshrnb_s,
7650 .vece = MO_32 },
7651 { .fniv = gen_uqshrnb_vec,
7652 .opt_opc = vec_list,
7653 .fno = gen_helper_sve2_uqshrnb_d,
7654 .vece = MO_64 },
7655 };
7656 return do_sve2_shr_narrow(s, a, ops);
7657}
7658
7659static void gen_uqshrnt_vec(unsigned vece, TCGv_vec d,
7660 TCGv_vec n, int64_t shr)
7661{
7662 TCGv_vec t = tcg_temp_new_vec_matching(d);
7663 int halfbits = 4 << vece;
7664
7665 tcg_gen_shri_vec(vece, n, n, shr);
7666 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
7667 tcg_gen_umin_vec(vece, n, n, t);
7668 tcg_gen_shli_vec(vece, n, n, halfbits);
7669 tcg_gen_bitsel_vec(vece, d, t, d, n);
7670 tcg_temp_free_vec(t);
7671}
7672
7673static bool trans_UQSHRNT(DisasContext *s, arg_rri_esz *a)
7674{
7675 static const TCGOpcode vec_list[] = {
7676 INDEX_op_shli_vec, INDEX_op_shri_vec, INDEX_op_umin_vec, 0
7677 };
7678 static const GVecGen2i ops[3] = {
7679 { .fniv = gen_uqshrnt_vec,
7680 .opt_opc = vec_list,
7681 .load_dest = true,
7682 .fno = gen_helper_sve2_uqshrnt_h,
7683 .vece = MO_16 },
7684 { .fniv = gen_uqshrnt_vec,
7685 .opt_opc = vec_list,
7686 .load_dest = true,
7687 .fno = gen_helper_sve2_uqshrnt_s,
7688 .vece = MO_32 },
7689 { .fniv = gen_uqshrnt_vec,
7690 .opt_opc = vec_list,
7691 .load_dest = true,
7692 .fno = gen_helper_sve2_uqshrnt_d,
7693 .vece = MO_64 },
7694 };
7695 return do_sve2_shr_narrow(s, a, ops);
7696}
7697
7698static bool trans_UQRSHRNB(DisasContext *s, arg_rri_esz *a)
7699{
7700 static const GVecGen2i ops[3] = {
7701 { .fno = gen_helper_sve2_uqrshrnb_h },
7702 { .fno = gen_helper_sve2_uqrshrnb_s },
7703 { .fno = gen_helper_sve2_uqrshrnb_d },
7704 };
7705 return do_sve2_shr_narrow(s, a, ops);
7706}
7707
7708static bool trans_UQRSHRNT(DisasContext *s, arg_rri_esz *a)
7709{
7710 static const GVecGen2i ops[3] = {
7711 { .fno = gen_helper_sve2_uqrshrnt_h },
7712 { .fno = gen_helper_sve2_uqrshrnt_s },
7713 { .fno = gen_helper_sve2_uqrshrnt_d },
7714 };
7715 return do_sve2_shr_narrow(s, a, ops);
5ff2838d 7716}
b87dbeeb 7717
40d5ea50
SL
7718#define DO_SVE2_ZZZ_NARROW(NAME, name) \
7719static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
7720{ \
7721 static gen_helper_gvec_3 * const fns[4] = { \
7722 NULL, gen_helper_sve2_##name##_h, \
7723 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7724 }; \
7725 return do_sve2_zzz_ool(s, a, fns[a->esz]); \
7726}
7727
7728DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb)
7729DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt)
0ea3ff02
SL
7730DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb)
7731DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt)
40d5ea50 7732
c3cd6766
SL
7733DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb)
7734DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt)
e9443d10
SL
7735DO_SVE2_ZZZ_NARROW(RSUBHNB, rsubhnb)
7736DO_SVE2_ZZZ_NARROW(RSUBHNT, rsubhnt)
c3cd6766 7737
e0ae6ec3
SL
7738static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
7739 gen_helper_gvec_flags_4 *fn)
7740{
7741 if (!dc_isar_feature(aa64_sve2, s)) {
7742 return false;
7743 }
7744 return do_ppzz_flags(s, a, fn);
7745}
7746
7747#define DO_SVE2_PPZZ_MATCH(NAME, name) \
7748static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7749{ \
7750 static gen_helper_gvec_flags_4 * const fns[4] = { \
7751 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
7752 NULL, NULL \
7753 }; \
7754 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
7755}
7756
7757DO_SVE2_PPZZ_MATCH(MATCH, match)
7758DO_SVE2_PPZZ_MATCH(NMATCH, nmatch)
7759
7d47ac94
SL
7760static bool trans_HISTCNT(DisasContext *s, arg_rprr_esz *a)
7761{
7762 static gen_helper_gvec_4 * const fns[2] = {
7763 gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d
7764 };
7765 if (a->esz < 2) {
7766 return false;
7767 }
7768 return do_sve2_zpzz_ool(s, a, fns[a->esz - 2]);
7769}
7770
7771static bool trans_HISTSEG(DisasContext *s, arg_rrr_esz *a)
7772{
7773 if (a->esz != 0) {
7774 return false;
7775 }
7776 return do_sve2_zzz_ool(s, a, gen_helper_sve2_histseg);
7777}
7778
b87dbeeb
SL
7779static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
7780 gen_helper_gvec_4_ptr *fn)
7781{
7782 if (!dc_isar_feature(aa64_sve2, s)) {
7783 return false;
7784 }
7785 return do_zpzz_fp(s, a, fn);
7786}
7787
7788#define DO_SVE2_ZPZZ_FP(NAME, name) \
7789static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7790{ \
7791 static gen_helper_gvec_4_ptr * const fns[4] = { \
7792 NULL, gen_helper_sve2_##name##_zpzz_h, \
7793 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
7794 }; \
7795 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
7796}
7797
7798DO_SVE2_ZPZZ_FP(FADDP, faddp)
7799DO_SVE2_ZPZZ_FP(FMAXNMP, fmaxnmp)
7800DO_SVE2_ZPZZ_FP(FMINNMP, fminnmp)
7801DO_SVE2_ZPZZ_FP(FMAXP, fmaxp)
7802DO_SVE2_ZPZZ_FP(FMINP, fminp)
bfc9307e
RH
7803
7804/*
7805 * SVE Integer Multiply-Add (unpredicated)
7806 */
7807
4f26756b
SL
7808static bool trans_FMMLA(DisasContext *s, arg_rrrr_esz *a)
7809{
7810 gen_helper_gvec_4_ptr *fn;
7811
7812 switch (a->esz) {
7813 case MO_32:
7814 if (!dc_isar_feature(aa64_sve_f32mm, s)) {
7815 return false;
7816 }
7817 fn = gen_helper_fmmla_s;
7818 break;
7819 case MO_64:
7820 if (!dc_isar_feature(aa64_sve_f64mm, s)) {
7821 return false;
7822 }
7823 fn = gen_helper_fmmla_d;
7824 break;
7825 default:
7826 return false;
7827 }
7828
7829 if (sve_access_check(s)) {
7830 unsigned vsz = vec_full_reg_size(s);
7831 TCGv_ptr status = fpstatus_ptr(FPST_FPCR);
7832 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
7833 vec_full_reg_offset(s, a->rn),
7834 vec_full_reg_offset(s, a->rm),
7835 vec_full_reg_offset(s, a->ra),
7836 status, vsz, vsz, 0, fn);
7837 tcg_temp_free_ptr(status);
7838 }
7839 return true;
7840}
7841
bfc9307e
RH
7842static bool do_sqdmlal_zzzw(DisasContext *s, arg_rrrr_esz *a,
7843 bool sel1, bool sel2)
7844{
7845 static gen_helper_gvec_4 * const fns[] = {
7846 NULL, gen_helper_sve2_sqdmlal_zzzw_h,
7847 gen_helper_sve2_sqdmlal_zzzw_s, gen_helper_sve2_sqdmlal_zzzw_d,
7848 };
7849 return do_sve2_zzzz_ool(s, a, fns[a->esz], (sel2 << 1) | sel1);
7850}
7851
7852static bool do_sqdmlsl_zzzw(DisasContext *s, arg_rrrr_esz *a,
7853 bool sel1, bool sel2)
7854{
7855 static gen_helper_gvec_4 * const fns[] = {
7856 NULL, gen_helper_sve2_sqdmlsl_zzzw_h,
7857 gen_helper_sve2_sqdmlsl_zzzw_s, gen_helper_sve2_sqdmlsl_zzzw_d,
7858 };
7859 return do_sve2_zzzz_ool(s, a, fns[a->esz], (sel2 << 1) | sel1);
7860}
7861
7862static bool trans_SQDMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a)
7863{
7864 return do_sqdmlal_zzzw(s, a, false, false);
7865}
7866
7867static bool trans_SQDMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a)
7868{
7869 return do_sqdmlal_zzzw(s, a, true, true);
7870}
7871
7872static bool trans_SQDMLALBT(DisasContext *s, arg_rrrr_esz *a)
7873{
7874 return do_sqdmlal_zzzw(s, a, false, true);
7875}
7876
7877static bool trans_SQDMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a)
7878{
7879 return do_sqdmlsl_zzzw(s, a, false, false);
7880}
7881
7882static bool trans_SQDMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a)
7883{
7884 return do_sqdmlsl_zzzw(s, a, true, true);
7885}
7886
7887static bool trans_SQDMLSLBT(DisasContext *s, arg_rrrr_esz *a)
7888{
7889 return do_sqdmlsl_zzzw(s, a, false, true);
7890}
ab3ddf31
RH
7891
7892static bool trans_SQRDMLAH_zzzz(DisasContext *s, arg_rrrr_esz *a)
7893{
7894 static gen_helper_gvec_4 * const fns[] = {
7895 gen_helper_sve2_sqrdmlah_b, gen_helper_sve2_sqrdmlah_h,
7896 gen_helper_sve2_sqrdmlah_s, gen_helper_sve2_sqrdmlah_d,
7897 };
7898 return do_sve2_zzzz_ool(s, a, fns[a->esz], 0);
7899}
7900
7901static bool trans_SQRDMLSH_zzzz(DisasContext *s, arg_rrrr_esz *a)
7902{
7903 static gen_helper_gvec_4 * const fns[] = {
7904 gen_helper_sve2_sqrdmlsh_b, gen_helper_sve2_sqrdmlsh_h,
7905 gen_helper_sve2_sqrdmlsh_s, gen_helper_sve2_sqrdmlsh_d,
7906 };
7907 return do_sve2_zzzz_ool(s, a, fns[a->esz], 0);
7908}
45a32e80
RH
7909
7910static bool do_smlal_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
7911{
7912 static gen_helper_gvec_4 * const fns[] = {
7913 NULL, gen_helper_sve2_smlal_zzzw_h,
7914 gen_helper_sve2_smlal_zzzw_s, gen_helper_sve2_smlal_zzzw_d,
7915 };
7916 return do_sve2_zzzz_ool(s, a, fns[a->esz], sel);
7917}
7918
7919static bool trans_SMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a)
7920{
7921 return do_smlal_zzzw(s, a, false);
7922}
7923
7924static bool trans_SMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a)
7925{
7926 return do_smlal_zzzw(s, a, true);
7927}
7928
7929static bool do_umlal_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
7930{
7931 static gen_helper_gvec_4 * const fns[] = {
7932 NULL, gen_helper_sve2_umlal_zzzw_h,
7933 gen_helper_sve2_umlal_zzzw_s, gen_helper_sve2_umlal_zzzw_d,
7934 };
7935 return do_sve2_zzzz_ool(s, a, fns[a->esz], sel);
7936}
7937
7938static bool trans_UMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a)
7939{
7940 return do_umlal_zzzw(s, a, false);
7941}
7942
7943static bool trans_UMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a)
7944{
7945 return do_umlal_zzzw(s, a, true);
7946}
7947
7948static bool do_smlsl_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
7949{
7950 static gen_helper_gvec_4 * const fns[] = {
7951 NULL, gen_helper_sve2_smlsl_zzzw_h,
7952 gen_helper_sve2_smlsl_zzzw_s, gen_helper_sve2_smlsl_zzzw_d,
7953 };
7954 return do_sve2_zzzz_ool(s, a, fns[a->esz], sel);
7955}
7956
7957static bool trans_SMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a)
7958{
7959 return do_smlsl_zzzw(s, a, false);
7960}
7961
7962static bool trans_SMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a)
7963{
7964 return do_smlsl_zzzw(s, a, true);
7965}
7966
7967static bool do_umlsl_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
7968{
7969 static gen_helper_gvec_4 * const fns[] = {
7970 NULL, gen_helper_sve2_umlsl_zzzw_h,
7971 gen_helper_sve2_umlsl_zzzw_s, gen_helper_sve2_umlsl_zzzw_d,
7972 };
7973 return do_sve2_zzzz_ool(s, a, fns[a->esz], sel);
7974}
7975
7976static bool trans_UMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a)
7977{
7978 return do_umlsl_zzzw(s, a, false);
7979}
7980
7981static bool trans_UMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a)
7982{
7983 return do_umlsl_zzzw(s, a, true);
7984}
d782d3ca
RH
7985
7986static bool trans_CMLA_zzzz(DisasContext *s, arg_CMLA_zzzz *a)
7987{
7988 static gen_helper_gvec_4 * const fns[] = {
7989 gen_helper_sve2_cmla_zzzz_b, gen_helper_sve2_cmla_zzzz_h,
7990 gen_helper_sve2_cmla_zzzz_s, gen_helper_sve2_cmla_zzzz_d,
7991 };
7992
7993 if (!dc_isar_feature(aa64_sve2, s)) {
7994 return false;
7995 }
7996 if (sve_access_check(s)) {
7997 gen_gvec_ool_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot);
7998 }
7999 return true;
8000}
8001
8002static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_SQRDCMLAH_zzzz *a)
8003{
8004 static gen_helper_gvec_4 * const fns[] = {
8005 gen_helper_sve2_sqrdcmlah_zzzz_b, gen_helper_sve2_sqrdcmlah_zzzz_h,
8006 gen_helper_sve2_sqrdcmlah_zzzz_s, gen_helper_sve2_sqrdcmlah_zzzz_d,
8007 };
8008
8009 if (!dc_isar_feature(aa64_sve2, s)) {
8010 return false;
8011 }
8012 if (sve_access_check(s)) {
8013 gen_gvec_ool_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot);
8014 }
8015 return true;
8016}