1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * QEMU LoongArch vector helper functions.
5 * Copyright (c) 2022-2023 Loongson Technology Corporation Limited
8 #include "qemu/osdep.h"
10 #include "exec/exec-all.h"
11 #include "exec/helper-proto.h"
12 #include "fpu/softfloat.h"
13 #include "internals.h"
16 #include "tcg/tcg-gvec-desc.h"
18 #define DO_ADD(a, b) (a + b)
19 #define DO_SUB(a, b) (a - b)
21 #define DO_ODD_EVEN(NAME, BIT, E1, E2, DO_OP) \
22 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
25 VReg *Vd = (VReg *)vd; \
26 VReg *Vj = (VReg *)vj; \
27 VReg *Vk = (VReg *)vk; \
28 typedef __typeof(Vd->E1(0)) TD; \
29 int oprsz = simd_oprsz(desc); \
31 for (i = 0; i < oprsz / (BIT / 8); i++) { \
32 Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i + 1), (TD)Vk->E2(2 * i)); \
36 DO_ODD_EVEN(vhaddw_h_b
, 16, H
, B
, DO_ADD
)
37 DO_ODD_EVEN(vhaddw_w_h
, 32, W
, H
, DO_ADD
)
38 DO_ODD_EVEN(vhaddw_d_w
, 64, D
, W
, DO_ADD
)
40 void HELPER(vhaddw_q_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
43 VReg
*Vd
= (VReg
*)vd
;
44 VReg
*Vj
= (VReg
*)vj
;
45 VReg
*Vk
= (VReg
*)vk
;
46 int oprsz
= simd_oprsz(desc
);
48 for (i
= 0; i
< oprsz
/ 16 ; i
++) {
49 Vd
->Q(i
) = int128_add(int128_makes64(Vj
->D(2 * i
+ 1)),
50 int128_makes64(Vk
->D(2 * i
)));
54 DO_ODD_EVEN(vhsubw_h_b
, 16, H
, B
, DO_SUB
)
55 DO_ODD_EVEN(vhsubw_w_h
, 32, W
, H
, DO_SUB
)
56 DO_ODD_EVEN(vhsubw_d_w
, 64, D
, W
, DO_SUB
)
58 void HELPER(vhsubw_q_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
61 VReg
*Vd
= (VReg
*)vd
;
62 VReg
*Vj
= (VReg
*)vj
;
63 VReg
*Vk
= (VReg
*)vk
;
64 int oprsz
= simd_oprsz(desc
);
66 for (i
= 0; i
< oprsz
/ 16; i
++) {
67 Vd
->Q(i
) = int128_sub(int128_makes64(Vj
->D(2 * i
+ 1)),
68 int128_makes64(Vk
->D(2 * i
)));
72 DO_ODD_EVEN(vhaddw_hu_bu
, 16, UH
, UB
, DO_ADD
)
73 DO_ODD_EVEN(vhaddw_wu_hu
, 32, UW
, UH
, DO_ADD
)
74 DO_ODD_EVEN(vhaddw_du_wu
, 64, UD
, UW
, DO_ADD
)
76 void HELPER(vhaddw_qu_du
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
79 VReg
*Vd
= (VReg
*)vd
;
80 VReg
*Vj
= (VReg
*)vj
;
81 VReg
*Vk
= (VReg
*)vk
;
82 int oprsz
= simd_oprsz(desc
);
84 for (i
= 0; i
< oprsz
/ 16; i
++) {
85 Vd
->Q(i
) = int128_add(int128_make64(Vj
->UD(2 * i
+ 1)),
86 int128_make64(Vk
->UD(2 * i
)));
90 DO_ODD_EVEN(vhsubw_hu_bu
, 16, UH
, UB
, DO_SUB
)
91 DO_ODD_EVEN(vhsubw_wu_hu
, 32, UW
, UH
, DO_SUB
)
92 DO_ODD_EVEN(vhsubw_du_wu
, 64, UD
, UW
, DO_SUB
)
94 void HELPER(vhsubw_qu_du
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
97 VReg
*Vd
= (VReg
*)vd
;
98 VReg
*Vj
= (VReg
*)vj
;
99 VReg
*Vk
= (VReg
*)vk
;
100 int oprsz
= simd_oprsz(desc
);
102 for (i
= 0; i
< oprsz
/ 16; i
++) {
103 Vd
->Q(i
) = int128_sub(int128_make64(Vj
->UD(2 * i
+ 1)),
104 int128_make64(Vk
->UD(2 * i
)));
108 #define DO_EVEN(NAME, BIT, E1, E2, DO_OP) \
109 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
112 VReg *Vd = (VReg *)vd; \
113 VReg *Vj = (VReg *)vj; \
114 VReg *Vk = (VReg *)vk; \
115 typedef __typeof(Vd->E1(0)) TD; \
116 int oprsz = simd_oprsz(desc); \
118 for (i = 0; i < oprsz / (BIT / 8); i++) { \
119 Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i) ,(TD)Vk->E2(2 * i)); \
123 #define DO_ODD(NAME, BIT, E1, E2, DO_OP) \
124 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
127 VReg *Vd = (VReg *)vd; \
128 VReg *Vj = (VReg *)vj; \
129 VReg *Vk = (VReg *)vk; \
130 typedef __typeof(Vd->E1(0)) TD; \
131 int oprsz = simd_oprsz(desc); \
133 for (i = 0; i < oprsz / (BIT / 8); i++) { \
134 Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i + 1), (TD)Vk->E2(2 * i + 1)); \
138 void HELPER(vaddwev_q_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
141 VReg
*Vd
= (VReg
*)vd
;
142 VReg
*Vj
= (VReg
*)vj
;
143 VReg
*Vk
= (VReg
*)vk
;
144 int oprsz
= simd_oprsz(desc
);
146 for (i
= 0; i
< oprsz
/ 16; i
++) {
147 Vd
->Q(i
) = int128_add(int128_makes64(Vj
->D(2 * i
)),
148 int128_makes64(Vk
->D(2 * i
)));
152 DO_EVEN(vaddwev_h_b
, 16, H
, B
, DO_ADD
)
153 DO_EVEN(vaddwev_w_h
, 32, W
, H
, DO_ADD
)
154 DO_EVEN(vaddwev_d_w
, 64, D
, W
, DO_ADD
)
156 void HELPER(vaddwod_q_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
159 VReg
*Vd
= (VReg
*)vd
;
160 VReg
*Vj
= (VReg
*)vj
;
161 VReg
*Vk
= (VReg
*)vk
;
162 int oprsz
= simd_oprsz(desc
);
164 for (i
= 0; i
< oprsz
/ 16; i
++) {
165 Vd
->Q(i
) = int128_add(int128_makes64(Vj
->D(2 * i
+1)),
166 int128_makes64(Vk
->D(2 * i
+1)));
170 DO_ODD(vaddwod_h_b
, 16, H
, B
, DO_ADD
)
171 DO_ODD(vaddwod_w_h
, 32, W
, H
, DO_ADD
)
172 DO_ODD(vaddwod_d_w
, 64, D
, W
, DO_ADD
)
174 void HELPER(vsubwev_q_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
177 VReg
*Vd
= (VReg
*)vd
;
178 VReg
*Vj
= (VReg
*)vj
;
179 VReg
*Vk
= (VReg
*)vk
;
180 int oprsz
= simd_oprsz(desc
);
182 for (i
= 0; i
< oprsz
/ 16; i
++) {
183 Vd
->Q(i
) = int128_sub(int128_makes64(Vj
->D(2 * i
)),
184 int128_makes64(Vk
->D(2 * i
)));
188 DO_EVEN(vsubwev_h_b
, 16, H
, B
, DO_SUB
)
189 DO_EVEN(vsubwev_w_h
, 32, W
, H
, DO_SUB
)
190 DO_EVEN(vsubwev_d_w
, 64, D
, W
, DO_SUB
)
192 void HELPER(vsubwod_q_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
195 VReg
*Vd
= (VReg
*)vd
;
196 VReg
*Vj
= (VReg
*)vj
;
197 VReg
*Vk
= (VReg
*)vk
;
198 int oprsz
= simd_oprsz(desc
);
200 for (i
= 0; i
< oprsz
/ 16; i
++) {
201 Vd
->Q(i
) = int128_sub(int128_makes64(Vj
->D(2 * i
+ 1)),
202 int128_makes64(Vk
->D(2 * i
+ 1)));
206 DO_ODD(vsubwod_h_b
, 16, H
, B
, DO_SUB
)
207 DO_ODD(vsubwod_w_h
, 32, W
, H
, DO_SUB
)
208 DO_ODD(vsubwod_d_w
, 64, D
, W
, DO_SUB
)
210 void HELPER(vaddwev_q_du
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
213 VReg
*Vd
= (VReg
*)vd
;
214 VReg
*Vj
= (VReg
*)vj
;
215 VReg
*Vk
= (VReg
*)vk
;
216 int oprsz
= simd_oprsz(desc
);
218 for (i
= 0; i
< oprsz
/ 16; i
++) {
219 Vd
->Q(i
) = int128_add(int128_make64(Vj
->UD(2 * i
)),
220 int128_make64(Vk
->UD(2 * i
)));
224 DO_EVEN(vaddwev_h_bu
, 16, UH
, UB
, DO_ADD
)
225 DO_EVEN(vaddwev_w_hu
, 32, UW
, UH
, DO_ADD
)
226 DO_EVEN(vaddwev_d_wu
, 64, UD
, UW
, DO_ADD
)
228 void HELPER(vaddwod_q_du
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
231 VReg
*Vd
= (VReg
*)vd
;
232 VReg
*Vj
= (VReg
*)vj
;
233 VReg
*Vk
= (VReg
*)vk
;
234 int oprsz
= simd_oprsz(desc
);
236 for (i
= 0; i
< oprsz
/ 16; i
++) {
237 Vd
->Q(i
) = int128_add(int128_make64(Vj
->UD(2 * i
+ 1)),
238 int128_make64(Vk
->UD(2 * i
+ 1)));
242 DO_ODD(vaddwod_h_bu
, 16, UH
, UB
, DO_ADD
)
243 DO_ODD(vaddwod_w_hu
, 32, UW
, UH
, DO_ADD
)
244 DO_ODD(vaddwod_d_wu
, 64, UD
, UW
, DO_ADD
)
246 void HELPER(vsubwev_q_du
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
249 VReg
*Vd
= (VReg
*)vd
;
250 VReg
*Vj
= (VReg
*)vj
;
251 VReg
*Vk
= (VReg
*)vk
;
252 int oprsz
= simd_oprsz(desc
);
254 for (i
= 0; i
< oprsz
/ 16; i
++) {
255 Vd
->Q(i
) = int128_sub(int128_make64(Vj
->UD(2 * i
)),
256 int128_make64(Vk
->UD(2 * i
)));
260 DO_EVEN(vsubwev_h_bu
, 16, UH
, UB
, DO_SUB
)
261 DO_EVEN(vsubwev_w_hu
, 32, UW
, UH
, DO_SUB
)
262 DO_EVEN(vsubwev_d_wu
, 64, UD
, UW
, DO_SUB
)
264 void HELPER(vsubwod_q_du
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
267 VReg
*Vd
= (VReg
*)vd
;
268 VReg
*Vj
= (VReg
*)vj
;
269 VReg
*Vk
= (VReg
*)vk
;
270 int oprsz
= simd_oprsz(desc
);
272 for (i
= 0; i
< oprsz
/ 16; i
++) {
273 Vd
->Q(i
) = int128_sub(int128_make64(Vj
->UD(2 * i
+ 1)),
274 int128_make64(Vk
->UD(2 * i
+ 1)));
278 DO_ODD(vsubwod_h_bu
, 16, UH
, UB
, DO_SUB
)
279 DO_ODD(vsubwod_w_hu
, 32, UW
, UH
, DO_SUB
)
280 DO_ODD(vsubwod_d_wu
, 64, UD
, UW
, DO_SUB
)
282 #define DO_EVEN_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \
283 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
286 VReg *Vd = (VReg *)vd; \
287 VReg *Vj = (VReg *)vj; \
288 VReg *Vk = (VReg *)vk; \
289 typedef __typeof(Vd->ES1(0)) TDS; \
290 typedef __typeof(Vd->EU1(0)) TDU; \
291 int oprsz = simd_oprsz(desc); \
293 for (i = 0; i < oprsz / (BIT / 8); i++) { \
294 Vd->ES1(i) = DO_OP((TDU)Vj->EU2(2 * i) ,(TDS)Vk->ES2(2 * i)); \
298 #define DO_ODD_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \
299 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
302 VReg *Vd = (VReg *)vd; \
303 VReg *Vj = (VReg *)vj; \
304 VReg *Vk = (VReg *)vk; \
305 typedef __typeof(Vd->ES1(0)) TDS; \
306 typedef __typeof(Vd->EU1(0)) TDU; \
307 int oprsz = simd_oprsz(desc); \
309 for (i = 0; i < oprsz / (BIT / 8); i++) { \
310 Vd->ES1(i) = DO_OP((TDU)Vj->EU2(2 * i + 1), (TDS)Vk->ES2(2 * i + 1)); \
314 void HELPER(vaddwev_q_du_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
317 VReg
*Vd
= (VReg
*)vd
;
318 VReg
*Vj
= (VReg
*)vj
;
319 VReg
*Vk
= (VReg
*)vk
;
320 int oprsz
= simd_oprsz(desc
);
322 for (i
= 0; i
< oprsz
/ 16; i
++) {
323 Vd
->Q(i
) = int128_add(int128_make64(Vj
->UD(2 * i
)),
324 int128_makes64(Vk
->D(2 * i
)));
328 DO_EVEN_U_S(vaddwev_h_bu_b
, 16, H
, UH
, B
, UB
, DO_ADD
)
329 DO_EVEN_U_S(vaddwev_w_hu_h
, 32, W
, UW
, H
, UH
, DO_ADD
)
330 DO_EVEN_U_S(vaddwev_d_wu_w
, 64, D
, UD
, W
, UW
, DO_ADD
)
332 void HELPER(vaddwod_q_du_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
335 VReg
*Vd
= (VReg
*)vd
;
336 VReg
*Vj
= (VReg
*)vj
;
337 VReg
*Vk
= (VReg
*)vk
;
338 int oprsz
= simd_oprsz(desc
);
340 for (i
= 0; i
< oprsz
/ 16; i
++) {
341 Vd
->Q(i
) = int128_add(int128_make64(Vj
->UD(2 * i
+ 1)),
342 int128_makes64(Vk
->D(2 * i
+ 1)));
346 DO_ODD_U_S(vaddwod_h_bu_b
, 16, H
, UH
, B
, UB
, DO_ADD
)
347 DO_ODD_U_S(vaddwod_w_hu_h
, 32, W
, UW
, H
, UH
, DO_ADD
)
348 DO_ODD_U_S(vaddwod_d_wu_w
, 64, D
, UD
, W
, UW
, DO_ADD
)
350 #define DO_VAVG(a, b) ((a >> 1) + (b >> 1) + (a & b & 1))
351 #define DO_VAVGR(a, b) ((a >> 1) + (b >> 1) + ((a | b) & 1))
353 #define DO_3OP(NAME, BIT, E, DO_OP) \
354 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
357 VReg *Vd = (VReg *)vd; \
358 VReg *Vj = (VReg *)vj; \
359 VReg *Vk = (VReg *)vk; \
360 int oprsz = simd_oprsz(desc); \
362 for (i = 0; i < oprsz / (BIT / 8); i++) { \
363 Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)); \
367 DO_3OP(vavg_b
, 8, B
, DO_VAVG
)
368 DO_3OP(vavg_h
, 16, H
, DO_VAVG
)
369 DO_3OP(vavg_w
, 32, W
, DO_VAVG
)
370 DO_3OP(vavg_d
, 64, D
, DO_VAVG
)
371 DO_3OP(vavgr_b
, 8, B
, DO_VAVGR
)
372 DO_3OP(vavgr_h
, 16, H
, DO_VAVGR
)
373 DO_3OP(vavgr_w
, 32, W
, DO_VAVGR
)
374 DO_3OP(vavgr_d
, 64, D
, DO_VAVGR
)
375 DO_3OP(vavg_bu
, 8, UB
, DO_VAVG
)
376 DO_3OP(vavg_hu
, 16, UH
, DO_VAVG
)
377 DO_3OP(vavg_wu
, 32, UW
, DO_VAVG
)
378 DO_3OP(vavg_du
, 64, UD
, DO_VAVG
)
379 DO_3OP(vavgr_bu
, 8, UB
, DO_VAVGR
)
380 DO_3OP(vavgr_hu
, 16, UH
, DO_VAVGR
)
381 DO_3OP(vavgr_wu
, 32, UW
, DO_VAVGR
)
382 DO_3OP(vavgr_du
, 64, UD
, DO_VAVGR
)
384 #define DO_VABSD(a, b) ((a > b) ? (a -b) : (b-a))
386 DO_3OP(vabsd_b
, 8, B
, DO_VABSD
)
387 DO_3OP(vabsd_h
, 16, H
, DO_VABSD
)
388 DO_3OP(vabsd_w
, 32, W
, DO_VABSD
)
389 DO_3OP(vabsd_d
, 64, D
, DO_VABSD
)
390 DO_3OP(vabsd_bu
, 8, UB
, DO_VABSD
)
391 DO_3OP(vabsd_hu
, 16, UH
, DO_VABSD
)
392 DO_3OP(vabsd_wu
, 32, UW
, DO_VABSD
)
393 DO_3OP(vabsd_du
, 64, UD
, DO_VABSD
)
395 #define DO_VABS(a) ((a < 0) ? (-a) : (a))
397 #define DO_VADDA(NAME, BIT, E) \
398 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
401 VReg *Vd = (VReg *)vd; \
402 VReg *Vj = (VReg *)vj; \
403 VReg *Vk = (VReg *)vk; \
404 int oprsz = simd_oprsz(desc); \
406 for (i = 0; i < oprsz / (BIT / 8); i++) { \
407 Vd->E(i) = DO_VABS(Vj->E(i)) + DO_VABS(Vk->E(i)); \
411 DO_VADDA(vadda_b
, 8, B
)
412 DO_VADDA(vadda_h
, 16, H
)
413 DO_VADDA(vadda_w
, 32, W
)
414 DO_VADDA(vadda_d
, 64, D
)
416 #define DO_MIN(a, b) (a < b ? a : b)
417 #define DO_MAX(a, b) (a > b ? a : b)
419 #define VMINMAXI(NAME, BIT, E, DO_OP) \
420 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
423 VReg *Vd = (VReg *)vd; \
424 VReg *Vj = (VReg *)vj; \
425 typedef __typeof(Vd->E(0)) TD; \
426 int oprsz = simd_oprsz(desc); \
428 for (i = 0; i < oprsz / (BIT / 8); i++) { \
429 Vd->E(i) = DO_OP(Vj->E(i), (TD)imm); \
433 VMINMAXI(vmini_b
, 8, B
, DO_MIN
)
434 VMINMAXI(vmini_h
, 16, H
, DO_MIN
)
435 VMINMAXI(vmini_w
, 32, W
, DO_MIN
)
436 VMINMAXI(vmini_d
, 64, D
, DO_MIN
)
437 VMINMAXI(vmaxi_b
, 8, B
, DO_MAX
)
438 VMINMAXI(vmaxi_h
, 16, H
, DO_MAX
)
439 VMINMAXI(vmaxi_w
, 32, W
, DO_MAX
)
440 VMINMAXI(vmaxi_d
, 64, D
, DO_MAX
)
441 VMINMAXI(vmini_bu
, 8, UB
, DO_MIN
)
442 VMINMAXI(vmini_hu
, 16, UH
, DO_MIN
)
443 VMINMAXI(vmini_wu
, 32, UW
, DO_MIN
)
444 VMINMAXI(vmini_du
, 64, UD
, DO_MIN
)
445 VMINMAXI(vmaxi_bu
, 8, UB
, DO_MAX
)
446 VMINMAXI(vmaxi_hu
, 16, UH
, DO_MAX
)
447 VMINMAXI(vmaxi_wu
, 32, UW
, DO_MAX
)
448 VMINMAXI(vmaxi_du
, 64, UD
, DO_MAX
)
450 #define DO_VMUH(NAME, BIT, E1, E2, DO_OP) \
451 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
454 VReg *Vd = (VReg *)vd; \
455 VReg *Vj = (VReg *)vj; \
456 VReg *Vk = (VReg *)vk; \
457 typedef __typeof(Vd->E1(0)) T; \
458 int oprsz = simd_oprsz(desc); \
460 for (i = 0; i < oprsz / (BIT / 8); i++) { \
461 Vd->E2(i) = ((T)Vj->E2(i)) * ((T)Vk->E2(i)) >> BIT; \
465 void HELPER(vmuh_d
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
469 VReg
*Vd
= (VReg
*)vd
;
470 VReg
*Vj
= (VReg
*)vj
;
471 VReg
*Vk
= (VReg
*)vk
;
472 int oprsz
= simd_oprsz(desc
);
474 for (i
= 0; i
< oprsz
/ 8; i
++) {
475 muls64(&l
, &h
, Vj
->D(i
), Vk
->D(i
));
480 DO_VMUH(vmuh_b
, 8, H
, B
, DO_MUH
)
481 DO_VMUH(vmuh_h
, 16, W
, H
, DO_MUH
)
482 DO_VMUH(vmuh_w
, 32, D
, W
, DO_MUH
)
484 void HELPER(vmuh_du
)(void *vd
, void *vj
, void *vk
, uint32_t desc
)
488 VReg
*Vd
= (VReg
*)vd
;
489 VReg
*Vj
= (VReg
*)vj
;
490 VReg
*Vk
= (VReg
*)vk
;
491 int oprsz
= simd_oprsz(desc
);
493 for (i
= 0; i
< oprsz
/ 8; i
++) {
494 mulu64(&l
, &h
, Vj
->D(i
), Vk
->D(i
));
499 DO_VMUH(vmuh_bu
, 8, UH
, UB
, DO_MUH
)
500 DO_VMUH(vmuh_hu
, 16, UW
, UH
, DO_MUH
)
501 DO_VMUH(vmuh_wu
, 32, UD
, UW
, DO_MUH
)
503 #define DO_MUL(a, b) (a * b)
505 DO_EVEN(vmulwev_h_b
, 16, H
, B
, DO_MUL
)
506 DO_EVEN(vmulwev_w_h
, 32, W
, H
, DO_MUL
)
507 DO_EVEN(vmulwev_d_w
, 64, D
, W
, DO_MUL
)
509 DO_ODD(vmulwod_h_b
, 16, H
, B
, DO_MUL
)
510 DO_ODD(vmulwod_w_h
, 32, W
, H
, DO_MUL
)
511 DO_ODD(vmulwod_d_w
, 64, D
, W
, DO_MUL
)
513 DO_EVEN(vmulwev_h_bu
, 16, UH
, UB
, DO_MUL
)
514 DO_EVEN(vmulwev_w_hu
, 32, UW
, UH
, DO_MUL
)
515 DO_EVEN(vmulwev_d_wu
, 64, UD
, UW
, DO_MUL
)
517 DO_ODD(vmulwod_h_bu
, 16, UH
, UB
, DO_MUL
)
518 DO_ODD(vmulwod_w_hu
, 32, UW
, UH
, DO_MUL
)
519 DO_ODD(vmulwod_d_wu
, 64, UD
, UW
, DO_MUL
)
521 DO_EVEN_U_S(vmulwev_h_bu_b
, 16, H
, UH
, B
, UB
, DO_MUL
)
522 DO_EVEN_U_S(vmulwev_w_hu_h
, 32, W
, UW
, H
, UH
, DO_MUL
)
523 DO_EVEN_U_S(vmulwev_d_wu_w
, 64, D
, UD
, W
, UW
, DO_MUL
)
525 DO_ODD_U_S(vmulwod_h_bu_b
, 16, H
, UH
, B
, UB
, DO_MUL
)
526 DO_ODD_U_S(vmulwod_w_hu_h
, 32, W
, UW
, H
, UH
, DO_MUL
)
527 DO_ODD_U_S(vmulwod_d_wu_w
, 64, D
, UD
, W
, UW
, DO_MUL
)
529 #define DO_MADD(a, b, c) (a + b * c)
530 #define DO_MSUB(a, b, c) (a - b * c)
532 #define VMADDSUB(NAME, BIT, E, DO_OP) \
533 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
536 VReg *Vd = (VReg *)vd; \
537 VReg *Vj = (VReg *)vj; \
538 VReg *Vk = (VReg *)vk; \
539 int oprsz = simd_oprsz(desc); \
541 for (i = 0; i < oprsz / (BIT / 8); i++) { \
542 Vd->E(i) = DO_OP(Vd->E(i), Vj->E(i) ,Vk->E(i)); \
546 VMADDSUB(vmadd_b
, 8, B
, DO_MADD
)
547 VMADDSUB(vmadd_h
, 16, H
, DO_MADD
)
548 VMADDSUB(vmadd_w
, 32, W
, DO_MADD
)
549 VMADDSUB(vmadd_d
, 64, D
, DO_MADD
)
550 VMADDSUB(vmsub_b
, 8, B
, DO_MSUB
)
551 VMADDSUB(vmsub_h
, 16, H
, DO_MSUB
)
552 VMADDSUB(vmsub_w
, 32, W
, DO_MSUB
)
553 VMADDSUB(vmsub_d
, 64, D
, DO_MSUB
)
555 #define VMADDWEV(NAME, BIT, E1, E2, DO_OP) \
556 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
559 VReg *Vd = (VReg *)vd; \
560 VReg *Vj = (VReg *)vj; \
561 VReg *Vk = (VReg *)vk; \
562 typedef __typeof(Vd->E1(0)) TD; \
563 int oprsz = simd_oprsz(desc); \
565 for (i = 0; i < oprsz / (BIT / 8); i++) { \
566 Vd->E1(i) += DO_OP((TD)Vj->E2(2 * i), (TD)Vk->E2(2 * i)); \
570 VMADDWEV(vmaddwev_h_b
, 16, H
, B
, DO_MUL
)
571 VMADDWEV(vmaddwev_w_h
, 32, W
, H
, DO_MUL
)
572 VMADDWEV(vmaddwev_d_w
, 64, D
, W
, DO_MUL
)
573 VMADDWEV(vmaddwev_h_bu
, 16, UH
, UB
, DO_MUL
)
574 VMADDWEV(vmaddwev_w_hu
, 32, UW
, UH
, DO_MUL
)
575 VMADDWEV(vmaddwev_d_wu
, 64, UD
, UW
, DO_MUL
)
577 #define VMADDWOD(NAME, BIT, E1, E2, DO_OP) \
578 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
581 VReg *Vd = (VReg *)vd; \
582 VReg *Vj = (VReg *)vj; \
583 VReg *Vk = (VReg *)vk; \
584 typedef __typeof(Vd->E1(0)) TD; \
585 int oprsz = simd_oprsz(desc); \
587 for (i = 0; i < oprsz / (BIT / 8); i++) { \
588 Vd->E1(i) += DO_OP((TD)Vj->E2(2 * i + 1), \
589 (TD)Vk->E2(2 * i + 1)); \
593 VMADDWOD(vmaddwod_h_b
, 16, H
, B
, DO_MUL
)
594 VMADDWOD(vmaddwod_w_h
, 32, W
, H
, DO_MUL
)
595 VMADDWOD(vmaddwod_d_w
, 64, D
, W
, DO_MUL
)
596 VMADDWOD(vmaddwod_h_bu
, 16, UH
, UB
, DO_MUL
)
597 VMADDWOD(vmaddwod_w_hu
, 32, UW
, UH
, DO_MUL
)
598 VMADDWOD(vmaddwod_d_wu
, 64, UD
, UW
, DO_MUL
)
600 #define VMADDWEV_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \
601 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
604 VReg *Vd = (VReg *)vd; \
605 VReg *Vj = (VReg *)vj; \
606 VReg *Vk = (VReg *)vk; \
607 typedef __typeof(Vd->ES1(0)) TS1; \
608 typedef __typeof(Vd->EU1(0)) TU1; \
609 int oprsz = simd_oprsz(desc); \
611 for (i = 0; i < oprsz / (BIT / 8); i++) { \
612 Vd->ES1(i) += DO_OP((TU1)Vj->EU2(2 * i), \
613 (TS1)Vk->ES2(2 * i)); \
617 VMADDWEV_U_S(vmaddwev_h_bu_b
, 16, H
, UH
, B
, UB
, DO_MUL
)
618 VMADDWEV_U_S(vmaddwev_w_hu_h
, 32, W
, UW
, H
, UH
, DO_MUL
)
619 VMADDWEV_U_S(vmaddwev_d_wu_w
, 64, D
, UD
, W
, UW
, DO_MUL
)
621 #define VMADDWOD_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \
622 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
625 VReg *Vd = (VReg *)vd; \
626 VReg *Vj = (VReg *)vj; \
627 VReg *Vk = (VReg *)vk; \
628 typedef __typeof(Vd->ES1(0)) TS1; \
629 typedef __typeof(Vd->EU1(0)) TU1; \
630 int oprsz = simd_oprsz(desc); \
632 for (i = 0; i < oprsz / (BIT / 8); i++) { \
633 Vd->ES1(i) += DO_OP((TU1)Vj->EU2(2 * i + 1), \
634 (TS1)Vk->ES2(2 * i + 1)); \
638 VMADDWOD_U_S(vmaddwod_h_bu_b
, 16, H
, UH
, B
, UB
, DO_MUL
)
639 VMADDWOD_U_S(vmaddwod_w_hu_h
, 32, W
, UW
, H
, UH
, DO_MUL
)
640 VMADDWOD_U_S(vmaddwod_d_wu_w
, 64, D
, UD
, W
, UW
, DO_MUL
)
642 #define DO_DIVU(N, M) (unlikely(M == 0) ? 0 : N / M)
643 #define DO_REMU(N, M) (unlikely(M == 0) ? 0 : N % M)
644 #define DO_DIV(N, M) (unlikely(M == 0) ? 0 :\
645 unlikely((N == -N) && (M == (__typeof(N))(-1))) ? N : N / M)
646 #define DO_REM(N, M) (unlikely(M == 0) ? 0 :\
647 unlikely((N == -N) && (M == (__typeof(N))(-1))) ? 0 : N % M)
649 #define VDIV(NAME, BIT, E, DO_OP) \
650 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
653 VReg *Vd = (VReg *)vd; \
654 VReg *Vj = (VReg *)vj; \
655 VReg *Vk = (VReg *)vk; \
656 int oprsz = simd_oprsz(desc); \
658 for (i = 0; i < oprsz / (BIT / 8); i++) { \
659 Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)); \
663 VDIV(vdiv_b
, 8, B
, DO_DIV
)
664 VDIV(vdiv_h
, 16, H
, DO_DIV
)
665 VDIV(vdiv_w
, 32, W
, DO_DIV
)
666 VDIV(vdiv_d
, 64, D
, DO_DIV
)
667 VDIV(vdiv_bu
, 8, UB
, DO_DIVU
)
668 VDIV(vdiv_hu
, 16, UH
, DO_DIVU
)
669 VDIV(vdiv_wu
, 32, UW
, DO_DIVU
)
670 VDIV(vdiv_du
, 64, UD
, DO_DIVU
)
671 VDIV(vmod_b
, 8, B
, DO_REM
)
672 VDIV(vmod_h
, 16, H
, DO_REM
)
673 VDIV(vmod_w
, 32, W
, DO_REM
)
674 VDIV(vmod_d
, 64, D
, DO_REM
)
675 VDIV(vmod_bu
, 8, UB
, DO_REMU
)
676 VDIV(vmod_hu
, 16, UH
, DO_REMU
)
677 VDIV(vmod_wu
, 32, UW
, DO_REMU
)
678 VDIV(vmod_du
, 64, UD
, DO_REMU
)
680 #define VSAT_S(NAME, BIT, E) \
681 void HELPER(NAME)(void *vd, void *vj, uint64_t max, uint32_t desc) \
684 VReg *Vd = (VReg *)vd; \
685 VReg *Vj = (VReg *)vj; \
686 typedef __typeof(Vd->E(0)) TD; \
687 int oprsz = simd_oprsz(desc); \
689 for (i = 0; i < oprsz / (BIT / 8); i++) { \
690 Vd->E(i) = Vj->E(i) > (TD)max ? (TD)max : \
691 Vj->E(i) < (TD)~max ? (TD)~max: Vj->E(i); \
696 VSAT_S(vsat_h
, 16, H
)
697 VSAT_S(vsat_w
, 32, W
)
698 VSAT_S(vsat_d
, 64, D
)
700 #define VSAT_U(NAME, BIT, E) \
701 void HELPER(NAME)(void *vd, void *vj, uint64_t max, uint32_t desc) \
704 VReg *Vd = (VReg *)vd; \
705 VReg *Vj = (VReg *)vj; \
706 typedef __typeof(Vd->E(0)) TD; \
707 int oprsz = simd_oprsz(desc); \
709 for (i = 0; i < oprsz / (BIT / 8); i++) { \
710 Vd->E(i) = Vj->E(i) > (TD)max ? (TD)max : Vj->E(i); \
714 VSAT_U(vsat_bu
, 8, UB
)
715 VSAT_U(vsat_hu
, 16, UH
)
716 VSAT_U(vsat_wu
, 32, UW
)
717 VSAT_U(vsat_du
, 64, UD
)
719 #define VEXTH(NAME, BIT, E1, E2) \
720 void HELPER(NAME)(void *vd, void *vj, uint32_t desc) \
723 VReg *Vd = (VReg *)vd; \
724 VReg *Vj = (VReg *)vj; \
725 int oprsz = simd_oprsz(desc); \
727 ofs = LSX_LEN / BIT; \
728 for (i = 0; i < oprsz / 16; i++) { \
729 for (j = 0; j < ofs; j++) { \
730 Vd->E1(j + i * ofs) = Vj->E2(j + ofs + ofs * 2 * i); \
735 void HELPER(vexth_q_d
)(void *vd
, void *vj
, uint32_t desc
)
738 VReg
*Vd
= (VReg
*)vd
;
739 VReg
*Vj
= (VReg
*)vj
;
740 int oprsz
= simd_oprsz(desc
);
742 for (i
= 0; i
< oprsz
/ 16; i
++) {
743 Vd
->Q(i
) = int128_makes64(Vj
->D(2 * i
+ 1));
747 void HELPER(vexth_qu_du
)(void *vd
, void *vj
, uint32_t desc
)
750 VReg
*Vd
= (VReg
*)vd
;
751 VReg
*Vj
= (VReg
*)vj
;
752 int oprsz
= simd_oprsz(desc
);
754 for (i
= 0; i
< oprsz
/ 16; i
++) {
755 Vd
->Q(i
) = int128_make64(Vj
->UD(2 * i
+ 1));
759 VEXTH(vexth_h_b
, 16, H
, B
)
760 VEXTH(vexth_w_h
, 32, W
, H
)
761 VEXTH(vexth_d_w
, 64, D
, W
)
762 VEXTH(vexth_hu_bu
, 16, UH
, UB
)
763 VEXTH(vexth_wu_hu
, 32, UW
, UH
)
764 VEXTH(vexth_du_wu
, 64, UD
, UW
)
766 #define VEXT2XV(NAME, BIT, E1, E2) \
767 void HELPER(NAME)(void *vd, void *vj, uint32_t desc) \
771 VReg *Vd = (VReg *)vd; \
772 VReg *Vj = (VReg *)vj; \
773 int oprsz = simd_oprsz(desc); \
775 for (i = 0; i < oprsz / (BIT / 8); i++) { \
776 temp.E1(i) = Vj->E2(i); \
781 VEXT2XV(vext2xv_h_b
, 16, H
, B
)
782 VEXT2XV(vext2xv_w_b
, 32, W
, B
)
783 VEXT2XV(vext2xv_d_b
, 64, D
, B
)
784 VEXT2XV(vext2xv_w_h
, 32, W
, H
)
785 VEXT2XV(vext2xv_d_h
, 64, D
, H
)
786 VEXT2XV(vext2xv_d_w
, 64, D
, W
)
787 VEXT2XV(vext2xv_hu_bu
, 16, UH
, UB
)
788 VEXT2XV(vext2xv_wu_bu
, 32, UW
, UB
)
789 VEXT2XV(vext2xv_du_bu
, 64, UD
, UB
)
790 VEXT2XV(vext2xv_wu_hu
, 32, UW
, UH
)
791 VEXT2XV(vext2xv_du_hu
, 64, UD
, UH
)
792 VEXT2XV(vext2xv_du_wu
, 64, UD
, UW
)
794 #define DO_SIGNCOV(a, b) (a == 0 ? 0 : a < 0 ? -b : b)
796 DO_3OP(vsigncov_b
, 8, B
, DO_SIGNCOV
)
797 DO_3OP(vsigncov_h
, 16, H
, DO_SIGNCOV
)
798 DO_3OP(vsigncov_w
, 32, W
, DO_SIGNCOV
)
799 DO_3OP(vsigncov_d
, 64, D
, DO_SIGNCOV
)
801 static uint64_t do_vmskltz_b(int64_t val
)
803 uint64_t m
= 0x8080808080808080ULL
;
804 uint64_t c
= val
& m
;
811 void HELPER(vmskltz_b
)(void *vd
, void *vj
, uint32_t desc
)
815 VReg
*Vd
= (VReg
*)vd
;
816 VReg
*Vj
= (VReg
*)vj
;
817 int oprsz
= simd_oprsz(desc
);
819 for (i
= 0; i
< oprsz
/ 16; i
++) {
821 temp
= do_vmskltz_b(Vj
->D(2 * i
));
822 temp
|= (do_vmskltz_b(Vj
->D(2 * i
+ 1)) << 8);
824 Vd
->D(2 * i
+ 1) = 0;
828 static uint64_t do_vmskltz_h(int64_t val
)
830 uint64_t m
= 0x8000800080008000ULL
;
831 uint64_t c
= val
& m
;
837 void HELPER(vmskltz_h
)(void *vd
, void *vj
, uint32_t desc
)
841 VReg
*Vd
= (VReg
*)vd
;
842 VReg
*Vj
= (VReg
*)vj
;
843 int oprsz
= simd_oprsz(desc
);
845 for (i
= 0; i
< oprsz
/ 16; i
++) {
847 temp
= do_vmskltz_h(Vj
->D(2 * i
));
848 temp
|= (do_vmskltz_h(Vj
->D(2 * i
+ 1)) << 4);
850 Vd
->D(2 * i
+ 1) = 0;
854 static uint64_t do_vmskltz_w(int64_t val
)
856 uint64_t m
= 0x8000000080000000ULL
;
857 uint64_t c
= val
& m
;
862 void HELPER(vmskltz_w
)(void *vd
, void *vj
, uint32_t desc
)
866 VReg
*Vd
= (VReg
*)vd
;
867 VReg
*Vj
= (VReg
*)vj
;
868 int oprsz
= simd_oprsz(desc
);
870 for (i
= 0; i
< oprsz
/ 16; i
++) {
872 temp
= do_vmskltz_w(Vj
->D(2 * i
));
873 temp
|= (do_vmskltz_w(Vj
->D(2 * i
+ 1)) << 2);
875 Vd
->D(2 * i
+ 1) = 0;
879 static uint64_t do_vmskltz_d(int64_t val
)
881 return (uint64_t)val
>> 63;
883 void HELPER(vmskltz_d
)(void *vd
, void *vj
, uint32_t desc
)
887 VReg
*Vd
= (VReg
*)vd
;
888 VReg
*Vj
= (VReg
*)vj
;
889 int oprsz
= simd_oprsz(desc
);
891 for (i
= 0; i
< oprsz
/ 16; i
++) {
893 temp
= do_vmskltz_d(Vj
->D(2 * i
));
894 temp
|= (do_vmskltz_d(Vj
->D(2 * i
+ 1)) << 1);
896 Vd
->D(2 * i
+ 1) = 0;
900 void HELPER(vmskgez_b
)(void *vd
, void *vj
, uint32_t desc
)
904 VReg
*Vd
= (VReg
*)vd
;
905 VReg
*Vj
= (VReg
*)vj
;
906 int oprsz
= simd_oprsz(desc
);
908 for (i
= 0; i
< oprsz
/ 16; i
++) {
910 temp
= do_vmskltz_b(Vj
->D(2 * i
));
911 temp
|= (do_vmskltz_b(Vj
->D(2 * i
+ 1)) << 8);
912 Vd
->D(2 * i
) = (uint16_t)(~temp
);
913 Vd
->D(2 * i
+ 1) = 0;
917 static uint64_t do_vmskez_b(uint64_t a
)
919 uint64_t m
= 0x7f7f7f7f7f7f7f7fULL
;
920 uint64_t c
= ~(((a
& m
) + m
) | a
| m
);
927 void HELPER(vmsknz_b
)(void *vd
, void *vj
, uint32_t desc
)
931 VReg
*Vd
= (VReg
*)vd
;
932 VReg
*Vj
= (VReg
*)vj
;
933 int oprsz
= simd_oprsz(desc
);
935 for (i
= 0; i
< oprsz
/ 16; i
++) {
937 temp
= do_vmskez_b(Vj
->D(2 * i
));
938 temp
|= (do_vmskez_b(Vj
->D(2 * i
+ 1)) << 8);
939 Vd
->D(2 * i
) = (uint16_t)(~temp
);
940 Vd
->D(2 * i
+ 1) = 0;
944 void HELPER(vnori_b
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
947 VReg
*Vd
= (VReg
*)vd
;
948 VReg
*Vj
= (VReg
*)vj
;
950 for (i
= 0; i
< simd_oprsz(desc
); i
++) {
951 Vd
->B(i
) = ~(Vj
->B(i
) | (uint8_t)imm
);
955 #define VSLLWIL(NAME, BIT, E1, E2) \
956 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
960 VReg *Vd = (VReg *)vd; \
961 VReg *Vj = (VReg *)vj; \
962 int oprsz = simd_oprsz(desc); \
963 typedef __typeof(temp.E1(0)) TD; \
965 ofs = LSX_LEN / BIT; \
966 for (i = 0; i < oprsz / 16; i++) { \
967 for (j = 0; j < ofs; j++) { \
968 temp.E1(j + ofs * i) = (TD)Vj->E2(j + ofs * 2 * i) << (imm % BIT); \
975 void HELPER(vextl_q_d
)(void *vd
, void *vj
, uint32_t desc
)
978 VReg
*Vd
= (VReg
*)vd
;
979 VReg
*Vj
= (VReg
*)vj
;
980 int oprsz
= simd_oprsz(desc
);
982 for (i
= 0; i
< oprsz
/ 16; i
++) {
983 Vd
->Q(i
) = int128_makes64(Vj
->D(2 * i
));
987 void HELPER(vextl_qu_du
)(void *vd
, void *vj
, uint32_t desc
)
990 VReg
*Vd
= (VReg
*)vd
;
991 VReg
*Vj
= (VReg
*)vj
;
992 int oprsz
= simd_oprsz(desc
);
994 for (i
= 0; i
< oprsz
/ 16; i
++) {
995 Vd
->Q(i
) = int128_make64(Vj
->UD(2 * i
));
999 VSLLWIL(vsllwil_h_b
, 16, H
, B
)
1000 VSLLWIL(vsllwil_w_h
, 32, W
, H
)
1001 VSLLWIL(vsllwil_d_w
, 64, D
, W
)
1002 VSLLWIL(vsllwil_hu_bu
, 16, UH
, UB
)
1003 VSLLWIL(vsllwil_wu_hu
, 32, UW
, UH
)
1004 VSLLWIL(vsllwil_du_wu
, 64, UD
, UW
)
1006 #define do_vsrlr(E, T) \
1007 static T do_vsrlr_ ##E(T s1, int sh) \
1012 return (s1 >> sh) + ((s1 >> (sh - 1)) & 0x1); \
1016 do_vsrlr(B
, uint8_t)
1017 do_vsrlr(H
, uint16_t)
1018 do_vsrlr(W
, uint32_t)
1019 do_vsrlr(D
, uint64_t)
1021 #define VSRLR(NAME, BIT, T, E) \
1022 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1025 VReg *Vd = (VReg *)vd; \
1026 VReg *Vj = (VReg *)vj; \
1027 VReg *Vk = (VReg *)vk; \
1028 int oprsz = simd_oprsz(desc); \
1030 for (i = 0; i < oprsz / (BIT / 8); i++) { \
1031 Vd->E(i) = do_vsrlr_ ## E(Vj->E(i), ((T)Vk->E(i))%BIT); \
1035 VSRLR(vsrlr_b
, 8, uint8_t, B
)
1036 VSRLR(vsrlr_h
, 16, uint16_t, H
)
1037 VSRLR(vsrlr_w
, 32, uint32_t, W
)
1038 VSRLR(vsrlr_d
, 64, uint64_t, D
)
1040 #define VSRLRI(NAME, BIT, E) \
1041 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1044 VReg *Vd = (VReg *)vd; \
1045 VReg *Vj = (VReg *)vj; \
1046 int oprsz = simd_oprsz(desc); \
1048 for (i = 0; i < oprsz / (BIT / 8); i++) { \
1049 Vd->E(i) = do_vsrlr_ ## E(Vj->E(i), imm); \
1053 VSRLRI(vsrlri_b
, 8, B
)
1054 VSRLRI(vsrlri_h
, 16, H
)
1055 VSRLRI(vsrlri_w
, 32, W
)
1056 VSRLRI(vsrlri_d
, 64, D
)
1058 #define do_vsrar(E, T) \
1059 static T do_vsrar_ ##E(T s1, int sh) \
1064 return (s1 >> sh) + ((s1 >> (sh - 1)) & 0x1); \
1069 do_vsrar(H
, int16_t)
1070 do_vsrar(W
, int32_t)
1071 do_vsrar(D
, int64_t)
1073 #define VSRAR(NAME, BIT, T, E) \
1074 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1077 VReg *Vd = (VReg *)vd; \
1078 VReg *Vj = (VReg *)vj; \
1079 VReg *Vk = (VReg *)vk; \
1080 int oprsz = simd_oprsz(desc); \
1082 for (i = 0; i < oprsz / (BIT / 8); i++) { \
1083 Vd->E(i) = do_vsrar_ ## E(Vj->E(i), ((T)Vk->E(i))%BIT); \
1087 VSRAR(vsrar_b
, 8, uint8_t, B
)
1088 VSRAR(vsrar_h
, 16, uint16_t, H
)
1089 VSRAR(vsrar_w
, 32, uint32_t, W
)
1090 VSRAR(vsrar_d
, 64, uint64_t, D
)
1092 #define VSRARI(NAME, BIT, E) \
1093 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1096 VReg *Vd = (VReg *)vd; \
1097 VReg *Vj = (VReg *)vj; \
1098 int oprsz = simd_oprsz(desc); \
1100 for (i = 0; i < oprsz / (BIT / 8); i++) { \
1101 Vd->E(i) = do_vsrar_ ## E(Vj->E(i), imm); \
1105 VSRARI(vsrari_b
, 8, B
)
1106 VSRARI(vsrari_h
, 16, H
)
1107 VSRARI(vsrari_w
, 32, W
)
1108 VSRARI(vsrari_d
, 64, D
)
1110 #define R_SHIFT(a, b) (a >> b)
1112 #define VSRLN(NAME, BIT, E1, E2) \
1113 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1116 VReg *Vd = (VReg *)vd; \
1117 VReg *Vj = (VReg *)vj; \
1118 VReg *Vk = (VReg *)vk; \
1119 int oprsz = simd_oprsz(desc); \
1121 ofs = LSX_LEN / BIT; \
1122 for (i = 0; i < oprsz / 16; i++) { \
1123 for (j = 0; j < ofs; j++) { \
1124 Vd->E1(j + ofs * 2 * i) = R_SHIFT(Vj->E2(j + ofs * i), \
1125 Vk->E2(j + ofs * i) % BIT); \
1127 Vd->D(2 * i + 1) = 0; \
1131 VSRLN(vsrln_b_h
, 16, B
, UH
)
1132 VSRLN(vsrln_h_w
, 32, H
, UW
)
1133 VSRLN(vsrln_w_d
, 64, W
, UD
)
1135 #define VSRAN(NAME, BIT, E1, E2, E3) \
1136 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1139 VReg *Vd = (VReg *)vd; \
1140 VReg *Vj = (VReg *)vj; \
1141 VReg *Vk = (VReg *)vk; \
1142 int oprsz = simd_oprsz(desc); \
1144 ofs = LSX_LEN / BIT; \
1145 for (i = 0; i < oprsz / 16; i++) { \
1146 for (j = 0; j < ofs; j++) { \
1147 Vd->E1(j + ofs * 2 * i) = R_SHIFT(Vj->E2(j + ofs * i), \
1148 Vk->E3(j + ofs * i) % BIT); \
1150 Vd->D(2 * i + 1) = 0; \
1154 VSRAN(vsran_b_h
, 16, B
, H
, UH
)
1155 VSRAN(vsran_h_w
, 32, H
, W
, UW
)
1156 VSRAN(vsran_w_d
, 64, W
, D
, UD
)
1158 #define VSRLNI(NAME, BIT, E1, E2) \
1159 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1163 VReg *Vd = (VReg *)vd; \
1164 VReg *Vj = (VReg *)vj; \
1165 int oprsz = simd_oprsz(desc); \
1167 ofs = LSX_LEN / BIT; \
1168 for (i = 0; i < oprsz / 16; i++) { \
1169 for (j = 0; j < ofs; j++) { \
1170 temp.E1(j + ofs * 2 * i) = R_SHIFT(Vj->E2(j + ofs * i), imm); \
1171 temp.E1(j + ofs * (2 * i + 1)) = R_SHIFT(Vd->E2(j + ofs * i), \
1178 void HELPER(vsrlni_d_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
1182 VReg
*Vd
= (VReg
*)vd
;
1183 VReg
*Vj
= (VReg
*)vj
;
1185 for (i
= 0; i
< 2; i
++) {
1186 temp
.D(2 * i
) = int128_getlo(int128_urshift(Vj
->Q(i
), imm
% 128));
1187 temp
.D(2 * i
+1) = int128_getlo(int128_urshift(Vd
->Q(i
), imm
% 128));
1192 VSRLNI(vsrlni_b_h
, 16, B
, UH
)
1193 VSRLNI(vsrlni_h_w
, 32, H
, UW
)
1194 VSRLNI(vsrlni_w_d
, 64, W
, UD
)
1196 #define VSRANI(NAME, BIT, E1, E2) \
1197 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1201 VReg *Vd = (VReg *)vd; \
1202 VReg *Vj = (VReg *)vj; \
1203 int oprsz = simd_oprsz(desc); \
1205 ofs = LSX_LEN / BIT; \
1206 for (i = 0; i < oprsz / 16; i++) { \
1207 for (j = 0; j < ofs; j++) { \
1208 temp.E1(j + ofs * 2 * i) = R_SHIFT(Vj->E2(j + ofs * i), imm); \
1209 temp.E1(j + ofs * (2 * i + 1)) = R_SHIFT(Vd->E2(j + ofs * i), \
1216 void HELPER(vsrani_d_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
1220 VReg
*Vd
= (VReg
*)vd
;
1221 VReg
*Vj
= (VReg
*)vj
;
1223 for (i
= 0; i
< 2; i
++) {
1224 temp
.D(2 * i
) = int128_getlo(int128_rshift(Vj
->Q(i
), imm
% 128));
1225 temp
.D(2 * i
+ 1) = int128_getlo(int128_rshift(Vd
->Q(i
), imm
% 128));
1230 VSRANI(vsrani_b_h
, 16, B
, H
)
1231 VSRANI(vsrani_h_w
, 32, H
, W
)
1232 VSRANI(vsrani_w_d
, 64, W
, D
)
1234 #define VSRLRN(NAME, BIT, E1, E2, E3) \
1235 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1238 VReg *Vd = (VReg *)vd; \
1239 VReg *Vj = (VReg *)vj; \
1240 VReg *Vk = (VReg *)vk; \
1241 int oprsz = simd_oprsz(desc); \
1243 ofs = LSX_LEN / BIT; \
1244 for (i = 0; i < oprsz / 16; i++) { \
1245 for (j = 0; j < ofs; j++) { \
1246 Vd->E1(j + ofs * 2 * i) = do_vsrlr_ ##E2(Vj->E2(j + ofs * i), \
1247 Vk->E3(j + ofs * i) % BIT); \
1249 Vd->D(2 * i + 1) = 0; \
1253 VSRLRN(vsrlrn_b_h
, 16, B
, H
, UH
)
1254 VSRLRN(vsrlrn_h_w
, 32, H
, W
, UW
)
1255 VSRLRN(vsrlrn_w_d
, 64, W
, D
, UD
)
1257 #define VSRARN(NAME, BIT, E1, E2, E3) \
1258 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1261 VReg *Vd = (VReg *)vd; \
1262 VReg *Vj = (VReg *)vj; \
1263 VReg *Vk = (VReg *)vk; \
1264 int oprsz = simd_oprsz(desc); \
1266 ofs = LSX_LEN / BIT; \
1267 for (i = 0; i < oprsz / 16; i++) { \
1268 for (j = 0; j < ofs; j++) { \
1269 Vd->E1(j + ofs * 2 * i) = do_vsrar_ ## E2(Vj->E2(j + ofs * i), \
1270 Vk->E3(j + ofs * i) % BIT); \
1272 Vd->D(2 * i + 1) = 0; \
1276 VSRARN(vsrarn_b_h
, 16, B
, H
, UH
)
1277 VSRARN(vsrarn_h_w
, 32, H
, W
, UW
)
1278 VSRARN(vsrarn_w_d
, 64, W
, D
, UD
)
1280 #define VSRLRNI(NAME, BIT, E1, E2) \
1281 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1285 VReg *Vd = (VReg *)vd; \
1286 VReg *Vj = (VReg *)vj; \
1287 int oprsz = simd_oprsz(desc); \
1289 ofs = LSX_LEN / BIT; \
1290 for (i = 0; i < oprsz / 16; i++) { \
1291 for (j = 0; j < ofs; j++) { \
1292 temp.E1(j + ofs * 2 * i) = do_vsrlr_ ## E2(Vj->E2(j + ofs * i), imm); \
1293 temp.E1(j + ofs * (2 * i + 1)) = do_vsrlr_ ## E2(Vd->E2(j + ofs * i), \
1300 void HELPER(vsrlrni_d_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
1304 VReg
*Vd
= (VReg
*)vd
;
1305 VReg
*Vj
= (VReg
*)vj
;
1307 int oprsz
= simd_oprsz(desc
);
1309 for (i
= 0; i
< oprsz
/ 16; i
++) {
1311 temp
.D(2 * i
) = int128_getlo(Vj
->Q(i
));
1312 temp
.D(2 * i
+ 1) = int128_getlo(Vd
->Q(i
));
1314 r
[2 * i
] = int128_and(int128_urshift(Vj
->Q(i
), (imm
- 1)),
1316 r
[2 * i
+ 1] = int128_and(int128_urshift(Vd
->Q(i
), (imm
- 1)),
1318 temp
.D(2 * i
) = int128_getlo(int128_add(int128_urshift(Vj
->Q(i
),
1320 temp
.D(2 * i
+ 1) = int128_getlo(int128_add(int128_urshift(Vd
->Q(i
),
1321 imm
), r
[ 2 * i
+ 1]));
1327 VSRLRNI(vsrlrni_b_h
, 16, B
, H
)
1328 VSRLRNI(vsrlrni_h_w
, 32, H
, W
)
1329 VSRLRNI(vsrlrni_w_d
, 64, W
, D
)
1331 #define VSRARNI(NAME, BIT, E1, E2) \
1332 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1336 VReg *Vd = (VReg *)vd; \
1337 VReg *Vj = (VReg *)vj; \
1338 int oprsz = simd_oprsz(desc); \
1340 ofs = LSX_LEN / BIT; \
1341 for (i = 0; i < oprsz / 16; i++) { \
1342 for (j = 0; j < ofs; j++) { \
1343 temp.E1(j + ofs * 2 * i) = do_vsrar_ ## E2(Vj->E2(j + ofs * i), imm); \
1344 temp.E1(j + ofs * (2 * i + 1)) = do_vsrar_ ## E2(Vd->E2(j + ofs * i), \
1351 void HELPER(vsrarni_d_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
1355 VReg
*Vd
= (VReg
*)vd
;
1356 VReg
*Vj
= (VReg
*)vj
;
1358 int oprsz
= simd_oprsz(desc
);
1360 for (i
= 0; i
< oprsz
/ 16; i
++) {
1362 temp
.D(2 * i
) = int128_getlo(Vj
->Q(i
));
1363 temp
.D(2 * i
+ 1) = int128_getlo(Vd
->Q(i
));
1365 r
[2 * i
] = int128_and(int128_rshift(Vj
->Q(i
), (imm
- 1)),
1367 r
[2 * i
+ 1] = int128_and(int128_rshift(Vd
->Q(i
), (imm
- 1)),
1369 temp
.D(2 * i
) = int128_getlo(int128_add(int128_rshift(Vj
->Q(i
),
1371 temp
.D(2 * i
+ 1) = int128_getlo(int128_add(int128_rshift(Vd
->Q(i
),
1372 imm
), r
[2 * i
+ 1]));
1378 VSRARNI(vsrarni_b_h
, 16, B
, H
)
1379 VSRARNI(vsrarni_h_w
, 32, H
, W
)
1380 VSRARNI(vsrarni_w_d
, 64, W
, D
)
1382 #define SSRLNS(NAME, T1, T2, T3) \
1383 static T1 do_ssrlns_ ## NAME(T2 e2, int sa, int sh) \
1389 shft_res = (((T1)e2) >> sa); \
1392 mask = (1ull << sh) -1; \
1393 if (shft_res > mask) { \
1400 SSRLNS(B
, uint16_t, int16_t, uint8_t)
1401 SSRLNS(H
, uint32_t, int32_t, uint16_t)
1402 SSRLNS(W
, uint64_t, int64_t, uint32_t)
1404 #define VSSRLN(NAME, BIT, E1, E2, E3) \
1405 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1408 VReg *Vd = (VReg *)vd; \
1409 VReg *Vj = (VReg *)vj; \
1410 VReg *Vk = (VReg *)vk; \
1411 int oprsz = simd_oprsz(desc); \
1413 ofs = LSX_LEN / BIT; \
1414 for (i = 0; i < oprsz / 16; i++) { \
1415 for (j = 0; j < ofs; j++) { \
1416 Vd->E1(j + ofs * 2 * i) = do_ssrlns_ ## E1(Vj->E2(j + ofs * i), \
1417 Vk->E3(j + ofs * i) % BIT, \
1420 Vd->D(2 * i + 1) = 0; \
1424 VSSRLN(vssrln_b_h
, 16, B
, H
, UH
)
1425 VSSRLN(vssrln_h_w
, 32, H
, W
, UW
)
1426 VSSRLN(vssrln_w_d
, 64, W
, D
, UD
)
1428 #define SSRANS(E, T1, T2) \
1429 static T1 do_ssrans_ ## E(T1 e2, int sa, int sh) \
1435 shft_res = e2 >> sa; \
1438 mask = (1ll << sh) - 1; \
1439 if (shft_res > mask) { \
1441 } else if (shft_res < -(mask + 1)) { \
1448 SSRANS(B
, int16_t, int8_t)
1449 SSRANS(H
, int32_t, int16_t)
1450 SSRANS(W
, int64_t, int32_t)
1452 #define VSSRAN(NAME, BIT, E1, E2, E3) \
1453 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1456 VReg *Vd = (VReg *)vd; \
1457 VReg *Vj = (VReg *)vj; \
1458 VReg *Vk = (VReg *)vk; \
1459 int oprsz = simd_oprsz(desc); \
1461 ofs = LSX_LEN / BIT; \
1462 for (i = 0; i < oprsz / 16; i++) { \
1463 for (j = 0; j < ofs; j++) { \
1464 Vd->E1(j + ofs * 2 * i) = do_ssrans_ ## E1(Vj->E2(j + ofs * i), \
1465 Vk->E3(j + ofs * i) % BIT, \
1468 Vd->D(2 * i + 1) = 0; \
1472 VSSRAN(vssran_b_h
, 16, B
, H
, UH
)
1473 VSSRAN(vssran_h_w
, 32, H
, W
, UW
)
1474 VSSRAN(vssran_w_d
, 64, W
, D
, UD
)
1476 #define SSRLNU(E, T1, T2, T3) \
1477 static T1 do_ssrlnu_ ## E(T3 e2, int sa, int sh) \
1483 shft_res = (((T1)e2) >> sa); \
1486 mask = (1ull << sh) - 1; \
1487 if (shft_res > mask) { \
1494 SSRLNU(B
, uint16_t, uint8_t, int16_t)
1495 SSRLNU(H
, uint32_t, uint16_t, int32_t)
1496 SSRLNU(W
, uint64_t, uint32_t, int64_t)
1498 #define VSSRLNU(NAME, BIT, E1, E2, E3) \
1499 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1502 VReg *Vd = (VReg *)vd; \
1503 VReg *Vj = (VReg *)vj; \
1504 VReg *Vk = (VReg *)vk; \
1505 int oprsz = simd_oprsz(desc); \
1507 ofs = LSX_LEN / BIT; \
1508 for (i = 0; i < oprsz / 16; i++) { \
1509 for (j = 0; j < ofs; j++) { \
1510 Vd->E1(j + ofs * 2 * i) = do_ssrlnu_ ## E1(Vj->E2(j + ofs * i), \
1511 Vk->E3(j + ofs * i) % BIT, \
1514 Vd->D(2 * i + 1) = 0; \
1518 VSSRLNU(vssrln_bu_h
, 16, B
, H
, UH
)
1519 VSSRLNU(vssrln_hu_w
, 32, H
, W
, UW
)
1520 VSSRLNU(vssrln_wu_d
, 64, W
, D
, UD
)
1522 #define SSRANU(E, T1, T2, T3) \
1523 static T1 do_ssranu_ ## E(T3 e2, int sa, int sh) \
1529 shft_res = e2 >> sa; \
1535 mask = (1ull << sh) - 1; \
1536 if (shft_res > mask) { \
1543 SSRANU(B
, uint16_t, uint8_t, int16_t)
1544 SSRANU(H
, uint32_t, uint16_t, int32_t)
1545 SSRANU(W
, uint64_t, uint32_t, int64_t)
1547 #define VSSRANU(NAME, BIT, E1, E2, E3) \
1548 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1551 VReg *Vd = (VReg *)vd; \
1552 VReg *Vj = (VReg *)vj; \
1553 VReg *Vk = (VReg *)vk; \
1554 int oprsz = simd_oprsz(desc); \
1556 ofs = LSX_LEN / BIT; \
1557 for (i = 0; i < oprsz / 16; i++) { \
1558 for (j = 0; j < ofs; j++) { \
1559 Vd->E1(j + ofs * 2 * i) = do_ssranu_ ## E1(Vj->E2(j + ofs * i), \
1560 Vk->E3(j + ofs * i) % BIT, \
1563 Vd->D(2 * i + 1) = 0; \
1567 VSSRANU(vssran_bu_h
, 16, B
, H
, UH
)
1568 VSSRANU(vssran_hu_w
, 32, H
, W
, UW
)
1569 VSSRANU(vssran_wu_d
, 64, W
, D
, UD
)
1571 #define VSSRLNI(NAME, BIT, E1, E2) \
1572 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1576 VReg *Vd = (VReg *)vd; \
1577 VReg *Vj = (VReg *)vj; \
1578 int oprsz = simd_oprsz(desc); \
1580 ofs = LSX_LEN / BIT; \
1581 for (i = 0; i < oprsz / 16; i++) { \
1582 for (j = 0; j < ofs; j++) { \
1583 temp.E1(j + ofs * 2 * i) = do_ssrlns_ ## E1(Vj->E2(j + ofs * i), \
1584 imm, BIT / 2 - 1); \
1585 temp.E1(j + ofs * (2 * i + 1)) = do_ssrlns_ ## E1(Vd->E2(j + ofs * i), \
1586 imm, BIT / 2 - 1); \
1592 static void do_vssrlni_q(VReg
*Vd
, VReg
*Vj
,
1593 uint64_t imm
, int idx
, Int128 mask
)
1595 Int128 shft_res1
, shft_res2
;
1598 shft_res1
= Vj
->Q(idx
);
1599 shft_res2
= Vd
->Q(idx
);
1601 shft_res1
= int128_urshift(Vj
->Q(idx
), imm
);
1602 shft_res2
= int128_urshift(Vd
->Q(idx
), imm
);
1605 if (int128_ult(mask
, shft_res1
)) {
1606 Vd
->D(idx
* 2) = int128_getlo(mask
);
1608 Vd
->D(idx
* 2) = int128_getlo(shft_res1
);
1611 if (int128_ult(mask
, shft_res2
)) {
1612 Vd
->D(idx
* 2 + 1) = int128_getlo(mask
);
1614 Vd
->D(idx
* 2 + 1) = int128_getlo(shft_res2
);
1618 void HELPER(vssrlni_d_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
1622 VReg
*Vd
= (VReg
*)vd
;
1623 VReg
*Vj
= (VReg
*)vj
;
1624 int oprsz
= simd_oprsz(desc
);
1626 mask
= int128_sub(int128_lshift(int128_one(), 63), int128_one());
1628 for (i
= 0; i
< oprsz
/ 16; i
++) {
1629 do_vssrlni_q(Vd
, Vj
, imm
, i
, mask
);
1633 VSSRLNI(vssrlni_b_h
, 16, B
, H
)
1634 VSSRLNI(vssrlni_h_w
, 32, H
, W
)
1635 VSSRLNI(vssrlni_w_d
, 64, W
, D
)
1637 #define VSSRANI(NAME, BIT, E1, E2) \
1638 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1642 VReg *Vd = (VReg *)vd; \
1643 VReg *Vj = (VReg *)vj; \
1644 int oprsz = simd_oprsz(desc); \
1646 ofs = LSX_LEN / BIT; \
1647 for (i = 0; i < oprsz / 16; i++) { \
1648 for (j = 0; j < ofs; j++) { \
1649 temp.E1(j + ofs * 2 * i) = do_ssrans_ ## E1(Vj->E2(j + ofs * i), \
1650 imm, BIT / 2 - 1); \
1651 temp.E1(j + ofs * (2 * i + 1)) = do_ssrans_ ## E1(Vd->E2(j + ofs * i), \
1652 imm, BIT / 2 - 1); \
1658 static void do_vssrani_d_q(VReg
*Vd
, VReg
*Vj
,
1659 uint64_t imm
, int idx
, Int128 mask
, Int128 min
)
1661 Int128 shft_res1
, shft_res2
;
1664 shft_res1
= Vj
->Q(idx
);
1665 shft_res2
= Vd
->Q(idx
);
1667 shft_res1
= int128_rshift(Vj
->Q(idx
), imm
);
1668 shft_res2
= int128_rshift(Vd
->Q(idx
), imm
);
1671 if (int128_gt(shft_res1
, mask
)) {
1672 Vd
->D(idx
* 2) = int128_getlo(mask
);
1673 } else if (int128_lt(shft_res1
, int128_neg(min
))) {
1674 Vd
->D(idx
* 2) = int128_getlo(min
);
1676 Vd
->D(idx
* 2) = int128_getlo(shft_res1
);
1679 if (int128_gt(shft_res2
, mask
)) {
1680 Vd
->D(idx
* 2 + 1) = int128_getlo(mask
);
1681 } else if (int128_lt(shft_res2
, int128_neg(min
))) {
1682 Vd
->D(idx
* 2 + 1) = int128_getlo(min
);
1684 Vd
->D(idx
* 2 + 1) = int128_getlo(shft_res2
);
1688 void HELPER(vssrani_d_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
1692 VReg
*Vd
= (VReg
*)vd
;
1693 VReg
*Vj
= (VReg
*)vj
;
1694 int oprsz
= simd_oprsz(desc
);
1696 mask
= int128_sub(int128_lshift(int128_one(), 63), int128_one());
1697 min
= int128_lshift(int128_one(), 63);
1699 for (i
= 0; i
< oprsz
/ 16; i
++) {
1700 do_vssrani_d_q(Vd
, Vj
, imm
, i
, mask
, min
);
1705 VSSRANI(vssrani_b_h
, 16, B
, H
)
1706 VSSRANI(vssrani_h_w
, 32, H
, W
)
1707 VSSRANI(vssrani_w_d
, 64, W
, D
)
1709 #define VSSRLNUI(NAME, BIT, E1, E2) \
1710 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1714 VReg *Vd = (VReg *)vd; \
1715 VReg *Vj = (VReg *)vj; \
1716 int oprsz = simd_oprsz(desc); \
1718 ofs = LSX_LEN / BIT; \
1719 for (i = 0; i < oprsz / 16; i++) { \
1720 for (j = 0; j < ofs; j++) { \
1721 temp.E1(j + ofs * 2 * i) = do_ssrlnu_ ## E1(Vj->E2(j + ofs * i), \
1723 temp.E1(j + ofs * (2 * i + 1)) = do_ssrlnu_ ## E1(Vd->E2(j + ofs * i), \
1730 void HELPER(vssrlni_du_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
1734 VReg
*Vd
= (VReg
*)vd
;
1735 VReg
*Vj
= (VReg
*)vj
;
1736 int oprsz
= simd_oprsz(desc
);
1738 mask
= int128_sub(int128_lshift(int128_one(), 64), int128_one());
1740 for (i
= 0; i
< oprsz
/ 16; i
++) {
1741 do_vssrlni_q(Vd
, Vj
, imm
, i
, mask
);
1745 VSSRLNUI(vssrlni_bu_h
, 16, B
, H
)
1746 VSSRLNUI(vssrlni_hu_w
, 32, H
, W
)
1747 VSSRLNUI(vssrlni_wu_d
, 64, W
, D
)
1749 #define VSSRANUI(NAME, BIT, E1, E2) \
1750 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
1754 VReg *Vd = (VReg *)vd; \
1755 VReg *Vj = (VReg *)vj; \
1756 int oprsz = simd_oprsz(desc); \
1758 ofs = LSX_LEN / BIT; \
1759 for (i = 0; i < oprsz / 16; i++) { \
1760 for (j = 0; j < ofs; j++) { \
1761 temp.E1(j + ofs * 2 * i) = do_ssranu_ ## E1(Vj->E2(j + ofs * i), \
1763 temp.E1(j + ofs * (2 * i + 1)) = do_ssranu_ ## E1(Vd->E2(j + ofs * i), \
1770 static void do_vssrani_du_q(VReg
*Vd
, VReg
*Vj
,
1771 uint64_t imm
, int idx
, Int128 mask
)
1773 Int128 shft_res1
, shft_res2
;
1776 shft_res1
= Vj
->Q(idx
);
1777 shft_res2
= Vd
->Q(idx
);
1779 shft_res1
= int128_rshift(Vj
->Q(idx
), imm
);
1780 shft_res2
= int128_rshift(Vd
->Q(idx
), imm
);
1783 if (int128_lt(Vj
->Q(idx
), int128_zero())) {
1784 shft_res1
= int128_zero();
1787 if (int128_lt(Vd
->Q(idx
), int128_zero())) {
1788 shft_res2
= int128_zero();
1790 if (int128_ult(mask
, shft_res1
)) {
1791 Vd
->D(idx
* 2) = int128_getlo(mask
);
1793 Vd
->D(idx
* 2) = int128_getlo(shft_res1
);
1796 if (int128_ult(mask
, shft_res2
)) {
1797 Vd
->D(idx
* 2 + 1) = int128_getlo(mask
);
1799 Vd
->D(idx
* 2 + 1) = int128_getlo(shft_res2
);
1804 void HELPER(vssrani_du_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
1808 VReg
*Vd
= (VReg
*)vd
;
1809 VReg
*Vj
= (VReg
*)vj
;
1810 int oprsz
= simd_oprsz(desc
);
1812 mask
= int128_sub(int128_lshift(int128_one(), 64), int128_one());
1814 for (i
= 0; i
< oprsz
/ 16; i
++) {
1815 do_vssrani_du_q(Vd
, Vj
, imm
, i
, mask
);
1819 VSSRANUI(vssrani_bu_h
, 16, B
, H
)
1820 VSSRANUI(vssrani_hu_w
, 32, H
, W
)
1821 VSSRANUI(vssrani_wu_d
, 64, W
, D
)
1823 #define SSRLRNS(E1, E2, T1, T2, T3) \
1824 static T1 do_ssrlrns_ ## E1(T2 e2, int sa, int sh) \
1828 shft_res = do_vsrlr_ ## E2(e2, sa); \
1830 mask = (1ull << sh) - 1; \
1831 if (shft_res > mask) { \
1838 SSRLRNS(B
, H
, uint16_t, int16_t, uint8_t)
1839 SSRLRNS(H
, W
, uint32_t, int32_t, uint16_t)
1840 SSRLRNS(W
, D
, uint64_t, int64_t, uint32_t)
1842 #define VSSRLRN(NAME, BIT, E1, E2, E3) \
1843 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1846 VReg *Vd = (VReg *)vd; \
1847 VReg *Vj = (VReg *)vj; \
1848 VReg *Vk = (VReg *)vk; \
1849 int oprsz = simd_oprsz(desc); \
1851 ofs = LSX_LEN / BIT; \
1852 for (i = 0; i < oprsz / 16; i++) { \
1853 for (j = 0; j < ofs; j++) { \
1854 Vd->E1(j + ofs * 2 * i) = do_ssrlrns_ ## E1(Vj->E2(j + ofs * i), \
1855 Vk->E3(j + ofs * i) % BIT, \
1858 Vd->D(2 * i + 1) = 0; \
1862 VSSRLRN(vssrlrn_b_h
, 16, B
, H
, UH
)
1863 VSSRLRN(vssrlrn_h_w
, 32, H
, W
, UW
)
1864 VSSRLRN(vssrlrn_w_d
, 64, W
, D
, UD
)
1866 #define SSRARNS(E1, E2, T1, T2) \
1867 static T1 do_ssrarns_ ## E1(T1 e2, int sa, int sh) \
1871 shft_res = do_vsrar_ ## E2(e2, sa); \
1873 mask = (1ll << sh) - 1; \
1874 if (shft_res > mask) { \
1876 } else if (shft_res < -(mask +1)) { \
1883 SSRARNS(B
, H
, int16_t, int8_t)
1884 SSRARNS(H
, W
, int32_t, int16_t)
1885 SSRARNS(W
, D
, int64_t, int32_t)
1887 #define VSSRARN(NAME, BIT, E1, E2, E3) \
1888 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1891 VReg *Vd = (VReg *)vd; \
1892 VReg *Vj = (VReg *)vj; \
1893 VReg *Vk = (VReg *)vk; \
1894 int oprsz = simd_oprsz(desc); \
1896 ofs = LSX_LEN / BIT; \
1897 for (i = 0; i < oprsz / 16; i++) { \
1898 for (j = 0; j < ofs; j++) { \
1899 Vd->E1(j + ofs * 2 * i) = do_ssrarns_ ## E1(Vj->E2(j + ofs * i), \
1900 Vk->E3(j + ofs * i) % BIT, \
1903 Vd->D(2 * i + 1) = 0; \
1907 VSSRARN(vssrarn_b_h
, 16, B
, H
, UH
)
1908 VSSRARN(vssrarn_h_w
, 32, H
, W
, UW
)
1909 VSSRARN(vssrarn_w_d
, 64, W
, D
, UD
)
1911 #define SSRLRNU(E1, E2, T1, T2, T3) \
1912 static T1 do_ssrlrnu_ ## E1(T3 e2, int sa, int sh) \
1916 shft_res = do_vsrlr_ ## E2(e2, sa); \
1919 mask = (1ull << sh) - 1; \
1920 if (shft_res > mask) { \
1927 SSRLRNU(B
, H
, uint16_t, uint8_t, int16_t)
1928 SSRLRNU(H
, W
, uint32_t, uint16_t, int32_t)
1929 SSRLRNU(W
, D
, uint64_t, uint32_t, int64_t)
1931 #define VSSRLRNU(NAME, BIT, E1, E2, E3) \
1932 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1935 VReg *Vd = (VReg *)vd; \
1936 VReg *Vj = (VReg *)vj; \
1937 VReg *Vk = (VReg *)vk; \
1938 int oprsz = simd_oprsz(desc); \
1940 ofs = LSX_LEN / BIT; \
1941 for (i = 0; i < oprsz / 16; i++) { \
1942 for (j = 0; j < ofs; j++) { \
1943 Vd->E1(j + ofs * 2 * i) = do_ssrlrnu_ ## E1(Vj->E2(j + ofs * i), \
1944 Vk->E3(j + ofs * i) % BIT, \
1947 Vd->D(2 * i + 1) = 0; \
1951 VSSRLRNU(vssrlrn_bu_h
, 16, B
, H
, UH
)
1952 VSSRLRNU(vssrlrn_hu_w
, 32, H
, W
, UW
)
1953 VSSRLRNU(vssrlrn_wu_d
, 64, W
, D
, UD
)
1955 #define SSRARNU(E1, E2, T1, T2, T3) \
1956 static T1 do_ssrarnu_ ## E1(T3 e2, int sa, int sh) \
1963 shft_res = do_vsrar_ ## E2(e2, sa); \
1966 mask = (1ull << sh) - 1; \
1967 if (shft_res > mask) { \
1974 SSRARNU(B
, H
, uint16_t, uint8_t, int16_t)
1975 SSRARNU(H
, W
, uint32_t, uint16_t, int32_t)
1976 SSRARNU(W
, D
, uint64_t, uint32_t, int64_t)
1978 #define VSSRARNU(NAME, BIT, E1, E2, E3) \
1979 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
1982 VReg *Vd = (VReg *)vd; \
1983 VReg *Vj = (VReg *)vj; \
1984 VReg *Vk = (VReg *)vk; \
1985 int oprsz = simd_oprsz(desc); \
1987 ofs = LSX_LEN / BIT; \
1988 for (i = 0; i < oprsz / 16; i++) { \
1989 for (j = 0; j < ofs; j++) { \
1990 Vd->E1(j + ofs * 2 * i) = do_ssrarnu_ ## E1(Vj->E2(j + ofs * i), \
1991 Vk->E3(j + ofs * i) % BIT, \
1994 Vd->D(2 * i + 1) = 0; \
1998 VSSRARNU(vssrarn_bu_h
, 16, B
, H
, UH
)
1999 VSSRARNU(vssrarn_hu_w
, 32, H
, W
, UW
)
2000 VSSRARNU(vssrarn_wu_d
, 64, W
, D
, UD
)
2002 #define VSSRLRNI(NAME, BIT, E1, E2) \
2003 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
2007 VReg *Vd = (VReg *)vd; \
2008 VReg *Vj = (VReg *)vj; \
2009 int oprsz = simd_oprsz(desc); \
2011 ofs = LSX_LEN / BIT; \
2012 for (i = 0; i < oprsz / 16; i++) { \
2013 for (j = 0; j < ofs; j++) { \
2014 temp.E1(j + ofs * 2 * i) = do_ssrlrns_ ## E1(Vj->E2(j + ofs * i), \
2015 imm, BIT / 2 - 1); \
2016 temp.E1(j + ofs * (2 * i + 1)) = do_ssrlrns_ ## E1(Vd->E2(j + ofs * i), \
2017 imm, BIT / 2 - 1); \
2023 static void do_vssrlrni_q(VReg
*Vd
, VReg
* Vj
,
2024 uint64_t imm
, int idx
, Int128 mask
)
2026 Int128 shft_res1
, shft_res2
, r1
, r2
;
2028 shft_res1
= Vj
->Q(idx
);
2029 shft_res2
= Vd
->Q(idx
);
2031 r1
= int128_and(int128_urshift(Vj
->Q(idx
), (imm
- 1)), int128_one());
2032 r2
= int128_and(int128_urshift(Vd
->Q(idx
), (imm
- 1)), int128_one());
2033 shft_res1
= (int128_add(int128_urshift(Vj
->Q(idx
), imm
), r1
));
2034 shft_res2
= (int128_add(int128_urshift(Vd
->Q(idx
), imm
), r2
));
2037 if (int128_ult(mask
, shft_res1
)) {
2038 Vd
->D(idx
* 2) = int128_getlo(mask
);
2040 Vd
->D(idx
* 2) = int128_getlo(shft_res1
);
2043 if (int128_ult(mask
, shft_res2
)) {
2044 Vd
->D(idx
* 2 + 1) = int128_getlo(mask
);
2046 Vd
->D(idx
* 2 + 1) = int128_getlo(shft_res2
);
2050 void HELPER(vssrlrni_d_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
2054 VReg
*Vd
= (VReg
*)vd
;
2055 VReg
*Vj
= (VReg
*)vj
;
2056 int oprsz
= simd_oprsz(desc
);
2058 mask
= int128_sub(int128_lshift(int128_one(), 63), int128_one());
2060 for (i
= 0; i
< oprsz
/ 16; i
++) {
2061 do_vssrlrni_q(Vd
, Vj
, imm
, i
, mask
);
2065 VSSRLRNI(vssrlrni_b_h
, 16, B
, H
)
2066 VSSRLRNI(vssrlrni_h_w
, 32, H
, W
)
2067 VSSRLRNI(vssrlrni_w_d
, 64, W
, D
)
2069 #define VSSRARNI(NAME, BIT, E1, E2) \
2070 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
2074 VReg *Vd = (VReg *)vd; \
2075 VReg *Vj = (VReg *)vj; \
2076 int oprsz = simd_oprsz(desc); \
2078 ofs = LSX_LEN / BIT; \
2079 for (i = 0; i < oprsz / 16; i++) { \
2080 for (j = 0; j < ofs; j++) { \
2081 temp.E1(j + ofs * 2 * i) = do_ssrarns_ ## E1(Vj->E2(j + ofs * i), \
2082 imm, BIT / 2 - 1); \
2083 temp.E1(j + ofs * (2 * i + 1)) = do_ssrarns_ ## E1(Vd->E2(j + ofs * i), \
2084 imm, BIT / 2 - 1); \
2090 static void do_vssrarni_d_q(VReg
*Vd
, VReg
*Vj
,
2091 uint64_t imm
, int idx
, Int128 mask1
, Int128 mask2
)
2093 Int128 shft_res1
, shft_res2
, r1
, r2
;
2096 shft_res1
= Vj
->Q(idx
);
2097 shft_res2
= Vd
->Q(idx
);
2099 r1
= int128_and(int128_rshift(Vj
->Q(idx
), (imm
- 1)), int128_one());
2100 r2
= int128_and(int128_rshift(Vd
->Q(idx
), (imm
- 1)), int128_one());
2101 shft_res1
= int128_add(int128_rshift(Vj
->Q(idx
), imm
), r1
);
2102 shft_res2
= int128_add(int128_rshift(Vd
->Q(idx
), imm
), r2
);
2104 if (int128_gt(shft_res1
, mask1
)) {
2105 Vd
->D(idx
* 2) = int128_getlo(mask1
);
2106 } else if (int128_lt(shft_res1
, int128_neg(mask2
))) {
2107 Vd
->D(idx
* 2) = int128_getlo(mask2
);
2109 Vd
->D(idx
* 2) = int128_getlo(shft_res1
);
2112 if (int128_gt(shft_res2
, mask1
)) {
2113 Vd
->D(idx
* 2 + 1) = int128_getlo(mask1
);
2114 } else if (int128_lt(shft_res2
, int128_neg(mask2
))) {
2115 Vd
->D(idx
* 2 + 1) = int128_getlo(mask2
);
2117 Vd
->D(idx
* 2 + 1) = int128_getlo(shft_res2
);
2121 void HELPER(vssrarni_d_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
2124 Int128 mask1
, mask2
;
2125 VReg
*Vd
= (VReg
*)vd
;
2126 VReg
*Vj
= (VReg
*)vj
;
2127 int oprsz
= simd_oprsz(desc
);
2129 mask1
= int128_sub(int128_lshift(int128_one(), 63), int128_one());
2130 mask2
= int128_lshift(int128_one(), 63);
2132 for (i
= 0; i
< oprsz
/ 16; i
++) {
2133 do_vssrarni_d_q(Vd
, Vj
, imm
, i
, mask1
, mask2
);
2137 VSSRARNI(vssrarni_b_h
, 16, B
, H
)
2138 VSSRARNI(vssrarni_h_w
, 32, H
, W
)
2139 VSSRARNI(vssrarni_w_d
, 64, W
, D
)
2141 #define VSSRLRNUI(NAME, BIT, E1, E2) \
2142 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
2146 VReg *Vd = (VReg *)vd; \
2147 VReg *Vj = (VReg *)vj; \
2148 int oprsz = simd_oprsz(desc); \
2150 ofs = LSX_LEN / BIT; \
2151 for (i = 0; i < oprsz / 16; i++) { \
2152 for (j = 0; j < ofs; j++) { \
2153 temp.E1(j + ofs * 2 * i) = do_ssrlrnu_ ## E1(Vj->E2(j + ofs * i), \
2155 temp.E1(j + ofs * (2 * i + 1)) = do_ssrlrnu_ ## E1(Vd->E2(j + ofs * i), \
2162 void HELPER(vssrlrni_du_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
2166 VReg
*Vd
= (VReg
*)vd
;
2167 VReg
*Vj
= (VReg
*)vj
;
2168 int oprsz
= simd_oprsz(desc
);
2170 mask
= int128_sub(int128_lshift(int128_one(), 64), int128_one());
2172 for (i
= 0; i
< oprsz
/ 16; i
++) {
2173 do_vssrlrni_q(Vd
, Vj
, imm
, i
, mask
);
2177 VSSRLRNUI(vssrlrni_bu_h
, 16, B
, H
)
2178 VSSRLRNUI(vssrlrni_hu_w
, 32, H
, W
)
2179 VSSRLRNUI(vssrlrni_wu_d
, 64, W
, D
)
2181 #define VSSRARNUI(NAME, BIT, E1, E2) \
2182 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
2186 VReg *Vd = (VReg *)vd; \
2187 VReg *Vj = (VReg *)vj; \
2188 int oprsz = simd_oprsz(desc); \
2190 ofs = LSX_LEN / BIT; \
2191 for (i = 0; i < oprsz / 16; i++) { \
2192 for (j = 0; j < ofs; j++) { \
2193 temp.E1(j + ofs * 2 * i) = do_ssrarnu_ ## E1(Vj->E2(j + ofs * i), \
2195 temp.E1(j + ofs * (2 * i + 1)) = do_ssrarnu_ ## E1(Vd->E2(j + ofs * i), \
2202 static void do_vssrarni_du_q(VReg
*Vd
, VReg
*Vj
,
2203 uint64_t imm
, int idx
, Int128 mask1
, Int128 mask2
)
2205 Int128 shft_res1
, shft_res2
, r1
, r2
;
2208 shft_res1
= Vj
->Q(idx
);
2209 shft_res2
= Vd
->Q(idx
);
2211 r1
= int128_and(int128_rshift(Vj
->Q(idx
), (imm
- 1)), int128_one());
2212 r2
= int128_and(int128_rshift(Vd
->Q(idx
), (imm
- 1)), int128_one());
2213 shft_res1
= int128_add(int128_rshift(Vj
->Q(idx
), imm
), r1
);
2214 shft_res2
= int128_add(int128_rshift(Vd
->Q(idx
), imm
), r2
);
2217 if (int128_lt(Vj
->Q(idx
), int128_zero())) {
2218 shft_res1
= int128_zero();
2220 if (int128_lt(Vd
->Q(idx
), int128_zero())) {
2221 shft_res2
= int128_zero();
2224 if (int128_gt(shft_res1
, mask1
)) {
2225 Vd
->D(idx
* 2) = int128_getlo(mask1
);
2226 } else if (int128_lt(shft_res1
, int128_neg(mask2
))) {
2227 Vd
->D(idx
* 2) = int128_getlo(mask2
);
2229 Vd
->D(idx
* 2) = int128_getlo(shft_res1
);
2232 if (int128_gt(shft_res2
, mask1
)) {
2233 Vd
->D(idx
* 2 + 1) = int128_getlo(mask1
);
2234 } else if (int128_lt(shft_res2
, int128_neg(mask2
))) {
2235 Vd
->D(idx
* 2 + 1) = int128_getlo(mask2
);
2237 Vd
->D(idx
* 2 + 1) = int128_getlo(shft_res2
);
2241 void HELPER(vssrarni_du_q
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
2244 Int128 mask1
, mask2
;
2245 VReg
*Vd
= (VReg
*)vd
;
2246 VReg
*Vj
= (VReg
*)vj
;
2247 int oprsz
= simd_oprsz(desc
);
2249 mask1
= int128_sub(int128_lshift(int128_one(), 64), int128_one());
2250 mask2
= int128_lshift(int128_one(), 64);
2252 for (i
= 0; i
< oprsz
/ 16; i
++) {
2253 do_vssrarni_du_q(Vd
, Vj
, imm
, i
, mask1
, mask2
);
2257 VSSRARNUI(vssrarni_bu_h
, 16, B
, H
)
2258 VSSRARNUI(vssrarni_hu_w
, 32, H
, W
)
2259 VSSRARNUI(vssrarni_wu_d
, 64, W
, D
)
2261 #define DO_2OP(NAME, BIT, E, DO_OP) \
2262 void HELPER(NAME)(void *vd, void *vj, uint32_t desc) \
2265 VReg *Vd = (VReg *)vd; \
2266 VReg *Vj = (VReg *)vj; \
2267 int oprsz = simd_oprsz(desc); \
2269 for (i = 0; i < oprsz / (BIT / 8); i++) \
2271 Vd->E(i) = DO_OP(Vj->E(i)); \
2275 #define DO_CLO_B(N) (clz32(~N & 0xff) - 24)
2276 #define DO_CLO_H(N) (clz32(~N & 0xffff) - 16)
2277 #define DO_CLO_W(N) (clz32(~N))
2278 #define DO_CLO_D(N) (clz64(~N))
2279 #define DO_CLZ_B(N) (clz32(N) - 24)
2280 #define DO_CLZ_H(N) (clz32(N) - 16)
2281 #define DO_CLZ_W(N) (clz32(N))
2282 #define DO_CLZ_D(N) (clz64(N))
2284 DO_2OP(vclo_b
, 8, UB
, DO_CLO_B
)
2285 DO_2OP(vclo_h
, 16, UH
, DO_CLO_H
)
2286 DO_2OP(vclo_w
, 32, UW
, DO_CLO_W
)
2287 DO_2OP(vclo_d
, 64, UD
, DO_CLO_D
)
2288 DO_2OP(vclz_b
, 8, UB
, DO_CLZ_B
)
2289 DO_2OP(vclz_h
, 16, UH
, DO_CLZ_H
)
2290 DO_2OP(vclz_w
, 32, UW
, DO_CLZ_W
)
2291 DO_2OP(vclz_d
, 64, UD
, DO_CLZ_D
)
2293 #define VPCNT(NAME, BIT, E, FN) \
2294 void HELPER(NAME)(void *vd, void *vj, uint32_t desc) \
2297 VReg *Vd = (VReg *)vd; \
2298 VReg *Vj = (VReg *)vj; \
2299 int oprsz = simd_oprsz(desc); \
2301 for (i = 0; i < oprsz / (BIT / 8); i++) \
2303 Vd->E(i) = FN(Vj->E(i)); \
2307 VPCNT(vpcnt_b
, 8, UB
, ctpop8
)
2308 VPCNT(vpcnt_h
, 16, UH
, ctpop16
)
2309 VPCNT(vpcnt_w
, 32, UW
, ctpop32
)
2310 VPCNT(vpcnt_d
, 64, UD
, ctpop64
)
2312 #define DO_BITCLR(a, bit) (a & ~(1ull << bit))
2313 #define DO_BITSET(a, bit) (a | 1ull << bit)
2314 #define DO_BITREV(a, bit) (a ^ (1ull << bit))
2316 #define DO_BIT(NAME, BIT, E, DO_OP) \
2317 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
2320 VReg *Vd = (VReg *)vd; \
2321 VReg *Vj = (VReg *)vj; \
2322 VReg *Vk = (VReg *)vk; \
2323 int oprsz = simd_oprsz(desc); \
2325 for (i = 0; i < oprsz / (BIT / 8); i++) { \
2326 Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)%BIT); \
2330 DO_BIT(vbitclr_b
, 8, UB
, DO_BITCLR
)
2331 DO_BIT(vbitclr_h
, 16, UH
, DO_BITCLR
)
2332 DO_BIT(vbitclr_w
, 32, UW
, DO_BITCLR
)
2333 DO_BIT(vbitclr_d
, 64, UD
, DO_BITCLR
)
2334 DO_BIT(vbitset_b
, 8, UB
, DO_BITSET
)
2335 DO_BIT(vbitset_h
, 16, UH
, DO_BITSET
)
2336 DO_BIT(vbitset_w
, 32, UW
, DO_BITSET
)
2337 DO_BIT(vbitset_d
, 64, UD
, DO_BITSET
)
2338 DO_BIT(vbitrev_b
, 8, UB
, DO_BITREV
)
2339 DO_BIT(vbitrev_h
, 16, UH
, DO_BITREV
)
2340 DO_BIT(vbitrev_w
, 32, UW
, DO_BITREV
)
2341 DO_BIT(vbitrev_d
, 64, UD
, DO_BITREV
)
2343 #define DO_BITI(NAME, BIT, E, DO_OP) \
2344 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
2347 VReg *Vd = (VReg *)vd; \
2348 VReg *Vj = (VReg *)vj; \
2349 int oprsz = simd_oprsz(desc); \
2351 for (i = 0; i < oprsz / (BIT / 8); i++) { \
2352 Vd->E(i) = DO_OP(Vj->E(i), imm); \
2356 DO_BITI(vbitclri_b
, 8, UB
, DO_BITCLR
)
2357 DO_BITI(vbitclri_h
, 16, UH
, DO_BITCLR
)
2358 DO_BITI(vbitclri_w
, 32, UW
, DO_BITCLR
)
2359 DO_BITI(vbitclri_d
, 64, UD
, DO_BITCLR
)
2360 DO_BITI(vbitseti_b
, 8, UB
, DO_BITSET
)
2361 DO_BITI(vbitseti_h
, 16, UH
, DO_BITSET
)
2362 DO_BITI(vbitseti_w
, 32, UW
, DO_BITSET
)
2363 DO_BITI(vbitseti_d
, 64, UD
, DO_BITSET
)
2364 DO_BITI(vbitrevi_b
, 8, UB
, DO_BITREV
)
2365 DO_BITI(vbitrevi_h
, 16, UH
, DO_BITREV
)
2366 DO_BITI(vbitrevi_w
, 32, UW
, DO_BITREV
)
2367 DO_BITI(vbitrevi_d
, 64, UD
, DO_BITREV
)
2369 #define VFRSTP(NAME, BIT, MASK, E) \
2370 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
2373 VReg *Vd = (VReg *)vd; \
2374 VReg *Vj = (VReg *)vj; \
2375 VReg *Vk = (VReg *)vk; \
2376 int oprsz = simd_oprsz(desc); \
2378 ofs = LSX_LEN / BIT; \
2379 for (i = 0; i < oprsz / 16; i++) { \
2380 m = Vk->E(i * ofs) & MASK; \
2381 for (j = 0; j < ofs; j++) { \
2382 if (Vj->E(j + ofs * i) < 0) { \
2386 Vd->E(m + i * ofs) = j; \
2390 VFRSTP(vfrstp_b
, 8, 0xf, B
)
2391 VFRSTP(vfrstp_h
, 16, 0x7, H
)
2393 #define VFRSTPI(NAME, BIT, E) \
2394 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
2397 VReg *Vd = (VReg *)vd; \
2398 VReg *Vj = (VReg *)vj; \
2399 int oprsz = simd_oprsz(desc); \
2401 ofs = LSX_LEN / BIT; \
2403 for (i = 0; i < oprsz / 16; i++) { \
2404 for (j = 0; j < ofs; j++) { \
2405 if (Vj->E(j + ofs * i) < 0) { \
2409 Vd->E(m + i * ofs) = j; \
2413 VFRSTPI(vfrstpi_b
, 8, B
)
2414 VFRSTPI(vfrstpi_h
, 16, H
)
2416 static void vec_update_fcsr0_mask(CPULoongArchState
*env
,
2417 uintptr_t pc
, int mask
)
2419 int flags
= get_float_exception_flags(&env
->fp_status
);
2421 set_float_exception_flags(0, &env
->fp_status
);
2426 flags
= ieee_ex_to_loongarch(flags
);
2427 UPDATE_FP_CAUSE(env
->fcsr0
, flags
);
2430 if (GET_FP_ENABLES(env
->fcsr0
) & flags
) {
2431 do_raise_exception(env
, EXCCODE_FPE
, pc
);
2433 UPDATE_FP_FLAGS(env
->fcsr0
, flags
);
2437 static void vec_update_fcsr0(CPULoongArchState
*env
, uintptr_t pc
)
2439 vec_update_fcsr0_mask(env
, pc
, 0);
2442 static inline void vec_clear_cause(CPULoongArchState
*env
)
2444 SET_FP_CAUSE(env
->fcsr0
, 0);
2447 #define DO_3OP_F(NAME, BIT, E, FN) \
2448 void HELPER(NAME)(void *vd, void *vj, void *vk, \
2449 CPULoongArchState *env, uint32_t desc) \
2452 VReg *Vd = (VReg *)vd; \
2453 VReg *Vj = (VReg *)vj; \
2454 VReg *Vk = (VReg *)vk; \
2455 int oprsz = simd_oprsz(desc); \
2457 vec_clear_cause(env); \
2458 for (i = 0; i < oprsz / (BIT / 8); i++) { \
2459 Vd->E(i) = FN(Vj->E(i), Vk->E(i), &env->fp_status); \
2460 vec_update_fcsr0(env, GETPC()); \
2464 DO_3OP_F(vfadd_s
, 32, UW
, float32_add
)
2465 DO_3OP_F(vfadd_d
, 64, UD
, float64_add
)
2466 DO_3OP_F(vfsub_s
, 32, UW
, float32_sub
)
2467 DO_3OP_F(vfsub_d
, 64, UD
, float64_sub
)
2468 DO_3OP_F(vfmul_s
, 32, UW
, float32_mul
)
2469 DO_3OP_F(vfmul_d
, 64, UD
, float64_mul
)
2470 DO_3OP_F(vfdiv_s
, 32, UW
, float32_div
)
2471 DO_3OP_F(vfdiv_d
, 64, UD
, float64_div
)
2472 DO_3OP_F(vfmax_s
, 32, UW
, float32_maxnum
)
2473 DO_3OP_F(vfmax_d
, 64, UD
, float64_maxnum
)
2474 DO_3OP_F(vfmin_s
, 32, UW
, float32_minnum
)
2475 DO_3OP_F(vfmin_d
, 64, UD
, float64_minnum
)
2476 DO_3OP_F(vfmaxa_s
, 32, UW
, float32_maxnummag
)
2477 DO_3OP_F(vfmaxa_d
, 64, UD
, float64_maxnummag
)
2478 DO_3OP_F(vfmina_s
, 32, UW
, float32_minnummag
)
2479 DO_3OP_F(vfmina_d
, 64, UD
, float64_minnummag
)
2481 #define DO_4OP_F(NAME, BIT, E, FN, flags) \
2482 void HELPER(NAME)(void *vd, void *vj, void *vk, void *va, \
2483 CPULoongArchState *env, uint32_t desc) \
2486 VReg *Vd = (VReg *)vd; \
2487 VReg *Vj = (VReg *)vj; \
2488 VReg *Vk = (VReg *)vk; \
2489 VReg *Va = (VReg *)va; \
2490 int oprsz = simd_oprsz(desc); \
2492 vec_clear_cause(env); \
2493 for (i = 0; i < oprsz / (BIT / 8); i++) { \
2494 Vd->E(i) = FN(Vj->E(i), Vk->E(i), Va->E(i), flags, &env->fp_status); \
2495 vec_update_fcsr0(env, GETPC()); \
2499 DO_4OP_F(vfmadd_s
, 32, UW
, float32_muladd
, 0)
2500 DO_4OP_F(vfmadd_d
, 64, UD
, float64_muladd
, 0)
2501 DO_4OP_F(vfmsub_s
, 32, UW
, float32_muladd
, float_muladd_negate_c
)
2502 DO_4OP_F(vfmsub_d
, 64, UD
, float64_muladd
, float_muladd_negate_c
)
2503 DO_4OP_F(vfnmadd_s
, 32, UW
, float32_muladd
, float_muladd_negate_result
)
2504 DO_4OP_F(vfnmadd_d
, 64, UD
, float64_muladd
, float_muladd_negate_result
)
2505 DO_4OP_F(vfnmsub_s
, 32, UW
, float32_muladd
,
2506 float_muladd_negate_c
| float_muladd_negate_result
)
2507 DO_4OP_F(vfnmsub_d
, 64, UD
, float64_muladd
,
2508 float_muladd_negate_c
| float_muladd_negate_result
)
2510 #define DO_2OP_F(NAME, BIT, E, FN) \
2511 void HELPER(NAME)(void *vd, void *vj, \
2512 CPULoongArchState *env, uint32_t desc) \
2515 VReg *Vd = (VReg *)vd; \
2516 VReg *Vj = (VReg *)vj; \
2517 int oprsz = simd_oprsz(desc); \
2519 vec_clear_cause(env); \
2520 for (i = 0; i < oprsz / (BIT / 8); i++) { \
2521 Vd->E(i) = FN(env, Vj->E(i)); \
2525 #define FLOGB(BIT, T) \
2526 static T do_flogb_## BIT(CPULoongArchState *env, T fj) \
2529 float_status *status = &env->fp_status; \
2530 FloatRoundMode old_mode = get_float_rounding_mode(status); \
2532 set_float_rounding_mode(float_round_down, status); \
2533 fp = float ## BIT ##_log2(fj, status); \
2534 fd = float ## BIT ##_round_to_int(fp, status); \
2535 set_float_rounding_mode(old_mode, status); \
2536 vec_update_fcsr0_mask(env, GETPC(), float_flag_inexact); \
2543 #define FCLASS(NAME, BIT, E, FN) \
2544 void HELPER(NAME)(void *vd, void *vj, \
2545 CPULoongArchState *env, uint32_t desc) \
2548 VReg *Vd = (VReg *)vd; \
2549 VReg *Vj = (VReg *)vj; \
2550 int oprsz = simd_oprsz(desc); \
2552 for (i = 0; i < oprsz / (BIT / 8); i++) { \
2553 Vd->E(i) = FN(env, Vj->E(i)); \
2557 FCLASS(vfclass_s
, 32, UW
, helper_fclass_s
)
2558 FCLASS(vfclass_d
, 64, UD
, helper_fclass_d
)
2560 #define FSQRT(BIT, T) \
2561 static T do_fsqrt_## BIT(CPULoongArchState *env, T fj) \
2564 fd = float ## BIT ##_sqrt(fj, &env->fp_status); \
2565 vec_update_fcsr0(env, GETPC()); \
2572 #define FRECIP(BIT, T) \
2573 static T do_frecip_## BIT(CPULoongArchState *env, T fj) \
2576 fd = float ## BIT ##_div(float ## BIT ##_one, fj, &env->fp_status); \
2577 vec_update_fcsr0(env, GETPC()); \
2581 FRECIP(32, uint32_t)
2582 FRECIP(64, uint64_t)
2584 #define FRSQRT(BIT, T) \
2585 static T do_frsqrt_## BIT(CPULoongArchState *env, T fj) \
2588 fp = float ## BIT ##_sqrt(fj, &env->fp_status); \
2589 fd = float ## BIT ##_div(float ## BIT ##_one, fp, &env->fp_status); \
2590 vec_update_fcsr0(env, GETPC()); \
2594 FRSQRT(32, uint32_t)
2595 FRSQRT(64, uint64_t)
2597 DO_2OP_F(vflogb_s
, 32, UW
, do_flogb_32
)
2598 DO_2OP_F(vflogb_d
, 64, UD
, do_flogb_64
)
2599 DO_2OP_F(vfsqrt_s
, 32, UW
, do_fsqrt_32
)
2600 DO_2OP_F(vfsqrt_d
, 64, UD
, do_fsqrt_64
)
2601 DO_2OP_F(vfrecip_s
, 32, UW
, do_frecip_32
)
2602 DO_2OP_F(vfrecip_d
, 64, UD
, do_frecip_64
)
2603 DO_2OP_F(vfrsqrt_s
, 32, UW
, do_frsqrt_32
)
2604 DO_2OP_F(vfrsqrt_d
, 64, UD
, do_frsqrt_64
)
2606 static uint32_t float16_cvt_float32(uint16_t h
, float_status
*status
)
2608 return float16_to_float32(h
, true, status
);
2610 static uint64_t float32_cvt_float64(uint32_t s
, float_status
*status
)
2612 return float32_to_float64(s
, status
);
2615 static uint16_t float32_cvt_float16(uint32_t s
, float_status
*status
)
2617 return float32_to_float16(s
, true, status
);
2619 static uint32_t float64_cvt_float32(uint64_t d
, float_status
*status
)
2621 return float64_to_float32(d
, status
);
2624 void HELPER(vfcvtl_s_h
)(void *vd
, void *vj
,
2625 CPULoongArchState
*env
, uint32_t desc
)
2629 VReg
*Vd
= (VReg
*)vd
;
2630 VReg
*Vj
= (VReg
*)vj
;
2631 int oprsz
= simd_oprsz(desc
);
2634 vec_clear_cause(env
);
2635 for (i
= 0; i
< oprsz
/ 16; i
++) {
2636 for (j
= 0; j
< ofs
; j
++) {
2637 temp
.UW(j
+ ofs
* i
) =float16_cvt_float32(Vj
->UH(j
+ ofs
* 2 * i
),
2640 vec_update_fcsr0(env
, GETPC());
2645 void HELPER(vfcvtl_d_s
)(void *vd
, void *vj
,
2646 CPULoongArchState
*env
, uint32_t desc
)
2650 VReg
*Vd
= (VReg
*)vd
;
2651 VReg
*Vj
= (VReg
*)vj
;
2652 int oprsz
= simd_oprsz(desc
);
2655 vec_clear_cause(env
);
2656 for (i
= 0; i
< oprsz
/ 16; i
++) {
2657 for (j
= 0; j
< ofs
; j
++) {
2658 temp
.UD(j
+ ofs
* i
) = float32_cvt_float64(Vj
->UW(j
+ ofs
* 2 * i
),
2661 vec_update_fcsr0(env
, GETPC());
2666 void HELPER(vfcvth_s_h
)(void *vd
, void *vj
,
2667 CPULoongArchState
*env
, uint32_t desc
)
2671 VReg
*Vd
= (VReg
*)vd
;
2672 VReg
*Vj
= (VReg
*)vj
;
2673 int oprsz
= simd_oprsz(desc
);
2676 vec_clear_cause(env
);
2677 for (i
= 0; i
< oprsz
/ 16; i
++) {
2678 for (j
= 0; j
< ofs
; j
++) {
2679 temp
.UW(j
+ ofs
* i
) = float16_cvt_float32(Vj
->UH(j
+ ofs
* (2 * i
+ 1)),
2682 vec_update_fcsr0(env
, GETPC());
2687 void HELPER(vfcvth_d_s
)(void *vd
, void *vj
,
2688 CPULoongArchState
*env
, uint32_t desc
)
2692 VReg
*Vd
= (VReg
*)vd
;
2693 VReg
*Vj
= (VReg
*)vj
;
2694 int oprsz
= simd_oprsz(desc
);
2697 vec_clear_cause(env
);
2698 for (i
= 0; i
< oprsz
/ 16; i
++) {
2699 for (j
= 0; j
< ofs
; j
++) {
2700 temp
.UD(j
+ ofs
* i
) = float32_cvt_float64(Vj
->UW(j
+ ofs
* (2 * i
+ 1)),
2703 vec_update_fcsr0(env
, GETPC());
2708 void HELPER(vfcvt_h_s
)(void *vd
, void *vj
, void *vk
,
2709 CPULoongArchState
*env
, uint32_t desc
)
2713 VReg
*Vd
= (VReg
*)vd
;
2714 VReg
*Vj
= (VReg
*)vj
;
2715 VReg
*Vk
= (VReg
*)vk
;
2716 int oprsz
= simd_oprsz(desc
);
2719 vec_clear_cause(env
);
2720 for(i
= 0; i
< oprsz
/ 16; i
++) {
2721 for (j
= 0; j
< ofs
; j
++) {
2722 temp
.UH(j
+ ofs
* (2 * i
+ 1)) = float32_cvt_float16(Vj
->UW(j
+ ofs
* i
),
2724 temp
.UH(j
+ ofs
* 2 * i
) = float32_cvt_float16(Vk
->UW(j
+ ofs
* i
),
2727 vec_update_fcsr0(env
, GETPC());
2732 void HELPER(vfcvt_s_d
)(void *vd
, void *vj
, void *vk
,
2733 CPULoongArchState
*env
, uint32_t desc
)
2737 VReg
*Vd
= (VReg
*)vd
;
2738 VReg
*Vj
= (VReg
*)vj
;
2739 VReg
*Vk
= (VReg
*)vk
;
2740 int oprsz
= simd_oprsz(desc
);
2743 vec_clear_cause(env
);
2744 for(i
= 0; i
< oprsz
/ 16; i
++) {
2745 for (j
= 0; j
< ofs
; j
++) {
2746 temp
.UW(j
+ ofs
* (2 * i
+ 1)) = float64_cvt_float32(Vj
->UD(j
+ ofs
* i
),
2748 temp
.UW(j
+ ofs
* 2 * i
) = float64_cvt_float32(Vk
->UD(j
+ ofs
* i
),
2751 vec_update_fcsr0(env
, GETPC());
2756 void HELPER(vfrint_s
)(void *vd
, void *vj
,
2757 CPULoongArchState
*env
, uint32_t desc
)
2760 VReg
*Vd
= (VReg
*)vd
;
2761 VReg
*Vj
= (VReg
*)vj
;
2762 int oprsz
= simd_oprsz(desc
);
2764 vec_clear_cause(env
);
2765 for (i
= 0; i
< oprsz
/ 4; i
++) {
2766 Vd
->W(i
) = float32_round_to_int(Vj
->UW(i
), &env
->fp_status
);
2767 vec_update_fcsr0(env
, GETPC());
2771 void HELPER(vfrint_d
)(void *vd
, void *vj
,
2772 CPULoongArchState
*env
, uint32_t desc
)
2775 VReg
*Vd
= (VReg
*)vd
;
2776 VReg
*Vj
= (VReg
*)vj
;
2777 int oprsz
= simd_oprsz(desc
);
2779 vec_clear_cause(env
);
2780 for (i
= 0; i
< oprsz
/ 8; i
++) {
2781 Vd
->D(i
) = float64_round_to_int(Vj
->UD(i
), &env
->fp_status
);
2782 vec_update_fcsr0(env
, GETPC());
2786 #define FCVT_2OP(NAME, BIT, E, MODE) \
2787 void HELPER(NAME)(void *vd, void *vj, \
2788 CPULoongArchState *env, uint32_t desc) \
2791 VReg *Vd = (VReg *)vd; \
2792 VReg *Vj = (VReg *)vj; \
2793 int oprsz = simd_oprsz(desc); \
2795 vec_clear_cause(env); \
2796 for (i = 0; i < oprsz / (BIT / 8); i++) { \
2797 FloatRoundMode old_mode = get_float_rounding_mode(&env->fp_status); \
2798 set_float_rounding_mode(MODE, &env->fp_status); \
2799 Vd->E(i) = float## BIT ## _round_to_int(Vj->E(i), &env->fp_status); \
2800 set_float_rounding_mode(old_mode, &env->fp_status); \
2801 vec_update_fcsr0(env, GETPC()); \
2805 FCVT_2OP(vfrintrne_s
, 32, UW
, float_round_nearest_even
)
2806 FCVT_2OP(vfrintrne_d
, 64, UD
, float_round_nearest_even
)
2807 FCVT_2OP(vfrintrz_s
, 32, UW
, float_round_to_zero
)
2808 FCVT_2OP(vfrintrz_d
, 64, UD
, float_round_to_zero
)
2809 FCVT_2OP(vfrintrp_s
, 32, UW
, float_round_up
)
2810 FCVT_2OP(vfrintrp_d
, 64, UD
, float_round_up
)
2811 FCVT_2OP(vfrintrm_s
, 32, UW
, float_round_down
)
2812 FCVT_2OP(vfrintrm_d
, 64, UD
, float_round_down
)
2814 #define FTINT(NAME, FMT1, FMT2, T1, T2, MODE) \
2815 static T2 do_ftint ## NAME(CPULoongArchState *env, T1 fj) \
2818 FloatRoundMode old_mode = get_float_rounding_mode(&env->fp_status); \
2820 set_float_rounding_mode(MODE, &env->fp_status); \
2821 fd = do_## FMT1 ##_to_## FMT2(env, fj); \
2822 set_float_rounding_mode(old_mode, &env->fp_status); \
2826 #define DO_FTINT(FMT1, FMT2, T1, T2) \
2827 static T2 do_## FMT1 ##_to_## FMT2(CPULoongArchState *env, T1 fj) \
2831 fd = FMT1 ##_to_## FMT2(fj, &env->fp_status); \
2832 if (get_float_exception_flags(&env->fp_status) & (float_flag_invalid)) { \
2833 if (FMT1 ##_is_any_nan(fj)) { \
2837 vec_update_fcsr0(env, GETPC()); \
2841 DO_FTINT(float32
, int32
, uint32_t, uint32_t)
2842 DO_FTINT(float64
, int64
, uint64_t, uint64_t)
2843 DO_FTINT(float32
, uint32
, uint32_t, uint32_t)
2844 DO_FTINT(float64
, uint64
, uint64_t, uint64_t)
2845 DO_FTINT(float64
, int32
, uint64_t, uint32_t)
2846 DO_FTINT(float32
, int64
, uint32_t, uint64_t)
2848 FTINT(rne_w_s
, float32
, int32
, uint32_t, uint32_t, float_round_nearest_even
)
2849 FTINT(rne_l_d
, float64
, int64
, uint64_t, uint64_t, float_round_nearest_even
)
2850 FTINT(rp_w_s
, float32
, int32
, uint32_t, uint32_t, float_round_up
)
2851 FTINT(rp_l_d
, float64
, int64
, uint64_t, uint64_t, float_round_up
)
2852 FTINT(rz_w_s
, float32
, int32
, uint32_t, uint32_t, float_round_to_zero
)
2853 FTINT(rz_l_d
, float64
, int64
, uint64_t, uint64_t, float_round_to_zero
)
2854 FTINT(rm_w_s
, float32
, int32
, uint32_t, uint32_t, float_round_down
)
2855 FTINT(rm_l_d
, float64
, int64
, uint64_t, uint64_t, float_round_down
)
2857 DO_2OP_F(vftintrne_w_s
, 32, UW
, do_ftintrne_w_s
)
2858 DO_2OP_F(vftintrne_l_d
, 64, UD
, do_ftintrne_l_d
)
2859 DO_2OP_F(vftintrp_w_s
, 32, UW
, do_ftintrp_w_s
)
2860 DO_2OP_F(vftintrp_l_d
, 64, UD
, do_ftintrp_l_d
)
2861 DO_2OP_F(vftintrz_w_s
, 32, UW
, do_ftintrz_w_s
)
2862 DO_2OP_F(vftintrz_l_d
, 64, UD
, do_ftintrz_l_d
)
2863 DO_2OP_F(vftintrm_w_s
, 32, UW
, do_ftintrm_w_s
)
2864 DO_2OP_F(vftintrm_l_d
, 64, UD
, do_ftintrm_l_d
)
2865 DO_2OP_F(vftint_w_s
, 32, UW
, do_float32_to_int32
)
2866 DO_2OP_F(vftint_l_d
, 64, UD
, do_float64_to_int64
)
2868 FTINT(rz_wu_s
, float32
, uint32
, uint32_t, uint32_t, float_round_to_zero
)
2869 FTINT(rz_lu_d
, float64
, uint64
, uint64_t, uint64_t, float_round_to_zero
)
2871 DO_2OP_F(vftintrz_wu_s
, 32, UW
, do_ftintrz_wu_s
)
2872 DO_2OP_F(vftintrz_lu_d
, 64, UD
, do_ftintrz_lu_d
)
2873 DO_2OP_F(vftint_wu_s
, 32, UW
, do_float32_to_uint32
)
2874 DO_2OP_F(vftint_lu_d
, 64, UD
, do_float64_to_uint64
)
2876 FTINT(rm_w_d
, float64
, int32
, uint64_t, uint32_t, float_round_down
)
2877 FTINT(rp_w_d
, float64
, int32
, uint64_t, uint32_t, float_round_up
)
2878 FTINT(rz_w_d
, float64
, int32
, uint64_t, uint32_t, float_round_to_zero
)
2879 FTINT(rne_w_d
, float64
, int32
, uint64_t, uint32_t, float_round_nearest_even
)
2881 #define FTINT_W_D(NAME, FN) \
2882 void HELPER(NAME)(void *vd, void *vj, void *vk, \
2883 CPULoongArchState *env, uint32_t desc) \
2887 VReg *Vd = (VReg *)vd; \
2888 VReg *Vj = (VReg *)vj; \
2889 VReg *Vk = (VReg *)vk; \
2890 int oprsz = simd_oprsz(desc); \
2892 ofs = LSX_LEN / 64; \
2893 vec_clear_cause(env); \
2894 for (i = 0; i < oprsz / 16; i++) { \
2895 for (j = 0; j < ofs; j++) { \
2896 temp.W(j + ofs * (2 * i + 1)) = FN(env, Vj->UD(j + ofs * i)); \
2897 temp.W(j + ofs * 2 * i) = FN(env, Vk->UD(j + ofs * i)); \
2903 FTINT_W_D(vftint_w_d
, do_float64_to_int32
)
2904 FTINT_W_D(vftintrm_w_d
, do_ftintrm_w_d
)
2905 FTINT_W_D(vftintrp_w_d
, do_ftintrp_w_d
)
2906 FTINT_W_D(vftintrz_w_d
, do_ftintrz_w_d
)
2907 FTINT_W_D(vftintrne_w_d
, do_ftintrne_w_d
)
2909 FTINT(rml_l_s
, float32
, int64
, uint32_t, uint64_t, float_round_down
)
2910 FTINT(rpl_l_s
, float32
, int64
, uint32_t, uint64_t, float_round_up
)
2911 FTINT(rzl_l_s
, float32
, int64
, uint32_t, uint64_t, float_round_to_zero
)
2912 FTINT(rnel_l_s
, float32
, int64
, uint32_t, uint64_t, float_round_nearest_even
)
2913 FTINT(rmh_l_s
, float32
, int64
, uint32_t, uint64_t, float_round_down
)
2914 FTINT(rph_l_s
, float32
, int64
, uint32_t, uint64_t, float_round_up
)
2915 FTINT(rzh_l_s
, float32
, int64
, uint32_t, uint64_t, float_round_to_zero
)
2916 FTINT(rneh_l_s
, float32
, int64
, uint32_t, uint64_t, float_round_nearest_even
)
2918 #define FTINTL_L_S(NAME, FN) \
2919 void HELPER(NAME)(void *vd, void *vj, \
2920 CPULoongArchState *env, uint32_t desc) \
2924 VReg *Vd = (VReg *)vd; \
2925 VReg *Vj = (VReg *)vj; \
2926 int oprsz = simd_oprsz(desc); \
2928 ofs = LSX_LEN / 64; \
2929 vec_clear_cause(env); \
2930 for (i = 0; i < oprsz / 16; i++) { \
2931 for (j = 0; j < ofs; j++) { \
2932 temp.D(j + ofs * i) = FN(env, Vj->UW(j + ofs * 2 * i)); \
2938 FTINTL_L_S(vftintl_l_s
, do_float32_to_int64
)
2939 FTINTL_L_S(vftintrml_l_s
, do_ftintrml_l_s
)
2940 FTINTL_L_S(vftintrpl_l_s
, do_ftintrpl_l_s
)
2941 FTINTL_L_S(vftintrzl_l_s
, do_ftintrzl_l_s
)
2942 FTINTL_L_S(vftintrnel_l_s
, do_ftintrnel_l_s
)
2944 #define FTINTH_L_S(NAME, FN) \
2945 void HELPER(NAME)(void *vd, void *vj, \
2946 CPULoongArchState *env, uint32_t desc) \
2950 VReg *Vd = (VReg *)vd; \
2951 VReg *Vj = (VReg *)vj; \
2952 int oprsz = simd_oprsz(desc); \
2954 ofs = LSX_LEN / 64; \
2955 vec_clear_cause(env); \
2956 for (i = 0; i < oprsz / 16; i++) { \
2957 for (j = 0; j < ofs; j++) { \
2958 temp.D(j + ofs * i) = FN(env, Vj->UW(j + ofs * (2 * i + 1))); \
2964 FTINTH_L_S(vftinth_l_s
, do_float32_to_int64
)
2965 FTINTH_L_S(vftintrmh_l_s
, do_ftintrmh_l_s
)
2966 FTINTH_L_S(vftintrph_l_s
, do_ftintrph_l_s
)
2967 FTINTH_L_S(vftintrzh_l_s
, do_ftintrzh_l_s
)
2968 FTINTH_L_S(vftintrneh_l_s
, do_ftintrneh_l_s
)
2970 #define FFINT(NAME, FMT1, FMT2, T1, T2) \
2971 static T2 do_ffint_ ## NAME(CPULoongArchState *env, T1 fj) \
2975 fd = FMT1 ##_to_## FMT2(fj, &env->fp_status); \
2976 vec_update_fcsr0(env, GETPC()); \
2980 FFINT(s_w
, int32
, float32
, int32_t, uint32_t)
2981 FFINT(d_l
, int64
, float64
, int64_t, uint64_t)
2982 FFINT(s_wu
, uint32
, float32
, uint32_t, uint32_t)
2983 FFINT(d_lu
, uint64
, float64
, uint64_t, uint64_t)
2985 DO_2OP_F(vffint_s_w
, 32, W
, do_ffint_s_w
)
2986 DO_2OP_F(vffint_d_l
, 64, D
, do_ffint_d_l
)
2987 DO_2OP_F(vffint_s_wu
, 32, UW
, do_ffint_s_wu
)
2988 DO_2OP_F(vffint_d_lu
, 64, UD
, do_ffint_d_lu
)
2990 void HELPER(vffintl_d_w
)(void *vd
, void *vj
,
2991 CPULoongArchState
*env
, uint32_t desc
)
2995 VReg
*Vd
= (VReg
*)vd
;
2996 VReg
*Vj
= (VReg
*)vj
;
2997 int oprsz
= simd_oprsz(desc
);
3000 vec_clear_cause(env
);
3001 for (i
= 0; i
< oprsz
/ 16; i
++) {
3002 for (j
= 0; j
< ofs
; j
++) {
3003 temp
.D(j
+ ofs
* i
) = int32_to_float64(Vj
->W(j
+ ofs
* 2 * i
),
3006 vec_update_fcsr0(env
, GETPC());
3011 void HELPER(vffinth_d_w
)(void *vd
, void *vj
,
3012 CPULoongArchState
*env
, uint32_t desc
)
3016 VReg
*Vd
= (VReg
*)vd
;
3017 VReg
*Vj
= (VReg
*)vj
;
3018 int oprsz
= simd_oprsz(desc
);
3021 vec_clear_cause(env
);
3022 for (i
= 0; i
< oprsz
/16; i
++) {
3023 for (j
= 0; j
< ofs
; j
++) {
3024 temp
.D(j
+ ofs
* i
) = int32_to_float64(Vj
->W(j
+ ofs
* (2 * i
+ 1)),
3027 vec_update_fcsr0(env
, GETPC());
3032 void HELPER(vffint_s_l
)(void *vd
, void *vj
, void *vk
,
3033 CPULoongArchState
*env
, uint32_t desc
)
3037 VReg
*Vd
= (VReg
*)vd
;
3038 VReg
*Vj
= (VReg
*)vj
;
3039 VReg
*Vk
= (VReg
*)vk
;
3040 int oprsz
= simd_oprsz(desc
);
3043 vec_clear_cause(env
);
3044 for (i
= 0; i
< oprsz
/ 16; i
++) {
3045 for (j
= 0; j
< ofs
; j
++) {
3046 temp
.W(j
+ ofs
* (2 * i
+ 1)) = int64_to_float32(Vj
->D(j
+ ofs
* i
),
3048 temp
.W(j
+ ofs
* 2 * i
) = int64_to_float32(Vk
->D(j
+ ofs
* i
),
3051 vec_update_fcsr0(env
, GETPC());
3056 #define VSEQ(a, b) (a == b ? -1 : 0)
3057 #define VSLE(a, b) (a <= b ? -1 : 0)
3058 #define VSLT(a, b) (a < b ? -1 : 0)
3060 #define VCMPI(NAME, BIT, E, DO_OP) \
3061 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
3064 VReg *Vd = (VReg *)vd; \
3065 VReg *Vj = (VReg *)vj; \
3066 typedef __typeof(Vd->E(0)) TD; \
3067 int oprsz = simd_oprsz(desc); \
3069 for (i = 0; i < oprsz / (BIT / 8); i++) { \
3070 Vd->E(i) = DO_OP(Vj->E(i), (TD)imm); \
3074 VCMPI(vseqi_b
, 8, B
, VSEQ
)
3075 VCMPI(vseqi_h
, 16, H
, VSEQ
)
3076 VCMPI(vseqi_w
, 32, W
, VSEQ
)
3077 VCMPI(vseqi_d
, 64, D
, VSEQ
)
3078 VCMPI(vslei_b
, 8, B
, VSLE
)
3079 VCMPI(vslei_h
, 16, H
, VSLE
)
3080 VCMPI(vslei_w
, 32, W
, VSLE
)
3081 VCMPI(vslei_d
, 64, D
, VSLE
)
3082 VCMPI(vslei_bu
, 8, UB
, VSLE
)
3083 VCMPI(vslei_hu
, 16, UH
, VSLE
)
3084 VCMPI(vslei_wu
, 32, UW
, VSLE
)
3085 VCMPI(vslei_du
, 64, UD
, VSLE
)
3086 VCMPI(vslti_b
, 8, B
, VSLT
)
3087 VCMPI(vslti_h
, 16, H
, VSLT
)
3088 VCMPI(vslti_w
, 32, W
, VSLT
)
3089 VCMPI(vslti_d
, 64, D
, VSLT
)
3090 VCMPI(vslti_bu
, 8, UB
, VSLT
)
3091 VCMPI(vslti_hu
, 16, UH
, VSLT
)
3092 VCMPI(vslti_wu
, 32, UW
, VSLT
)
3093 VCMPI(vslti_du
, 64, UD
, VSLT
)
3095 static uint64_t vfcmp_common(CPULoongArchState
*env
,
3096 FloatRelation cmp
, uint32_t flags
)
3101 case float_relation_less
:
3102 ret
= (flags
& FCMP_LT
);
3104 case float_relation_equal
:
3105 ret
= (flags
& FCMP_EQ
);
3107 case float_relation_greater
:
3108 ret
= (flags
& FCMP_GT
);
3110 case float_relation_unordered
:
3111 ret
= (flags
& FCMP_UN
);
3114 g_assert_not_reached();
3124 #define VFCMP(NAME, BIT, E, FN) \
3125 void HELPER(NAME)(CPULoongArchState *env, uint32_t oprsz, \
3126 uint32_t vd, uint32_t vj, uint32_t vk, uint32_t flags) \
3130 VReg *Vd = &(env->fpr[vd].vreg); \
3131 VReg *Vj = &(env->fpr[vj].vreg); \
3132 VReg *Vk = &(env->fpr[vk].vreg); \
3134 vec_clear_cause(env); \
3135 for (i = 0; i < oprsz / (BIT / 8); i++) { \
3136 FloatRelation cmp; \
3137 cmp = FN(Vj->E(i), Vk->E(i), &env->fp_status); \
3138 t.E(i) = vfcmp_common(env, cmp, flags); \
3139 vec_update_fcsr0(env, GETPC()); \
3144 VFCMP(vfcmp_c_s
, 32, UW
, float32_compare_quiet
)
3145 VFCMP(vfcmp_s_s
, 32, UW
, float32_compare
)
3146 VFCMP(vfcmp_c_d
, 64, UD
, float64_compare_quiet
)
3147 VFCMP(vfcmp_s_d
, 64, UD
, float64_compare
)
3149 void HELPER(vbitseli_b
)(void *vd
, void *vj
, uint64_t imm
, uint32_t v
)
3152 VReg
*Vd
= (VReg
*)vd
;
3153 VReg
*Vj
= (VReg
*)vj
;
3155 for (i
= 0; i
< 16; i
++) {
3156 Vd
->B(i
) = (~Vd
->B(i
) & Vj
->B(i
)) | (Vd
->B(i
) & imm
);
3160 /* Copy from target/arm/tcg/sve_helper.c */
3161 static inline bool do_match2(uint64_t n
, uint64_t m0
, uint64_t m1
, int esz
)
3163 uint64_t bits
= 8 << esz
;
3164 uint64_t ones
= dup_const(esz
, 1);
3165 uint64_t signs
= ones
<< (bits
- 1);
3166 uint64_t cmp0
, cmp1
;
3168 cmp1
= dup_const(esz
, n
);
3171 cmp0
= (cmp0
- ones
) & ~cmp0
;
3172 cmp1
= (cmp1
- ones
) & ~cmp1
;
3173 return (cmp0
| cmp1
) & signs
;
3176 #define SETANYEQZ(NAME, MO) \
3177 void HELPER(NAME)(CPULoongArchState *env, uint32_t cd, uint32_t vj) \
3179 VReg *Vj = &(env->fpr[vj].vreg); \
3181 env->cf[cd & 0x7] = do_match2(0, Vj->D(0), Vj->D(1), MO); \
3183 SETANYEQZ(vsetanyeqz_b
, MO_8
)
3184 SETANYEQZ(vsetanyeqz_h
, MO_16
)
3185 SETANYEQZ(vsetanyeqz_w
, MO_32
)
3186 SETANYEQZ(vsetanyeqz_d
, MO_64
)
3188 #define SETALLNEZ(NAME, MO) \
3189 void HELPER(NAME)(CPULoongArchState *env, uint32_t cd, uint32_t vj) \
3191 VReg *Vj = &(env->fpr[vj].vreg); \
3193 env->cf[cd & 0x7]= !do_match2(0, Vj->D(0), Vj->D(1), MO); \
3195 SETALLNEZ(vsetallnez_b
, MO_8
)
3196 SETALLNEZ(vsetallnez_h
, MO_16
)
3197 SETALLNEZ(vsetallnez_w
, MO_32
)
3198 SETALLNEZ(vsetallnez_d
, MO_64
)
3200 #define VPACKEV(NAME, BIT, E) \
3201 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
3205 VReg *Vd = (VReg *)vd; \
3206 VReg *Vj = (VReg *)vj; \
3207 VReg *Vk = (VReg *)vk; \
3209 for (i = 0; i < LSX_LEN/BIT; i++) { \
3210 temp.E(2 * i + 1) = Vj->E(2 * i); \
3211 temp.E(2 *i) = Vk->E(2 * i); \
3216 VPACKEV(vpackev_b
, 16, B
)
3217 VPACKEV(vpackev_h
, 32, H
)
3218 VPACKEV(vpackev_w
, 64, W
)
3219 VPACKEV(vpackev_d
, 128, D
)
3221 #define VPACKOD(NAME, BIT, E) \
3222 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
3226 VReg *Vd = (VReg *)vd; \
3227 VReg *Vj = (VReg *)vj; \
3228 VReg *Vk = (VReg *)vk; \
3230 for (i = 0; i < LSX_LEN/BIT; i++) { \
3231 temp.E(2 * i + 1) = Vj->E(2 * i + 1); \
3232 temp.E(2 * i) = Vk->E(2 * i + 1); \
3237 VPACKOD(vpackod_b
, 16, B
)
3238 VPACKOD(vpackod_h
, 32, H
)
3239 VPACKOD(vpackod_w
, 64, W
)
3240 VPACKOD(vpackod_d
, 128, D
)
3242 #define VPICKEV(NAME, BIT, E) \
3243 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
3247 VReg *Vd = (VReg *)vd; \
3248 VReg *Vj = (VReg *)vj; \
3249 VReg *Vk = (VReg *)vk; \
3251 for (i = 0; i < LSX_LEN/BIT; i++) { \
3252 temp.E(i + LSX_LEN/BIT) = Vj->E(2 * i); \
3253 temp.E(i) = Vk->E(2 * i); \
3258 VPICKEV(vpickev_b
, 16, B
)
3259 VPICKEV(vpickev_h
, 32, H
)
3260 VPICKEV(vpickev_w
, 64, W
)
3261 VPICKEV(vpickev_d
, 128, D
)
3263 #define VPICKOD(NAME, BIT, E) \
3264 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
3268 VReg *Vd = (VReg *)vd; \
3269 VReg *Vj = (VReg *)vj; \
3270 VReg *Vk = (VReg *)vk; \
3272 for (i = 0; i < LSX_LEN/BIT; i++) { \
3273 temp.E(i + LSX_LEN/BIT) = Vj->E(2 * i + 1); \
3274 temp.E(i) = Vk->E(2 * i + 1); \
3279 VPICKOD(vpickod_b
, 16, B
)
3280 VPICKOD(vpickod_h
, 32, H
)
3281 VPICKOD(vpickod_w
, 64, W
)
3282 VPICKOD(vpickod_d
, 128, D
)
3284 #define VILVL(NAME, BIT, E) \
3285 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
3289 VReg *Vd = (VReg *)vd; \
3290 VReg *Vj = (VReg *)vj; \
3291 VReg *Vk = (VReg *)vk; \
3293 for (i = 0; i < LSX_LEN/BIT; i++) { \
3294 temp.E(2 * i + 1) = Vj->E(i); \
3295 temp.E(2 * i) = Vk->E(i); \
3300 VILVL(vilvl_b
, 16, B
)
3301 VILVL(vilvl_h
, 32, H
)
3302 VILVL(vilvl_w
, 64, W
)
3303 VILVL(vilvl_d
, 128, D
)
3305 #define VILVH(NAME, BIT, E) \
3306 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
3310 VReg *Vd = (VReg *)vd; \
3311 VReg *Vj = (VReg *)vj; \
3312 VReg *Vk = (VReg *)vk; \
3314 for (i = 0; i < LSX_LEN/BIT; i++) { \
3315 temp.E(2 * i + 1) = Vj->E(i + LSX_LEN/BIT); \
3316 temp.E(2 * i) = Vk->E(i + LSX_LEN/BIT); \
3321 VILVH(vilvh_b
, 16, B
)
3322 VILVH(vilvh_h
, 32, H
)
3323 VILVH(vilvh_w
, 64, W
)
3324 VILVH(vilvh_d
, 128, D
)
3326 void HELPER(vshuf_b
)(void *vd
, void *vj
, void *vk
, void *va
, uint32_t desc
)
3330 VReg
*Vd
= (VReg
*)vd
;
3331 VReg
*Vj
= (VReg
*)vj
;
3332 VReg
*Vk
= (VReg
*)vk
;
3333 VReg
*Va
= (VReg
*)va
;
3336 for (i
= 0; i
< m
; i
++) {
3337 uint64_t k
= (uint8_t)Va
->B(i
) % (2 * m
);
3338 temp
.B(i
) = k
< m
? Vk
->B(k
) : Vj
->B(k
- m
);
3343 #define VSHUF(NAME, BIT, E) \
3344 void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \
3348 VReg *Vd = (VReg *)vd; \
3349 VReg *Vj = (VReg *)vj; \
3350 VReg *Vk = (VReg *)vk; \
3353 for (i = 0; i < m; i++) { \
3354 uint64_t k = ((uint8_t) Vd->E(i)) % (2 * m); \
3355 temp.E(i) = k < m ? Vk->E(k) : Vj->E(k - m); \
3360 VSHUF(vshuf_h
, 16, H
)
3361 VSHUF(vshuf_w
, 32, W
)
3362 VSHUF(vshuf_d
, 64, D
)
3364 #define VSHUF4I(NAME, BIT, E) \
3365 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
3369 VReg *Vd = (VReg *)vd; \
3370 VReg *Vj = (VReg *)vj; \
3372 for (i = 0; i < LSX_LEN/BIT; i++) { \
3373 temp.E(i) = Vj->E(((i) & 0xfc) + (((imm) >> \
3374 (2 * ((i) & 0x03))) & 0x03)); \
3379 VSHUF4I(vshuf4i_b
, 8, B
)
3380 VSHUF4I(vshuf4i_h
, 16, H
)
3381 VSHUF4I(vshuf4i_w
, 32, W
)
3383 void HELPER(vshuf4i_d
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
3385 VReg
*Vd
= (VReg
*)vd
;
3386 VReg
*Vj
= (VReg
*)vj
;
3389 temp
.D(0) = (imm
& 2 ? Vj
: Vd
)->D(imm
& 1);
3390 temp
.D(1) = (imm
& 8 ? Vj
: Vd
)->D((imm
>> 2) & 1);
3394 void HELPER(vpermi_w
)(void *vd
, void *vj
, uint64_t imm
, uint32_t desc
)
3397 VReg
*Vd
= (VReg
*)vd
;
3398 VReg
*Vj
= (VReg
*)vj
;
3400 temp
.W(0) = Vj
->W(imm
& 0x3);
3401 temp
.W(1) = Vj
->W((imm
>> 2) & 0x3);
3402 temp
.W(2) = Vd
->W((imm
>> 4) & 0x3);
3403 temp
.W(3) = Vd
->W((imm
>> 6) & 0x3);
3407 #define VEXTRINS(NAME, BIT, E, MASK) \
3408 void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \
3411 VReg *Vd = (VReg *)vd; \
3412 VReg *Vj = (VReg *)vj; \
3414 ins = (imm >> 4) & MASK; \
3415 extr = imm & MASK; \
3416 Vd->E(ins) = Vj->E(extr); \
3419 VEXTRINS(vextrins_b
, 8, B
, 0xf)
3420 VEXTRINS(vextrins_h
, 16, H
, 0x7)
3421 VEXTRINS(vextrins_w
, 32, W
, 0x3)
3422 VEXTRINS(vextrins_d
, 64, D
, 0x1)