]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/aarch64/tcg-target.inc.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg: Restart TB generation after constant pool overflow
[mirror_qemu.git] / tcg / aarch64 / tcg-target.inc.c
CommitLineData
4a136e0a
CF		 1/*
2 * Initial TCG Implementation for aarch64
3 *
4 * Copyright (c) 2013 Huawei Technologies Duesseldorf GmbH
5 * Written by Claudio Fontana
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or
8 * (at your option) any later version.
9 *
10 * See the COPYING file in the top-level directory for details.
11 */
12
55129955 13#include "tcg-pool.inc.c"
4a136e0a
CF		 14#include "qemu/bitops.h"
15
7763ffa0
RH		 16/* We're going to re-use TCGType in setting of the SF bit, which controls
17 the size of the operation performed. If we know the values match, it
18 makes things much cleaner. */
19QEMU_BUILD_BUG_ON(TCG_TYPE_I32 != 0 || TCG_TYPE_I64 != 1);
20
8d8fdbae 21#ifdef CONFIG_DEBUG_TCG
4a136e0a 22static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
14e4c1e2
RH		 23 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
24 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
25 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
26 "x24", "x25", "x26", "x27", "x28", "fp", "x30", "sp",
27
28 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
29 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
30 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
31 "v24", "v25", "v26", "v27", "v28", "fp", "v30", "v31",
4a136e0a 32};
8d8fdbae 33#endif /* CONFIG_DEBUG_TCG */
4a136e0a
CF		 34
35static const int tcg_target_reg_alloc_order[] = {
36 TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
37 TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
b76f21a7 38 TCG_REG_X28, /* we will reserve this for guest_base if configured */
4a136e0a 39
d82b78e4
RH		 40 TCG_REG_X8, TCG_REG_X9, TCG_REG_X10, TCG_REG_X11,
41 TCG_REG_X12, TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
4a136e0a
CF		 42 TCG_REG_X16, TCG_REG_X17,
43
4a136e0a
CF		 44 TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
45 TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7,
46
d82b78e4
RH		 47 /* X18 reserved by system */
48 /* X19 reserved for AREG0 */
49 /* X29 reserved as fp */
50 /* X30 reserved as temporary */
14e4c1e2
RH		 51
52 TCG_REG_V0, TCG_REG_V1, TCG_REG_V2, TCG_REG_V3,
53 TCG_REG_V4, TCG_REG_V5, TCG_REG_V6, TCG_REG_V7,
54 /* V8 - V15 are call-saved, and skipped. */
55 TCG_REG_V16, TCG_REG_V17, TCG_REG_V18, TCG_REG_V19,
56 TCG_REG_V20, TCG_REG_V21, TCG_REG_V22, TCG_REG_V23,
57 TCG_REG_V24, TCG_REG_V25, TCG_REG_V26, TCG_REG_V27,
58 TCG_REG_V28, TCG_REG_V29, TCG_REG_V30, TCG_REG_V31,
4a136e0a
CF		 59};
60
61static const int tcg_target_call_iarg_regs[8] = {
62 TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
63 TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7
64};
65static const int tcg_target_call_oarg_regs[1] = {
66 TCG_REG_X0
67};
68
d82b78e4 69#define TCG_REG_TMP TCG_REG_X30
14e4c1e2 70#define TCG_VEC_TMP TCG_REG_V31
4a136e0a 71
6a91c7c9 72#ifndef CONFIG_SOFTMMU
352bcb0a
RH		 73/* Note that XZR cannot be encoded in the address base register slot,
74 as that actaully encodes SP. So if we need to zero-extend the guest
75 address, via the address index register slot, we need to load even
76 a zero guest base into a register. */
77#define USE_GUEST_BASE (guest_base != 0 || TARGET_LONG_BITS == 32)
4cbea598 78#define TCG_REG_GUEST_BASE TCG_REG_X28
6a91c7c9
JK		 79#endif
80
214bfe83 81static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
4a136e0a 82{
8587c30c 83 ptrdiff_t offset = target - code_ptr;
214bfe83
RH		 84 if (offset == sextract64(offset, 0, 26)) {
85 /* read instruction, mask away previous PC_REL26 parameter contents,
86 set the proper offset, then write back the instruction. */
87 *code_ptr = deposit32(*code_ptr, 0, 26, offset);
88 return true;
89 }
90 return false;
4a136e0a
CF		 91}
92
214bfe83 93static inline bool reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
4a136e0a 94{
8587c30c 95 ptrdiff_t offset = target - code_ptr;
214bfe83
RH		 96 if (offset == sextract64(offset, 0, 19)) {
97 *code_ptr = deposit32(*code_ptr, 5, 19, offset);
98 return true;
99 }
100 return false;
4a136e0a
CF		 101}
102
6ac17786 103static inline bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 104 intptr_t value, intptr_t addend)
4a136e0a 105{
eabb7b91 106 tcg_debug_assert(addend == 0);
4a136e0a
CF		 107 switch (type) {
108 case R_AARCH64_JUMP26:
109 case R_AARCH64_CALL26:
214bfe83 110 return reloc_pc26(code_ptr, (tcg_insn_unit *)value);
4a136e0a 111 case R_AARCH64_CONDBR19:
214bfe83 112 return reloc_pc19(code_ptr, (tcg_insn_unit *)value);
4a136e0a 113 default:
214bfe83 114 g_assert_not_reached();
4a136e0a
CF		 115 }
116}
117
170bf931
RH		 118#define TCG_CT_CONST_AIMM 0x100
119#define TCG_CT_CONST_LIMM 0x200
120#define TCG_CT_CONST_ZERO 0x400
121#define TCG_CT_CONST_MONE 0x800
90f1cd91 122
4a136e0a 123/* parse target specific constraints */
069ea736
RH		 124static const char *target_parse_constraint(TCGArgConstraint *ct,
125 const char *ct_str, TCGType type)
4a136e0a 126{
069ea736 127 switch (*ct_str++) {
14e4c1e2 128 case 'r': /* general registers */
4a136e0a 129 ct->ct |= TCG_CT_REG;
14e4c1e2
RH		 130 ct->u.regs |= 0xffffffffu;
131 break;
132 case 'w': /* advsimd registers */
133 ct->ct |= TCG_CT_REG;
134 ct->u.regs |= 0xffffffff00000000ull;
4a136e0a
CF		 135 break;
136 case 'l': /* qemu_ld / qemu_st address, data_reg */
137 ct->ct |= TCG_CT_REG;
f46934df 138 ct->u.regs = 0xffffffffu;
4a136e0a
CF		 139#ifdef CONFIG_SOFTMMU
140 /* x0 and x1 will be overwritten when reading the tlb entry,
141 and x2, and x3 for helper args, better to avoid using them. */
142 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X0);
143 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X1);
144 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X2);
145 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X3);
146#endif
147 break;
90f1cd91
RH		 148 case 'A': /* Valid for arithmetic immediate (positive or negative). */
149 ct->ct |= TCG_CT_CONST_AIMM;
150 break;
e029f293
RH		 151 case 'L': /* Valid for logical immediate. */
152 ct->ct |= TCG_CT_CONST_LIMM;
153 break;
c6e929e7
RH		 154 case 'M': /* minus one */
155 ct->ct |= TCG_CT_CONST_MONE;
156 break;
04ce397b
RH		 157 case 'Z': /* zero */
158 ct->ct |= TCG_CT_CONST_ZERO;
159 break;
4a136e0a 160 default:
069ea736 161 return NULL;
4a136e0a 162 }
069ea736 163 return ct_str;
4a136e0a
CF		 164}
165
14e4c1e2 166/* Match a constant valid for addition (12-bit, optionally shifted). */
90f1cd91
RH		 167static inline bool is_aimm(uint64_t val)
168{
169 return (val & ~0xfff) == 0 || (val & ~0xfff000) == 0;
170}
171
14e4c1e2 172/* Match a constant valid for logical operations. */
e029f293
RH		 173static inline bool is_limm(uint64_t val)
174{
175 /* Taking a simplified view of the logical immediates for now, ignoring
176 the replication that can happen across the field. Match bit patterns
177 of the forms
178 0....01....1
179 0..01..10..0
180 and their inverses. */
181
182 /* Make things easier below, by testing the form with msb clear. */
183 if ((int64_t)val < 0) {
184 val = ~val;
185 }
186 if (val == 0) {
187 return false;
188 }
189 val += val & -val;
190 return (val & (val - 1)) == 0;
191}
192
14e4c1e2
RH		 193/* Match a constant that is valid for vectors. */
194static bool is_fimm(uint64_t v64, int *op, int *cmode, int *imm8)
195{
196 int i;
197
198 *op = 0;
199 /* Match replication across 8 bits. */
200 if (v64 == dup_const(MO_8, v64)) {
201 *cmode = 0xe;
202 *imm8 = v64 & 0xff;
203 return true;
204 }
205 /* Match replication across 16 bits. */
206 if (v64 == dup_const(MO_16, v64)) {
207 uint16_t v16 = v64;
208
209 if (v16 == (v16 & 0xff)) {
210 *cmode = 0x8;
211 *imm8 = v16 & 0xff;
212 return true;
213 } else if (v16 == (v16 & 0xff00)) {
214 *cmode = 0xa;
215 *imm8 = v16 >> 8;
216 return true;
217 }
218 }
219 /* Match replication across 32 bits. */
220 if (v64 == dup_const(MO_32, v64)) {
221 uint32_t v32 = v64;
222
223 if (v32 == (v32 & 0xff)) {
224 *cmode = 0x0;
225 *imm8 = v32 & 0xff;
226 return true;
227 } else if (v32 == (v32 & 0xff00)) {
228 *cmode = 0x2;
229 *imm8 = (v32 >> 8) & 0xff;
230 return true;
231 } else if (v32 == (v32 & 0xff0000)) {
232 *cmode = 0x4;
233 *imm8 = (v32 >> 16) & 0xff;
234 return true;
235 } else if (v32 == (v32 & 0xff000000)) {
236 *cmode = 0x6;
237 *imm8 = v32 >> 24;
238 return true;
239 } else if ((v32 & 0xffff00ff) == 0xff) {
240 *cmode = 0xc;
241 *imm8 = (v32 >> 8) & 0xff;
242 return true;
243 } else if ((v32 & 0xff00ffff) == 0xffff) {
244 *cmode = 0xd;
245 *imm8 = (v32 >> 16) & 0xff;
246 return true;
247 }
248 /* Match forms of a float32. */
249 if (extract32(v32, 0, 19) == 0
250 && (extract32(v32, 25, 6) == 0x20
251 || extract32(v32, 25, 6) == 0x1f)) {
252 *cmode = 0xf;
253 *imm8 = (extract32(v32, 31, 1) << 7)
254 | (extract32(v32, 25, 1) << 6)
255 | extract32(v32, 19, 6);
256 return true;
257 }
258 }
259 /* Match forms of a float64. */
260 if (extract64(v64, 0, 48) == 0
261 && (extract64(v64, 54, 9) == 0x100
262 || extract64(v64, 54, 9) == 0x0ff)) {
263 *cmode = 0xf;
264 *op = 1;
265 *imm8 = (extract64(v64, 63, 1) << 7)
266 | (extract64(v64, 54, 1) << 6)
267 | extract64(v64, 48, 6);
268 return true;
269 }
270 /* Match bytes of 0x00 and 0xff. */
271 for (i = 0; i < 64; i += 8) {
272 uint64_t byte = extract64(v64, i, 8);
273 if (byte != 0 && byte != 0xff) {
274 break;
275 }
276 }
277 if (i == 64) {
278 *cmode = 0xe;
279 *op = 1;
280 *imm8 = (extract64(v64, 0, 1) << 0)
281 | (extract64(v64, 8, 1) << 1)
282 | (extract64(v64, 16, 1) << 2)
283 | (extract64(v64, 24, 1) << 3)
284 | (extract64(v64, 32, 1) << 4)
285 | (extract64(v64, 40, 1) << 5)
286 | (extract64(v64, 48, 1) << 6)
287 | (extract64(v64, 56, 1) << 7);
288 return true;
289 }
290 return false;
291}
292
f6c6afc1 293static int tcg_target_const_match(tcg_target_long val, TCGType type,
90f1cd91 294 const TCGArgConstraint *arg_ct)
4a136e0a
CF		 295{
296 int ct = arg_ct->ct;
297
298 if (ct & TCG_CT_CONST) {
299 return 1;
300 }
170bf931 301 if (type == TCG_TYPE_I32) {
90f1cd91
RH		 302 val = (int32_t)val;
303 }
304 if ((ct & TCG_CT_CONST_AIMM) && (is_aimm(val) || is_aimm(-val))) {
305 return 1;
306 }
e029f293
RH		 307 if ((ct & TCG_CT_CONST_LIMM) && is_limm(val)) {
308 return 1;
309 }
04ce397b
RH		 310 if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
311 return 1;
312 }
c6e929e7
RH		 313 if ((ct & TCG_CT_CONST_MONE) && val == -1) {
314 return 1;
315 }
4a136e0a
CF		 316
317 return 0;
318}
319
320enum aarch64_cond_code {
321 COND_EQ = 0x0,
322 COND_NE = 0x1,
323 COND_CS = 0x2, /* Unsigned greater or equal */
324 COND_HS = COND_CS, /* ALIAS greater or equal */
325 COND_CC = 0x3, /* Unsigned less than */
326 COND_LO = COND_CC, /* ALIAS Lower */
327 COND_MI = 0x4, /* Negative */
328 COND_PL = 0x5, /* Zero or greater */
329 COND_VS = 0x6, /* Overflow */
330 COND_VC = 0x7, /* No overflow */
331 COND_HI = 0x8, /* Unsigned greater than */
332 COND_LS = 0x9, /* Unsigned less or equal */
333 COND_GE = 0xa,
334 COND_LT = 0xb,
335 COND_GT = 0xc,
336 COND_LE = 0xd,
337 COND_AL = 0xe,
338 COND_NV = 0xf, /* behaves like COND_AL here */
339};
340
341static const enum aarch64_cond_code tcg_cond_to_aarch64[] = {
342 [TCG_COND_EQ] = COND_EQ,
343 [TCG_COND_NE] = COND_NE,
344 [TCG_COND_LT] = COND_LT,
345 [TCG_COND_GE] = COND_GE,
346 [TCG_COND_LE] = COND_LE,
347 [TCG_COND_GT] = COND_GT,
348 /* unsigned */
349 [TCG_COND_LTU] = COND_LO,
350 [TCG_COND_GTU] = COND_HI,
351 [TCG_COND_GEU] = COND_HS,
352 [TCG_COND_LEU] = COND_LS,
353};
354
3d4299f4
RH		 355typedef enum {
356 LDST_ST = 0, /* store */
357 LDST_LD = 1, /* load */
358 LDST_LD_S_X = 2, /* load and sign-extend into Xt */
359 LDST_LD_S_W = 3, /* load and sign-extend into Wt */
360} AArch64LdstType;
4a136e0a 361
50573c66
RH		 362/* We encode the format of the insn into the beginning of the name, so that
363 we can have the preprocessor help "typecheck" the insn vs the output
364 function. Arm didn't provide us with nice names for the formats, so we
365 use the section number of the architecture reference manual in which the
366 instruction group is described. */
367typedef enum {
3d9e69a2
RH		 368 /* Compare and branch (immediate). */
369 I3201_CBZ = 0x34000000,
370 I3201_CBNZ = 0x35000000,
371
81d8a5ee
RH		 372 /* Conditional branch (immediate). */
373 I3202_B_C = 0x54000000,
374
375 /* Unconditional branch (immediate). */
376 I3206_B = 0x14000000,
377 I3206_BL = 0x94000000,
378
379 /* Unconditional branch (register). */
380 I3207_BR = 0xd61f0000,
381 I3207_BLR = 0xd63f0000,
382 I3207_RET = 0xd65f0000,
383
2acee8b2
PK		 384 /* Load literal for loading the address at pc-relative offset */
385 I3305_LDR = 0x58000000,
14e4c1e2
RH		 386 I3305_LDR_v64 = 0x5c000000,
387 I3305_LDR_v128 = 0x9c000000,
388
3d4299f4
RH		 389 /* Load/store register. Described here as 3.3.12, but the helper
390 that emits them can transform to 3.3.10 or 3.3.13. */
391 I3312_STRB = 0x38000000 | LDST_ST << 22 | MO_8 << 30,
392 I3312_STRH = 0x38000000 | LDST_ST << 22 | MO_16 << 30,
393 I3312_STRW = 0x38000000 | LDST_ST << 22 | MO_32 << 30,
394 I3312_STRX = 0x38000000 | LDST_ST << 22 | MO_64 << 30,
395
396 I3312_LDRB = 0x38000000 | LDST_LD << 22 | MO_8 << 30,
397 I3312_LDRH = 0x38000000 | LDST_LD << 22 | MO_16 << 30,
398 I3312_LDRW = 0x38000000 | LDST_LD << 22 | MO_32 << 30,
399 I3312_LDRX = 0x38000000 | LDST_LD << 22 | MO_64 << 30,
400
401 I3312_LDRSBW = 0x38000000 | LDST_LD_S_W << 22 | MO_8 << 30,
402 I3312_LDRSHW = 0x38000000 | LDST_LD_S_W << 22 | MO_16 << 30,
403
404 I3312_LDRSBX = 0x38000000 | LDST_LD_S_X << 22 | MO_8 << 30,
405 I3312_LDRSHX = 0x38000000 | LDST_LD_S_X << 22 | MO_16 << 30,
406 I3312_LDRSWX = 0x38000000 | LDST_LD_S_X << 22 | MO_32 << 30,
407
14e4c1e2
RH		 408 I3312_LDRVS = 0x3c000000 | LDST_LD << 22 | MO_32 << 30,
409 I3312_STRVS = 0x3c000000 | LDST_ST << 22 | MO_32 << 30,
410
411 I3312_LDRVD = 0x3c000000 | LDST_LD << 22 | MO_64 << 30,
412 I3312_STRVD = 0x3c000000 | LDST_ST << 22 | MO_64 << 30,
413
414 I3312_LDRVQ = 0x3c000000 | 3 << 22 | 0 << 30,
415 I3312_STRVQ = 0x3c000000 | 2 << 22 | 0 << 30,
416
6c0f0c0f 417 I3312_TO_I3310 = 0x00200800,
3d4299f4
RH		 418 I3312_TO_I3313 = 0x01000000,
419
95f72aa9
RH		 420 /* Load/store register pair instructions. */
421 I3314_LDP = 0x28400000,
422 I3314_STP = 0x28000000,
423
096c46c0
RH		 424 /* Add/subtract immediate instructions. */
425 I3401_ADDI = 0x11000000,
426 I3401_ADDSI = 0x31000000,
427 I3401_SUBI = 0x51000000,
428 I3401_SUBSI = 0x71000000,
429
b3c56df7
RH		 430 /* Bitfield instructions. */
431 I3402_BFM = 0x33000000,
432 I3402_SBFM = 0x13000000,
433 I3402_UBFM = 0x53000000,
434
435 /* Extract instruction. */
436 I3403_EXTR = 0x13800000,
437
e029f293
RH		 438 /* Logical immediate instructions. */
439 I3404_ANDI = 0x12000000,
440 I3404_ORRI = 0x32000000,
441 I3404_EORI = 0x52000000,
442
582ab779
RH		 443 /* Move wide immediate instructions. */
444 I3405_MOVN = 0x12800000,
445 I3405_MOVZ = 0x52800000,
446 I3405_MOVK = 0x72800000,
447
c6e310d9
RH		 448 /* PC relative addressing instructions. */
449 I3406_ADR = 0x10000000,
450 I3406_ADRP = 0x90000000,
451
50573c66
RH		 452 /* Add/subtract shifted register instructions (without a shift). */
453 I3502_ADD = 0x0b000000,
454 I3502_ADDS = 0x2b000000,
455 I3502_SUB = 0x4b000000,
456 I3502_SUBS = 0x6b000000,
457
458 /* Add/subtract shifted register instructions (with a shift). */
459 I3502S_ADD_LSL = I3502_ADD,
460
c6e929e7
RH		 461 /* Add/subtract with carry instructions. */
462 I3503_ADC = 0x1a000000,
463 I3503_SBC = 0x5a000000,
464
04ce397b
RH		 465 /* Conditional select instructions. */
466 I3506_CSEL = 0x1a800000,
467 I3506_CSINC = 0x1a800400,
53c76c19
RH		 468 I3506_CSINV = 0x5a800000,
469 I3506_CSNEG = 0x5a800400,
04ce397b 470
edd8824c 471 /* Data-processing (1 source) instructions. */
53c76c19
RH		 472 I3507_CLZ = 0x5ac01000,
473 I3507_RBIT = 0x5ac00000,
edd8824c
RH		 474 I3507_REV16 = 0x5ac00400,
475 I3507_REV32 = 0x5ac00800,
476 I3507_REV64 = 0x5ac00c00,
477
df9351e3
RH		 478 /* Data-processing (2 source) instructions. */
479 I3508_LSLV = 0x1ac02000,
480 I3508_LSRV = 0x1ac02400,
481 I3508_ASRV = 0x1ac02800,
482 I3508_RORV = 0x1ac02c00,
1fcc9ddf
RH		 483 I3508_SMULH = 0x9b407c00,
484 I3508_UMULH = 0x9bc07c00,
8678b71c
RH		 485 I3508_UDIV = 0x1ac00800,
486 I3508_SDIV = 0x1ac00c00,
487
488 /* Data-processing (3 source) instructions. */
489 I3509_MADD = 0x1b000000,
490 I3509_MSUB = 0x1b008000,
df9351e3 491
50573c66
RH		 492 /* Logical shifted register instructions (without a shift). */
493 I3510_AND = 0x0a000000,
14b155dd 494 I3510_BIC = 0x0a200000,
50573c66 495 I3510_ORR = 0x2a000000,
14b155dd 496 I3510_ORN = 0x2a200000,
50573c66 497 I3510_EOR = 0x4a000000,
14b155dd 498 I3510_EON = 0x4a200000,
50573c66 499 I3510_ANDS = 0x6a000000,
c7a59c2a 500
f7bcd966
RH		 501 /* Logical shifted register instructions (with a shift). */
502 I3502S_AND_LSR = I3510_AND | (1 << 22),
503
14e4c1e2
RH		 504 /* AdvSIMD copy */
505 I3605_DUP = 0x0e000400,
506 I3605_INS = 0x4e001c00,
507 I3605_UMOV = 0x0e003c00,
508
509 /* AdvSIMD modified immediate */
510 I3606_MOVI = 0x0f000400,
511
512 /* AdvSIMD shift by immediate */
513 I3614_SSHR = 0x0f000400,
514 I3614_SSRA = 0x0f001400,
515 I3614_SHL = 0x0f005400,
516 I3614_USHR = 0x2f000400,
517 I3614_USRA = 0x2f001400,
518
519 /* AdvSIMD three same. */
520 I3616_ADD = 0x0e208400,
521 I3616_AND = 0x0e201c00,
522 I3616_BIC = 0x0e601c00,
523 I3616_EOR = 0x2e201c00,
524 I3616_MUL = 0x0e209c00,
525 I3616_ORR = 0x0ea01c00,
526 I3616_ORN = 0x0ee01c00,
527 I3616_SUB = 0x2e208400,
528 I3616_CMGT = 0x0e203400,
529 I3616_CMGE = 0x0e203c00,
530 I3616_CMTST = 0x0e208c00,
531 I3616_CMHI = 0x2e203400,
532 I3616_CMHS = 0x2e203c00,
533 I3616_CMEQ = 0x2e208c00,
93f332a5
RH		 534 I3616_SMAX = 0x0e206400,
535 I3616_SMIN = 0x0e206c00,
d32648d4
RH		 536 I3616_SQADD = 0x0e200c00,
537 I3616_SQSUB = 0x0e202c00,
93f332a5
RH		 538 I3616_UMAX = 0x2e206400,
539 I3616_UMIN = 0x2e206c00,
d32648d4
RH		 540 I3616_UQADD = 0x2e200c00,
541 I3616_UQSUB = 0x2e202c00,
14e4c1e2
RH		 542
543 /* AdvSIMD two-reg misc. */
544 I3617_CMGT0 = 0x0e208800,
545 I3617_CMEQ0 = 0x0e209800,
546 I3617_CMLT0 = 0x0e20a800,
547 I3617_CMGE0 = 0x2e208800,
548 I3617_CMLE0 = 0x2e20a800,
549 I3617_NOT = 0x2e205800,
550 I3617_NEG = 0x2e20b800,
551
c7a59c2a 552 /* System instructions. */
14e4c1e2 553 NOP = 0xd503201f,
c7a59c2a
PK		 554 DMB_ISH = 0xd50338bf,
555 DMB_LD = 0x00000100,
556 DMB_ST = 0x00000200,
50573c66 557} AArch64Insn;
4a136e0a 558
4a136e0a
CF		 559static inline uint32_t tcg_in32(TCGContext *s)
560{
561 uint32_t v = *(uint32_t *)s->code_ptr;
562 return v;
563}
564
50573c66
RH		 565/* Emit an opcode with "type-checking" of the format. */
566#define tcg_out_insn(S, FMT, OP, ...) \
567 glue(tcg_out_insn_,FMT)(S, glue(glue(glue(I,FMT),_),OP), ## __VA_ARGS__)
568
2acee8b2
PK		 569static void tcg_out_insn_3305(TCGContext *s, AArch64Insn insn, int imm19, TCGReg rt)
570{
571 tcg_out32(s, insn | (imm19 & 0x7ffff) << 5 | rt);
572}
573
3d9e69a2
RH		 574static void tcg_out_insn_3201(TCGContext *s, AArch64Insn insn, TCGType ext,
575 TCGReg rt, int imm19)
576{
577 tcg_out32(s, insn | ext << 31 | (imm19 & 0x7ffff) << 5 | rt);
578}
579
81d8a5ee
RH		 580static void tcg_out_insn_3202(TCGContext *s, AArch64Insn insn,
581 TCGCond c, int imm19)
582{
583 tcg_out32(s, insn | tcg_cond_to_aarch64[c] | (imm19 & 0x7ffff) << 5);
584}
585
586static void tcg_out_insn_3206(TCGContext *s, AArch64Insn insn, int imm26)
587{
588 tcg_out32(s, insn | (imm26 & 0x03ffffff));
589}
590
591static void tcg_out_insn_3207(TCGContext *s, AArch64Insn insn, TCGReg rn)
592{
593 tcg_out32(s, insn | rn << 5);
594}
595
95f72aa9
RH		 596static void tcg_out_insn_3314(TCGContext *s, AArch64Insn insn,
597 TCGReg r1, TCGReg r2, TCGReg rn,
598 tcg_target_long ofs, bool pre, bool w)
599{
600 insn |= 1u << 31; /* ext */
601 insn |= pre << 24;
602 insn |= w << 23;
603
eabb7b91 604 tcg_debug_assert(ofs >= -0x200 && ofs < 0x200 && (ofs & 7) == 0);
95f72aa9
RH		 605 insn |= (ofs & (0x7f << 3)) << (15 - 3);
606
607 tcg_out32(s, insn | r2 << 10 | rn << 5 | r1);
608}
609
096c46c0
RH		 610static void tcg_out_insn_3401(TCGContext *s, AArch64Insn insn, TCGType ext,
611 TCGReg rd, TCGReg rn, uint64_t aimm)
612{
613 if (aimm > 0xfff) {
eabb7b91 614 tcg_debug_assert((aimm & 0xfff) == 0);
096c46c0 615 aimm >>= 12;
eabb7b91 616 tcg_debug_assert(aimm <= 0xfff);
096c46c0
RH		 617 aimm |= 1 << 12; /* apply LSL 12 */
618 }
619 tcg_out32(s, insn | ext << 31 | aimm << 10 | rn << 5 | rd);
620}
621
e029f293
RH		 622/* This function can be used for both 3.4.2 (Bitfield) and 3.4.4
623 (Logical immediate). Both insn groups have N, IMMR and IMMS fields
624 that feed the DecodeBitMasks pseudo function. */
625static void tcg_out_insn_3402(TCGContext *s, AArch64Insn insn, TCGType ext,
626 TCGReg rd, TCGReg rn, int n, int immr, int imms)
627{
628 tcg_out32(s, insn | ext << 31 | n << 22 | immr << 16 | imms << 10
629 | rn << 5 | rd);
630}
631
632#define tcg_out_insn_3404 tcg_out_insn_3402
633
b3c56df7
RH		 634static void tcg_out_insn_3403(TCGContext *s, AArch64Insn insn, TCGType ext,
635 TCGReg rd, TCGReg rn, TCGReg rm, int imms)
636{
637 tcg_out32(s, insn | ext << 31 | ext << 22 | rm << 16 | imms << 10
638 | rn << 5 | rd);
639}
640
582ab779
RH		 641/* This function is used for the Move (wide immediate) instruction group.
642 Note that SHIFT is a full shift count, not the 2 bit HW field. */
643static void tcg_out_insn_3405(TCGContext *s, AArch64Insn insn, TCGType ext,
644 TCGReg rd, uint16_t half, unsigned shift)
645{
eabb7b91 646 tcg_debug_assert((shift & ~0x30) == 0);
582ab779
RH		 647 tcg_out32(s, insn | ext << 31 | shift << (21 - 4) | half << 5 | rd);
648}
649
c6e310d9
RH		 650static void tcg_out_insn_3406(TCGContext *s, AArch64Insn insn,
651 TCGReg rd, int64_t disp)
652{
653 tcg_out32(s, insn | (disp & 3) << 29 | (disp & 0x1ffffc) << (5 - 2) | rd);
654}
655
50573c66
RH		 656/* This function is for both 3.5.2 (Add/Subtract shifted register), for
657 the rare occasion when we actually want to supply a shift amount. */
658static inline void tcg_out_insn_3502S(TCGContext *s, AArch64Insn insn,
659 TCGType ext, TCGReg rd, TCGReg rn,
660 TCGReg rm, int imm6)
661{
662 tcg_out32(s, insn | ext << 31 | rm << 16 | imm6 << 10 | rn << 5 | rd);
663}
664
665/* This function is for 3.5.2 (Add/subtract shifted register),
666 and 3.5.10 (Logical shifted register), for the vast majorty of cases
667 when we don't want to apply a shift. Thus it can also be used for
668 3.5.3 (Add/subtract with carry) and 3.5.8 (Data processing 2 source). */
669static void tcg_out_insn_3502(TCGContext *s, AArch64Insn insn, TCGType ext,
670 TCGReg rd, TCGReg rn, TCGReg rm)
671{
672 tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd);
673}
674
675#define tcg_out_insn_3503 tcg_out_insn_3502
676#define tcg_out_insn_3508 tcg_out_insn_3502
677#define tcg_out_insn_3510 tcg_out_insn_3502
678
04ce397b
RH		 679static void tcg_out_insn_3506(TCGContext *s, AArch64Insn insn, TCGType ext,
680 TCGReg rd, TCGReg rn, TCGReg rm, TCGCond c)
681{
682 tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd
683 | tcg_cond_to_aarch64[c] << 12);
684}
685
edd8824c
RH		 686static void tcg_out_insn_3507(TCGContext *s, AArch64Insn insn, TCGType ext,
687 TCGReg rd, TCGReg rn)
688{
689 tcg_out32(s, insn | ext << 31 | rn << 5 | rd);
690}
691
8678b71c
RH		 692static void tcg_out_insn_3509(TCGContext *s, AArch64Insn insn, TCGType ext,
693 TCGReg rd, TCGReg rn, TCGReg rm, TCGReg ra)
694{
695 tcg_out32(s, insn | ext << 31 | rm << 16 | ra << 10 | rn << 5 | rd);
696}
697
14e4c1e2
RH		 698static void tcg_out_insn_3605(TCGContext *s, AArch64Insn insn, bool q,
699 TCGReg rd, TCGReg rn, int dst_idx, int src_idx)
700{
701 /* Note that bit 11 set means general register input. Therefore
702 we can handle both register sets with one function. */
703 tcg_out32(s, insn | q << 30 | (dst_idx << 16) | (src_idx << 11)
704 | (rd & 0x1f) | (~rn & 0x20) << 6 | (rn & 0x1f) << 5);
705}
706
707static void tcg_out_insn_3606(TCGContext *s, AArch64Insn insn, bool q,
708 TCGReg rd, bool op, int cmode, uint8_t imm8)
709{
710 tcg_out32(s, insn | q << 30 | op << 29 | cmode << 12 | (rd & 0x1f)
711 | (imm8 & 0xe0) << (16 - 5) | (imm8 & 0x1f) << 5);
712}
713
714static void tcg_out_insn_3614(TCGContext *s, AArch64Insn insn, bool q,
715 TCGReg rd, TCGReg rn, unsigned immhb)
716{
717 tcg_out32(s, insn | q << 30 | immhb << 16
718 | (rn & 0x1f) << 5 | (rd & 0x1f));
719}
720
721static void tcg_out_insn_3616(TCGContext *s, AArch64Insn insn, bool q,
722 unsigned size, TCGReg rd, TCGReg rn, TCGReg rm)
723{
724 tcg_out32(s, insn | q << 30 | (size << 22) | (rm & 0x1f) << 16
725 | (rn & 0x1f) << 5 | (rd & 0x1f));
726}
727
728static void tcg_out_insn_3617(TCGContext *s, AArch64Insn insn, bool q,
729 unsigned size, TCGReg rd, TCGReg rn)
730{
731 tcg_out32(s, insn | q << 30 | (size << 22)
732 | (rn & 0x1f) << 5 | (rd & 0x1f));
733}
734
3d4299f4 735static void tcg_out_insn_3310(TCGContext *s, AArch64Insn insn,
6c0f0c0f
PB		 736 TCGReg rd, TCGReg base, TCGType ext,
737 TCGReg regoff)
3d4299f4
RH		 738{
739 /* Note the AArch64Insn constants above are for C3.3.12. Adjust. */
6c0f0c0f 740 tcg_out32(s, insn | I3312_TO_I3310 | regoff << 16 |
14e4c1e2 741 0x4000 | ext << 13 | base << 5 | (rd & 0x1f));
3d4299f4 742}
50573c66 743
3d4299f4
RH		 744static void tcg_out_insn_3312(TCGContext *s, AArch64Insn insn,
745 TCGReg rd, TCGReg rn, intptr_t offset)
4a136e0a 746{
14e4c1e2 747 tcg_out32(s, insn | (offset & 0x1ff) << 12 | rn << 5 | (rd & 0x1f));
4a136e0a
CF		 748}
749
3d4299f4
RH		 750static void tcg_out_insn_3313(TCGContext *s, AArch64Insn insn,
751 TCGReg rd, TCGReg rn, uintptr_t scaled_uimm)
b1f6dc0d 752{
3d4299f4 753 /* Note the AArch64Insn constants above are for C3.3.12. Adjust. */
14e4c1e2
RH		 754 tcg_out32(s, insn | I3312_TO_I3313 | scaled_uimm << 10
755 | rn << 5 | (rd & 0x1f));
b1f6dc0d
CF		 756}
757
7d11fc7c
RH		 758/* Register to register move using ORR (shifted register with no shift). */
759static void tcg_out_movr(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rm)
4a136e0a 760{
7d11fc7c
RH		 761 tcg_out_insn(s, 3510, ORR, ext, rd, TCG_REG_XZR, rm);
762}
763
764/* Register to register move using ADDI (move to/from SP). */
765static void tcg_out_movr_sp(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rn)
766{
767 tcg_out_insn(s, 3401, ADDI, ext, rd, rn, 0);
4a136e0a
CF		 768}
769
4ec4f0bd
RH		 770/* This function is used for the Logical (immediate) instruction group.
771 The value of LIMM must satisfy IS_LIMM. See the comment above about
772 only supporting simplified logical immediates. */
773static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext,
774 TCGReg rd, TCGReg rn, uint64_t limm)
775{
776 unsigned h, l, r, c;
777
eabb7b91 778 tcg_debug_assert(is_limm(limm));
4ec4f0bd
RH		 779
780 h = clz64(limm);
781 l = ctz64(limm);
782 if (l == 0) {
783 r = 0; /* form 0....01....1 */
784 c = ctz64(~limm) - 1;
785 if (h == 0) {
786 r = clz64(~limm); /* form 1..10..01..1 */
787 c += r;
788 }
789 } else {
790 r = 64 - l; /* form 1....10....0 or 0..01..10..0 */
791 c = r - h - 1;
792 }
793 if (ext == TCG_TYPE_I32) {
794 r &= 31;
795 c &= 31;
796 }
797
798 tcg_out_insn_3404(s, insn, ext, rd, rn, ext, r, c);
799}
800
14e4c1e2
RH		 801static void tcg_out_dupi_vec(TCGContext *s, TCGType type,
802 TCGReg rd, uint64_t v64)
803{
804 int op, cmode, imm8;
805
806 if (is_fimm(v64, &op, &cmode, &imm8)) {
807 tcg_out_insn(s, 3606, MOVI, type == TCG_TYPE_V128, rd, op, cmode, imm8);
808 } else if (type == TCG_TYPE_V128) {
809 new_pool_l2(s, R_AARCH64_CONDBR19, s->code_ptr, 0, v64, v64);
810 tcg_out_insn(s, 3305, LDR_v128, 0, rd);
811 } else {
812 new_pool_label(s, v64, R_AARCH64_CONDBR19, s->code_ptr, 0);
813 tcg_out_insn(s, 3305, LDR_v64, 0, rd);
814 }
815}
816
582ab779
RH		 817static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
818 tcg_target_long value)
4a136e0a 819{
dfeb5fe7
RH		 820 tcg_target_long svalue = value;
821 tcg_target_long ivalue = ~value;
55129955
RH		 822 tcg_target_long t0, t1, t2;
823 int s0, s1;
824 AArch64Insn opc;
dfeb5fe7 825
14e4c1e2
RH		 826 switch (type) {
827 case TCG_TYPE_I32:
828 case TCG_TYPE_I64:
829 tcg_debug_assert(rd < 32);
830 break;
831
832 case TCG_TYPE_V64:
833 case TCG_TYPE_V128:
834 tcg_debug_assert(rd >= 32);
835 tcg_out_dupi_vec(s, type, rd, value);
836 return;
837
838 default:
839 g_assert_not_reached();
840 }
841
dfeb5fe7
RH		 842 /* For 32-bit values, discard potential garbage in value. For 64-bit
843 values within [2**31, 2**32-1], we can create smaller sequences by
844 interpreting this as a negative 32-bit number, while ensuring that
845 the high 32 bits are cleared by setting SF=0. */
846 if (type == TCG_TYPE_I32 || (value & ~0xffffffffull) == 0) {
847 svalue = (int32_t)value;
582ab779 848 value = (uint32_t)value;
dfeb5fe7
RH		 849 ivalue = (uint32_t)ivalue;
850 type = TCG_TYPE_I32;
851 }
852
d8918df5
RH		 853 /* Speed things up by handling the common case of small positive
854 and negative values specially. */
855 if ((value & ~0xffffull) == 0) {
856 tcg_out_insn(s, 3405, MOVZ, type, rd, value, 0);
857 return;
858 } else if ((ivalue & ~0xffffull) == 0) {
859 tcg_out_insn(s, 3405, MOVN, type, rd, ivalue, 0);
860 return;
861 }
862
4ec4f0bd
RH		 863 /* Check for bitfield immediates. For the benefit of 32-bit quantities,
864 use the sign-extended value. That lets us match rotated values such
865 as 0xff0000ff with the same 64-bit logic matching 0xffffffffff0000ff. */
866 if (is_limm(svalue)) {
867 tcg_out_logicali(s, I3404_ORRI, type, rd, TCG_REG_XZR, svalue);
868 return;
869 }
870
c6e310d9
RH		 871 /* Look for host pointer values within 4G of the PC. This happens
872 often when loading pointers to QEMU's own data structures. */
873 if (type == TCG_TYPE_I64) {
cc74d332
RH		 874 tcg_target_long disp = value - (intptr_t)s->code_ptr;
875 if (disp == sextract64(disp, 0, 21)) {
876 tcg_out_insn(s, 3406, ADR, rd, disp);
877 return;
878 }
879 disp = (value >> 12) - ((intptr_t)s->code_ptr >> 12);
c6e310d9
RH		 880 if (disp == sextract64(disp, 0, 21)) {
881 tcg_out_insn(s, 3406, ADRP, rd, disp);
882 if (value & 0xfff) {
883 tcg_out_insn(s, 3401, ADDI, type, rd, rd, value & 0xfff);
884 }
885 return;
886 }
887 }
888
55129955
RH		 889 /* Would it take fewer insns to begin with MOVN? */
890 if (ctpop64(value) >= 32) {
891 t0 = ivalue;
892 opc = I3405_MOVN;
8cf9a3d3 893 } else {
55129955
RH		 894 t0 = value;
895 opc = I3405_MOVZ;
896 }
897 s0 = ctz64(t0) & (63 & -16);
898 t1 = t0 & ~(0xffffUL << s0);
899 s1 = ctz64(t1) & (63 & -16);
900 t2 = t1 & ~(0xffffUL << s1);
901 if (t2 == 0) {
902 tcg_out_insn_3405(s, opc, type, rd, t0 >> s0, s0);
903 if (t1 != 0) {
904 tcg_out_insn(s, 3405, MOVK, type, rd, value >> s1, s1);
8cf9a3d3 905 }
55129955 906 return;
dfeb5fe7 907 }
55129955
RH		 908
909 /* For more than 2 insns, dump it into the constant pool. */
910 new_pool_label(s, value, R_AARCH64_CONDBR19, s->code_ptr, 0);
911 tcg_out_insn(s, 3305, LDR, 0, rd);
4a136e0a
CF		 912}
913
3d4299f4
RH		 914/* Define something more legible for general use. */
915#define tcg_out_ldst_r tcg_out_insn_3310
4a136e0a 916
14e4c1e2
RH		 917static void tcg_out_ldst(TCGContext *s, AArch64Insn insn, TCGReg rd,
918 TCGReg rn, intptr_t offset, int lgsize)
4a136e0a 919{
3d4299f4
RH		 920 /* If the offset is naturally aligned and in range, then we can
921 use the scaled uimm12 encoding */
14e4c1e2
RH		 922 if (offset >= 0 && !(offset & ((1 << lgsize) - 1))) {
923 uintptr_t scaled_uimm = offset >> lgsize;
3d4299f4
RH		 924 if (scaled_uimm <= 0xfff) {
925 tcg_out_insn_3313(s, insn, rd, rn, scaled_uimm);
926 return;
b1f6dc0d
CF		 927 }
928 }
929
a056c9fa
RH		 930 /* Small signed offsets can use the unscaled encoding. */
931 if (offset >= -256 && offset < 256) {
932 tcg_out_insn_3312(s, insn, rd, rn, offset);
933 return;
934 }
935
3d4299f4 936 /* Worst-case scenario, move offset to temp register, use reg offset. */
b1f6dc0d 937 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, offset);
6c0f0c0f 938 tcg_out_ldst_r(s, insn, rd, rn, TCG_TYPE_I64, TCG_REG_TMP);
4a136e0a
CF		 939}
940
14e4c1e2 941static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
4a136e0a 942{
14e4c1e2
RH		 943 if (ret == arg) {
944 return;
945 }
946 switch (type) {
947 case TCG_TYPE_I32:
948 case TCG_TYPE_I64:
949 if (ret < 32 && arg < 32) {
950 tcg_out_movr(s, type, ret, arg);
951 break;
952 } else if (ret < 32) {
953 tcg_out_insn(s, 3605, UMOV, type, ret, arg, 0, 0);
954 break;
955 } else if (arg < 32) {
956 tcg_out_insn(s, 3605, INS, 0, ret, arg, 4 << type, 0);
957 break;
958 }
959 /* FALLTHRU */
960
961 case TCG_TYPE_V64:
962 tcg_debug_assert(ret >= 32 && arg >= 32);
963 tcg_out_insn(s, 3616, ORR, 0, 0, ret, arg, arg);
964 break;
965 case TCG_TYPE_V128:
966 tcg_debug_assert(ret >= 32 && arg >= 32);
967 tcg_out_insn(s, 3616, ORR, 1, 0, ret, arg, arg);
968 break;
969
970 default:
971 g_assert_not_reached();
4a136e0a
CF		 972 }
973}
974
14e4c1e2
RH		 975static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
976 TCGReg base, intptr_t ofs)
4a136e0a 977{
14e4c1e2
RH		 978 AArch64Insn insn;
979 int lgsz;
980
981 switch (type) {
982 case TCG_TYPE_I32:
983 insn = (ret < 32 ? I3312_LDRW : I3312_LDRVS);
984 lgsz = 2;
985 break;
986 case TCG_TYPE_I64:
987 insn = (ret < 32 ? I3312_LDRX : I3312_LDRVD);
988 lgsz = 3;
989 break;
990 case TCG_TYPE_V64:
991 insn = I3312_LDRVD;
992 lgsz = 3;
993 break;
994 case TCG_TYPE_V128:
995 insn = I3312_LDRVQ;
996 lgsz = 4;
997 break;
998 default:
999 g_assert_not_reached();
1000 }
1001 tcg_out_ldst(s, insn, ret, base, ofs, lgsz);
4a136e0a
CF		 1002}
1003
14e4c1e2
RH		 1004static void tcg_out_st(TCGContext *s, TCGType type, TCGReg src,
1005 TCGReg base, intptr_t ofs)
4a136e0a 1006{
14e4c1e2
RH		 1007 AArch64Insn insn;
1008 int lgsz;
1009
1010 switch (type) {
1011 case TCG_TYPE_I32:
1012 insn = (src < 32 ? I3312_STRW : I3312_STRVS);
1013 lgsz = 2;
1014 break;
1015 case TCG_TYPE_I64:
1016 insn = (src < 32 ? I3312_STRX : I3312_STRVD);
1017 lgsz = 3;
1018 break;
1019 case TCG_TYPE_V64:
1020 insn = I3312_STRVD;
1021 lgsz = 3;
1022 break;
1023 case TCG_TYPE_V128:
1024 insn = I3312_STRVQ;
1025 lgsz = 4;
1026 break;
1027 default:
1028 g_assert_not_reached();
1029 }
1030 tcg_out_ldst(s, insn, src, base, ofs, lgsz);
4a136e0a
CF		 1031}
1032
59d7c14e
RH		 1033static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1034 TCGReg base, intptr_t ofs)
1035{
14e4c1e2 1036 if (type <= TCG_TYPE_I64 && val == 0) {
59d7c14e
RH		 1037 tcg_out_st(s, type, TCG_REG_XZR, base, ofs);
1038 return true;
1039 }
1040 return false;
1041}
1042
b3c56df7
RH		 1043static inline void tcg_out_bfm(TCGContext *s, TCGType ext, TCGReg rd,
1044 TCGReg rn, unsigned int a, unsigned int b)
1045{
1046 tcg_out_insn(s, 3402, BFM, ext, rd, rn, ext, a, b);
1047}
1048
7763ffa0
RH		 1049static inline void tcg_out_ubfm(TCGContext *s, TCGType ext, TCGReg rd,
1050 TCGReg rn, unsigned int a, unsigned int b)
4a136e0a 1051{
b3c56df7 1052 tcg_out_insn(s, 3402, UBFM, ext, rd, rn, ext, a, b);
4a136e0a
CF		 1053}
1054
7763ffa0
RH		 1055static inline void tcg_out_sbfm(TCGContext *s, TCGType ext, TCGReg rd,
1056 TCGReg rn, unsigned int a, unsigned int b)
4a136e0a 1057{
b3c56df7 1058 tcg_out_insn(s, 3402, SBFM, ext, rd, rn, ext, a, b);
4a136e0a
CF		 1059}
1060
7763ffa0 1061static inline void tcg_out_extr(TCGContext *s, TCGType ext, TCGReg rd,
4a136e0a
CF		 1062 TCGReg rn, TCGReg rm, unsigned int a)
1063{
b3c56df7 1064 tcg_out_insn(s, 3403, EXTR, ext, rd, rn, rm, a);
4a136e0a
CF		 1065}
1066
7763ffa0 1067static inline void tcg_out_shl(TCGContext *s, TCGType ext,
4a136e0a
CF		 1068 TCGReg rd, TCGReg rn, unsigned int m)
1069{
b3c56df7
RH		 1070 int bits = ext ? 64 : 32;
1071 int max = bits - 1;
4a136e0a
CF		 1072 tcg_out_ubfm(s, ext, rd, rn, bits - (m & max), max - (m & max));
1073}
1074
7763ffa0 1075static inline void tcg_out_shr(TCGContext *s, TCGType ext,
4a136e0a
CF		 1076 TCGReg rd, TCGReg rn, unsigned int m)
1077{
1078 int max = ext ? 63 : 31;
1079 tcg_out_ubfm(s, ext, rd, rn, m & max, max);
1080}
1081
7763ffa0 1082static inline void tcg_out_sar(TCGContext *s, TCGType ext,
4a136e0a
CF		 1083 TCGReg rd, TCGReg rn, unsigned int m)
1084{
1085 int max = ext ? 63 : 31;
1086 tcg_out_sbfm(s, ext, rd, rn, m & max, max);
1087}
1088
7763ffa0 1089static inline void tcg_out_rotr(TCGContext *s, TCGType ext,
4a136e0a
CF		 1090 TCGReg rd, TCGReg rn, unsigned int m)
1091{
1092 int max = ext ? 63 : 31;
1093 tcg_out_extr(s, ext, rd, rn, rn, m & max);
1094}
1095
7763ffa0 1096static inline void tcg_out_rotl(TCGContext *s, TCGType ext,
4a136e0a
CF		 1097 TCGReg rd, TCGReg rn, unsigned int m)
1098{
b3c56df7
RH		 1099 int bits = ext ? 64 : 32;
1100 int max = bits - 1;
4a136e0a
CF		 1101 tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
1102}
1103
b3c56df7
RH		 1104static inline void tcg_out_dep(TCGContext *s, TCGType ext, TCGReg rd,
1105 TCGReg rn, unsigned lsb, unsigned width)
1106{
1107 unsigned size = ext ? 64 : 32;
1108 unsigned a = (size - lsb) & (size - 1);
1109 unsigned b = width - 1;
1110 tcg_out_bfm(s, ext, rd, rn, a, b);
1111}
1112
90f1cd91
RH		 1113static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
1114 tcg_target_long b, bool const_b)
4a136e0a 1115{
90f1cd91
RH		 1116 if (const_b) {
1117 /* Using CMP or CMN aliases. */
1118 if (b >= 0) {
1119 tcg_out_insn(s, 3401, SUBSI, ext, TCG_REG_XZR, a, b);
1120 } else {
1121 tcg_out_insn(s, 3401, ADDSI, ext, TCG_REG_XZR, a, -b);
1122 }
1123 } else {
1124 /* Using CMP alias SUBS wzr, Wn, Wm */
1125 tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b);
1126 }
4a136e0a
CF		 1127}
1128
8587c30c 1129static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
4a136e0a 1130{
8587c30c 1131 ptrdiff_t offset = target - s->code_ptr;
eabb7b91 1132 tcg_debug_assert(offset == sextract64(offset, 0, 26));
81d8a5ee 1133 tcg_out_insn(s, 3206, B, offset);
4a136e0a
CF		 1134}
1135
23b7aa1d
PK		 1136static inline void tcg_out_goto_long(TCGContext *s, tcg_insn_unit *target)
1137{
1138 ptrdiff_t offset = target - s->code_ptr;
1139 if (offset == sextract64(offset, 0, 26)) {
1140 tcg_out_insn(s, 3206, BL, offset);
1141 } else {
1142 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target);
1143 tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
1144 }
1145}
1146
4a136e0a
CF		 1147static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
1148{
81d8a5ee 1149 tcg_out_insn(s, 3207, BLR, reg);
4a136e0a
CF		 1150}
1151
8587c30c 1152static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
4a136e0a 1153{
8587c30c
RH		 1154 ptrdiff_t offset = target - s->code_ptr;
1155 if (offset == sextract64(offset, 0, 26)) {
81d8a5ee 1156 tcg_out_insn(s, 3206, BL, offset);
8587c30c
RH		 1157 } else {
1158 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target);
1159 tcg_out_callr(s, TCG_REG_TMP);
4a136e0a
CF		 1160 }
1161}
1162
a8583393
RH		 1163void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
1164 uintptr_t addr)
4a136e0a 1165{
b68686bd
PK		 1166 tcg_insn_unit i1, i2;
1167 TCGType rt = TCG_TYPE_I64;
1168 TCGReg rd = TCG_REG_TMP;
1169 uint64_t pair;
4a136e0a 1170
b68686bd
PK		 1171 ptrdiff_t offset = addr - jmp_addr;
1172
1173 if (offset == sextract64(offset, 0, 26)) {
1174 i1 = I3206_B | ((offset >> 2) & 0x3ffffff);
1175 i2 = NOP;
1176 } else {
1177 offset = (addr >> 12) - (jmp_addr >> 12);
1178
1179 /* patch ADRP */
1180 i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd;
1181 /* patch ADDI */
1182 i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
1183 }
1184 pair = (uint64_t)i2 << 32 | i1;
1185 atomic_set((uint64_t *)jmp_addr, pair);
1186 flush_icache_range(jmp_addr, jmp_addr + 8);
4a136e0a
CF		 1187}
1188
bec16311 1189static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
4a136e0a 1190{
4a136e0a 1191 if (!l->has_value) {
bec16311 1192 tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, l, 0);
733589b3 1193 tcg_out_insn(s, 3206, B, 0);
4a136e0a 1194 } else {
8587c30c 1195 tcg_out_goto(s, l->u.value_ptr);
4a136e0a
CF		 1196 }
1197}
1198
dc1eccd6 1199static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a,
bec16311 1200 TCGArg b, bool b_const, TCGLabel *l)
4a136e0a 1201{
cae1f6f3 1202 intptr_t offset;
3d9e69a2 1203 bool need_cmp;
cae1f6f3 1204
3d9e69a2
RH		 1205 if (b_const && b == 0 && (c == TCG_COND_EQ || c == TCG_COND_NE)) {
1206 need_cmp = false;
1207 } else {
1208 need_cmp = true;
1209 tcg_out_cmp(s, ext, a, b, b_const);
1210 }
4a136e0a
CF		 1211
1212 if (!l->has_value) {
bec16311 1213 tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0);
cae1f6f3 1214 offset = tcg_in32(s) >> 5;
4a136e0a 1215 } else {
8587c30c 1216 offset = l->u.value_ptr - s->code_ptr;
eabb7b91 1217 tcg_debug_assert(offset == sextract64(offset, 0, 19));
4a136e0a 1218 }
cae1f6f3 1219
3d9e69a2
RH		 1220 if (need_cmp) {
1221 tcg_out_insn(s, 3202, B_C, c, offset);
1222 } else if (c == TCG_COND_EQ) {
1223 tcg_out_insn(s, 3201, CBZ, ext, a, offset);
1224 } else {
1225 tcg_out_insn(s, 3201, CBNZ, ext, a, offset);
1226 }
4a136e0a
CF		 1227}
1228
edd8824c 1229static inline void tcg_out_rev64(TCGContext *s, TCGReg rd, TCGReg rn)
9c4a059d 1230{
edd8824c 1231 tcg_out_insn(s, 3507, REV64, TCG_TYPE_I64, rd, rn);
9c4a059d
CF		 1232}
1233
edd8824c 1234static inline void tcg_out_rev32(TCGContext *s, TCGReg rd, TCGReg rn)
9c4a059d 1235{
edd8824c
RH		 1236 tcg_out_insn(s, 3507, REV32, TCG_TYPE_I32, rd, rn);
1237}
1238
1239static inline void tcg_out_rev16(TCGContext *s, TCGReg rd, TCGReg rn)
1240{
1241 tcg_out_insn(s, 3507, REV16, TCG_TYPE_I32, rd, rn);
9c4a059d
CF		 1242}
1243
929f8b55 1244static inline void tcg_out_sxt(TCGContext *s, TCGType ext, TCGMemOp s_bits,
31f1275b
CF		 1245 TCGReg rd, TCGReg rn)
1246{
b3c56df7 1247 /* Using ALIASes SXTB, SXTH, SXTW, of SBFM Xd, Xn, #0, #7|15|31 */
929f8b55 1248 int bits = (8 << s_bits) - 1;
31f1275b
CF		 1249 tcg_out_sbfm(s, ext, rd, rn, 0, bits);
1250}
1251
929f8b55 1252static inline void tcg_out_uxt(TCGContext *s, TCGMemOp s_bits,
31f1275b
CF		 1253 TCGReg rd, TCGReg rn)
1254{
b3c56df7 1255 /* Using ALIASes UXTB, UXTH of UBFM Wd, Wn, #0, #7|15 */
929f8b55 1256 int bits = (8 << s_bits) - 1;
31f1275b
CF		 1257 tcg_out_ubfm(s, 0, rd, rn, 0, bits);
1258}
1259
90f1cd91
RH		 1260static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
1261 TCGReg rn, int64_t aimm)
1262{
1263 if (aimm >= 0) {
1264 tcg_out_insn(s, 3401, ADDI, ext, rd, rn, aimm);
1265 } else {
1266 tcg_out_insn(s, 3401, SUBI, ext, rd, rn, -aimm);
1267 }
1268}
1269
dc1eccd6 1270static inline void tcg_out_addsub2(TCGContext *s, TCGType ext, TCGReg rl,
c6e929e7
RH		 1271 TCGReg rh, TCGReg al, TCGReg ah,
1272 tcg_target_long bl, tcg_target_long bh,
1273 bool const_bl, bool const_bh, bool sub)
1274{
1275 TCGReg orig_rl = rl;
1276 AArch64Insn insn;
1277
1278 if (rl == ah || (!const_bh && rl == bh)) {
1279 rl = TCG_REG_TMP;
1280 }
1281
1282 if (const_bl) {
1283 insn = I3401_ADDSI;
1284 if ((bl < 0) ^ sub) {
1285 insn = I3401_SUBSI;
1286 bl = -bl;
1287 }
b1eb20da
RH		 1288 if (unlikely(al == TCG_REG_XZR)) {
1289 /* ??? We want to allow al to be zero for the benefit of
1290 negation via subtraction. However, that leaves open the
1291 possibility of adding 0+const in the low part, and the
1292 immediate add instructions encode XSP not XZR. Don't try
1293 anything more elaborate here than loading another zero. */
1294 al = TCG_REG_TMP;
1295 tcg_out_movi(s, ext, al, 0);
1296 }
c6e929e7
RH		 1297 tcg_out_insn_3401(s, insn, ext, rl, al, bl);
1298 } else {
1299 tcg_out_insn_3502(s, sub ? I3502_SUBS : I3502_ADDS, ext, rl, al, bl);
1300 }
1301
1302 insn = I3503_ADC;
1303 if (const_bh) {
1304 /* Note that the only two constants we support are 0 and -1, and
1305 that SBC = rn + ~rm + c, so adc -1 is sbc 0, and vice-versa. */
1306 if ((bh != 0) ^ sub) {
1307 insn = I3503_SBC;
1308 }
1309 bh = TCG_REG_XZR;
1310 } else if (sub) {
1311 insn = I3503_SBC;
1312 }
1313 tcg_out_insn_3503(s, insn, ext, rh, ah, bh);
1314
b825025f 1315 tcg_out_mov(s, ext, orig_rl, rl);
c6e929e7
RH		 1316}
1317
c7a59c2a
PK		 1318static inline void tcg_out_mb(TCGContext *s, TCGArg a0)
1319{
1320 static const uint32_t sync[] = {
1321 [0 ... TCG_MO_ALL] = DMB_ISH | DMB_LD | DMB_ST,
1322 [TCG_MO_ST_ST] = DMB_ISH | DMB_ST,
1323 [TCG_MO_LD_LD] = DMB_ISH | DMB_LD,
1324 [TCG_MO_LD_ST] = DMB_ISH | DMB_LD,
1325 [TCG_MO_LD_ST | TCG_MO_LD_LD] = DMB_ISH | DMB_LD,
1326 };
1327 tcg_out32(s, sync[a0 & TCG_MO_ALL]);
1328}
1329
53c76c19
RH		 1330static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d,
1331 TCGReg a0, TCGArg b, bool const_b, bool is_ctz)
1332{
1333 TCGReg a1 = a0;
1334 if (is_ctz) {
1335 a1 = TCG_REG_TMP;
1336 tcg_out_insn(s, 3507, RBIT, ext, a1, a0);
1337 }
1338 if (const_b && b == (ext ? 64 : 32)) {
1339 tcg_out_insn(s, 3507, CLZ, ext, d, a1);
1340 } else {
1341 AArch64Insn sel = I3506_CSEL;
1342
1343 tcg_out_cmp(s, ext, a0, 0, 1);
1344 tcg_out_insn(s, 3507, CLZ, ext, TCG_REG_TMP, a1);
1345
1346 if (const_b) {
1347 if (b == -1) {
1348 b = TCG_REG_XZR;
1349 sel = I3506_CSINV;
1350 } else if (b == 0) {
1351 b = TCG_REG_XZR;
1352 } else {
1353 tcg_out_movi(s, ext, d, b);
1354 b = d;
1355 }
1356 }
1357 tcg_out_insn_3506(s, sel, ext, d, TCG_REG_TMP, b, TCG_COND_NE);
1358 }
1359}
1360
4a136e0a 1361#ifdef CONFIG_SOFTMMU
659ef5cb
RH		 1362#include "tcg-ldst.inc.c"
1363
023261ef 1364/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
3972ef6f 1365 * TCGMemOpIdx oi, uintptr_t ra)
023261ef 1366 */
8587c30c 1367static void * const qemu_ld_helpers[16] = {
de61d14f
RH		 1368 [MO_UB] = helper_ret_ldub_mmu,
1369 [MO_LEUW] = helper_le_lduw_mmu,
1370 [MO_LEUL] = helper_le_ldul_mmu,
1371 [MO_LEQ] = helper_le_ldq_mmu,
1372 [MO_BEUW] = helper_be_lduw_mmu,
1373 [MO_BEUL] = helper_be_ldul_mmu,
1374 [MO_BEQ] = helper_be_ldq_mmu,
4a136e0a
CF		 1375};
1376
023261ef 1377/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
3972ef6f
RH		 1378 * uintxx_t val, TCGMemOpIdx oi,
1379 * uintptr_t ra)
023261ef 1380 */
8587c30c 1381static void * const qemu_st_helpers[16] = {
de61d14f
RH		 1382 [MO_UB] = helper_ret_stb_mmu,
1383 [MO_LEUW] = helper_le_stw_mmu,
1384 [MO_LEUL] = helper_le_stl_mmu,
1385 [MO_LEQ] = helper_le_stq_mmu,
1386 [MO_BEUW] = helper_be_stw_mmu,
1387 [MO_BEUL] = helper_be_stl_mmu,
1388 [MO_BEQ] = helper_be_stq_mmu,
4a136e0a
CF		 1389};
1390
8587c30c 1391static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target)
dc0c8aaf 1392{
8587c30c 1393 ptrdiff_t offset = tcg_pcrel_diff(s, target);
eabb7b91 1394 tcg_debug_assert(offset == sextract64(offset, 0, 21));
8587c30c 1395 tcg_out_insn(s, 3406, ADR, rd, offset);
dc0c8aaf
RH		 1396}
1397
c6d8ed24
JK		 1398static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1399{
3972ef6f
RH		 1400 TCGMemOpIdx oi = lb->oi;
1401 TCGMemOp opc = get_memop(oi);
929f8b55
RH		 1402 TCGMemOp size = opc & MO_SIZE;
1403
214bfe83
RH		 1404 bool ok = reloc_pc19(lb->label_ptr[0], s->code_ptr);
1405 tcg_debug_assert(ok);
017a86f7 1406
3972ef6f 1407 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
b825025f 1408 tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
3972ef6f 1409 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, oi);
8587c30c 1410 tcg_out_adr(s, TCG_REG_X3, lb->raddr);
2b7ec66f 1411 tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
929f8b55 1412 if (opc & MO_SIGN) {
9c53889b 1413 tcg_out_sxt(s, lb->type, size, lb->datalo_reg, TCG_REG_X0);
c6d8ed24 1414 } else {
b825025f 1415 tcg_out_mov(s, size == MO_64, lb->datalo_reg, TCG_REG_X0);
c6d8ed24
JK		 1416 }
1417
8587c30c 1418 tcg_out_goto(s, lb->raddr);
c6d8ed24
JK		 1419}
1420
1421static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
1422{
3972ef6f
RH		 1423 TCGMemOpIdx oi = lb->oi;
1424 TCGMemOp opc = get_memop(oi);
de61d14f 1425 TCGMemOp size = opc & MO_SIZE;
929f8b55 1426
214bfe83
RH		 1427 bool ok = reloc_pc19(lb->label_ptr[0], s->code_ptr);
1428 tcg_debug_assert(ok);
c6d8ed24 1429
3972ef6f 1430 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
b825025f
RH		 1431 tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
1432 tcg_out_mov(s, size == MO_64, TCG_REG_X2, lb->datalo_reg);
3972ef6f 1433 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, oi);
8587c30c 1434 tcg_out_adr(s, TCG_REG_X4, lb->raddr);
2b7ec66f 1435 tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
8587c30c 1436 tcg_out_goto(s, lb->raddr);
c6d8ed24
JK		 1437}
1438
3972ef6f 1439static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
9c53889b 1440 TCGType ext, TCGReg data_reg, TCGReg addr_reg,
3972ef6f 1441 tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
c6d8ed24 1442{
9ecefc84 1443 TCGLabelQemuLdst *label = new_ldst_label(s);
c6d8ed24 1444
c6d8ed24 1445 label->is_ld = is_ld;
3972ef6f 1446 label->oi = oi;
9c53889b 1447 label->type = ext;
c6d8ed24
JK		 1448 label->datalo_reg = data_reg;
1449 label->addrlo_reg = addr_reg;
c6d8ed24
JK		 1450 label->raddr = raddr;
1451 label->label_ptr[0] = label_ptr;
1452}
1453
f7bcd966
RH		 1454/* We expect tlb_mask to be before tlb_table. */
1455QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table) <
1456 offsetof(CPUArchState, tlb_mask));
1457
1458/* We expect to use a 24-bit unsigned offset from ENV. */
1459QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1])
1460 > 0xffffff);
1461
c6d8ed24
JK		 1462/* Load and compare a TLB entry, emitting the conditional jump to the
1463 slow path for the failure case, which will be patched later when finalizing
1464 the slow path. Generated code returns the host addend in X1,
1465 clobbers X0,X2,X3,TMP. */
9ee14902 1466static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp opc,
8587c30c
RH		 1467 tcg_insn_unit **label_ptr, int mem_index,
1468 bool is_read)
c6d8ed24 1469{
f7bcd966
RH		 1470 int mask_ofs = offsetof(CPUArchState, tlb_mask[mem_index]);
1471 int table_ofs = offsetof(CPUArchState, tlb_table[mem_index]);
85aa8081
RH		 1472 unsigned a_bits = get_alignment_bits(opc);
1473 unsigned s_bits = opc & MO_SIZE;
1474 unsigned a_mask = (1u << a_bits) - 1;
1475 unsigned s_mask = (1u << s_bits) - 1;
f7bcd966
RH		 1476 TCGReg mask_base = TCG_AREG0, table_base = TCG_AREG0, x3;
1477 TCGType mask_type;
1478 uint64_t compare_mask;
1479
1480 if (table_ofs > 0xfff) {
1481 int table_hi = table_ofs & ~0xfff;
1482 int mask_hi = mask_ofs & ~0xfff;
1483
1484 table_base = TCG_REG_X1;
1485 if (mask_hi == table_hi) {
1486 mask_base = table_base;
1487 } else if (mask_hi) {
1488 mask_base = TCG_REG_X0;
1489 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64,
1490 mask_base, TCG_AREG0, mask_hi);
1491 }
1492 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64,
1493 table_base, TCG_AREG0, table_hi);
1494 mask_ofs -= mask_hi;
1495 table_ofs -= table_hi;
1496 }
1497
1498 mask_type = (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32
1499 ? TCG_TYPE_I64 : TCG_TYPE_I32);
1500
1501 /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */
1502 tcg_out_ld(s, mask_type, TCG_REG_X0, mask_base, mask_ofs);
1503 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, table_base, table_ofs);
1504
1505 /* Extract the TLB index from the address into X0. */
1506 tcg_out_insn(s, 3502S, AND_LSR, mask_type == TCG_TYPE_I64,
1507 TCG_REG_X0, TCG_REG_X0, addr_reg,
1508 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1509
1510 /* Add the tlb_table pointer, creating the CPUTLBEntry address into X1. */
1511 tcg_out_insn(s, 3502, ADD, 1, TCG_REG_X1, TCG_REG_X1, TCG_REG_X0);
1512
1513 /* Load the tlb comparator into X0, and the fast path addend into X1. */
1514 tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1, is_read
1515 ? offsetof(CPUTLBEntry, addr_read)
1516 : offsetof(CPUTLBEntry, addr_write));
1517 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, TCG_REG_X1,
1518 offsetof(CPUTLBEntry, addend));
9ee14902
RH		 1519
1520 /* For aligned accesses, we check the first byte and include the alignment
1521 bits within the address. For unaligned access, we check that we don't
1522 cross pages using the address of the last byte of the access. */
85aa8081 1523 if (a_bits >= s_bits) {
9ee14902
RH		 1524 x3 = addr_reg;
1525 } else {
1526 tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,
85aa8081 1527 TCG_REG_X3, addr_reg, s_mask - a_mask);
9ee14902
RH		 1528 x3 = TCG_REG_X3;
1529 }
f7bcd966 1530 compare_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
6f472467 1531
9ee14902
RH		 1532 /* Store the page mask part of the address into X3. */
1533 tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
f7bcd966 1534 TCG_REG_X3, x3, compare_mask);
6f472467 1535
c6d8ed24 1536 /* Perform the address comparison. */
f7bcd966 1537 tcg_out_cmp(s, TARGET_LONG_BITS == 64, TCG_REG_X0, TCG_REG_X3, 0);
6f472467 1538
c6d8ed24 1539 /* If not equal, we jump to the slow path. */
6f472467 1540 *label_ptr = s->code_ptr;
733589b3 1541 tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
c6d8ed24
JK		 1542}
1543
1544#endif /* CONFIG_SOFTMMU */
6a91c7c9 1545
9c53889b 1546static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp memop, TCGType ext,
ffc63728
PB		 1547 TCGReg data_r, TCGReg addr_r,
1548 TCGType otype, TCGReg off_r)
6a91c7c9 1549{
9e4177ad
RH		 1550 const TCGMemOp bswap = memop & MO_BSWAP;
1551
1552 switch (memop & MO_SSIZE) {
1553 case MO_UB:
6c0f0c0f 1554 tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, otype, off_r);
6a91c7c9 1555 break;
9e4177ad 1556 case MO_SB:
9c53889b 1557 tcg_out_ldst_r(s, ext ? I3312_LDRSBX : I3312_LDRSBW,
6c0f0c0f 1558 data_r, addr_r, otype, off_r);
6a91c7c9 1559 break;
9e4177ad 1560 case MO_UW:
6c0f0c0f 1561 tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r);
9e4177ad 1562 if (bswap) {
edd8824c 1563 tcg_out_rev16(s, data_r, data_r);
6a91c7c9
JK		 1564 }
1565 break;
9e4177ad
RH		 1566 case MO_SW:
1567 if (bswap) {
6c0f0c0f 1568 tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r);
edd8824c 1569 tcg_out_rev16(s, data_r, data_r);
9c53889b 1570 tcg_out_sxt(s, ext, MO_16, data_r, data_r);
6a91c7c9 1571 } else {
6c0f0c0f
PB		 1572 tcg_out_ldst_r(s, (ext ? I3312_LDRSHX : I3312_LDRSHW),
1573 data_r, addr_r, otype, off_r);
6a91c7c9
JK		 1574 }
1575 break;
9e4177ad 1576 case MO_UL:
6c0f0c0f 1577 tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r);
9e4177ad 1578 if (bswap) {
edd8824c 1579 tcg_out_rev32(s, data_r, data_r);
6a91c7c9
JK		 1580 }
1581 break;
9e4177ad
RH		 1582 case MO_SL:
1583 if (bswap) {
6c0f0c0f 1584 tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r);
edd8824c 1585 tcg_out_rev32(s, data_r, data_r);
929f8b55 1586 tcg_out_sxt(s, TCG_TYPE_I64, MO_32, data_r, data_r);
6a91c7c9 1587 } else {
6c0f0c0f 1588 tcg_out_ldst_r(s, I3312_LDRSWX, data_r, addr_r, otype, off_r);
6a91c7c9
JK		 1589 }
1590 break;
9e4177ad 1591 case MO_Q:
6c0f0c0f 1592 tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r);
9e4177ad 1593 if (bswap) {
edd8824c 1594 tcg_out_rev64(s, data_r, data_r);
6a91c7c9
JK		 1595 }
1596 break;
1597 default:
1598 tcg_abort();
1599 }
1600}
1601
9e4177ad 1602static void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp memop,
ffc63728
PB		 1603 TCGReg data_r, TCGReg addr_r,
1604 TCGType otype, TCGReg off_r)
6a91c7c9 1605{
9e4177ad
RH		 1606 const TCGMemOp bswap = memop & MO_BSWAP;
1607
1608 switch (memop & MO_SIZE) {
1609 case MO_8:
6c0f0c0f 1610 tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, otype, off_r);
6a91c7c9 1611 break;
9e4177ad 1612 case MO_16:
e81864a1 1613 if (bswap && data_r != TCG_REG_XZR) {
edd8824c 1614 tcg_out_rev16(s, TCG_REG_TMP, data_r);
9e4177ad 1615 data_r = TCG_REG_TMP;
6a91c7c9 1616 }
6c0f0c0f 1617 tcg_out_ldst_r(s, I3312_STRH, data_r, addr_r, otype, off_r);
6a91c7c9 1618 break;
9e4177ad 1619 case MO_32:
e81864a1 1620 if (bswap && data_r != TCG_REG_XZR) {
edd8824c 1621 tcg_out_rev32(s, TCG_REG_TMP, data_r);
9e4177ad 1622 data_r = TCG_REG_TMP;
6a91c7c9 1623 }
6c0f0c0f 1624 tcg_out_ldst_r(s, I3312_STRW, data_r, addr_r, otype, off_r);
6a91c7c9 1625 break;
9e4177ad 1626 case MO_64:
e81864a1 1627 if (bswap && data_r != TCG_REG_XZR) {
edd8824c 1628 tcg_out_rev64(s, TCG_REG_TMP, data_r);
9e4177ad 1629 data_r = TCG_REG_TMP;
6a91c7c9 1630 }
6c0f0c0f 1631 tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r);
6a91c7c9
JK		 1632 break;
1633 default:
1634 tcg_abort();
1635 }
1636}
4a136e0a 1637
667b1cdd 1638static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
59227d5d 1639 TCGMemOpIdx oi, TCGType ext)
4a136e0a 1640{
59227d5d 1641 TCGMemOp memop = get_memop(oi);
80adb8fc 1642 const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
4a136e0a 1643#ifdef CONFIG_SOFTMMU
59227d5d 1644 unsigned mem_index = get_mmuidx(oi);
8587c30c 1645 tcg_insn_unit *label_ptr;
4a136e0a 1646
9ee14902 1647 tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1);
80adb8fc
RH		 1648 tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
1649 TCG_REG_X1, otype, addr_reg);
3972ef6f
RH		 1650 add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg,
1651 s->code_ptr, label_ptr);
4a136e0a 1652#else /* !CONFIG_SOFTMMU */
352bcb0a
RH		 1653 if (USE_GUEST_BASE) {
1654 tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
1655 TCG_REG_GUEST_BASE, otype, addr_reg);
1656 } else {
1657 tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
1658 addr_reg, TCG_TYPE_I64, TCG_REG_XZR);
1659 }
6a91c7c9 1660#endif /* CONFIG_SOFTMMU */
4a136e0a
CF		 1661}
1662
667b1cdd 1663static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
59227d5d 1664 TCGMemOpIdx oi)
4a136e0a 1665{
59227d5d 1666 TCGMemOp memop = get_memop(oi);
80adb8fc 1667 const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
4a136e0a 1668#ifdef CONFIG_SOFTMMU
59227d5d 1669 unsigned mem_index = get_mmuidx(oi);
8587c30c 1670 tcg_insn_unit *label_ptr;
4a136e0a 1671
9ee14902 1672 tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0);
80adb8fc
RH		 1673 tcg_out_qemu_st_direct(s, memop, data_reg,
1674 TCG_REG_X1, otype, addr_reg);
9ee14902
RH		 1675 add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64,
1676 data_reg, addr_reg, s->code_ptr, label_ptr);
4a136e0a 1677#else /* !CONFIG_SOFTMMU */
352bcb0a
RH		 1678 if (USE_GUEST_BASE) {
1679 tcg_out_qemu_st_direct(s, memop, data_reg,
1680 TCG_REG_GUEST_BASE, otype, addr_reg);
1681 } else {
1682 tcg_out_qemu_st_direct(s, memop, data_reg,
1683 addr_reg, TCG_TYPE_I64, TCG_REG_XZR);
1684 }
6a91c7c9 1685#endif /* CONFIG_SOFTMMU */
4a136e0a
CF		 1686}
1687
8587c30c 1688static tcg_insn_unit *tb_ret_addr;
4a136e0a 1689
4a136e0a 1690static void tcg_out_op(TCGContext *s, TCGOpcode opc,
8d8db193
RH		 1691 const TCGArg args[TCG_MAX_OP_ARGS],
1692 const int const_args[TCG_MAX_OP_ARGS])
4a136e0a 1693{
f0293414
RH		 1694 /* 99% of the time, we can signal the use of extension registers
1695 by looking to see if the opcode handles 64-bit data. */
1696 TCGType ext = (tcg_op_defs[opc].flags & TCG_OPF_64BIT) != 0;
4a136e0a 1697
8d8db193
RH		 1698 /* Hoist the loads of the most common arguments. */
1699 TCGArg a0 = args[0];
1700 TCGArg a1 = args[1];
1701 TCGArg a2 = args[2];
1702 int c2 = const_args[2];
1703
04ce397b
RH		 1704 /* Some operands are defined with "rZ" constraint, a register or
1705 the zero register. These need not actually test args[I] == 0. */
1706#define REG0(I) (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
1707
4a136e0a
CF		 1708 switch (opc) {
1709 case INDEX_op_exit_tb:
b19f0c2e
RH		 1710 /* Reuse the zeroing that exists for goto_ptr. */
1711 if (a0 == 0) {
23b7aa1d 1712 tcg_out_goto_long(s, s->code_gen_epilogue);
b19f0c2e
RH		 1713 } else {
1714 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
23b7aa1d 1715 tcg_out_goto_long(s, tb_ret_addr);
b19f0c2e 1716 }
4a136e0a
CF		 1717 break;
1718
1719 case INDEX_op_goto_tb:
2acee8b2 1720 if (s->tb_jmp_insn_offset != NULL) {
a8583393 1721 /* TCG_TARGET_HAS_direct_jump */
2acee8b2
PK		 1722 /* Ensure that ADRP+ADD are 8-byte aligned so that an atomic
1723 write can be used to patch the target address. */
1724 if ((uintptr_t)s->code_ptr & 7) {
1725 tcg_out32(s, NOP);
1726 }
1727 s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
1728 /* actual branch destination will be patched by
a8583393 1729 tb_target_set_jmp_target later. */
2acee8b2
PK		 1730 tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
1731 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
1732 } else {
a8583393 1733 /* !TCG_TARGET_HAS_direct_jump */
2acee8b2
PK		 1734 tcg_debug_assert(s->tb_jmp_target_addr != NULL);
1735 intptr_t offset = tcg_pcrel_diff(s, (s->tb_jmp_target_addr + a0)) >> 2;
1736 tcg_out_insn(s, 3305, LDR, offset, TCG_REG_TMP);
b68686bd 1737 }
b68686bd 1738 tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
9f754620 1739 set_jmp_reset_offset(s, a0);
4a136e0a
CF		 1740 break;
1741
b19f0c2e
RH		 1742 case INDEX_op_goto_ptr:
1743 tcg_out_insn(s, 3207, BR, a0);
1744 break;
1745
4a136e0a 1746 case INDEX_op_br:
bec16311 1747 tcg_out_goto_label(s, arg_label(a0));
4a136e0a
CF		 1748 break;
1749
4a136e0a 1750 case INDEX_op_ld8u_i32:
4a136e0a 1751 case INDEX_op_ld8u_i64:
14e4c1e2 1752 tcg_out_ldst(s, I3312_LDRB, a0, a1, a2, 0);
dc73dfd4
RH		 1753 break;
1754 case INDEX_op_ld8s_i32:
14e4c1e2 1755 tcg_out_ldst(s, I3312_LDRSBW, a0, a1, a2, 0);
dc73dfd4 1756 break;
4a136e0a 1757 case INDEX_op_ld8s_i64:
14e4c1e2 1758 tcg_out_ldst(s, I3312_LDRSBX, a0, a1, a2, 0);
dc73dfd4
RH		 1759 break;
1760 case INDEX_op_ld16u_i32:
4a136e0a 1761 case INDEX_op_ld16u_i64:
14e4c1e2 1762 tcg_out_ldst(s, I3312_LDRH, a0, a1, a2, 1);
dc73dfd4
RH		 1763 break;
1764 case INDEX_op_ld16s_i32:
14e4c1e2 1765 tcg_out_ldst(s, I3312_LDRSHW, a0, a1, a2, 1);
dc73dfd4 1766 break;
4a136e0a 1767 case INDEX_op_ld16s_i64:
14e4c1e2 1768 tcg_out_ldst(s, I3312_LDRSHX, a0, a1, a2, 1);
dc73dfd4
RH		 1769 break;
1770 case INDEX_op_ld_i32:
4a136e0a 1771 case INDEX_op_ld32u_i64:
14e4c1e2 1772 tcg_out_ldst(s, I3312_LDRW, a0, a1, a2, 2);
dc73dfd4 1773 break;
4a136e0a 1774 case INDEX_op_ld32s_i64:
14e4c1e2 1775 tcg_out_ldst(s, I3312_LDRSWX, a0, a1, a2, 2);
e81864a1 1776 break;
dc73dfd4 1777 case INDEX_op_ld_i64:
14e4c1e2 1778 tcg_out_ldst(s, I3312_LDRX, a0, a1, a2, 3);
dc73dfd4
RH		 1779 break;
1780
4a136e0a
CF		 1781 case INDEX_op_st8_i32:
1782 case INDEX_op_st8_i64:
14e4c1e2 1783 tcg_out_ldst(s, I3312_STRB, REG0(0), a1, a2, 0);
dc73dfd4 1784 break;
4a136e0a
CF		 1785 case INDEX_op_st16_i32:
1786 case INDEX_op_st16_i64:
14e4c1e2 1787 tcg_out_ldst(s, I3312_STRH, REG0(0), a1, a2, 1);
dc73dfd4
RH		 1788 break;
1789 case INDEX_op_st_i32:
4a136e0a 1790 case INDEX_op_st32_i64:
14e4c1e2 1791 tcg_out_ldst(s, I3312_STRW, REG0(0), a1, a2, 2);
dc73dfd4
RH		 1792 break;
1793 case INDEX_op_st_i64:
14e4c1e2 1794 tcg_out_ldst(s, I3312_STRX, REG0(0), a1, a2, 3);
4a136e0a
CF		 1795 break;
1796
4a136e0a 1797 case INDEX_op_add_i32:
90f1cd91
RH		 1798 a2 = (int32_t)a2;
1799 /* FALLTHRU */
1800 case INDEX_op_add_i64:
1801 if (c2) {
1802 tcg_out_addsubi(s, ext, a0, a1, a2);
1803 } else {
1804 tcg_out_insn(s, 3502, ADD, ext, a0, a1, a2);
1805 }
4a136e0a
CF		 1806 break;
1807
4a136e0a 1808 case INDEX_op_sub_i32:
90f1cd91
RH		 1809 a2 = (int32_t)a2;
1810 /* FALLTHRU */
1811 case INDEX_op_sub_i64:
1812 if (c2) {
1813 tcg_out_addsubi(s, ext, a0, a1, -a2);
1814 } else {
1815 tcg_out_insn(s, 3502, SUB, ext, a0, a1, a2);
1816 }
4a136e0a
CF		 1817 break;
1818
14b155dd
RH		 1819 case INDEX_op_neg_i64:
1820 case INDEX_op_neg_i32:
1821 tcg_out_insn(s, 3502, SUB, ext, a0, TCG_REG_XZR, a1);
1822 break;
1823
4a136e0a 1824 case INDEX_op_and_i32:
e029f293
RH		 1825 a2 = (int32_t)a2;
1826 /* FALLTHRU */
1827 case INDEX_op_and_i64:
1828 if (c2) {
1829 tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, a2);
1830 } else {
1831 tcg_out_insn(s, 3510, AND, ext, a0, a1, a2);
1832 }
4a136e0a
CF		 1833 break;
1834
14b155dd
RH		 1835 case INDEX_op_andc_i32:
1836 a2 = (int32_t)a2;
1837 /* FALLTHRU */
1838 case INDEX_op_andc_i64:
1839 if (c2) {
1840 tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, ~a2);
1841 } else {
1842 tcg_out_insn(s, 3510, BIC, ext, a0, a1, a2);
1843 }
1844 break;
1845
4a136e0a 1846 case INDEX_op_or_i32:
e029f293
RH		 1847 a2 = (int32_t)a2;
1848 /* FALLTHRU */
1849 case INDEX_op_or_i64:
1850 if (c2) {
1851 tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, a2);
1852 } else {
1853 tcg_out_insn(s, 3510, ORR, ext, a0, a1, a2);
1854 }
4a136e0a
CF		 1855 break;
1856
14b155dd
RH		 1857 case INDEX_op_orc_i32:
1858 a2 = (int32_t)a2;
1859 /* FALLTHRU */
1860 case INDEX_op_orc_i64:
1861 if (c2) {
1862 tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, ~a2);
1863 } else {
1864 tcg_out_insn(s, 3510, ORN, ext, a0, a1, a2);
1865 }
1866 break;
1867
4a136e0a 1868 case INDEX_op_xor_i32:
e029f293
RH		 1869 a2 = (int32_t)a2;
1870 /* FALLTHRU */
1871 case INDEX_op_xor_i64:
1872 if (c2) {
1873 tcg_out_logicali(s, I3404_EORI, ext, a0, a1, a2);
1874 } else {
1875 tcg_out_insn(s, 3510, EOR, ext, a0, a1, a2);
1876 }
4a136e0a
CF		 1877 break;
1878
14b155dd
RH		 1879 case INDEX_op_eqv_i32:
1880 a2 = (int32_t)a2;
1881 /* FALLTHRU */
1882 case INDEX_op_eqv_i64:
1883 if (c2) {
1884 tcg_out_logicali(s, I3404_EORI, ext, a0, a1, ~a2);
1885 } else {
1886 tcg_out_insn(s, 3510, EON, ext, a0, a1, a2);
1887 }
1888 break;
1889
1890 case INDEX_op_not_i64:
1891 case INDEX_op_not_i32:
1892 tcg_out_insn(s, 3510, ORN, ext, a0, TCG_REG_XZR, a1);
1893 break;
1894
4a136e0a 1895 case INDEX_op_mul_i64:
4a136e0a 1896 case INDEX_op_mul_i32:
8678b71c
RH		 1897 tcg_out_insn(s, 3509, MADD, ext, a0, a1, a2, TCG_REG_XZR);
1898 break;
1899
1900 case INDEX_op_div_i64:
1901 case INDEX_op_div_i32:
1902 tcg_out_insn(s, 3508, SDIV, ext, a0, a1, a2);
1903 break;
1904 case INDEX_op_divu_i64:
1905 case INDEX_op_divu_i32:
1906 tcg_out_insn(s, 3508, UDIV, ext, a0, a1, a2);
1907 break;
1908
1909 case INDEX_op_rem_i64:
1910 case INDEX_op_rem_i32:
1911 tcg_out_insn(s, 3508, SDIV, ext, TCG_REG_TMP, a1, a2);
1912 tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
1913 break;
1914 case INDEX_op_remu_i64:
1915 case INDEX_op_remu_i32:
1916 tcg_out_insn(s, 3508, UDIV, ext, TCG_REG_TMP, a1, a2);
1917 tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
4a136e0a
CF		 1918 break;
1919
1920 case INDEX_op_shl_i64:
4a136e0a 1921 case INDEX_op_shl_i32:
df9351e3 1922 if (c2) {
8d8db193 1923 tcg_out_shl(s, ext, a0, a1, a2);
df9351e3
RH		 1924 } else {
1925 tcg_out_insn(s, 3508, LSLV, ext, a0, a1, a2);
4a136e0a
CF		 1926 }
1927 break;
1928
1929 case INDEX_op_shr_i64:
4a136e0a 1930 case INDEX_op_shr_i32:
df9351e3 1931 if (c2) {
8d8db193 1932 tcg_out_shr(s, ext, a0, a1, a2);
df9351e3
RH		 1933 } else {
1934 tcg_out_insn(s, 3508, LSRV, ext, a0, a1, a2);
4a136e0a
CF		 1935 }
1936 break;
1937
1938 case INDEX_op_sar_i64:
4a136e0a 1939 case INDEX_op_sar_i32:
df9351e3 1940 if (c2) {
8d8db193 1941 tcg_out_sar(s, ext, a0, a1, a2);
df9351e3
RH		 1942 } else {
1943 tcg_out_insn(s, 3508, ASRV, ext, a0, a1, a2);
4a136e0a
CF		 1944 }
1945 break;
1946
1947 case INDEX_op_rotr_i64:
4a136e0a 1948 case INDEX_op_rotr_i32:
df9351e3 1949 if (c2) {
8d8db193 1950 tcg_out_rotr(s, ext, a0, a1, a2);
df9351e3
RH		 1951 } else {
1952 tcg_out_insn(s, 3508, RORV, ext, a0, a1, a2);
4a136e0a
CF		 1953 }
1954 break;
1955
1956 case INDEX_op_rotl_i64:
df9351e3
RH		 1957 case INDEX_op_rotl_i32:
1958 if (c2) {
8d8db193 1959 tcg_out_rotl(s, ext, a0, a1, a2);
4a136e0a 1960 } else {
50573c66 1961 tcg_out_insn(s, 3502, SUB, 0, TCG_REG_TMP, TCG_REG_XZR, a2);
df9351e3 1962 tcg_out_insn(s, 3508, RORV, ext, a0, a1, TCG_REG_TMP);
4a136e0a
CF		 1963 }
1964 break;
1965
53c76c19
RH		 1966 case INDEX_op_clz_i64:
1967 case INDEX_op_clz_i32:
1968 tcg_out_cltz(s, ext, a0, a1, a2, c2, false);
1969 break;
1970 case INDEX_op_ctz_i64:
1971 case INDEX_op_ctz_i32:
1972 tcg_out_cltz(s, ext, a0, a1, a2, c2, true);
1973 break;
1974
8d8db193 1975 case INDEX_op_brcond_i32:
90f1cd91
RH		 1976 a1 = (int32_t)a1;
1977 /* FALLTHRU */
1978 case INDEX_op_brcond_i64:
bec16311 1979 tcg_out_brcond(s, ext, a2, a0, a1, const_args[1], arg_label(args[3]));
4a136e0a
CF		 1980 break;
1981
4a136e0a 1982 case INDEX_op_setcond_i32:
90f1cd91
RH		 1983 a2 = (int32_t)a2;
1984 /* FALLTHRU */
1985 case INDEX_op_setcond_i64:
1986 tcg_out_cmp(s, ext, a1, a2, c2);
ed7a0aa8
RH		 1987 /* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond). */
1988 tcg_out_insn(s, 3506, CSINC, TCG_TYPE_I32, a0, TCG_REG_XZR,
1989 TCG_REG_XZR, tcg_invert_cond(args[3]));
4a136e0a
CF		 1990 break;
1991
04ce397b
RH		 1992 case INDEX_op_movcond_i32:
1993 a2 = (int32_t)a2;
1994 /* FALLTHRU */
1995 case INDEX_op_movcond_i64:
1996 tcg_out_cmp(s, ext, a1, a2, c2);
1997 tcg_out_insn(s, 3506, CSEL, ext, a0, REG0(3), REG0(4), args[5]);
1998 break;
1999
de61d14f
RH		 2000 case INDEX_op_qemu_ld_i32:
2001 case INDEX_op_qemu_ld_i64:
59227d5d 2002 tcg_out_qemu_ld(s, a0, a1, a2, ext);
4a136e0a 2003 break;
de61d14f
RH		 2004 case INDEX_op_qemu_st_i32:
2005 case INDEX_op_qemu_st_i64:
59227d5d 2006 tcg_out_qemu_st(s, REG0(0), a1, a2);
4a136e0a
CF		 2007 break;
2008
f0293414 2009 case INDEX_op_bswap64_i64:
edd8824c
RH		 2010 tcg_out_rev64(s, a0, a1);
2011 break;
2012 case INDEX_op_bswap32_i64:
9c4a059d 2013 case INDEX_op_bswap32_i32:
edd8824c 2014 tcg_out_rev32(s, a0, a1);
9c4a059d
CF		 2015 break;
2016 case INDEX_op_bswap16_i64:
2017 case INDEX_op_bswap16_i32:
edd8824c 2018 tcg_out_rev16(s, a0, a1);
9c4a059d
CF		 2019 break;
2020
31f1275b 2021 case INDEX_op_ext8s_i64:
31f1275b 2022 case INDEX_op_ext8s_i32:
929f8b55 2023 tcg_out_sxt(s, ext, MO_8, a0, a1);
31f1275b
CF		 2024 break;
2025 case INDEX_op_ext16s_i64:
31f1275b 2026 case INDEX_op_ext16s_i32:
929f8b55 2027 tcg_out_sxt(s, ext, MO_16, a0, a1);
31f1275b 2028 break;
4f2331e5 2029 case INDEX_op_ext_i32_i64:
31f1275b 2030 case INDEX_op_ext32s_i64:
929f8b55 2031 tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
31f1275b
CF		 2032 break;
2033 case INDEX_op_ext8u_i64:
2034 case INDEX_op_ext8u_i32:
929f8b55 2035 tcg_out_uxt(s, MO_8, a0, a1);
31f1275b
CF		 2036 break;
2037 case INDEX_op_ext16u_i64:
2038 case INDEX_op_ext16u_i32:
929f8b55 2039 tcg_out_uxt(s, MO_16, a0, a1);
31f1275b 2040 break;
4f2331e5 2041 case INDEX_op_extu_i32_i64:
31f1275b 2042 case INDEX_op_ext32u_i64:
929f8b55 2043 tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
31f1275b
CF		 2044 break;
2045
b3c56df7
RH		 2046 case INDEX_op_deposit_i64:
2047 case INDEX_op_deposit_i32:
2048 tcg_out_dep(s, ext, a0, REG0(2), args[3], args[4]);
2049 break;
2050
e2179f94
RH		 2051 case INDEX_op_extract_i64:
2052 case INDEX_op_extract_i32:
2053 tcg_out_ubfm(s, ext, a0, a1, a2, a2 + args[3] - 1);
2054 break;
2055
2056 case INDEX_op_sextract_i64:
2057 case INDEX_op_sextract_i32:
2058 tcg_out_sbfm(s, ext, a0, a1, a2, a2 + args[3] - 1);
2059 break;
2060
464c2969
RH		 2061 case INDEX_op_extract2_i64:
2062 case INDEX_op_extract2_i32:
2063 tcg_out_extr(s, ext, a0, a1, a2, args[3]);
2064 break;
2065
c6e929e7
RH		 2066 case INDEX_op_add2_i32:
2067 tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
2068 (int32_t)args[4], args[5], const_args[4],
2069 const_args[5], false);
2070 break;
2071 case INDEX_op_add2_i64:
2072 tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
2073 args[5], const_args[4], const_args[5], false);
2074 break;
2075 case INDEX_op_sub2_i32:
2076 tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
2077 (int32_t)args[4], args[5], const_args[4],
2078 const_args[5], true);
2079 break;
2080 case INDEX_op_sub2_i64:
2081 tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
2082 args[5], const_args[4], const_args[5], true);
2083 break;
2084
1fcc9ddf
RH		 2085 case INDEX_op_muluh_i64:
2086 tcg_out_insn(s, 3508, UMULH, TCG_TYPE_I64, a0, a1, a2);
2087 break;
2088 case INDEX_op_mulsh_i64:
2089 tcg_out_insn(s, 3508, SMULH, TCG_TYPE_I64, a0, a1, a2);
2090 break;
2091
c7a59c2a
PK		 2092 case INDEX_op_mb:
2093 tcg_out_mb(s, a0);
2094 break;
2095
96d0ee7f 2096 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
a51a6b6a 2097 case INDEX_op_mov_i64:
14e4c1e2 2098 case INDEX_op_mov_vec:
96d0ee7f 2099 case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */
a51a6b6a 2100 case INDEX_op_movi_i64:
14e4c1e2 2101 case INDEX_op_dupi_vec:
96d0ee7f 2102 case INDEX_op_call: /* Always emitted via tcg_out_call. */
4a136e0a 2103 default:
14e4c1e2 2104 g_assert_not_reached();
4a136e0a 2105 }
04ce397b
RH		 2106
2107#undef REG0
4a136e0a
CF		 2108}
2109
14e4c1e2
RH		 2110static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2111 unsigned vecl, unsigned vece,
2112 const TCGArg *args, const int *const_args)
2113{
2114 static const AArch64Insn cmp_insn[16] = {
2115 [TCG_COND_EQ] = I3616_CMEQ,
2116 [TCG_COND_GT] = I3616_CMGT,
2117 [TCG_COND_GE] = I3616_CMGE,
2118 [TCG_COND_GTU] = I3616_CMHI,
2119 [TCG_COND_GEU] = I3616_CMHS,
2120 };
2121 static const AArch64Insn cmp0_insn[16] = {
2122 [TCG_COND_EQ] = I3617_CMEQ0,
2123 [TCG_COND_GT] = I3617_CMGT0,
2124 [TCG_COND_GE] = I3617_CMGE0,
2125 [TCG_COND_LT] = I3617_CMLT0,
2126 [TCG_COND_LE] = I3617_CMLE0,
2127 };
2128
2129 TCGType type = vecl + TCG_TYPE_V64;
2130 unsigned is_q = vecl;
2131 TCGArg a0, a1, a2;
2132
2133 a0 = args[0];
2134 a1 = args[1];
2135 a2 = args[2];
2136
2137 switch (opc) {
2138 case INDEX_op_ld_vec:
2139 tcg_out_ld(s, type, a0, a1, a2);
2140 break;
2141 case INDEX_op_st_vec:
2142 tcg_out_st(s, type, a0, a1, a2);
2143 break;
2144 case INDEX_op_add_vec:
2145 tcg_out_insn(s, 3616, ADD, is_q, vece, a0, a1, a2);
2146 break;
2147 case INDEX_op_sub_vec:
2148 tcg_out_insn(s, 3616, SUB, is_q, vece, a0, a1, a2);
2149 break;
2150 case INDEX_op_mul_vec:
2151 tcg_out_insn(s, 3616, MUL, is_q, vece, a0, a1, a2);
2152 break;
2153 case INDEX_op_neg_vec:
2154 tcg_out_insn(s, 3617, NEG, is_q, vece, a0, a1);
2155 break;
2156 case INDEX_op_and_vec:
2157 tcg_out_insn(s, 3616, AND, is_q, 0, a0, a1, a2);
2158 break;
2159 case INDEX_op_or_vec:
2160 tcg_out_insn(s, 3616, ORR, is_q, 0, a0, a1, a2);
2161 break;
2162 case INDEX_op_xor_vec:
2163 tcg_out_insn(s, 3616, EOR, is_q, 0, a0, a1, a2);
2164 break;
2165 case INDEX_op_andc_vec:
2166 tcg_out_insn(s, 3616, BIC, is_q, 0, a0, a1, a2);
2167 break;
2168 case INDEX_op_orc_vec:
2169 tcg_out_insn(s, 3616, ORN, is_q, 0, a0, a1, a2);
2170 break;
d32648d4
RH		 2171 case INDEX_op_ssadd_vec:
2172 tcg_out_insn(s, 3616, SQADD, is_q, vece, a0, a1, a2);
2173 break;
2174 case INDEX_op_sssub_vec:
2175 tcg_out_insn(s, 3616, SQSUB, is_q, vece, a0, a1, a2);
2176 break;
2177 case INDEX_op_usadd_vec:
2178 tcg_out_insn(s, 3616, UQADD, is_q, vece, a0, a1, a2);
2179 break;
2180 case INDEX_op_ussub_vec:
2181 tcg_out_insn(s, 3616, UQSUB, is_q, vece, a0, a1, a2);
2182 break;
93f332a5
RH		 2183 case INDEX_op_smax_vec:
2184 tcg_out_insn(s, 3616, SMAX, is_q, vece, a0, a1, a2);
2185 break;
2186 case INDEX_op_smin_vec:
2187 tcg_out_insn(s, 3616, SMIN, is_q, vece, a0, a1, a2);
2188 break;
2189 case INDEX_op_umax_vec:
2190 tcg_out_insn(s, 3616, UMAX, is_q, vece, a0, a1, a2);
2191 break;
2192 case INDEX_op_umin_vec:
2193 tcg_out_insn(s, 3616, UMIN, is_q, vece, a0, a1, a2);
2194 break;
14e4c1e2
RH		 2195 case INDEX_op_not_vec:
2196 tcg_out_insn(s, 3617, NOT, is_q, 0, a0, a1);
2197 break;
2198 case INDEX_op_dup_vec:
2199 tcg_out_insn(s, 3605, DUP, is_q, a0, a1, 1 << vece, 0);
2200 break;
2201 case INDEX_op_shli_vec:
2202 tcg_out_insn(s, 3614, SHL, is_q, a0, a1, a2 + (8 << vece));
2203 break;
2204 case INDEX_op_shri_vec:
2205 tcg_out_insn(s, 3614, USHR, is_q, a0, a1, (16 << vece) - a2);
2206 break;
2207 case INDEX_op_sari_vec:
2208 tcg_out_insn(s, 3614, SSHR, is_q, a0, a1, (16 << vece) - a2);
2209 break;
2210 case INDEX_op_cmp_vec:
2211 {
2212 TCGCond cond = args[3];
2213 AArch64Insn insn;
2214
2215 if (cond == TCG_COND_NE) {
2216 if (const_args[2]) {
2217 tcg_out_insn(s, 3616, CMTST, is_q, vece, a0, a1, a1);
2218 } else {
2219 tcg_out_insn(s, 3616, CMEQ, is_q, vece, a0, a1, a2);
2220 tcg_out_insn(s, 3617, NOT, is_q, 0, a0, a0);
2221 }
2222 } else {
2223 if (const_args[2]) {
2224 insn = cmp0_insn[cond];
2225 if (insn) {
2226 tcg_out_insn_3617(s, insn, is_q, vece, a0, a1);
2227 break;
2228 }
2229 tcg_out_dupi_vec(s, type, TCG_VEC_TMP, 0);
2230 a2 = TCG_VEC_TMP;
2231 }
2232 insn = cmp_insn[cond];
2233 if (insn == 0) {
2234 TCGArg t;
2235 t = a1, a1 = a2, a2 = t;
2236 cond = tcg_swap_cond(cond);
2237 insn = cmp_insn[cond];
2238 tcg_debug_assert(insn != 0);
2239 }
2240 tcg_out_insn_3616(s, insn, is_q, vece, a0, a1, a2);
2241 }
2242 }
2243 break;
2244 default:
2245 g_assert_not_reached();
2246 }
2247}
2248
2249int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
2250{
2251 switch (opc) {
2252 case INDEX_op_add_vec:
2253 case INDEX_op_sub_vec:
14e4c1e2
RH		 2254 case INDEX_op_and_vec:
2255 case INDEX_op_or_vec:
2256 case INDEX_op_xor_vec:
2257 case INDEX_op_andc_vec:
2258 case INDEX_op_orc_vec:
2259 case INDEX_op_neg_vec:
2260 case INDEX_op_not_vec:
2261 case INDEX_op_cmp_vec:
2262 case INDEX_op_shli_vec:
2263 case INDEX_op_shri_vec:
2264 case INDEX_op_sari_vec:
d32648d4
RH		 2265 case INDEX_op_ssadd_vec:
2266 case INDEX_op_sssub_vec:
2267 case INDEX_op_usadd_vec:
2268 case INDEX_op_ussub_vec:
93f332a5
RH		 2269 case INDEX_op_smax_vec:
2270 case INDEX_op_smin_vec:
2271 case INDEX_op_umax_vec:
2272 case INDEX_op_umin_vec:
14e4c1e2 2273 return 1;
e65a5f22
AB		 2274 case INDEX_op_mul_vec:
2275 return vece < MO_64;
14e4c1e2
RH		 2276
2277 default:
2278 return 0;
2279 }
2280}
2281
2282void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
2283 TCGArg a0, ...)
2284{
2285}
2286
f69d277e
RH		 2287static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
2288{
1897cc2e
RH		 2289 static const TCGTargetOpDef r = { .args_ct_str = { "r" } };
2290 static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } };
14e4c1e2
RH		 2291 static const TCGTargetOpDef w_w = { .args_ct_str = { "w", "w" } };
2292 static const TCGTargetOpDef w_r = { .args_ct_str = { "w", "r" } };
2293 static const TCGTargetOpDef w_wr = { .args_ct_str = { "w", "wr" } };
1897cc2e
RH		 2294 static const TCGTargetOpDef r_l = { .args_ct_str = { "r", "l" } };
2295 static const TCGTargetOpDef r_rA = { .args_ct_str = { "r", "rA" } };
2296 static const TCGTargetOpDef rZ_r = { .args_ct_str = { "rZ", "r" } };
2297 static const TCGTargetOpDef lZ_l = { .args_ct_str = { "lZ", "l" } };
2298 static const TCGTargetOpDef r_r_r = { .args_ct_str = { "r", "r", "r" } };
14e4c1e2
RH		 2299 static const TCGTargetOpDef w_w_w = { .args_ct_str = { "w", "w", "w" } };
2300 static const TCGTargetOpDef w_w_wZ = { .args_ct_str = { "w", "w", "wZ" } };
1897cc2e
RH		 2301 static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } };
2302 static const TCGTargetOpDef r_r_rA = { .args_ct_str = { "r", "r", "rA" } };
2303 static const TCGTargetOpDef r_r_rL = { .args_ct_str = { "r", "r", "rL" } };
2304 static const TCGTargetOpDef r_r_rAL
2305 = { .args_ct_str = { "r", "r", "rAL" } };
2306 static const TCGTargetOpDef dep
2307 = { .args_ct_str = { "r", "0", "rZ" } };
464c2969
RH		 2308 static const TCGTargetOpDef ext2
2309 = { .args_ct_str = { "r", "rZ", "rZ" } };
1897cc2e
RH		 2310 static const TCGTargetOpDef movc
2311 = { .args_ct_str = { "r", "r", "rA", "rZ", "rZ" } };
2312 static const TCGTargetOpDef add2
2313 = { .args_ct_str = { "r", "r", "rZ", "rZ", "rA", "rMZ" } };
2314
2315 switch (op) {
2316 case INDEX_op_goto_ptr:
2317 return &r;
f69d277e 2318
1897cc2e
RH		 2319 case INDEX_op_ld8u_i32:
2320 case INDEX_op_ld8s_i32:
2321 case INDEX_op_ld16u_i32:
2322 case INDEX_op_ld16s_i32:
2323 case INDEX_op_ld_i32:
2324 case INDEX_op_ld8u_i64:
2325 case INDEX_op_ld8s_i64:
2326 case INDEX_op_ld16u_i64:
2327 case INDEX_op_ld16s_i64:
2328 case INDEX_op_ld32u_i64:
2329 case INDEX_op_ld32s_i64:
2330 case INDEX_op_ld_i64:
2331 case INDEX_op_neg_i32:
2332 case INDEX_op_neg_i64:
2333 case INDEX_op_not_i32:
2334 case INDEX_op_not_i64:
2335 case INDEX_op_bswap16_i32:
2336 case INDEX_op_bswap32_i32:
2337 case INDEX_op_bswap16_i64:
2338 case INDEX_op_bswap32_i64:
2339 case INDEX_op_bswap64_i64:
2340 case INDEX_op_ext8s_i32:
2341 case INDEX_op_ext16s_i32:
2342 case INDEX_op_ext8u_i32:
2343 case INDEX_op_ext16u_i32:
2344 case INDEX_op_ext8s_i64:
2345 case INDEX_op_ext16s_i64:
2346 case INDEX_op_ext32s_i64:
2347 case INDEX_op_ext8u_i64:
2348 case INDEX_op_ext16u_i64:
2349 case INDEX_op_ext32u_i64:
2350 case INDEX_op_ext_i32_i64:
2351 case INDEX_op_extu_i32_i64:
2352 case INDEX_op_extract_i32:
2353 case INDEX_op_extract_i64:
2354 case INDEX_op_sextract_i32:
2355 case INDEX_op_sextract_i64:
2356 return &r_r;
2357
2358 case INDEX_op_st8_i32:
2359 case INDEX_op_st16_i32:
2360 case INDEX_op_st_i32:
2361 case INDEX_op_st8_i64:
2362 case INDEX_op_st16_i64:
2363 case INDEX_op_st32_i64:
2364 case INDEX_op_st_i64:
2365 return &rZ_r;
2366
2367 case INDEX_op_add_i32:
2368 case INDEX_op_add_i64:
2369 case INDEX_op_sub_i32:
2370 case INDEX_op_sub_i64:
2371 case INDEX_op_setcond_i32:
2372 case INDEX_op_setcond_i64:
2373 return &r_r_rA;
2374
2375 case INDEX_op_mul_i32:
2376 case INDEX_op_mul_i64:
2377 case INDEX_op_div_i32:
2378 case INDEX_op_div_i64:
2379 case INDEX_op_divu_i32:
2380 case INDEX_op_divu_i64:
2381 case INDEX_op_rem_i32:
2382 case INDEX_op_rem_i64:
2383 case INDEX_op_remu_i32:
2384 case INDEX_op_remu_i64:
2385 case INDEX_op_muluh_i64:
2386 case INDEX_op_mulsh_i64:
2387 return &r_r_r;
2388
2389 case INDEX_op_and_i32:
2390 case INDEX_op_and_i64:
2391 case INDEX_op_or_i32:
2392 case INDEX_op_or_i64:
2393 case INDEX_op_xor_i32:
2394 case INDEX_op_xor_i64:
2395 case INDEX_op_andc_i32:
2396 case INDEX_op_andc_i64:
2397 case INDEX_op_orc_i32:
2398 case INDEX_op_orc_i64:
2399 case INDEX_op_eqv_i32:
2400 case INDEX_op_eqv_i64:
2401 return &r_r_rL;
2402
2403 case INDEX_op_shl_i32:
2404 case INDEX_op_shr_i32:
2405 case INDEX_op_sar_i32:
2406 case INDEX_op_rotl_i32:
2407 case INDEX_op_rotr_i32:
2408 case INDEX_op_shl_i64:
2409 case INDEX_op_shr_i64:
2410 case INDEX_op_sar_i64:
2411 case INDEX_op_rotl_i64:
2412 case INDEX_op_rotr_i64:
2413 return &r_r_ri;
2414
2415 case INDEX_op_clz_i32:
2416 case INDEX_op_ctz_i32:
2417 case INDEX_op_clz_i64:
2418 case INDEX_op_ctz_i64:
2419 return &r_r_rAL;
2420
2421 case INDEX_op_brcond_i32:
2422 case INDEX_op_brcond_i64:
2423 return &r_rA;
2424
2425 case INDEX_op_movcond_i32:
2426 case INDEX_op_movcond_i64:
2427 return &movc;
2428
2429 case INDEX_op_qemu_ld_i32:
2430 case INDEX_op_qemu_ld_i64:
2431 return &r_l;
2432 case INDEX_op_qemu_st_i32:
2433 case INDEX_op_qemu_st_i64:
2434 return &lZ_l;
2435
2436 case INDEX_op_deposit_i32:
2437 case INDEX_op_deposit_i64:
2438 return &dep;
2439
464c2969
RH		 2440 case INDEX_op_extract2_i32:
2441 case INDEX_op_extract2_i64:
2442 return &ext2;
2443
1897cc2e
RH		 2444 case INDEX_op_add2_i32:
2445 case INDEX_op_add2_i64:
2446 case INDEX_op_sub2_i32:
2447 case INDEX_op_sub2_i64:
2448 return &add2;
2449
14e4c1e2
RH		 2450 case INDEX_op_add_vec:
2451 case INDEX_op_sub_vec:
2452 case INDEX_op_mul_vec:
2453 case INDEX_op_and_vec:
2454 case INDEX_op_or_vec:
2455 case INDEX_op_xor_vec:
2456 case INDEX_op_andc_vec:
2457 case INDEX_op_orc_vec:
d32648d4
RH		 2458 case INDEX_op_ssadd_vec:
2459 case INDEX_op_sssub_vec:
2460 case INDEX_op_usadd_vec:
2461 case INDEX_op_ussub_vec:
93f332a5
RH		 2462 case INDEX_op_smax_vec:
2463 case INDEX_op_smin_vec:
2464 case INDEX_op_umax_vec:
2465 case INDEX_op_umin_vec:
14e4c1e2
RH		 2466 return &w_w_w;
2467 case INDEX_op_not_vec:
2468 case INDEX_op_neg_vec:
2469 case INDEX_op_shli_vec:
2470 case INDEX_op_shri_vec:
2471 case INDEX_op_sari_vec:
2472 return &w_w;
2473 case INDEX_op_ld_vec:
2474 case INDEX_op_st_vec:
2475 return &w_r;
2476 case INDEX_op_dup_vec:
2477 return &w_wr;
2478 case INDEX_op_cmp_vec:
2479 return &w_w_wZ;
2480
1897cc2e
RH		 2481 default:
2482 return NULL;
f69d277e 2483 }
f69d277e
RH		 2484}
2485
4a136e0a
CF		 2486static void tcg_target_init(TCGContext *s)
2487{
f46934df
RH		 2488 tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffffu;
2489 tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffffu;
14e4c1e2
RH		 2490 tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
2491 tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
f46934df 2492
14e4c1e2 2493 tcg_target_call_clobber_regs = -1ull;
f46934df
RH		 2494 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X19);
2495 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X20);
2496 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X21);
2497 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X22);
2498 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X23);
2499 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X24);
2500 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X25);
2501 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X26);
2502 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X27);
2503 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X28);
2504 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X29);
14e4c1e2
RH		 2505 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V8);
2506 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V9);
2507 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V10);
2508 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V11);
2509 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V12);
2510 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V13);
2511 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V14);
2512 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V15);
4a136e0a 2513
ccb1bb66 2514 s->reserved_regs = 0;
4a136e0a
CF		 2515 tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
2516 tcg_regset_set_reg(s->reserved_regs, TCG_REG_FP);
2517 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP);
2518 tcg_regset_set_reg(s->reserved_regs, TCG_REG_X18); /* platform register */
14e4c1e2 2519 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP);
4a136e0a
CF		 2520}
2521
38d195aa
RH		 2522/* Saving pairs: (X19, X20) .. (X27, X28), (X29(fp), X30(lr)). */
2523#define PUSH_SIZE ((30 - 19 + 1) * 8)
2524
2525#define FRAME_SIZE \
2526 ((PUSH_SIZE \
2527 + TCG_STATIC_CALL_ARGS_SIZE \
2528 + CPU_TEMP_BUF_NLONGS * sizeof(long) \
2529 + TCG_TARGET_STACK_ALIGN - 1) \
2530 & ~(TCG_TARGET_STACK_ALIGN - 1))
2531
2532/* We're expecting a 2 byte uleb128 encoded value. */
2533QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
2534
2535/* We're expecting to use a single ADDI insn. */
2536QEMU_BUILD_BUG_ON(FRAME_SIZE - PUSH_SIZE > 0xfff);
2537
4a136e0a
CF		 2538static void tcg_target_qemu_prologue(TCGContext *s)
2539{
4a136e0a
CF		 2540 TCGReg r;
2541
95f72aa9
RH		 2542 /* Push (FP, LR) and allocate space for all saved registers. */
2543 tcg_out_insn(s, 3314, STP, TCG_REG_FP, TCG_REG_LR,
38d195aa 2544 TCG_REG_SP, -PUSH_SIZE, 1, 1);
4a136e0a 2545
d82b78e4 2546 /* Set up frame pointer for canonical unwinding. */
929f8b55 2547 tcg_out_movr_sp(s, TCG_TYPE_I64, TCG_REG_FP, TCG_REG_SP);
4a136e0a 2548
d82b78e4 2549 /* Store callee-preserved regs x19..x28. */
4a136e0a 2550 for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
95f72aa9
RH		 2551 int ofs = (r - TCG_REG_X19 + 2) * 8;
2552 tcg_out_insn(s, 3314, STP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
4a136e0a
CF		 2553 }
2554
096c46c0
RH		 2555 /* Make stack space for TCG locals. */
2556 tcg_out_insn(s, 3401, SUBI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
38d195aa 2557 FRAME_SIZE - PUSH_SIZE);
096c46c0 2558
95f72aa9 2559 /* Inform TCG about how to find TCG locals with register, offset, size. */
4a136e0a
CF		 2560 tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
2561 CPU_TEMP_BUF_NLONGS * sizeof(long));
2562
4cbea598 2563#if !defined(CONFIG_SOFTMMU)
352bcb0a 2564 if (USE_GUEST_BASE) {
b76f21a7 2565 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base);
6a91c7c9
JK		 2566 tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
2567 }
2568#endif
2569
4a136e0a 2570 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
81d8a5ee 2571 tcg_out_insn(s, 3207, BR, tcg_target_call_iarg_regs[1]);
4a136e0a 2572
b19f0c2e
RH		 2573 /*
2574 * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
2575 * and fall through to the rest of the epilogue.
2576 */
2577 s->code_gen_epilogue = s->code_ptr;
2578 tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0);
2579
2580 /* TB epilogue */
4a136e0a
CF		 2581 tb_ret_addr = s->code_ptr;
2582
096c46c0
RH		 2583 /* Remove TCG locals stack space. */
2584 tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
38d195aa 2585 FRAME_SIZE - PUSH_SIZE);
4a136e0a 2586
95f72aa9 2587 /* Restore registers x19..x28. */
4a136e0a 2588 for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
95f72aa9
RH		 2589 int ofs = (r - TCG_REG_X19 + 2) * 8;
2590 tcg_out_insn(s, 3314, LDP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
4a136e0a
CF		 2591 }
2592
95f72aa9
RH		 2593 /* Pop (FP, LR), restore SP to previous frame. */
2594 tcg_out_insn(s, 3314, LDP, TCG_REG_FP, TCG_REG_LR,
38d195aa 2595 TCG_REG_SP, PUSH_SIZE, 0, 1);
81d8a5ee 2596 tcg_out_insn(s, 3207, RET, TCG_REG_LR);
4a136e0a 2597}
38d195aa 2598
55129955
RH		 2599static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
2600{
2601 int i;
2602 for (i = 0; i < count; ++i) {
2603 p[i] = NOP;
2604 }
2605}
2606
38d195aa 2607typedef struct {
3d9bddb3 2608 DebugFrameHeader h;
38d195aa
RH		 2609 uint8_t fde_def_cfa[4];
2610 uint8_t fde_reg_ofs[24];
2611} DebugFrame;
2612
2613#define ELF_HOST_MACHINE EM_AARCH64
2614
3d9bddb3
RH		 2615static const DebugFrame debug_frame = {
2616 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
2617 .h.cie.id = -1,
2618 .h.cie.version = 1,
2619 .h.cie.code_align = 1,
2620 .h.cie.data_align = 0x78, /* sleb128 -8 */
2621 .h.cie.return_column = TCG_REG_LR,
38d195aa
RH		 2622
2623 /* Total FDE size does not include the "len" member. */
3d9bddb3 2624 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
38d195aa
RH		 2625
2626 .fde_def_cfa = {
2627 12, TCG_REG_SP, /* DW_CFA_def_cfa sp, ... */
2628 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
2629 (FRAME_SIZE >> 7)
2630 },
2631 .fde_reg_ofs = {
2632 0x80 + 28, 1, /* DW_CFA_offset, x28, -8 */
2633 0x80 + 27, 2, /* DW_CFA_offset, x27, -16 */
2634 0x80 + 26, 3, /* DW_CFA_offset, x26, -24 */
2635 0x80 + 25, 4, /* DW_CFA_offset, x25, -32 */
2636 0x80 + 24, 5, /* DW_CFA_offset, x24, -40 */
2637 0x80 + 23, 6, /* DW_CFA_offset, x23, -48 */
2638 0x80 + 22, 7, /* DW_CFA_offset, x22, -56 */
2639 0x80 + 21, 8, /* DW_CFA_offset, x21, -64 */
2640 0x80 + 20, 9, /* DW_CFA_offset, x20, -72 */
2641 0x80 + 19, 10, /* DW_CFA_offset, x1p, -80 */
2642 0x80 + 30, 11, /* DW_CFA_offset, lr, -88 */
2643 0x80 + 29, 12, /* DW_CFA_offset, fp, -96 */
2644 }
2645};
2646
2647void tcg_register_jit(void *buf, size_t buf_size)
2648{
38d195aa
RH		 2649 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
2650}
QEMU mirror